AMBARI-8995. Service to advertise HDP version of component during START in response's structuredOut (alejandro)

ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+import re
+import tempfile
+
+from resource_management.core.logger import Logger
+from resource_management.core import shell
+
+
+def get_component_version(stack_name, component_name):
+  """
+  For any stack name, returns the version currently installed for a given component.
+  Because each stack name may have different logic, the input is a generic dictionary.
+  :param stack_name: one of HDP, HDPWIN, BIGTOP, PHD, etc. usually retrieved from
+  the command-#.json file's ["hostLevelParams"]["stack_name"]
+  :param component_name: Component name as a string necessary to get the version
+  :return: Returns a string if found, e.g., 2.2.1.0-2175, otherwise, returns None
+  """
+  version = None
+  if stack_name is None or component_name is None:
+    Logger.error("Could not determine component version because of the parameters is empty. " \
+                 "stack_name: %s, component_name: %s" % (str(stack_name), str(component_name)))
+    return version
+
+  out = None
+  code = -1
+  if stack_name == "HDP":
+    tmpfile = tempfile.NamedTemporaryFile()
+
+    get_hdp_comp_version_cmd = ""
+    try:
+      # This is necessary because Ubuntu returns "stdin: is not a tty", see AMBARI-8088
+      with open(tmpfile.name, 'r') as file:
+        get_hdp_comp_version_cmd = '/usr/bin/hdp-select status %s > %s' % (component_name, tmpfile.name)
+        code, stdoutdata = shell.call(get_hdp_comp_version_cmd)
+        out = file.read()
+
+      if code != 0 or out is None:
+        raise Exception("Code is nonzero or output is empty")
+
+      Logger.info("Command: %s\nOutput: %s" % (get_hdp_comp_version_cmd, str(out)))
+      matches = re.findall(r"([\d\.]+\-\d+)", out)
+      version = matches[0] if matches and len(matches) > 0 else None
+    except Exception, e:
+      Logger.error("Could not determine HDP version for component %s by calling '%s'. Return Code: %s, Output: %s." %
+                   (component_name, get_hdp_comp_version_cmd, str(code), str(out)))
+  elif stack_name == "HDPWIN":
+    pass
+  elif stack_name == "GlusterFS":
+    pass
+  elif stack_name == "PHD":
+    pass
+  elif stack_name == "BIGTOP":
+    pass
+  else:
+    Logger.error("Could not find a stack for stack name: %s" % str(stack_name))
+
+  return version

ambari-common/src/main/python/resource_management/libraries/script/script.py

@@ -35,6 +35,8 @@ from resource_management.core.environment import Environment
 from resource_management.core.logger import Logger
 from resource_management.core.exceptions import Fail, ClientComponentHasNoStatus, ComponentIsNotRunning
 from resource_management.core.resources.packaging import Package
+from resource_management.libraries.functions.version_select_util import get_component_version
+from resource_management.libraries.functions.version import compare_versions
 from resource_management.libraries.script.config_dictionary import ConfigDictionary, UnknownConfiguration
 
 IS_WINDOWS = platform.system() == "Windows"
@@ -91,6 +93,13 @@ class Script(object):
   # Class variable
   tmp_dir = ""
 
+  def get_stack_to_component(self):
+    """
+    To be overridden by subclasses.
+    Returns a dictionary where the key is a stack name, and the value is the component name used in selecting the version.
+    """
+    return {}
+
   def put_structured_out(self, sout):
     Script.structuredOut.update(sout)
     try:
@@ -99,6 +108,23 @@ class Script(object):
     except IOError:
       Script.structuredOut.update({"errMsg" : "Unable to write to " + self.stroutfile})
 
+  def save_component_version_to_structured_out(self, stack_name):
+    """
+    :param stack_name: One of HDP, HDPWIN, PHD, BIGTOP.
+    :return: Append the version number to the structured out.
+    """
+    import params
+    component_version = None
+
+    stack_to_component = self.get_stack_to_component()
+    if stack_to_component:
+      if stack_name == "HDP" and params.hdp_stack_version != "" and compare_versions(params.hdp_stack_version, '2.2') >= 0:
+        component_name = stack_to_component[stack_name] if stack_name in stack_to_component else None
+        component_version = get_component_version(stack_name, component_name)
+
+      if component_version:
+        self.put_structured_out({"version": component_version})
+
   def execute(self):
     """
     Sets up logging;

ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_client.py

@@ -20,23 +20,27 @@ limitations under the License.
 from resource_management import *
 from falcon import falcon
 
+
 class FalconClient(Script):
+
+  def get_stack_to_component(self):
+    return {"HDP": "falcon-client"}
+
   def install(self, env):
     self.install_packages(env)
     self.configure(env)
-
-
+  
   def configure(self, env):
     import params
 
     env.set_params(params)
     falcon('client', action='config')
 
+    self.save_component_version_to_structured_out(params.stack_name)
 
   def status(self, env):
     raise ClientComponentHasNoStatus()
 
-
   def pre_rolling_restart(self, env):
     import params
     env.set_params(params)
@@ -49,7 +53,6 @@ class FalconClient(Script):
     Logger.info("Executing Falcon Client Rolling Upgrade pre-restart")
     Execute(format("hdp-select set hadoop-client {version}"))
 
-
   def security_status(self, env):
     import status_params
     env.set_params(status_params)

ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server.py

@@ -27,6 +27,10 @@ from resource_management.libraries.functions.security_commons import build_expec
 from falcon import falcon
 
 class FalconServer(Script):
+
+  def get_stack_to_component(self):
+    return {"HDP": "falcon-server"}
+
   def install(self, env):
     import params
 
@@ -42,6 +46,8 @@ class FalconServer(Script):
 
     falcon('server', action='start')
 
+    self.save_component_version_to_structured_out(params.stack_name)
+
 
   def stop(self, env, rolling_restart=False):
     import params

ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params.py

@@ -18,12 +18,15 @@ limitations under the License.
 """
 
 from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
+from resource_management.libraries.functions.default import default
 from resource_management import *
 
 from status_params import *
 
 config = Script.get_config()
 
+stack_name = default("/hostLevelParams/stack_name", None)
+
 # New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade
 version = default("/commandParams/version", None)
 

ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_handler.py

@@ -25,6 +25,10 @@ from resource_management.libraries.functions.flume_agent_helper import find_expe
 from resource_management.libraries.functions.flume_agent_helper import get_flume_status
 
 class FlumeHandler(Script):
+
+  def get_stack_to_component(self):
+    return {"HDP": "flume-server"}
+
   def install(self, env):
     import params
 
@@ -39,6 +43,8 @@ class FlumeHandler(Script):
 
     flume(action='start')
 
+    self.save_component_version_to_structured_out(params.stack_name)
+
   def stop(self, env, rolling_restart=False):
     import params
 

ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py

@@ -18,10 +18,13 @@ limitations under the License.
 """
 
 from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
+from resource_management.libraries.functions.default import default
 from resource_management import *
 
 config = Script.get_config()
 
+stack_name = default("/hostLevelParams/stack_name", None)
+
 user_group = config['configurations']['cluster-env']['user_group']
 proxyuser_group =  config['configurations']['hadoop-env']['proxyuser_group']
 

ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_client.py

@@ -26,6 +26,9 @@ from hbase import hbase
          
 class HbaseClient(Script):
 
+  def get_stack_to_component(self):
+    return {"HDP": "hbase-client"}
+
   def pre_rolling_restart(self, env):
     import params
     env.set_params(params)
@@ -43,6 +46,8 @@ class HbaseClient(Script):
     
     hbase(name='client')
 
+    self.save_component_version_to_structured_out(params.stack_name)
+
   def status(self, env):
     raise ClientComponentHasNoStatus()
 

ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py

@@ -30,6 +30,10 @@ import upgrade
 
          
 class HbaseMaster(Script):
+
+  def get_stack_to_component(self):
+    return {"HDP": "hbase-master"}
+
   def install(self, env):
     self.install_packages(env)
     
@@ -52,6 +56,8 @@ class HbaseMaster(Script):
     hbase_service( 'master',
       action = 'start'
     )
+
+    self.save_component_version_to_structured_out(params.stack_name)
     
   def stop(self, env, rolling_restart=False):
     import params

ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py

@@ -29,6 +29,10 @@ import upgrade
 
          
 class HbaseRegionServer(Script):
+
+  def get_stack_to_component(self):
+    return {"HDP": "hbase-regionserver"}
+
   def install(self, env):
     self.install_packages(env)
     
@@ -57,7 +61,8 @@ class HbaseRegionServer(Script):
       action = 'start'
     )
 
-    
+    self.save_component_version_to_structured_out(params.stack_name)
+
   def stop(self, env, rolling_restart=False):
     import params
     env.set_params(params)

ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params.py

@@ -20,6 +20,7 @@ limitations under the License.
 
 from functions import calc_xmn_from_xms
 from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
+from resource_management.libraries.functions.default import default
 from resource_management import *
 import status_params
 
@@ -27,6 +28,8 @@ import status_params
 config = Script.get_config()
 exec_tmp_dir = Script.get_tmp_dir()
 
+stack_name = default("/hostLevelParams/stack_name", None)
+
 version = default("/commandParams/version", None)
 
 stack_version_unformatted = str(config['hostLevelParams']['stack_version'])

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/datanode.py

@@ -19,14 +19,17 @@ limitations under the License.
 import datanode_upgrade
 from hdfs_datanode import datanode
 from resource_management import *
-from resource_management.libraries.functions.version import compare_versions, \
-  format_hdp_stack_version
+from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version
 from resource_management.libraries.functions.security_commons import build_expectations, \
   cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, FILE_TYPE_XML
 from hdfs import hdfs
 
 
 class DataNode(Script):
+
+  def get_stack_to_component(self):
+    return {"HDP": "hadoop-hdfs-datanode"}
+
   def install(self, env):
     import params
 
@@ -59,6 +62,8 @@ class DataNode(Script):
     self.configure(env)
     datanode(action="start")
 
+    self.save_component_version_to_structured_out(params.stack_name)
+
 
   def stop(self, env, rolling_restart=False):
     import params

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py

@@ -30,6 +30,10 @@ from hdfs import hdfs
 
 
 class JournalNode(Script):
+
+  def get_stack_to_component(self):
+    return {"HDP": "hadoop-hdfs-journalnode"}
+
   def install(self, env):
     import params
 
@@ -60,6 +64,8 @@ class JournalNode(Script):
       create_log_dir=True
     )
 
+    self.save_component_version_to_structured_out(params.stack_name)
+
   def stop(self, env, rolling_restart=False):
     import params
 

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py

@@ -39,6 +39,10 @@ from utils import failover_namenode
 
 
 class NameNode(Script):
+
+  def get_stack_to_component(self):
+    return {"HDP": "hadoop-hdfs-namenode"}
+
   def install(self, env):
     import params
 
@@ -68,6 +72,8 @@ class NameNode(Script):
     self.configure(env)
     namenode(action="start", rolling_restart=rolling_restart, env=env)
 
+    self.save_component_version_to_structured_out(params.stack_name)
+
   def post_rolling_restart(self, env):
     Logger.info("Executing Rolling Upgrade post-restart")
     import params

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params.py

@@ -18,6 +18,7 @@ limitations under the License.
 """
 
 from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
+from resource_management.libraries.functions.default import default
 from resource_management import *
 import status_params
 import utils
@@ -28,6 +29,8 @@ import re
 config = Script.get_config()
 tmp_dir = Script.get_tmp_dir()
 
+stack_name = default("/hostLevelParams/stack_name", None)
+
 stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
 hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
 

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/snamenode.py

@@ -27,6 +27,9 @@ from hdfs import hdfs
 
 class SNameNode(Script):
 
+  def get_stack_to_component(self):
+    return {"HDP": "hadoop-hdfs-secondarynamenode"}
+
   def install(self, env):
     import params
 
@@ -46,6 +49,8 @@ class SNameNode(Script):
     self.configure(env)
     snamenode(action="start")
 
+    self.save_component_version_to_structured_out(params.stack_name)
+
   def stop(self, env, rolling_restart=False):
     import params
     env.set_params(params)

ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py

@@ -28,6 +28,10 @@ from hive_service import hive_service
 
 
 class HiveMetastore(Script):
+
+  def get_stack_to_component(self):
+    return {"HDP": "hive-metastore"}
+
   def install(self, env):
     import params
 
@@ -49,6 +53,8 @@ class HiveMetastore(Script):
     self.configure(env)  # FOR SECURITY
     hive_service('metastore', action = 'start')
 
+    self.save_component_version_to_structured_out(params.stack_name)
+
 
   def stop(self, env, rolling_restart = False):
     import params

ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py

@@ -30,6 +30,9 @@ from install_jars import install_tez_jars
 
 class HiveServer(Script):
 
+  def get_stack_to_component(self):
+    return {"HDP": "hive-server2"}
+
   def install(self, env):
     import params
     self.install_packages(env, exclude_packages=params.hive_exclude_packages)
@@ -56,6 +59,8 @@ class HiveServer(Script):
     hive_service( 'hiveserver2', action = 'start',
       rolling_restart=rolling_restart )
 
+    self.save_component_version_to_structured_out(params.stack_name)
+
 
   def stop(self, env, rolling_restart=False):
     import params

ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params.py

@@ -19,6 +19,7 @@ limitations under the License.
 """
 
 from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
+from resource_management.libraries.functions.default import default
 from resource_management import *
 import status_params
 import os
@@ -27,6 +28,8 @@ import os
 config = Script.get_config()
 tmp_dir = Script.get_tmp_dir()
 
+stack_name = default("/hostLevelParams/stack_name", None)
+
 # node hostname
 hostname = config["hostname"]
 

ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_server.py

@@ -26,6 +26,10 @@ from webhcat import webhcat
 from webhcat_service import webhcat_service
 
 class WebHCatServer(Script):
+
+  def get_stack_to_component(self):
+    return {"HDP": "hive-webhcat"}
+
   def install(self, env):
     self.install_packages(env)
 
@@ -42,6 +46,8 @@ class WebHCatServer(Script):
     self.configure(env) # FOR SECURITY
     webhcat_service(action = 'start')
 
+    self.save_component_version_to_structured_out(params.stack_name)
+
 
   def stop(self, env, rolling_restart=False):
     import params

ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/kafka_broker.py

@@ -24,6 +24,10 @@ import upgrade
 from kafka import kafka
 
 class KafkaBroker(Script):
+
+  def get_stack_to_component(self):
+    return {"HDP": "kafka-broker"}
+
   def install(self, env):
     self.install_packages(env)
     self.configure(env)
@@ -48,6 +52,8 @@ class KafkaBroker(Script):
             user=params.kafka_user,
             not_if=no_op_test
     )
+    
+    self.save_component_version_to_structured_out(params.stack_name)
 
   def stop(self, env, rolling_restart=False):
     import params

ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py

@@ -19,12 +19,15 @@ limitations under the License.
 """
 
 from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
+from resource_management.libraries.functions.default import default
 from resource_management import *
 import status_params
 
 # server configurations
 config = Script.get_config()
 
+stack_name = default("/hostLevelParams/stack_name", None)
+
 version = default("/commandParams/version", None)
 
 stack_version_unformatted = str(config['hostLevelParams']['stack_version'])

ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox_gateway.py

@@ -27,6 +27,10 @@ from knox import knox
 from ldap import ldap
 
 class KnoxGateway(Script):
+
+  def get_stack_to_component(self):
+    return {"HDP": "knox-server"}
+
   def install(self, env):
     self.install_packages(env)
     import params
@@ -54,6 +58,8 @@ class KnoxGateway(Script):
             not_if=no_op_test
     )
 
+    self.save_component_version_to_structured_out(params.stack_name)
+
   def stop(self, env):
     import params
     env.set_params(params)

ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params.py

@@ -20,12 +20,15 @@ Ambari Agent
 """
 
 from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
+from resource_management.libraries.functions.default import default
 from resource_management import *
 import status_params
 
 config = Script.get_config()
 tmp_dir = Script.get_tmp_dir()
 
+stack_name = default("/hostLevelParams/stack_name", None)
+
 stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
 hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
 

ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_client.py

@@ -26,6 +26,10 @@ from oozie_service import oozie_service
 
          
 class OozieClient(Script):
+
+  def get_stack_to_component(self):
+    return {"HDP": "oozie-client"}
+
   def install(self, env):
     self.install_packages(env)
     self.configure(env)
@@ -36,6 +40,8 @@ class OozieClient(Script):
 
     oozie(is_server=False)
 
+    self.save_component_version_to_structured_out(params.stack_name)
+
   def status(self, env):
     raise ClientComponentHasNoStatus()
     

ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py

@@ -29,6 +29,10 @@ from oozie_service import oozie_service
 
          
 class OozieServer(Script):
+
+  def get_stack_to_component(self):
+    return {"HDP": "oozie-server"}
+
   def install(self, env):
     self.install_packages(env)
     
@@ -44,6 +48,8 @@ class OozieServer(Script):
     #TODO remove this when config command will be implemented
     self.configure(env)
     oozie_service(action='start')
+
+    self.save_component_version_to_structured_out(params.stack_name)
     
   def stop(self, env, rolling_restart=False):
     import params

ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params.py

@@ -19,6 +19,7 @@ limitations under the License.
 """
 
 from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
+from resource_management.libraries.functions.default import default
 from resource_management import *
 import status_params
 import itertools
@@ -28,6 +29,8 @@ import os
 config = Script.get_config()
 tmp_dir = Script.get_tmp_dir()
 
+stack_name = default("/hostLevelParams/stack_name", None)
+
 stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
 hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
 

ambari-server/src/main/resources/common-services/SLIDER/0.60.0.2.2/package/scripts/params.py

@@ -19,11 +19,14 @@ limitations under the License.
 """
 
 from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
+from resource_management.libraries.functions.default import default
 from resource_management import *
 
 # server configurations
 config = Script.get_config()
 
+stack_name = default("/hostLevelParams/stack_name", None)
+
 stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
 hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
 

ambari-server/src/main/resources/common-services/SLIDER/0.60.0.2.2/package/scripts/slider_client.py

@@ -24,6 +24,10 @@ from slider import slider
 
 
 class SliderClient(Script):
+
+  def get_stack_to_component(self):
+    return {"HDP": "slider-client"}
+
   def install(self, env):
     self.install_packages(env)
     self.configure(env)
@@ -35,6 +39,8 @@ class SliderClient(Script):
 
     slider()
 
+    self.save_component_version_to_structured_out(params.stack_name)
+
   def status(self, env):
     raise ClientComponentHasNoStatus()
 

ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params.py

@@ -18,10 +18,13 @@ limitations under the License.
 """
 
 from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
+from resource_management.libraries.functions.default import default
 from resource_management import *
 
 config = Script.get_config()
 
+stack_name = default("/hostLevelParams/stack_name", None)
+
 stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
 hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
 

ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/service_check.py

@@ -23,6 +23,10 @@ from resource_management import *
 
 
 class SqoopServiceCheck(Script):
+
+  def get_stack_to_component(self):
+    return {"HDP": "sqoop-server"}
+
   def service_check(self, env):
     import params
     env.set_params(params)
@@ -36,5 +40,7 @@ class SqoopServiceCheck(Script):
             logoutput = True
     )
 
+    self.save_component_version_to_structured_out(params.stack_name)
+
 if __name__ == "__main__":
   SqoopServiceCheck().execute()

ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop_client.py

@@ -25,6 +25,10 @@ from sqoop import sqoop
 
 
 class SqoopClient(Script):
+
+  def get_stack_to_component(self):
+    return {"HDP": "sqoop-client"}
+
   def install(self, env):
     self.install_packages(env)
     self.configure(env)
@@ -34,6 +38,8 @@ class SqoopClient(Script):
     env.set_params(params)
     sqoop(type='client')
 
+    self.save_component_version_to_structured_out(params.stack_name)
+
   def status(self, env):
     raise ClientComponentHasNoStatus()
 

ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/nimbus.py

@@ -20,11 +20,16 @@ limitations under the License.
 
 import sys
 from resource_management import *
+
 from storm import storm
 from service import service
 
 
 class Nimbus(Script):
+
+  def get_stack_to_component(self):
+    return {"HDP": "storm-nimbus"}
+
   def install(self, env):
     self.install_packages(env)
     self.configure(env)
@@ -42,6 +47,8 @@ class Nimbus(Script):
 
     service("nimbus", action="start")
 
+    self.save_component_version_to_structured_out(params.stack_name)
+
   def stop(self, env):
     import params
     env.set_params(params)

ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params.py

@@ -19,12 +19,15 @@ limitations under the License.
 """
 
 from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
+from resource_management.libraries.functions.default import default
 from resource_management import *
 import status_params
 
 # server configurations
 config = Script.get_config()
 
+stack_name = default("/hostLevelParams/stack_name", None)
+
 stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
 hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
 stack_is_hdp22_or_further = hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0

ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/rest_api.py

@@ -20,12 +20,17 @@ limitations under the License.
 
 import sys
 from resource_management import *
+
 from storm import storm
 from service import service
 from service_check import ServiceCheck
 
 
 class StormRestApi(Script):
+
+  def get_stack_to_component(self):
+    return {"HDP": "storm-client"}
+
   def install(self, env):
     self.install_packages(env)
     self.configure(env)
@@ -43,6 +48,8 @@ class StormRestApi(Script):
 
     service("rest_api", action="start")
 
+    self.save_component_version_to_structured_out(params.stack_name)
+
   def stop(self, env):
     import params
     env.set_params(params)

ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/supervisor.py

@@ -20,11 +20,16 @@ limitations under the License.
 
 import sys
 from resource_management import *
+
 from storm import storm
 from service import service
 
 
 class Supervisor(Script):
+
+  def get_stack_to_component(self):
+    return {"HDP": "storm-supervisor"}
+
   def install(self, env):
     self.install_packages(env)
     self.configure(env)
@@ -42,6 +47,8 @@ class Supervisor(Script):
     service("supervisor", action="start")
     service("logviewer", action="start")
 
+    self.save_component_version_to_structured_out(params.stack_name)
+
   def stop(self, env):
     import params
     env.set_params(params)

ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/application_timeline_server.py

@@ -31,6 +31,9 @@ from service import service
 
 class ApplicationTimelineServer(Script):
 
+  def get_stack_to_component(self):
+    return {"HDP": "hadoop-yarn-timelineserver"}
+
   def install(self, env):
     self.install_packages(env)
     #self.configure(env)
@@ -54,6 +57,8 @@ class ApplicationTimelineServer(Script):
     self.configure(env) # FOR SECURITY
     service('timelineserver', action='start')
 
+    self.save_component_version_to_structured_out(params.stack_name)
+
   def stop(self, env, rolling_restart=False):
     import params
     env.set_params(params)

ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py

@@ -31,6 +31,10 @@ from yarn import yarn
 from service import service
 
 class HistoryServer(Script):
+
+  def get_stack_to_component(self):
+    return {"HDP": "hadoop-mapreduce-historyserver"}
+
   def install(self, env):
     self.install_packages(env)
 
@@ -55,6 +59,8 @@ class HistoryServer(Script):
     copy_tarballs_to_hdfs('mapreduce', params.mapred_user, params.hdfs_user, params.user_group)
     service('historyserver', action='start', serviceName='mapreduce')
 
+    self.save_component_version_to_structured_out(params.stack_name)
+
   def stop(self, env, rolling_restart=False):
     import params
     env.set_params(params)

ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py

@@ -32,6 +32,10 @@ from yarn import yarn
 from service import service
 
 class Nodemanager(Script):
+
+  def get_stack_to_component(self):
+    return {"HDP": "hadoop-yarn-nodemanager"}
+
   def install(self, env):
     self.install_packages(env)
 
@@ -54,6 +58,8 @@ class Nodemanager(Script):
     self.configure(env) # FOR SECURITY
     service('nodemanager',action='start')
 
+    self.save_component_version_to_structured_out(params.stack_name)
+
   def post_rolling_restart(self, env):
     Logger.info("Executing NodeManager Rolling Upgrade post-restart")
     import params

ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params.py

@@ -28,6 +28,8 @@ import status_params
 config = Script.get_config()
 tmp_dir = Script.get_tmp_dir()
 
+stack_name = default("/hostLevelParams/stack_name", None)
+
 # This is expected to be of the form #.#.#.#
 stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
 hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)

ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py

@@ -30,6 +30,10 @@ from service import service
 
 
 class Resourcemanager(Script):
+
+  def get_stack_to_component(self):
+    return {"HDP": "hadoop-yarn-resourcemanager"}
+
   def install(self, env):
     self.install_packages(env)
 
@@ -56,6 +60,8 @@ class Resourcemanager(Script):
             action='start'
     )
 
+    self.save_component_version_to_structured_out(params.stack_name)
+
   def stop(self, env, rolling_restart=False):
     import params
 

ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/params.py

@@ -31,6 +31,8 @@ tmp_dir = Script.get_tmp_dir()
 stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
 hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
 
+stack_name = default("/hostLevelParams/stack_name", None)
+
 # New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade
 version = default("/commandParams/version", None)
 

ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/zookeeper_client.py

@@ -25,6 +25,10 @@ from resource_management import *
 from zookeeper import zookeeper
 
 class ZookeeperClient(Script):
+
+  def get_stack_to_component(self):
+    return {"HDP": "zookeeper-client"}
+
   def install(self, env):
     self.install_packages(env)
     self.configure(env)
@@ -35,6 +39,8 @@ class ZookeeperClient(Script):
 
     zookeeper(type='client')
 
+    self.save_component_version_to_structured_out(params.stack_name)
+
   def status(self, env):
     raise ClientComponentHasNoStatus()
 

ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/zookeeper_server.py

@@ -50,6 +50,10 @@ def call_and_match_output(command, regex_expression, err_message):
 
 
 class ZookeeperServer(Script):
+
+  def get_stack_to_component(self):
+    return {"HDP": "zookeeper-server"}
+
   def install(self, env):
     self.install_packages(env)
     self.configure(env)
@@ -73,6 +77,8 @@ class ZookeeperServer(Script):
     self.configure(env)
     zookeeper_service(action = 'start')
 
+    self.save_component_version_to_structured_out(params.stack_name)
+
   def post_rolling_restart(self, env):
     Logger.info("Executing Rolling Upgrade post-restart")
     import params

ambari-server/src/test/python/TestVersionSelectUtil.py

@@ -0,0 +1,93 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+from unittest import TestCase
+import os
+from mock.mock import patch, MagicMock
+
+from resource_management.core.logger import Logger
+
+class TestVersionSelectUtil(TestCase):
+  """
+  Class that tests the method of the version_select_util.py file.
+  """
+  def setUp(self):
+    import imp
+
+    Logger.logger = MagicMock()
+
+    self.test_directory = os.path.dirname(os.path.abspath(__file__))
+    test_file_path = os.path.join(self.test_directory, '../../../../ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py')
+    with open(test_file_path, 'rb') as fp:
+        self.module = imp.load_module('module', fp, test_file_path, ('.py', 'rb', imp.PY_SOURCE))
+
+  @patch('__builtin__.open')
+  @patch("resource_management.core.shell.call")
+  def test_get_component_version(self, call_mock, open_mock):
+    hdp_expected_version = "2.2.1.0-2175"
+
+    # Mock classes for reading from a file
+    class MagicFile(object):
+      allowed_names = set(["hadoop-hdfs-namenode",
+                           "hadoop-hdfs-datanode",
+                           "zookeeper-server",
+                           "zookeeper-client"
+                           ])
+      def read(self, value):
+        return (value + " - " + hdp_expected_version) if value in self.allowed_names else ("ERROR: Invalid package - " + value)
+
+      def __exit__(self, exc_type, exc_val, exc_tb):
+        pass
+
+      def __enter__(self):
+        return self
+    pass
+
+    class MagicFile1(MagicFile):
+      def read(self):
+        return super(MagicFile1, self).read("hadoop-nonexistent-component-name")
+    class MagicFile2(MagicFile):
+      def read(self):
+        return super(MagicFile2, self).read("hadoop-hdfs-namenode")
+    class MagicFile3(MagicFile):
+      def read(self):
+        return super(MagicFile3, self).read("hadoop-hdfs-datanode")
+
+    open_mock.side_effect = [MagicFile1(), MagicFile2(), MagicFile3()]
+    call_mock.side_effect = [(0, "value will come from MagicFile"), ] * 3
+
+
+    # Missing Stack
+    version = self.module.get_component_version(None, "hadoop-hdfs-datanode")
+    self.assertEquals(version, None)
+    version = self.module.get_component_version("StackDoesNotExist", "hadoop-hdfs-datanode")
+    self.assertEquals(version, None)
+
+    # Invalid request
+    version = self.module.get_component_version("HDP", None)
+    self.assertEquals(version, None)
+
+    # Incorrect name
+    version = self.module.get_component_version("HDP", "hadoop-nonexistent-component-name")
+    self.assertEquals(version, None)
+
+    # Pass
+    version = self.module.get_component_version("HDP", "hadoop-hdfs-namenode")
+    self.assertEquals(version, hdp_expected_version)
+    version = self.module.get_component_version("HDP", "hadoop-hdfs-datanode")
+    self.assertEquals(version, hdp_expected_version)