AMBARI-9081. Rolling Upgrades: clients do not send information about their version on restart (alejandro)

ambari-agent/src/main/python/ambari_agent/PythonExecutor.py

@@ -71,15 +71,9 @@ class PythonExecutor:
     Timeout meaning: how many seconds should pass before script execution
     is forcibly terminated
     override_output_files option defines whether stdout/stderr files will be
-    recreated or appended
+    recreated or appended.
+    The structured out file, however, is preserved during multiple invocations that use the same file.
     """
-    # need to remove this file for the following case:
-    # status call 1 does not write to file; call 2 writes to file;
-    # call 3 does not write to file, so contents are still call 2's result
-    try:
-      os.unlink(tmpstructedoutfile)
-    except OSError:
-      pass # no error
 
     script_params += [tmpstructedoutfile, logger_level, tmp_dir]
     pythonCommand = self.python_command(script, script_params)

ambari-agent/src/test/python/ambari_agent/TestPythonExecutor.py

@@ -113,10 +113,9 @@ class TestPythonExecutor(TestCase):
     executor = PythonExecutor("/tmp", AmbariConfig().getConfig())
     _, tmpoutfile = tempfile.mkstemp()
     _, tmperrfile = tempfile.mkstemp()
-    # PythonExecutor need only filename and will try to delete this file before execution, this will fail on windows
-    # so we will use NamedTemporaryFile only to generate temp file and close it immediately after creation
-    tmp_file = tempfile.NamedTemporaryFile(delete=False)
-    tmpstroutfile = tmp_file.name
+    
+    tmp_file = tempfile.NamedTemporaryFile()    # the structured out file should be preserved across calls to the hooks and script.
+    tmpstructuredoutfile = tmp_file.name
     tmp_file.close()
 
     PYTHON_TIMEOUT_SECONDS =  5
@@ -134,7 +133,7 @@ class TestPythonExecutor(TestCase):
     callback_method = MagicMock()
     result = executor.run_file("file", ["arg1", "arg2"], "/fake_tmp_dir",
                                tmpoutfile, tmperrfile, PYTHON_TIMEOUT_SECONDS,
-                               tmpstroutfile, "INFO", callback_method, "1-1")
+                               tmpstructuredoutfile, "INFO", callback_method, "1-1")
     self.assertEquals(result, {'exitcode': 0, 'stderr': 'Dummy err', 'stdout': 'Dummy output',
                                'structuredOut': {}})
     self.assertTrue(callback_method.called)

ambari-agent/src/test/python/resource_management/TestScript.py

@@ -85,6 +85,7 @@ class TestScript(TestCase):
 
     # Testing installing of a list of packages
     with Environment(".", test_mode=True) as env:
+      script = Script()
       Script.config = dummy_config
       script.install_packages("env")
     resource_dump = pprint.pformat(env.resource_list)
@@ -94,7 +95,6 @@ class TestScript(TestCase):
   def test_structured_out(self, open_mock):
     script = Script()
     script.stroutfile = ''
-
     self.assertEqual(Script.structuredOut, {})
 
     script.put_structured_out({"1": "1"})
@@ -110,8 +110,7 @@ class TestScript(TestCase):
     self.assertEqual(open_mock.call_count, 3)
     self.assertEqual(Script.structuredOut, {"1": "3", "2": "2"})
 
+
   def tearDown(self):
     # enable stdout
     sys.stdout = sys.__stdout__
-
-

ambari-common/src/main/python/resource_management/libraries/script/hook.py

@@ -26,7 +26,7 @@ import sys
 
 class Hook(Script):
   """
-  Executes a hook for acommand for custom service. stdout and stderr are written to
+  Executes a hook for a command for custom service. stdout and stderr are written to
   tmpoutfile and to tmperrfile respectively.
   """
 

ambari-common/src/main/python/resource_management/libraries/script/script.py

@@ -99,13 +99,20 @@ class Script(object):
     Returns a dictionary where the key is a stack name, and the value is the component name used in selecting the version.
     """
     return {}
-
+    
+  def load_structured_out(self):
+    Script.structuredOut = {}
+    if os.path.exists(self.stroutfile):
+      with open(self.stroutfile, 'r') as fp:
+        Script.structuredOut = json.load(fp)
+  
   def put_structured_out(self, sout):
+    curr_content = Script.structuredOut.copy()
     Script.structuredOut.update(sout)
     try:
       with open(self.stroutfile, 'w') as fp:
         json.dump(Script.structuredOut, fp)
-    except IOError:
+    except IOError, err:
       Script.structuredOut.update({"errMsg" : "Unable to write to " + self.stroutfile})
 
   def save_component_version_to_structured_out(self, stack_name):
@@ -142,6 +149,7 @@ class Script(object):
     self.command_data_file = sys.argv[2]
     self.basedir = sys.argv[3]
     self.stroutfile = sys.argv[4]
+    self.load_structured_out()
     self.logging_level = sys.argv[5]
     Script.tmp_dir = sys.argv[6]
 
@@ -162,17 +170,28 @@ class Script(object):
         #load passwords here(used on windows to impersonate different users)
         Script.passwords = {}
         for k, v in _PASSWORD_MAP.iteritems():
-          if get_path_form_configuration(k,Script.config) and get_path_form_configuration(v,Script.config ):
-            Script.passwords[get_path_form_configuration(k,Script.config)] = get_path_form_configuration(v,Script.config)
+          if get_path_form_configuration(k,Script.config) and get_path_form_configuration(v, Script.config):
+            Script.passwords[get_path_form_configuration(k,Script.config)] = get_path_form_configuration(v, Script.config)
 
     except IOError:
       logger.exception("Can not read json file with command parameters: ")
       sys.exit(1)
+
     # Run class method depending on a command type
     try:
       method = self.choose_method_to_execute(command_name)
       with Environment(self.basedir) as env:
         method(env)
+
+        # For start actions, try to advertise the component's version
+        if command_name.lower() == "start" or command_name.lower() == "install":
+          try:
+            import params
+            # This is to support older stacks
+            if hasattr(params, "stack_name"):
+              self.save_component_version_to_structured_out(params.stack_name)
+          except ImportError:
+            logger.error("Executing command %s could not import params" % str(command_name))
     except ClientComponentHasNoStatus or ComponentIsNotRunning:
       # Support of component status checks.
       # Non-zero exit code is interpreted as an INSTALLED status of a component
@@ -333,6 +352,14 @@ class Script(object):
       if rolling_restart:
         self.post_rolling_restart(env)
 
+    try:
+      import params
+      if hasattr(params, "stack_name"):
+        self.save_component_version_to_structured_out(params.stack_name)
+    except ImportError:
+      logger.error("Restart command could not import params")
+
+
   def post_rolling_restart(self, env):
     """
     To be overridden by subclasses

ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_client.py

@@ -36,8 +36,6 @@ class FalconClient(Script):
     env.set_params(params)
     falcon('client', action='config')
 
-    self.save_component_version_to_structured_out(params.stack_name)
-
   def status(self, env):
     raise ClientComponentHasNoStatus()
 
@@ -51,7 +49,7 @@ class FalconClient(Script):
       return
 
     Logger.info("Executing Falcon Client Rolling Upgrade pre-restart")
-    Execute(format("hdp-select set hadoop-client {version}"))
+    Execute(format("hdp-select set falcon-client {version}"))
 
   def security_status(self, env):
     import status_params

ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server.py

@@ -46,8 +46,6 @@ class FalconServer(Script):
 
     falcon('server', action='start')
 
-    self.save_component_version_to_structured_out(params.stack_name)
-
 
   def stop(self, env, rolling_restart=False):
     import params

ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_handler.py

@@ -43,8 +43,6 @@ class FlumeHandler(Script):
 
     flume(action='start')
 
-    self.save_component_version_to_structured_out(params.stack_name)
-
   def stop(self, env, rolling_restart=False):
     import params
 

ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_client.py

@@ -46,8 +46,6 @@ class HbaseClient(Script):
     
     hbase(name='client')
 
-    self.save_component_version_to_structured_out(params.stack_name)
-
   def status(self, env):
     raise ClientComponentHasNoStatus()
 

ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py

@@ -56,8 +56,6 @@ class HbaseMaster(Script):
     hbase_service( 'master',
       action = 'start'
     )
-
-    self.save_component_version_to_structured_out(params.stack_name)
     
   def stop(self, env, rolling_restart=False):
     import params

ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py

@@ -61,8 +61,6 @@ class HbaseRegionServer(Script):
       action = 'start'
     )
 
-    self.save_component_version_to_structured_out(params.stack_name)
-
   def stop(self, env, rolling_restart=False):
     import params
     env.set_params(params)

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/datanode.py

@@ -62,8 +62,6 @@ class DataNode(Script):
     self.configure(env)
     datanode(action="start")
 
-    self.save_component_version_to_structured_out(params.stack_name)
-
 
   def stop(self, env, rolling_restart=False):
     import params

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_client.py

@@ -26,6 +26,10 @@ from utils import service
 
 
 class HdfsClient(Script):
+
+  def get_stack_to_component(self):
+    return {"HDP": "hadoop-client"}
+
   def install(self, env):
     import params
 
@@ -55,7 +59,6 @@ class HdfsClient(Script):
   def config(self, env):
     import params
     hdfs()
-    pass
 
   def security_status(self, env):
     import status_params

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py

@@ -64,8 +64,6 @@ class JournalNode(Script):
       create_log_dir=True
     )
 
-    self.save_component_version_to_structured_out(params.stack_name)
-
   def stop(self, env, rolling_restart=False):
     import params
 

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py

@@ -72,8 +72,6 @@ class NameNode(Script):
     self.configure(env)
     namenode(action="start", rolling_restart=rolling_restart, env=env)
 
-    self.save_component_version_to_structured_out(params.stack_name)
-
   def post_rolling_restart(self, env):
     Logger.info("Executing Rolling Upgrade post-restart")
     import params

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/snamenode.py

@@ -49,8 +49,6 @@ class SNameNode(Script):
     self.configure(env)
     snamenode(action="start")
 
-    self.save_component_version_to_structured_out(params.stack_name)
-
   def stop(self, env, rolling_restart=False):
     import params
     env.set_params(params)

ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hcat_client.py

@@ -23,6 +23,10 @@ from resource_management import *
 from hcat import hcat
 
 class HCatClient(Script):
+
+  def get_stack_to_component(self):
+    return {"HDP": "hadoop-client"}
+
   def install(self, env):
     import params
     self.install_packages(env, exclude_packages=params.hive_exclude_packages)
@@ -30,12 +34,9 @@ class HCatClient(Script):
 
   def configure(self, env):
     import params
-
     env.set_params(params)
-
     hcat()
 
-
   def status(self, env):
     raise ClientComponentHasNoStatus()
 

ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_client.py

@@ -24,6 +24,9 @@ from hive import hive
 
 class HiveClient(Script):
 
+  def get_stack_to_component(self):
+    return {"HDP": "hadoop-client"}
+
   def pre_rolling_restart(self, env):
     import params
     env.set_params(params)
@@ -42,7 +45,6 @@ class HiveClient(Script):
 
     hive(name='client')
 
-
   def status(self, env):
     raise ClientComponentHasNoStatus()
 

ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py

@@ -53,8 +53,6 @@ class HiveMetastore(Script):
     self.configure(env)  # FOR SECURITY
     hive_service('metastore', action = 'start')
 
-    self.save_component_version_to_structured_out(params.stack_name)
-
 
   def stop(self, env, rolling_restart = False):
     import params

ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py

@@ -59,8 +59,6 @@ class HiveServer(Script):
     hive_service( 'hiveserver2', action = 'start',
       rolling_restart=rolling_restart )
 
-    self.save_component_version_to_structured_out(params.stack_name)
-
 
   def stop(self, env, rolling_restart=False):
     import params

ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_server.py

@@ -46,8 +46,6 @@ class WebHCatServer(Script):
     self.configure(env) # FOR SECURITY
     webhcat_service(action = 'start')
 
-    self.save_component_version_to_structured_out(params.stack_name)
-
 
   def stop(self, env, rolling_restart=False):
     import params

ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/kafka_broker.py

@@ -52,8 +52,6 @@ class KafkaBroker(Script):
             user=params.kafka_user,
             not_if=no_op_test
     )
-    
-    self.save_component_version_to_structured_out(params.stack_name)
 
   def stop(self, env, rolling_restart=False):
     import params

ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox_gateway.py

@@ -58,8 +58,6 @@ class KnoxGateway(Script):
             not_if=no_op_test
     )
 
-    self.save_component_version_to_structured_out(params.stack_name)
-
   def stop(self, env):
     import params
     env.set_params(params)

ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_client.py

@@ -41,8 +41,6 @@ class OozieClient(Script):
 
     oozie(is_server=False)
 
-    self.save_component_version_to_structured_out(params.stack_name)
-
   def status(self, env):
     raise ClientComponentHasNoStatus()
 

ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py

@@ -61,7 +61,6 @@ class OozieServer(Script):
 
     oozie_service(action='start', rolling_restart=rolling_restart)
 
-    self.save_component_version_to_structured_out(params.stack_name)
     
   def stop(self, env, rolling_restart=False):
     import params

ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params.py

@@ -26,6 +26,8 @@ from resource_management import *
 config = Script.get_config()
 tmp_dir = Script.get_tmp_dir()
 
+stack_name = default("/hostLevelParams/stack_name", None)
+
 stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
 hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
 

ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig_client.py

@@ -26,6 +26,9 @@ from pig import pig
 
 class PigClient(Script):
 
+  def get_stack_to_component(self):
+    return {"HDP": "hadoop-client"}
+
   def pre_rolling_restart(self, env):
     import params
     env.set_params(params)

ambari-server/src/main/resources/common-services/SLIDER/0.60.0.2.2/package/scripts/slider_client.py

@@ -46,8 +46,6 @@ class SliderClient(Script):
 
     slider()
 
-    self.save_component_version_to_structured_out(params.stack_name)
-
   def status(self, env):
     raise ClientComponentHasNoStatus()
 

ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/service_check.py

@@ -40,7 +40,5 @@ class SqoopServiceCheck(Script):
             logoutput = True
     )
 
-    self.save_component_version_to_structured_out(params.stack_name)
-
 if __name__ == "__main__":
   SqoopServiceCheck().execute()

ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop_client.py

@@ -38,8 +38,6 @@ class SqoopClient(Script):
     env.set_params(params)
     sqoop(type='client')
 
-    self.save_component_version_to_structured_out(params.stack_name)
-
   def status(self, env):
     raise ClientComponentHasNoStatus()
 

ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/nimbus.py

@@ -60,8 +60,6 @@ class Nimbus(Script):
 
     service("nimbus", action="start")
 
-    self.save_component_version_to_structured_out(params.stack_name)
-
   def stop(self, env, rolling_restart=False):
     import params
     env.set_params(params)

ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/rest_api.py

@@ -59,8 +59,6 @@ class StormRestApi(Script):
 
     service("rest_api", action="start")
 
-    self.save_component_version_to_structured_out(params.stack_name)
-
   def stop(self, env, rolling_restart=False):
     import params
     env.set_params(params)

ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/supervisor.py

@@ -58,8 +58,6 @@ class Supervisor(Script):
     service("supervisor", action="start")
     service("logviewer", action="start")
 
-    self.save_component_version_to_structured_out(params.stack_name)
-
   def stop(self, env, rolling_restart=False):
     import params
     env.set_params(params)

ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params.py

@@ -24,6 +24,8 @@ from resource_management import *
 # server configurations
 config = Script.get_config()
 
+stack_name = default("/hostLevelParams/stack_name", None)
+
 # This is expected to be of the form #.#.#.#
 stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
 hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)

ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez_client.py

@@ -25,6 +25,9 @@ from tez import tez
 
 class TezClient(Script):
 
+  def get_stack_to_component(self):
+    return {"HDP": "hadoop-client"}
+
   def pre_rolling_restart(self, env):
     import params
     env.set_params(params)

ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/application_timeline_server.py

@@ -57,8 +57,6 @@ class ApplicationTimelineServer(Script):
     self.configure(env) # FOR SECURITY
     service('timelineserver', action='start')
 
-    self.save_component_version_to_structured_out(params.stack_name)
-
   def stop(self, env, rolling_restart=False):
     import params
     env.set_params(params)

ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py

@@ -59,7 +59,6 @@ class HistoryServer(Script):
     copy_tarballs_to_hdfs('mapreduce', 'hadoop-mapreduce-historyserver', params.mapred_user, params.hdfs_user, params.user_group)
     service('historyserver', action='start', serviceName='mapreduce')
 
-    self.save_component_version_to_structured_out(params.stack_name)
 
   def stop(self, env, rolling_restart=False):
     import params

ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/mapreduce2_client.py

@@ -26,6 +26,9 @@ from yarn import yarn
 
 class MapReduce2Client(Script):
 
+  def get_stack_to_component(self):
+    return {"HDP": "hadoop-client"}
+
   def pre_rolling_restart(self, env):
     import params
     env.set_params(params)

ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py

@@ -58,8 +58,6 @@ class Nodemanager(Script):
     self.configure(env) # FOR SECURITY
     service('nodemanager',action='start')
 
-    self.save_component_version_to_structured_out(params.stack_name)
-
   def post_rolling_restart(self, env):
     Logger.info("Executing NodeManager Rolling Upgrade post-restart")
     import params

ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py

@@ -60,8 +60,6 @@ class Resourcemanager(Script):
             action='start'
     )
 
-    self.save_component_version_to_structured_out(params.stack_name)
-
   def stop(self, env, rolling_restart=False):
     import params
 

ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn_client.py

@@ -26,6 +26,9 @@ from yarn import yarn
 
 class YarnClient(Script):
 
+  def get_stack_to_component(self):
+    return {"HDP": "hadoop-client"}
+
   def pre_rolling_restart(self, env):
     import params
     env.set_params(params)

ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/zookeeper_client.py

@@ -39,8 +39,6 @@ class ZookeeperClient(Script):
 
     zookeeper(type='client')
 
-    self.save_component_version_to_structured_out(params.stack_name)
-
   def status(self, env):
     raise ClientComponentHasNoStatus()
 

ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/zookeeper_server.py

@@ -77,8 +77,6 @@ class ZookeeperServer(Script):
     self.configure(env)
     zookeeper_service(action = 'start')
 
-    self.save_component_version_to_structured_out(params.stack_name)
-
   def post_rolling_restart(self, env):
     Logger.info("Executing Rolling Upgrade post-restart")
     import params