AMBARI-13664: Express Upgrade: Handle Pig, Tez, Sqoop in all upgrade paths and downgrades (Dmitro Lisnichenko via jluniya)

ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_client.py

@@ -53,7 +53,7 @@ class HbaseClientDefault(HbaseClient):
   def get_stack_to_component(self):
     return {"HDP": "hbase-client"}
 
-  def pre_rolling_restart(self, env):
+  def pre_upgrade_restart(self, env, upgrade_type=None):
     import params
     env.set_params(params)
 

ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py

@@ -72,19 +72,19 @@ class HbaseMasterDefault(HbaseMaster):
   def get_stack_to_component(self):
     return {"HDP": "hbase-master"}
 
-  def pre_rolling_restart(self, env):
+  def pre_upgrade_restart(self, env, upgrade_type=None):
     import params
     env.set_params(params)
     upgrade.prestart(env, "hbase-master")
 
-  def start(self, env, rolling_restart=False):
+  def start(self, env, upgrade_type=None):
     import params
     env.set_params(params)
     self.configure(env) # for security
-    setup_ranger_hbase(rolling_upgrade=rolling_restart)  
+    setup_ranger_hbase(upgrade_type=upgrade_type)
     hbase_service('master', action = 'start')
     
-  def stop(self, env, rolling_restart=False):
+  def stop(self, env, upgrade_type=None):
     import params
     env.set_params(params)
     hbase_service('master', action = 'stop')

ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py

@@ -70,26 +70,26 @@ class HbaseRegionServerDefault(HbaseRegionServer):
   def get_stack_to_component(self):
     return {"HDP": "hbase-regionserver"}
 
-  def pre_rolling_restart(self, env):
+  def pre_upgrade_restart(self, env, upgrade_type=None):
     import params
     env.set_params(params)
     upgrade.prestart(env, "hbase-regionserver")
 
-  def post_rolling_restart(self, env):
+  def post_upgrade_restart(self, env):
     import params
     env.set_params(params)
     upgrade.post_regionserver(env)
 
-  def start(self, env, rolling_restart=False):
+  def start(self, env, upgrade_type=None):
     import params
     env.set_params(params)
     self.configure(env) # for security
-    setup_ranger_hbase(rolling_upgrade=rolling_restart)  
+    setup_ranger_hbase(upgrade_type=upgrade_type)
     hbase_service( 'regionserver',
       action = 'start'
     )
 
-  def stop(self, env, rolling_restart=False):
+  def stop(self, env, upgrade_type=None):
     import params
     env.set_params(params)
 

ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/phoenix_queryserver.py

@@ -41,20 +41,20 @@ class PhoenixQueryServer(Script):
     hbase(name='queryserver')
 
 
-  def start(self, env, rolling_restart=False):
+  def start(self, env, upgrade_type=None):
     import params
     env.set_params(params)
     self.configure(env)
     phoenix_service('start')
 
 
-  def stop(self, env, rolling_restart=False):
+  def stop(self, env, upgrade_type=None):
     import params
     env.set_params(params)
     phoenix_service('stop')
 
 
-  def pre_rolling_restart(self, env):
+  def pre_upgrade_restart(self, env, upgrade_type=None):
     import params
     env.set_params(params)
 

ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py

@@ -19,7 +19,7 @@ limitations under the License.
 """
 from resource_management.core.logger import Logger
 
-def setup_ranger_hbase(rolling_upgrade = False):
+def setup_ranger_hbase(upgrade_type=None):
   import params
   
   if params.has_ranger_admin:
@@ -31,7 +31,7 @@ def setup_ranger_hbase(rolling_upgrade = False):
     
     hdp_version = None
 
-    if rolling_upgrade:
+    if upgrade_type is not None:
       hdp_version = params.version
 
     if params.retryAble:

ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig_client.py

@@ -42,7 +42,7 @@ class PigClientLinux(PigClient):
   def get_stack_to_component(self):
     return {"HDP": "hadoop-client"}
 
-  def pre_rolling_restart(self, env):
+  def pre_upgrade_restart(self, env, upgrade_type=None):
     import params
     env.set_params(params)
 

ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop_client.py

@@ -47,7 +47,7 @@ class SqoopClientDefault(SqoopClient):
   def get_stack_to_component(self):
     return {"HDP": "sqoop-client"}
 
-  def pre_rolling_restart(self, env):
+  def pre_upgrade_restart(self, env, upgrade_type=None):
     import params
     env.set_params(params)
 

ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/pre_upgrade.py

@@ -30,13 +30,13 @@ class TezPreUpgrade(Script):
 
   def prepare(self, env):
     """
-    During the "Upgrade" direction of a Rolling Upgrade, it is necessary to ensure that the older tez tarball
+    During the "Upgrade" direction of a Stack Upgrade, it is necessary to ensure that the older tez tarball
     has been copied to HDFS. This is an additional check for added robustness.
     """
     import params
     env.set_params(params)
 
-    Logger.info("Before starting Rolling Upgrade, check if tez tarball has been copied to HDFS.")
+    Logger.info("Before starting Stack Upgrade, check if tez tarball has been copied to HDFS.")
 
     if params.hdp_stack_version and compare_versions(params.hdp_stack_version, '2.2.0.0') >= 0:
       Logger.info("Stack version {0} is sufficient to check if need to copy tez.tar.gz to HDFS.".format(params.hdp_stack_version))

ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez_client.py

@@ -51,7 +51,7 @@ class TezClientLinux(TezClient):
   def get_stack_to_component(self):
     return {"HDP": "hadoop-client"}
 
-  def pre_rolling_restart(self, env):
+  def pre_upgrade_restart(self, env, upgrade_type=None):
     import params
     env.set_params(params)
 

ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml

@@ -44,6 +44,13 @@
           <message>Before continuing, please deactivate and kill any currently running topologies.</message>
         </task>
       </execute-stage>
+
+      <execute-stage service="TEZ" component="TEZ_CLIENT" title="Check Tez Tarball">
+        <task xsi:type="execute" hosts="any">
+          <script>scripts/pre_upgrade.py</script>
+          <function>prepare</function>
+        </task>
+      </execute-stage>
     </group>
 
     <group xsi:type="stop" name="STOP_HIGH_LEVEL_SERVICE_COMPONENTS" title="Stop Components for High-Level Services">
@@ -186,7 +193,7 @@
 
     <group xsi:type="cluster" name="Upgrade service configs" title="Upgrade service configs">
       <direction>UPGRADE</direction>   <!--  prevent config changes on downgrade -->
-      <skippable>false</skippable>
+      <skippable>true</skippable>  <!-- May fix configuration problems manually -->
 
       <!--Changes for stack 2.2-->
       <!--YARN-->

ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml

@@ -44,6 +44,13 @@
           <message>Before continuing, please deactivate and kill any currently running topologies.</message>
         </task>
       </execute-stage>
+
+      <execute-stage service="TEZ" component="TEZ_CLIENT" title="Check Tez Tarball">
+        <task xsi:type="execute" hosts="any">
+          <script>scripts/pre_upgrade.py</script>
+          <function>prepare</function>
+        </task>
+      </execute-stage>
     </group>
 
     <group xsi:type="stop" name="STOP_HIGH_LEVEL_SERVICE_COMPONENTS" title="Stop Components for High-Level Services">

ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml

@@ -44,6 +44,13 @@
           <message>Before continuing, please deactivate and kill any currently running topologies.</message>
         </task>
       </execute-stage>
+
+      <execute-stage service="TEZ" component="TEZ_CLIENT" title="Check Tez Tarball">
+        <task xsi:type="execute" hosts="any">
+          <script>scripts/pre_upgrade.py</script>
+          <function>prepare</function>
+        </task>
+      </execute-stage>
     </group>
 
     <!--
@@ -258,7 +265,7 @@
 
     <group xsi:type="cluster" name="Upgrade service configs" title="Upgrade service configs">
       <direction>UPGRADE</direction>   <!--  prevent config changes on downgrade -->
-      <skippable>false</skippable>
+      <skippable>true</skippable>  <!-- May fix configuration problems manually -->
 
       <execute-stage service="RANGER" component="RANGER_ADMIN" title="Apply config changes for Ranger">
         <task xsi:type="configure" id="hdp_2_3_0_0_update_ranger_env"/>

ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml

@@ -44,6 +44,13 @@
           <message>Before continuing, please deactivate and kill any currently running topologies.</message>
         </task>
       </execute-stage>
+
+      <execute-stage service="TEZ" component="TEZ_CLIENT" title="Check Tez Tarball">
+        <task xsi:type="execute" hosts="any">
+          <script>scripts/pre_upgrade.py</script>
+          <function>prepare</function>
+        </task>
+      </execute-stage>
     </group>
 
     <group xsi:type="stop" name="STOP_HIGH_LEVEL_SERVICE_COMPONENTS" title="Stop Components for High-Level Services">

ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py

@@ -747,7 +747,7 @@ class TestHBaseMaster(RMFTestCase):
     self.assertNoMoreResources()
 
   @patch("resource_management.core.shell.call")
-  def test_pre_rolling_restart(self, call_mock):
+  def test_pre_upgrade_restart(self, call_mock):
     call_mock.side_effects = [(0, None), (0, None)]
 
     config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
@@ -759,7 +759,7 @@ class TestHBaseMaster(RMFTestCase):
     mocks_dict = {}
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hbase_master.py",
                        classname = "HbaseMaster",
-                       command = "pre_rolling_restart",
+                       command = "pre_upgrade_restart",
                        config_dict = json_content,
                        hdp_stack_version = self.STACK_VERSION,
                        target = RMFTestCase.TARGET_COMMON_SERVICES,
@@ -782,7 +782,7 @@ class TestHBaseMaster(RMFTestCase):
     mocks_dict = {}
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hbase_master.py",
                        classname = "HbaseMaster",
-                       command = "pre_rolling_restart",
+                       command = "pre_upgrade_restart",
                        config_dict = json_content,
                        hdp_stack_version = self.STACK_VERSION,
                        target = RMFTestCase.TARGET_COMMON_SERVICES,

ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py

@@ -621,7 +621,7 @@ class TestHbaseRegionServer(RMFTestCase):
     )
     put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
 
-  def test_pre_rolling_restart(self):
+  def test_pre_upgrade_restart(self):
     config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
     with open(config_file, "r") as f:
       json_content = json.load(f)
@@ -629,7 +629,7 @@ class TestHbaseRegionServer(RMFTestCase):
     json_content['commandParams']['version'] = version
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hbase_regionserver.py",
                        classname = "HbaseRegionServer",
-                       command = "pre_rolling_restart",
+                       command = "pre_upgrade_restart",
                        config_dict = json_content,
                        hdp_stack_version = self.STACK_VERSION,
                        target = RMFTestCase.TARGET_COMMON_SERVICES)
@@ -647,7 +647,7 @@ class TestHbaseRegionServer(RMFTestCase):
     mocks_dict = {}
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hbase_regionserver.py",
                        classname = "HbaseRegionServer",
-                       command = "post_rolling_restart",
+                       command = "post_upgrade_restart",
                        config_dict = json_content,
                        hdp_stack_version = self.STACK_VERSION,
                        call_mocks = [(0, "Dummy output c6401.ambari.apache.org:")],
@@ -669,7 +669,7 @@ class TestHbaseRegionServer(RMFTestCase):
     mocks_dict = {}
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hbase_regionserver.py",
                        classname = "HbaseRegionServer",
-                       command = "pre_rolling_restart",
+                       command = "pre_upgrade_restart",
                        config_dict = json_content,
                        hdp_stack_version = self.STACK_VERSION,
                        target = RMFTestCase.TARGET_COMMON_SERVICES,

ambari-server/src/test/python/stacks/2.0.6/HBASE/test_phoenix_queryserver.py

@@ -424,7 +424,7 @@ class TestPhoenixQueryServer(RMFTestCase):
     self.executeScript(
       self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/phoenix_queryserver.py",
       classname = "PhoenixQueryServer",
-      command = "pre_rolling_restart",
+      command = "pre_upgrade_restart",
       config_dict = json_content,
       call_mocks = [(0, "/etc/hbase/2.3.0.0-1234/0")],
       hdp_stack_version = self.STACK_VERSION,

ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_client.py

@@ -133,7 +133,7 @@ class TestPigClient(RMFTestCase):
     )
     self.assertNoMoreResources()
 
-  def test_pre_rolling_restart(self):
+  def test_pre_upgrade_restart(self):
     config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
     with open(config_file, "r") as f:
       json_content = json.load(f)
@@ -141,7 +141,7 @@ class TestPigClient(RMFTestCase):
     json_content['commandParams']['version'] = version
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/pig_client.py",
                        classname = "PigClient",
-                       command = "pre_rolling_restart",
+                       command = "pre_upgrade_restart",
                        config_dict = json_content,
                        hdp_stack_version = self.STACK_VERSION,
                        target = RMFTestCase.TARGET_COMMON_SERVICES)
@@ -149,7 +149,7 @@ class TestPigClient(RMFTestCase):
                               ('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'hadoop-client', version), sudo=True)
     self.assertNoMoreResources()
 
-  def test_pre_rolling_restart_23(self):
+  def test_pre_upgrade_restart_23(self):
     config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
     with open(config_file, "r") as f:
       json_content = json.load(f)
@@ -159,7 +159,7 @@ class TestPigClient(RMFTestCase):
     mocks_dict = {}
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/pig_client.py",
                        classname = "PigClient",
-                       command = "pre_rolling_restart",
+                       command = "pre_upgrade_restart",
                        config_dict = json_content,
                        hdp_stack_version = self.STACK_VERSION,
                        target = RMFTestCase.TARGET_COMMON_SERVICES,

ambari-server/src/test/python/stacks/2.0.6/SQOOP/test_sqoop.py

@@ -112,7 +112,7 @@ class TestSqoop(RMFTestCase):
     self.assertNoMoreResources()
 
 
-  def test_pre_rolling_restart_23(self):
+  def test_pre_upgrade_restart_23(self):
     config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
     with open(config_file, "r") as f:
       json_content = json.load(f)
@@ -122,7 +122,7 @@ class TestSqoop(RMFTestCase):
     mocks_dict = {}
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/sqoop_client.py",
                        classname = "SqoopClient",
-                       command = "pre_rolling_restart",
+                       command = "pre_upgrade_restart",
                        config_dict = json_content,
                        hdp_stack_version = self.STACK_VERSION,
                        target = RMFTestCase.TARGET_COMMON_SERVICES,

ambari-server/src/test/python/stacks/2.1/TEZ/test_tez_client.py

@@ -91,7 +91,7 @@ class TestTezClient(RMFTestCase):
 
     # for now, it's enough that hdp-select is confirmed
 
-  def test_pre_rolling_restart_23(self):
+  def test_pre_upgrade_restart_23(self):
     config_file = self.get_src_folder()+"/test/python/stacks/2.1/configs/client-upgrade.json"
     with open(config_file, "r") as f:
       json_content = json.load(f)
@@ -101,7 +101,7 @@ class TestTezClient(RMFTestCase):
     mocks_dict = {}
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/tez_client.py",
                        classname = "TezClient",
-                       command = "pre_rolling_restart",
+                       command = "pre_upgrade_restart",
                        config_dict = json_content,
                        hdp_stack_version = self.STACK_VERSION,
                        target = RMFTestCase.TARGET_COMMON_SERVICES,