AMBARI-9339 - Oozie Downgrade Fails Because Upgrade Behavior Is Used (jonathanhurley)

ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py

@@ -27,6 +27,8 @@ from resource_management.core.logger import Logger
 from resource_management.core.exceptions import Fail
 from resource_management.core.resources.system import Execute
 from resource_management.libraries.functions import format
+from resource_management.libraries.functions import compare_versions
+from resource_management.libraries.functions import format_hdp_stack_version
 
 BACKUP_TEMP_DIR = "oozie-upgrade-backup"
 BACKUP_CONF_ARCHIVE = "oozie-conf-backup.tar"
@@ -110,6 +112,10 @@ def prepare_libext_directory():
   """
   import params
 
+  # some versions of HDP don't need the lzo compression libraries
+  target_version_needs_compression_libraries = compare_versions(
+    format_hdp_stack_version(params.version), '2.2.1.0') >= 0
+
   if not os.path.isdir(params.oozie_libext_customer_dir):
     os.makedirs(params.oozie_libext_customer_dir, 0o777)
 
@@ -118,27 +124,31 @@ def prepare_libext_directory():
 
   # get all hadooplzo* JAR files
   # hdp-select set hadoop-client has not run yet, therefore we cannot use
-  # /usr/hdp/current/hadoop-client ; we must use params.current directly
-  hadoop_lzo_pattern = 'hadoop-lzo*.jar'
-  hadoop_client_new_lib_dir = format("/usr/hdp/{version}/hadoop/lib")
-
-  files = glob.iglob(os.path.join(hadoop_client_new_lib_dir, hadoop_lzo_pattern))
-  if not files:
-    raise Fail("There are no files at {0} matching {1}".format(
-      hadoop_client_new_lib_dir, hadoop_lzo_pattern))
-
-  # copy files into libext
-  files_copied = False
-  for file in files:
-    if os.path.isfile(file):
-      files_copied = True
-      Logger.info("Copying {0} to {1}".format(str(file), params.oozie_libext_customer_dir))
-      shutil.copy(file, params.oozie_libext_customer_dir)
-
-  if not files_copied:
-    raise Fail("There are no files at {0} matching {1}".format(
-      hadoop_client_new_lib_dir, hadoop_lzo_pattern))
-
+  # /usr/hdp/current/hadoop-client ; we must use params.version directly
+  # however, this only works when upgrading beyond 2.2.0.0; don't do this
+  # for downgrade to 2.2.0.0 since hadoop-lzo will not be present
+  if params.upgrade_direction == "upgrade" or target_version_needs_compression_libraries:
+    hadoop_lzo_pattern = 'hadoop-lzo*.jar'
+    hadoop_client_new_lib_dir = format("/usr/hdp/{version}/hadoop/lib")
+
+    files = glob.iglob(os.path.join(hadoop_client_new_lib_dir, hadoop_lzo_pattern))
+    if not files:
+      raise Fail("There are no files at {0} matching {1}".format(
+        hadoop_client_new_lib_dir, hadoop_lzo_pattern))
+
+    # copy files into libext
+    files_copied = False
+    for file in files:
+      if os.path.isfile(file):
+        files_copied = True
+        Logger.info("Copying {0} to {1}".format(str(file), params.oozie_libext_customer_dir))
+        shutil.copy(file, params.oozie_libext_customer_dir)
+
+    if not files_copied:
+      raise Fail("There are no files at {0} matching {1}".format(
+        hadoop_client_new_lib_dir, hadoop_lzo_pattern))
+
+  # copy ext ZIP to customer dir
   oozie_ext_zip_file = '/usr/share/HDP-oozie/ext-2.2.zip'
   if not os.path.isfile(oozie_ext_zip_file):
     raise Fail("Unable to copy {0} because it does not exist".format(oozie_ext_zip_file))

ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params.py

@@ -39,6 +39,7 @@ tmp_dir = Script.get_tmp_dir()
 # New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade
 version = default("/commandParams/version", None)
 stack_name = default("/hostLevelParams/stack_name", None)
+upgrade_direction = default("/commandParams/upgrade_direction", None)
 
 stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
 hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)

ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py

@@ -656,6 +656,56 @@ class TestOozieServer(RMFTestCase):
 
     self.assertNoMoreResources()
 
+
+  @patch("tarfile.open")
+  @patch("os.path.isdir")
+  @patch("os.path.exists")
+  @patch("os.path.isfile")
+  @patch("os.remove")
+  @patch("os.chmod")
+  @patch("shutil.rmtree", new = MagicMock())
+  @patch("shutil.copy", new = MagicMock())
+  @patch.object(shell, "call")
+  def test_downgrade_no_compression_library_copy(self, call_mock, chmod_mock, remove_mock,
+      isfile_mock, exists_mock, isdir_mock, tarfile_open_mock):
+
+    isdir_mock.return_value = True
+    exists_mock.side_effect = [False,False,True]
+    isfile_mock.return_value = True
+
+    prepare_war_stdout = """INFO: Adding extension: libext/mysql-connector-java.jar
+    New Oozie WAR file with added 'JARs' at /var/lib/oozie/oozie-server/webapps/oozie.war"""
+
+    call_mock.return_value = (0, prepare_war_stdout)
+
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/oozie_server.py",
+     classname = "OozieServer", command = "pre_rolling_restart", config_file = "oozie-downgrade.json",
+     hdp_stack_version = self.UPGRADE_STACK_VERSION,
+     target = RMFTestCase.TARGET_COMMON_SERVICES )
+
+    # 2 calls to tarfile.open (1 directories, read + write)
+    self.assertTrue(tarfile_open_mock.called)
+    self.assertEqual(tarfile_open_mock.call_count,2)
+
+    self.assertTrue(chmod_mock.called)
+    self.assertEqual(chmod_mock.call_count,1)
+    chmod_mock.assert_called_once_with('/usr/hdp/current/oozie-server/libext-customer', 511)
+
+    self.assertTrue(isfile_mock.called)
+    self.assertEqual(isfile_mock.call_count,3)
+    isfile_mock.assert_called_with('/usr/share/HDP-oozie/ext-2.2.zip')
+
+    self.assertTrue(remove_mock.called)
+    self.assertEqual(remove_mock.call_count,1)
+    remove_mock.assert_called_with('/usr/bin/oozie')
+
+    self.assertResourceCalled('Execute', 'hdp-select set oozie-server 2.2.0.0-0000')
+    self.assertResourceCalled('Execute', 'hdfs dfs -chown oozie:hadoop /user/oozie/share', user='oozie')
+    self.assertResourceCalled('Execute', 'hdfs dfs -chmod -R 755 /user/oozie/share', user='oozie')
+    self.assertResourceCalled('Execute', '/usr/hdp/current/oozie-server/bin/ooziedb.sh upgrade -run', user='oozie')
+    self.assertResourceCalled('Execute', '/usr/hdp/current/oozie-server/bin/oozie-setup.sh sharelib create -fs hdfs://c6401.ambari.apache.org:8020', user='oozie')
+
+
   @patch("tarfile.open")
   @patch("os.path.isdir")
   @patch("os.path.exists")