AMBARI-12726 - Oozie RU Downgrade Fails When Agents Run As Non-Root (jonathanhurley)

ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py

@@ -19,13 +19,14 @@ limitations under the License.
 import glob
 import os
 import shutil
-import tarfile
 import tempfile
 
 from resource_management.core import shell
+from resource_management.core import sudo
 from resource_management.core.logger import Logger
 from resource_management.core.exceptions import Fail
-from resource_management.core.resources.system import Execute, Directory
+from resource_management.core.resources.system import Execute
+from resource_management.core.resources.system import Directory
 from resource_management.libraries.functions import Direction
 from resource_management.libraries.functions import format
 from resource_management.libraries.functions import compare_versions
@@ -143,11 +144,17 @@ class OozieUpgrade(Script):
 
     # copy ext ZIP to libext dir
     oozie_ext_zip_file = '/usr/share/HDP-oozie/ext-2.2.zip'
+
+    # something like /usr/hdp/current/oozie-server/libext/ext-2.2.zip
+    oozie_ext_zip_target_path = os.path.join(params.oozie_libext_dir, "ext-2.2.zip")
+
     if not os.path.isfile(oozie_ext_zip_file):
       raise Fail("Unable to copy {0} because it does not exist".format(oozie_ext_zip_file))
 
     Logger.info("Copying {0} to {1}".format(oozie_ext_zip_file, params.oozie_libext_dir))
-    shutil.copy2(oozie_ext_zip_file, params.oozie_libext_dir)
+    Execute(("cp", oozie_ext_zip_file, params.oozie_libext_dir), sudo=True)
+    Execute(("chown", format("{oozie_user}:{user_group}"), oozie_ext_zip_target_path), sudo=True)
+    sudo.chmod(oozie_ext_zip_target_path, 0644)
 
     # Redownload jdbc driver to a new current location
     oozie.download_database_library_if_needed()

ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py

@@ -209,11 +209,11 @@ class TestOozieServer(RMFTestCase):
         not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1'",
         sudo = True,
     )
-    self.assertResourceCalled('Execute', ('chown', u'oozie:hadoop', '/usr/lib/oozie/libext/ext-2.2.zip'),
+    self.assertResourceCalled('Execute', ('chown', 'oozie:hadoop', '/usr/lib/oozie/libext/ext-2.2.zip'),
         not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1'",
         sudo = True,
     )
-    self.assertResourceCalled('Execute', ('chown', '-RL', u'oozie:hadoop', '/var/lib/oozie/oozie-server/conf'),
+    self.assertResourceCalled('Execute', ('chown', '-RL', 'oozie:hadoop', '/var/lib/oozie/oozie-server/conf'),
         not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1'",
         sudo = True,
     )
@@ -249,7 +249,7 @@ class TestOozieServer(RMFTestCase):
         content = 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-setup.sh prepare-war ',
         mode = 0644,
     )
-    self.assertResourceCalled('Execute', ('chown', '-R', u'oozie:hadoop', '/var/lib/oozie/oozie-server'),
+    self.assertResourceCalled('Execute', ('chown', '-R', 'oozie:hadoop', '/var/lib/oozie/oozie-server'),
         sudo = True,
     )
 
@@ -558,11 +558,11 @@ class TestOozieServer(RMFTestCase):
         not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1'",
         sudo = True,
     )
-    self.assertResourceCalled('Execute', ('chown', u'oozie:hadoop', '/usr/lib/oozie/libext/ext-2.2.zip'),
+    self.assertResourceCalled('Execute', ('chown', 'oozie:hadoop', '/usr/lib/oozie/libext/ext-2.2.zip'),
         not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1'",
         sudo = True,
     )
-    self.assertResourceCalled('Execute', ('chown', '-RL', u'oozie:hadoop', '/var/lib/oozie/oozie-server/conf'),
+    self.assertResourceCalled('Execute', ('chown', '-RL', 'oozie:hadoop', '/var/lib/oozie/oozie-server/conf'),
         not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1'",
         sudo = True,
     )
@@ -584,7 +584,7 @@ class TestOozieServer(RMFTestCase):
         content = 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-setup.sh prepare-war ',
         mode = 0644,
     )
-    self.assertResourceCalled('Execute', ('chown', '-R', u'oozie:hadoop', '/var/lib/oozie/oozie-server'),
+    self.assertResourceCalled('Execute', ('chown', '-R', 'oozie:hadoop', '/var/lib/oozie/oozie-server'),
         sudo = True,
     )
 
@@ -734,11 +734,11 @@ class TestOozieServer(RMFTestCase):
         not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1'",
         sudo = True,
     )
-    self.assertResourceCalled('Execute', ('chown', u'oozie:hadoop', '/usr/lib/oozie/libext/ext-2.2.zip'),
+    self.assertResourceCalled('Execute', ('chown', 'oozie:hadoop', '/usr/lib/oozie/libext/ext-2.2.zip'),
         not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1'",
         sudo = True,
     )
-    self.assertResourceCalled('Execute', ('chown', '-RL', u'oozie:hadoop', '/var/lib/oozie/oozie-server/conf'),
+    self.assertResourceCalled('Execute', ('chown', '-RL', 'oozie:hadoop', '/var/lib/oozie/oozie-server/conf'),
         not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1'",
         sudo = True,
     )
@@ -760,7 +760,7 @@ class TestOozieServer(RMFTestCase):
         content = 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-setup.sh prepare-war -secure',
         mode = 0644,
     )
-    self.assertResourceCalled('Execute', ('chown', '-R', u'oozie:hadoop', '/var/lib/oozie/oozie-server'),
+    self.assertResourceCalled('Execute', ('chown', '-R', 'oozie:hadoop', '/var/lib/oozie/oozie-server'),
         sudo = True,
     )
 
@@ -943,8 +943,7 @@ class TestOozieServer(RMFTestCase):
      classname = "OozieServer", command = "pre_rolling_restart", config_file = "oozie-upgrade.json",
      hdp_stack_version = self.UPGRADE_STACK_VERSION,
      target = RMFTestCase.TARGET_COMMON_SERVICES,
-     call_mocks = [(0, prepare_war_stdout)]
-    )
+     call_mocks = [(0, prepare_war_stdout)])
     
     self.assertTrue(isfile_mock.called)
     self.assertEqual(isfile_mock.call_count,3)
@@ -954,28 +953,21 @@ class TestOozieServer(RMFTestCase):
     self.assertEqual(glob_mock.call_count,1)
     glob_mock.assert_called_with('/usr/hdp/2.2.1.0-2135/hadoop/lib/hadoop-lzo*.jar')
 
-    self.assertResourceCalled('Execute', ('tar',
-     '-zcvhf',
-     '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar',
-     '/usr/hdp/current/oozie-server/conf/'),
-        sudo = True,
-    )
+    self.assertResourceCalled('Execute',
+      ('tar', '-zcvhf', '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar', '/usr/hdp/current/oozie-server/conf/'),
+      sudo = True)
+
     self.assertResourceCalled('Execute', ('hdp-select', 'set', 'oozie-server', u'2.2.1.0-2135'),
-        sudo = True,
-    )
-    self.assertResourceCalled('Execute', ('tar',
-     '-xvf',
-     '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar',
-     '-C',
-     '/usr/hdp/current/oozie-server/conf//'),
-        sudo = True,
-    )
-    self.assertResourceCalled('Directory', '/tmp/oozie-upgrade-backup',
-        action = ['delete'],
-    )
-    self.assertResourceCalled('Directory', '/usr/hdp/current/oozie-server/libext',
-        mode = 0777,
-    )
+      sudo = True )
+
+    self.assertResourceCalled('Execute',
+      ('tar', '-xvf', '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar', '-C', '/usr/hdp/current/oozie-server/conf//'),
+        sudo = True)
+
+    self.assertResourceCalled('Directory', '/tmp/oozie-upgrade-backup', action = ['delete'])
+    self.assertResourceCalled('Directory', '/usr/hdp/current/oozie-server/libext', mode = 0777)
+    self.assertResourceCalled('Execute', ('cp', '/usr/share/HDP-oozie/ext-2.2.zip', '/usr/hdp/current/oozie-server/libext'), sudo=True)
+    self.assertResourceCalled('Execute', ('chown', 'oozie:hadoop', '/usr/hdp/current/oozie-server/libext/ext-2.2.zip'), sudo=True)
     self.assertNoMoreResources()
 
   @patch("os.path.isdir")
@@ -1026,32 +1018,26 @@ class TestOozieServer(RMFTestCase):
     self.assertEqual(glob_mock.call_count,1)
     glob_mock.assert_called_with('/usr/hdp/2.3.0.0-1234/hadoop/lib/hadoop-lzo*.jar')
 
-    self.assertResourceCalled('Execute', ('tar',
-     '-zcvhf',
-     '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar',
-     '/usr/hdp/current/oozie-server/conf/'),
-        sudo = True)
+    self.assertResourceCalled('Execute',
+      ('tar', '-zcvhf', '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar', '/usr/hdp/current/oozie-server/conf/'),
+      sudo = True)
 
-    self.assertResourceCalled('Execute', ('hdp-select', 'set', 'oozie-server', '2.3.0.0-1234'),
-        sudo = True)
+    self.assertResourceCalled('Execute', ('hdp-select', 'set', 'oozie-server', '2.3.0.0-1234'), sudo = True)
 
-    self.assertResourceCalled('Execute', ('tar',
-     '-xvf',
-     '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar',
-     '-C',
-     '/usr/hdp/current/oozie-server/conf//'),
-        sudo = True)
+    self.assertResourceCalled('Execute',
+      ('tar', '-xvf', '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar', '-C', '/usr/hdp/current/oozie-server/conf//'),
+      sudo = True)
 
-    self.assertResourceCalled('Directory', '/tmp/oozie-upgrade-backup',
-        action = ['delete'])
+    self.assertResourceCalled('Directory', '/tmp/oozie-upgrade-backup', action = ['delete'])
+    self.assertResourceCalled('Directory', '/usr/hdp/current/oozie-server/libext', mode = 0777)
 
-    self.assertResourceCalled('Directory', '/usr/hdp/current/oozie-server/libext',
-        mode = 0777)
+    self.assertResourceCalled('Execute', ('cp', '/usr/share/HDP-oozie/ext-2.2.zip', '/usr/hdp/current/oozie-server/libext'), sudo=True)
+    self.assertResourceCalled('Execute', ('chown', 'oozie:hadoop', '/usr/hdp/current/oozie-server/libext/ext-2.2.zip'), sudo=True)
 
     self.assertNoMoreResources()
 
     self.assertEquals(1, mocks_dict['call'].call_count)
-    self.assertEquals(1, mocks_dict['checked_call'].call_count)
+    self.assertEquals(2, mocks_dict['checked_call'].call_count)
 
     self.assertEquals(
       ('conf-select', 'set-conf-dir', '--package', 'oozie', '--stack-version', '2.3.0.0-1234', '--conf-version', '0'),
@@ -1087,28 +1073,21 @@ class TestOozieServer(RMFTestCase):
     self.assertTrue(isfile_mock.called)
     self.assertEqual(isfile_mock.call_count,2)
     isfile_mock.assert_called_with('/usr/share/HDP-oozie/ext-2.2.zip')
-    self.assertResourceCalled('Execute', ('tar',
-     '-zcvhf',
-     '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar',
-     '/usr/hdp/current/oozie-server/conf/'),
-        sudo = True,
-    )
-    self.assertResourceCalled('Execute', ('hdp-select', 'set', 'oozie-server', u'2.2.0.0-0000'),
-        sudo = True,
-    )
-    self.assertResourceCalled('Execute', ('tar',
-     '-xvf',
-     '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar',
-     '-C',
-     '/usr/hdp/current/oozie-server/conf//'),
-        sudo = True,
-    )
-    self.assertResourceCalled('Directory', '/tmp/oozie-upgrade-backup',
-        action = ['delete'],
-    )
-    self.assertResourceCalled('Directory', '/usr/hdp/current/oozie-server/libext',
-        mode = 0777,
-    )
+
+    self.assertResourceCalled('Execute',
+      ('tar', '-zcvhf', '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar', '/usr/hdp/current/oozie-server/conf/'),
+      sudo = True)
+
+    self.assertResourceCalled('Execute', ('hdp-select', 'set', 'oozie-server', u'2.2.0.0-0000'), sudo = True)
+
+    self.assertResourceCalled('Execute',
+      ('tar', '-xvf', '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar', '-C', '/usr/hdp/current/oozie-server/conf//'),
+      sudo = True)
+
+    self.assertResourceCalled('Directory', '/tmp/oozie-upgrade-backup', action = ['delete'])
+    self.assertResourceCalled('Directory', '/usr/hdp/current/oozie-server/libext',mode = 0777)
+    self.assertResourceCalled('Execute', ('cp', '/usr/share/HDP-oozie/ext-2.2.zip', '/usr/hdp/current/oozie-server/libext'), sudo=True)
+    self.assertResourceCalled('Execute', ('chown', 'oozie:hadoop', '/usr/hdp/current/oozie-server/libext/ext-2.2.zip'), sudo=True)
     self.assertNoMoreResources()
 
 
@@ -1288,27 +1267,21 @@ class TestOozieServer(RMFTestCase):
     self.assertEqual(glob_mock.call_count,1)
     glob_mock.assert_called_with('/usr/hdp/2.3.0.0-1234/hadoop/lib/hadoop-lzo*.jar')
 
-    self.assertResourceCalled('Execute', ('tar',
-     '-zcvhf',
-     '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar',
-     '/usr/hdp/current/oozie-server/conf/'),
-        sudo = True)
+    self.assertResourceCalled('Execute',
+      ('tar', '-zcvhf', '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar', '/usr/hdp/current/oozie-server/conf/'),
+      sudo = True)
 
-    self.assertResourceCalled('Execute', ('hdp-select', 'set', 'oozie-server', '2.3.0.0-1234'),
-        sudo = True)
+    self.assertResourceCalled('Execute', ('hdp-select', 'set', 'oozie-server', '2.3.0.0-1234'), sudo = True)
 
-    self.assertResourceCalled('Execute', ('tar',
-     '-xvf',
-     '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar',
-     '-C',
-     '/usr/hdp/current/oozie-server/conf//'),
-        sudo = True)
+    self.assertResourceCalled('Execute',
+      ('tar', '-xvf', '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar', '-C', '/usr/hdp/current/oozie-server/conf//'),
+      sudo = True)
 
-    self.assertResourceCalled('Directory', '/tmp/oozie-upgrade-backup',
-        action = ['delete'])
+    self.assertResourceCalled('Directory', '/tmp/oozie-upgrade-backup', action = ['delete'])
+    self.assertResourceCalled('Directory', '/usr/hdp/current/oozie-server/libext', mode = 0777)
 
-    self.assertResourceCalled('Directory', '/usr/hdp/current/oozie-server/libext',
-        mode = 0777)
+    self.assertResourceCalled('Execute', ('cp', '/usr/share/HDP-oozie/ext-2.2.zip', '/usr/hdp/current/oozie-server/libext'), sudo=True)
+    self.assertResourceCalled('Execute', ('chown', 'oozie:hadoop', '/usr/hdp/current/oozie-server/libext/ext-2.2.zip'), sudo=True)
 
     self.assertResourceCalled('Execute', 'ambari-sudo.sh cp /usr/hdp/2.3.0.0-1234/falcon/oozie/ext/falcon-oozie-el-extension-*.jar /usr/hdp/current/oozie-server/libext')
     self.assertResourceCalled('Execute', 'ambari-sudo.sh chown oozie:hadoop /usr/hdp/current/oozie-server/libext/falcon-oozie-el-extension-*.jar')
@@ -1316,7 +1289,7 @@ class TestOozieServer(RMFTestCase):
     self.assertNoMoreResources()
 
     self.assertEquals(1, mocks_dict['call'].call_count)
-    self.assertEquals(1, mocks_dict['checked_call'].call_count)
+    self.assertEquals(2, mocks_dict['checked_call'].call_count)
 
     self.assertEquals(
       ('conf-select', 'set-conf-dir', '--package', 'oozie', '--stack-version', '2.3.0.0-1234', '--conf-version', '0'),