AMBARI-15057. Oozie untar and prepare-war should be only done exclusively in preupload.py (aonishuk)

ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py

@@ -17,7 +17,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 
 """
-import hashlib
 import os
 
 from resource_management.core.resources.service import ServiceConfig
@@ -228,11 +227,7 @@ def prepare_war():
 
   if run_prepare_war:
     # Time-consuming to run
-    Execute(command,
-            user=params.oozie_user
-    )
-
-    return_code, output = shell.call(command, user=params.oozie_user, logoutput=False, quiet=False)
+    return_code, output = shell.call(command, user=params.oozie_user)
     if output is None:
       output = ""
 
@@ -273,8 +268,7 @@ def oozie_server_specific():
   )
   
   hashcode_file = format("{oozie_home}/.hashcode")
-  hashcode = hashlib.md5(format('{oozie_home}/oozie-sharelib.tar.gz')).hexdigest()
-  skip_recreate_sharelib = format("test -f {hashcode_file} && test -d {oozie_home}/share && [[ `cat {hashcode_file}` == '{hashcode}' ]]")
+  skip_recreate_sharelib = format("test -f {hashcode_file} && test -d {oozie_home}/share")
 
   untar_sharelib = ('tar','-xvf',format('{oozie_home}/oozie-sharelib.tar.gz'),'-C',params.oozie_home)
 
@@ -319,7 +313,6 @@ def oozie_server_specific():
   prepare_war()
 
   File(hashcode_file,
-       content = hashcode,
        mode = 0644,
   )
 

ambari-server/src/main/resources/scripts/Ambaripreupload.py

@@ -26,7 +26,6 @@ sys.path.append("/usr/lib/python2.6/site-packages")
 import glob
 from logging import thread
 import re
-import hashlib
 import tempfile
 import time
 import functools
@@ -143,6 +142,7 @@ with Environment() as env:
     hdfs_site = ConfigDictionary({'dfs.webhdfs.enabled':False, 
     })
     fs_default = get_fs_root()
+    oozie_secure = ''
     oozie_env_sh_template = \
   '''
   #!/bin/bash
@@ -232,14 +232,13 @@ with Environment() as env:
     source_and_dest_pairs = [(component_tar_source_file, destination_file), ]
     return _copy_files(source_and_dest_pairs, file_owner, group_owner, kinit_if_needed)
 
-
-
   env.set_params(params)
   hadoop_conf_dir = params.hadoop_conf_dir
    
   oozie_libext_dir = format("/usr/hdp/{hdp_version}/oozie/libext")
   oozie_home=format("/usr/hdp/{hdp_version}/oozie")
   oozie_setup_sh=format("/usr/hdp/{hdp_version}/oozie/bin/oozie-setup.sh")
+  oozie_setup_sh_current="/usr/hdp/current/oozie-server/bin/oozie-setup.sh"
   oozie_tmp_dir = "/var/tmp/oozie"
   configure_cmds = []
   configure_cmds.append(('tar','-xvf', oozie_home + '/oozie-sharelib.tar.gz','-C', oozie_home))
@@ -254,22 +253,62 @@ with Environment() as env:
   )
 
   hashcode_file = format("{oozie_home}/.hashcode")
-  hashcode = hashlib.md5(format('{oozie_home}/oozie-sharelib.tar.gz')).hexdigest()
-  skip_recreate_sharelib = format("test -f {hashcode_file} && test -d {oozie_home}/share && [[ `cat {hashcode_file}` == '{hashcode}' ]]")
+  skip_recreate_sharelib = format("test -f {hashcode_file} && test -d {oozie_home}/share")
 
   Execute( configure_cmds,
            not_if  = format("{no_op_test} || {skip_recreate_sharelib}"), 
            sudo = True,
            )
-  Execute(format("cd {oozie_tmp_dir} && {oozie_setup_sh} prepare-war"),
-    user = params.oozie_user,
-    not_if  = format("{no_op_test} || {skip_recreate_sharelib}")
-  )
+  
   File(hashcode_file,
-       content = hashcode,
        mode = 0644,
   )
+  
+  ###############################################
+  # PREPARE-WAR [BEGIN]
+  ###############################################
+  prepare_war_cmd_file = format("{oozie_home}/.prepare_war_cmd")
+
+  # DON'T CHANGE THE VALUE SINCE IT'S USED TO DETERMINE WHETHER TO RUN THE COMMAND OR NOT BY READING THE MARKER FILE.
+  # Oozie tmp dir should be /var/tmp/oozie and is already created by a function above.
+  command = format("cd {oozie_tmp_dir} && {oozie_setup_sh} prepare-war {oozie_secure} ")
+  command_to_file = format("cd {oozie_tmp_dir} && {oozie_setup_sh_current} prepare-war {oozie_secure} ")
+
+  run_prepare_war = False
+  if os.path.exists(prepare_war_cmd_file):
+    cmd = ""
+    with open(prepare_war_cmd_file, "r") as f:
+      cmd = f.readline().strip()
 
+    if command_to_file != cmd:
+      run_prepare_war = True
+      Logger.info(format("Will run prepare war cmd since marker file {prepare_war_cmd_file} has contents which differ.\n" \
+      "Expected: {command_to_file}.\nActual: {cmd}."))
+  else:
+    run_prepare_war = True
+    Logger.info(format("Will run prepare war cmd since marker file {prepare_war_cmd_file} is missing."))
+
+  if run_prepare_war:
+    # Time-consuming to run
+    return_code, output = shell.call(command, user=params.oozie_user)
+    if output is None:
+      output = ""
+
+    if return_code != 0 or "New Oozie WAR file with added".lower() not in output.lower():
+      message = "Unexpected Oozie WAR preparation output {0}".format(output)
+      Logger.error(message)
+      raise Fail(message)
+
+    # Generate marker file
+    File(prepare_war_cmd_file,
+         content=command_to_file,
+         mode=0644,
+    )
+  else:
+    Logger.info(format("No need to run prepare-war since marker file {prepare_war_cmd_file} already exists."))
+  ###############################################
+  # PREPARE-WAR END [BEGIN]
+  ###############################################
   oozie_shared_lib = format("/usr/hdp/{hdp_version}/oozie/share")
   oozie_user = 'oozie'
   oozie_hdfs_user_dir = format("{hdfs_path_prefix}/user/{oozie_user}")

ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py

@@ -25,14 +25,9 @@ from resource_management.core import shell
 from resource_management.core.exceptions import Fail
 from resource_management.libraries import functions
 from resource_management.libraries.providers.hdfs_resource import WebHDFSUtil
-import hashlib
 import tempfile
 
-md5_mock = MagicMock()
-md5_mock.hexdigest.return_value = "abc123hash"
-
 @patch("platform.linux_distribution", new = MagicMock(return_value="Linux"))
-@patch.object(hashlib, "md5", new=MagicMock(return_value=md5_mock))
 @patch.object(WebHDFSUtil, "run_command", new=MagicMock(return_value={}))
 @patch.object(tempfile, "gettempdir", new=MagicMock(return_value="/tmp"))
 class TestOozieServer(RMFTestCase):
@@ -221,7 +216,7 @@ class TestOozieServer(RMFTestCase):
                               create_parents = True,
                               )
     self.assertResourceCalled('Execute', ('tar', '-xvf', '/usr/lib/oozie/oozie-sharelib.tar.gz', '-C', '/usr/lib/oozie'),
-        not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1' || test -f /usr/lib/oozie/.hashcode && test -d /usr/lib/oozie/share && [[ `cat /usr/lib/oozie/.hashcode` == 'abc123hash' ]]",
+        not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1' || test -f /usr/lib/oozie/.hashcode && test -d /usr/lib/oozie/share",
         sudo = True,
     )
     self.assertResourceCalled('Execute', ('cp', '/usr/share/HDP-oozie/ext-2.2.zip', '/usr/lib/oozie/libext'),
@@ -259,15 +254,11 @@ class TestOozieServer(RMFTestCase):
         not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1'",
     )
 
-    self.assertResourceCalled('Execute', 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-setup.sh prepare-war',
-                              user = "oozie"
-    )
     self.assertResourceCalled('File', '/usr/lib/oozie/.prepare_war_cmd',
                               content = 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-setup.sh prepare-war',
                               mode = 0644,
     )
     self.assertResourceCalled('File', '/usr/lib/oozie/.hashcode',
-                              content = 'abc123hash',
                               mode = 0644,
     )
     self.assertResourceCalled('Directory', '/var/lib/oozie/oozie-server',
@@ -443,9 +434,9 @@ class TestOozieServer(RMFTestCase):
                               create_parents = True,
                               )
     self.assertResourceCalled('Execute', ('tar', '-xvf', '/usr/lib/oozie/oozie-sharelib.tar.gz', '-C', '/usr/lib/oozie'),
-                              not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1' || test -f /usr/lib/oozie/.hashcode && test -d /usr/lib/oozie/share && [[ `cat /usr/lib/oozie/.hashcode` == 'abc123hash' ]]",
-                              sudo = True,
-                              )
+        not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1' || test -f /usr/lib/oozie/.hashcode && test -d /usr/lib/oozie/share",
+        sudo = True,
+    )
     self.assertResourceCalled('Execute', ('cp', '/usr/share/HDP-oozie/ext-2.2.zip', '/usr/lib/oozie/libext'),
                               not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1'",
                               sudo = True,
@@ -482,15 +473,11 @@ class TestOozieServer(RMFTestCase):
     self.assertResourceCalled('Execute', 'ambari-sudo.sh chown oozie:hadoop /usr/lib/oozie/libext/falcon-oozie-el-extension-*.jar',
                               not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1'",
                               )
-    self.assertResourceCalled('Execute', 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-setup.sh prepare-war',
-                              user = "oozie"
-    )
     self.assertResourceCalled('File', '/usr/lib/oozie/.prepare_war_cmd',
                               content = 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-setup.sh prepare-war',
                               mode = 0644,
     )
     self.assertResourceCalled('File', '/usr/lib/oozie/.hashcode',
-                              content = 'abc123hash',
                               mode = 0644,
     )
     self.assertResourceCalled('Directory', '/var/lib/oozie/oozie-server',
@@ -822,7 +809,7 @@ class TestOozieServer(RMFTestCase):
         create_parents = True,
     )
     self.assertResourceCalled('Execute', ('tar', '-xvf', '/usr/lib/oozie/oozie-sharelib.tar.gz', '-C', '/usr/lib/oozie'),
-        not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1' || test -f /usr/lib/oozie/.hashcode && test -d /usr/lib/oozie/share && [[ `cat /usr/lib/oozie/.hashcode` == 'abc123hash' ]]",
+        not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1' || test -f /usr/lib/oozie/.hashcode && test -d /usr/lib/oozie/share",
         sudo = True,
     )
     self.assertResourceCalled('Execute', ('cp', '/usr/share/HDP-oozie/ext-2.2.zip', '/usr/lib/oozie/libext'),
@@ -846,15 +833,11 @@ class TestOozieServer(RMFTestCase):
         not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1'",
     )
 
-    self.assertResourceCalled('Execute', 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-setup.sh prepare-war',
-                              user="oozie")
-
     self.assertResourceCalled('File', '/usr/lib/oozie/.prepare_war_cmd',
                               content = 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-setup.sh prepare-war',
                               mode = 0644,
     )
     self.assertResourceCalled('File', '/usr/lib/oozie/.hashcode',
-                              content = 'abc123hash',
                               mode = 0644,
     )
     self.assertResourceCalled('Directory', '/var/lib/oozie/oozie-server',
@@ -1015,7 +998,7 @@ class TestOozieServer(RMFTestCase):
         create_parents = True,
     )
     self.assertResourceCalled('Execute', ('tar', '-xvf', '/usr/lib/oozie/oozie-sharelib.tar.gz', '-C', '/usr/lib/oozie'),
-        not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1' || test -f /usr/lib/oozie/.hashcode && test -d /usr/lib/oozie/share && [[ `cat /usr/lib/oozie/.hashcode` == 'abc123hash' ]]",
+        not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1' || test -f /usr/lib/oozie/.hashcode && test -d /usr/lib/oozie/share",
         sudo = True,
     )
     self.assertResourceCalled('Execute', ('cp', '/usr/share/HDP-oozie/ext-2.2.zip', '/usr/lib/oozie/libext'),
@@ -1039,15 +1022,11 @@ class TestOozieServer(RMFTestCase):
         not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1'",
     )
 
-    self.assertResourceCalled('Execute', 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-setup.sh prepare-war -secure',
-                              user="oozie")
-
     self.assertResourceCalled('File', '/usr/lib/oozie/.prepare_war_cmd',
                               content = 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-setup.sh prepare-war -secure',
                               mode = 0644,
     )
     self.assertResourceCalled('File', '/usr/lib/oozie/.hashcode',
-                              content = 'abc123hash',
                               mode = 0644,
     )
     self.assertResourceCalled('Directory', '/var/lib/oozie/oozie-server',