AMBARI-12969. sys_prepped clusters should not have to copy tarballs again

ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py

@@ -96,7 +96,7 @@ def _get_single_version_from_hdp_select():
   return hdp_version
 
 def copy_to_hdfs(name, user_group, owner, file_mode=0444, custom_source_file=None, custom_dest_file=None, force_execute=False,
-                 use_ru_version_during_ru=True, replace_existing_files=False):
+                 use_ru_version_during_ru=True, replace_existing_files=False, host_sys_prepped=False):
   """
   :param name: Tarball name, e.g., tez, hive, pig, sqoop.
   :param user_group: Group to own the directory.
@@ -106,6 +106,7 @@ def copy_to_hdfs(name, user_group, owner, file_mode=0444, custom_source_file=Non
   :param custom_dest_file: Override the destination file path
   :param force_execute: If true, will execute the HDFS commands immediately, otherwise, will defer to the calling function.
   :param use_ru_version_during_ru: If true, will use the version going to during RU. Otherwise, use the CURRENT (source) version.
+  :param host_sys_prepped: If true, tarballs will not be copied as the cluster deployment uses prepped VMs.
   :return: Will return True if successful, otherwise, False.
   """
   import params
@@ -127,6 +128,10 @@ def copy_to_hdfs(name, user_group, owner, file_mode=0444, custom_source_file=Non
   if custom_dest_file is not None:
     dest_file = custom_dest_file
 
+  if host_sys_prepped:
+    Logger.info("Skipping copying {0} to {1} for {2} as its a sys_prepped host.".format(str(source_file), str(dest_file), str(name)))
+    return True
+
   upgrade_direction = default("/commandParams/upgrade_direction", None)
   is_rolling_upgrade = upgrade_direction is not None
   current_version = default("/hostLevelParams/current_version", None)
@@ -200,4 +205,4 @@ def copy_to_hdfs(name, user_group, owner, file_mode=0444, custom_source_file=Non
   if force_execute:
     params.HdfsResource(None, action="execute")
 
-  return True
+  return True

ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py

@@ -132,8 +132,8 @@ def hive(name=None):
     # *********************************
     # HDP 2.2 or higher, copy mapreduce.tar.gz to HDFS
     if params.hdp_stack_version_major != "" and compare_versions(params.hdp_stack_version_major, '2.2') >= 0:
-      copy_to_hdfs("mapreduce", params.user_group, params.hdfs_user)
-      copy_to_hdfs("tez", params.user_group, params.hdfs_user)
+      copy_to_hdfs("mapreduce", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped)
+      copy_to_hdfs("tez", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped)
 
     # Always copy pig.tar.gz and hive.tar.gz using the appropriate mode.
     # This can use a different source and dest location to account for both HDP 2.1 and 2.2
@@ -142,13 +142,15 @@ def hive(name=None):
                  params.hdfs_user,
                  file_mode=params.tarballs_mode,
                  custom_source_file=params.pig_tar_source,
-                 custom_dest_file=params.pig_tar_dest_file)
+                 custom_dest_file=params.pig_tar_dest_file,
+                 host_sys_prepped=params.host_sys_prepped)
     copy_to_hdfs("hive",
                  params.user_group,
                  params.hdfs_user,
                  file_mode=params.tarballs_mode,
                  custom_source_file=params.hive_tar_source,
-                 custom_dest_file=params.hive_tar_dest_file)
+                 custom_dest_file=params.hive_tar_dest_file,
+                 host_sys_prepped=params.host_sys_prepped)
 
     wildcard_tarballs = ["sqoop", "hadoop_streaming"]
     for tarball_name in wildcard_tarballs:
@@ -168,7 +170,8 @@ def hive(name=None):
                      params.hdfs_user,
                      file_mode=params.tarballs_mode,
                      custom_source_file=source_file,
-                     custom_dest_file=dest_file)
+                     custom_dest_file=dest_file,
+                     host_sys_prepped=params.host_sys_prepped)
     # ******* End Copy Tarballs *******
     # *********************************
 

ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py

@@ -115,8 +115,16 @@ class HiveServerDefault(HiveServer):
       hdp_select.select("hive-server2", params.version)
 
       # Copy mapreduce.tar.gz and tez.tar.gz to HDFS
-      resource_created = copy_to_hdfs("mapreduce", params.user_group, params.hdfs_user)
-      resource_created = copy_to_hdfs("tez", params.user_group, params.hdfs_user) or resource_created
+      resource_created = copy_to_hdfs(
+        "mapreduce",
+        params.user_group,
+        params.hdfs_user,
+        host_sys_prepped=params.host_sys_prepped)
+      resource_created = copy_to_hdfs(
+        "tez",
+        params.user_group,
+        params.hdfs_user,
+        host_sys_prepped=params.host_sys_prepped) or resource_created
       if resource_created:
         params.HdfsResource(None, action="execute")
 

ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/service_check.py

@@ -95,7 +95,10 @@ class PigServiceCheckLinux(PigServiceCheck):
       )
 
       # Check for Pig-on-Tez
-      resource_created = copy_to_hdfs("tez", params.user_group, params.hdfs_user)
+      resource_created = copy_to_hdfs(
+        "tez", params.user_group,
+        params.hdfs_user,
+        host_sys_prepped=params.host_sys_prepped)
       if resource_created:
         params.HdfsResource(None, action="execute")
 

ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py

@@ -82,7 +82,11 @@ class JobHistoryServer(Script):
       # need to copy the tarball, otherwise, copy it.
 
       if params.version and compare_versions(format_hdp_stack_version(params.version), '2.3.0.0') < 0:
-        resource_created = copy_to_hdfs("tez", params.user_group, params.hdfs_user)
+        resource_created = copy_to_hdfs(
+          "tez",
+          params.user_group,
+          params.hdfs_user,
+          host_sys_prepped=params.host_sys_prepped)
         if resource_created:
           params.HdfsResource(None, action="execute")
 

ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_service.py

@@ -36,7 +36,7 @@ def spark_service(action):
     # Spark 1.3.1.2.3, and higher, which was included in HDP 2.3, does not have a dependency on Tez, so it does not
     # need to copy the tarball, otherwise, copy it.
     if params.hdp_stack_version and compare_versions(params.hdp_stack_version, '2.3.0.0') < 0:
-      resource_created = copy_to_hdfs("tez", params.user_group, params.hdfs_user)
+      resource_created = copy_to_hdfs("tez", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped)
       if resource_created:
         params.HdfsResource(None, action="execute")
 

ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/pre_upgrade.py

@@ -42,7 +42,12 @@ class TezPreUpgrade(Script):
       Logger.info("Stack version {0} is sufficient to check if need to copy tez.tar.gz to HDFS.".format(params.hdp_stack_version))
 
       # Force it to copy the current version of the tez tarball, rather than the version the RU will go to.
-      resource_created = copy_to_hdfs("tez", params.user_group, params.hdfs_user, use_ru_version_during_ru=False)
+      resource_created = copy_to_hdfs(
+        "tez",
+        params.user_group,
+        params.hdfs_user,
+        use_ru_version_during_ru=False,
+        host_sys_prepped=params.host_sys_prepped)
       if resource_created:
         params.HdfsResource(None, action="execute")
       else:

ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/service_check.py

@@ -63,7 +63,7 @@ class TezServiceCheckLinux(TezServiceCheck):
     )
 
     if params.hdp_stack_version and compare_versions(params.hdp_stack_version, '2.2.0.0') >= 0:
-      copy_to_hdfs("tez", params.user_group, params.hdfs_user)
+      copy_to_hdfs("tez", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped)
 
     params.HdfsResource(None, action = "execute")
 

ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py

@@ -81,8 +81,8 @@ class HistoryServerDefault(HistoryServer):
       conf_select.select(params.stack_name, "hadoop", params.version)
       hdp_select.select("hadoop-mapreduce-historyserver", params.version)
       # MC Hammer said, "Can't touch this"
-      copy_to_hdfs("mapreduce", params.user_group, params.hdfs_user)
-      copy_to_hdfs("tez", params.user_group, params.hdfs_user)
+      copy_to_hdfs("mapreduce", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped)
+      copy_to_hdfs("tez", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped)
       params.HdfsResource(None, action="execute")
 
   def start(self, env, rolling_restart=False):
@@ -92,8 +92,16 @@ class HistoryServerDefault(HistoryServer):
 
     if params.hdp_stack_version_major and compare_versions(params.hdp_stack_version_major, '2.2.0.0') >= 0:
       # MC Hammer said, "Can't touch this"
-      resource_created = copy_to_hdfs("mapreduce", params.user_group, params.hdfs_user)
-      resource_created = copy_to_hdfs("tez", params.user_group, params.hdfs_user) or resource_created
+      resource_created = copy_to_hdfs(
+        "mapreduce",
+        params.user_group,
+        params.hdfs_user,
+        host_sys_prepped=params.host_sys_prepped)
+      resource_created = copy_to_hdfs(
+        "tez",
+        params.user_group,
+        params.hdfs_user,
+        host_sys_prepped=params.host_sys_prepped) or resource_created
       if resource_created:
         params.HdfsResource(None, action="execute")
     else:

ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py

@@ -867,8 +867,8 @@ From source with checksum 150f554beae04f76f814f59549dead8b"""
     self.assertResourceCalled('Execute',
                               ('hdp-select', 'set', 'hive-server2', version), sudo=True,)
 
-    copy_to_hdfs_mock.assert_any_call("mapreduce", "hadoop", "hdfs")
-    copy_to_hdfs_mock.assert_any_call("tez", "hadoop", "hdfs")
+    copy_to_hdfs_mock.assert_any_call("mapreduce", "hadoop", "hdfs", host_sys_prepped=False)
+    copy_to_hdfs_mock.assert_any_call("tez", "hadoop", "hdfs", host_sys_prepped=False)
     self.assertEquals(2, copy_to_hdfs_mock.call_count)
     self.assertResourceCalled('HdfsResource', None,
         security_enabled = False,
@@ -905,8 +905,8 @@ From source with checksum 150f554beae04f76f814f59549dead8b"""
     self.assertResourceCalled('Execute',
 
                               ('hdp-select', 'set', 'hive-server2', version), sudo=True,)
-    copy_to_hdfs_mock.assert_any_call("mapreduce", "hadoop", "hdfs")
-    copy_to_hdfs_mock.assert_any_call("tez", "hadoop", "hdfs")
+    copy_to_hdfs_mock.assert_any_call("mapreduce", "hadoop", "hdfs", host_sys_prepped=False)
+    copy_to_hdfs_mock.assert_any_call("tez", "hadoop", "hdfs", host_sys_prepped=False)
     self.assertEquals(2, copy_to_hdfs_mock.call_count)
     self.assertResourceCalled('HdfsResource', None,
         security_enabled = False,

ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py

@@ -767,7 +767,7 @@ class TestHistoryServer(RMFTestCase):
                        mocks_dict = mocks_dict)
 
     self.assertResourceCalled('Execute', ('hdp-select', 'set', 'hadoop-mapreduce-historyserver', version), sudo=True)
-    copy_to_hdfs_mock.assert_called_with("tez", "hadoop", "hdfs")
+    copy_to_hdfs_mock.assert_called_with("tez", "hadoop", "hdfs", host_sys_prepped=False)
 
     self.assertResourceCalled('HdfsResource', None,
         security_enabled = False,

ambari-server/src/test/python/stacks/2.2/PIG/test_pig_service_check.py

@@ -125,7 +125,7 @@ class TestPigServiceCheck(RMFTestCase):
         action = ['create_on_execute'],
     )
 
-    copy_to_hdfs_mock.assert_called_with("tez", "hadoop", "hdfs")
+    copy_to_hdfs_mock.assert_called_with("tez", "hadoop", "hdfs", host_sys_prepped=False)
     self.assertResourceCalled('HdfsResource', None,
         security_enabled = True,
         hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',