AMBARI-7333. Tez deployment changes for Champlain (aonishuk)

ambari-common/src/main/python/resource_management/libraries/providers/copy_from_local.py

@@ -27,6 +27,7 @@ class CopyFromLocalProvider(Provider):
   def action_run(self):
     path = self.resource.path
     dest_dir = self.resource.dest_dir
+    dest_file = self.resource.dest_file
     kinnit_if_needed = self.resource.kinnit_if_needed
     owner = self.resource.owner
     group = self.resource.group
@@ -34,9 +35,14 @@ class CopyFromLocalProvider(Provider):
     hdfs_usr=self.resource.hdfs_user
     hadoop_conf_path = self.resource.hadoop_conf_dir
 
-    copy_cmd = format("fs -copyFromLocal {path} {dest_dir}")
-    dest_file_name = os.path.split(path)[1]
-    dest_path = dest_dir + dest_file_name if dest_dir.endswith(os.sep) else dest_dir + os.sep + dest_file_name
+
+    if dest_file:
+      copy_cmd = format("fs -copyFromLocal {path} {dest_dir}/{dest_file}")
+      dest_path = dest_dir + dest_file if dest_dir.endswith(os.sep) else dest_dir + os.sep + dest_file
+    else:
+      dest_file_name = os.path.split(path)[1]
+      copy_cmd = format("fs -copyFromLocal {path} {dest_dir}")
+      dest_path = dest_dir + os.sep + dest_file_name
     # Need to run unless as resource user
     su_cmd = 'su - {0} -c'.format(owner)
     unless_cmd = format("{su_cmd} '{kinnit_if_needed} hadoop fs -ls {dest_path}' >/dev/null 2>&1")

ambari-common/src/main/python/resource_management/libraries/resources/copy_from_local.py

@@ -28,6 +28,7 @@ class CopyFromLocal(Resource):
 
   path = ResourceArgument(default=lambda obj: obj.name)
   dest_dir = ResourceArgument(required=True)
+  dest_file = ResourceArgument()
   owner = ResourceArgument(required=True)
   group = ResourceArgument()
   mode = ResourceArgument()
@@ -35,4 +36,4 @@ class CopyFromLocal(Resource):
   hadoop_conf_dir = ResourceArgument(default='/etc/hadoop/conf')
   hdfs_user = ResourceArgument(default='hdfs')
 
-  actions = Resource.actions + ["run"]
+  actions = Resource.actions + ["run"]

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/install_jars.py

@@ -72,6 +72,16 @@ def install_tez_jars():
                     hdfs_user=params.hdfs_user,
                     hadoop_conf_dir=params.hadoop_conf_dir
       )
+
+      CopyFromLocal(params.tez_tar_file,
+                    owner=params.tez_user,
+                    mode=0755,
+                    dest_dir=app_dir_path,
+                    dest_file="tez.tar.gz",
+                    kinnit_if_needed=kinit_if_needed,
+                    hdfs_user=params.hdfs_user,
+                    hadoop_conf_dir=params.hadoop_conf_dir
+      )
     pass
 
     if lib_dir_path:
@@ -92,10 +102,11 @@ def get_tez_hdfs_dir_paths(tez_lib_uris = None):
   if tez_lib_uris and tez_lib_uris.strip().find(hdfs_path_prefix, 0) != -1:
     dir_paths = tez_lib_uris.split(',')
     for path in dir_paths:
-      lib_dir_path = path.replace(hdfs_path_prefix, '')
-      lib_dir_path = lib_dir_path if lib_dir_path.endswith(os.sep) else lib_dir_path + os.sep
-      lib_dir_paths.append(lib_dir_path)
+      if not "tez.tar.gz" in path:
+        lib_dir_path = path.replace(hdfs_path_prefix, '')
+        lib_dir_path = lib_dir_path if lib_dir_path.endswith(os.sep) else lib_dir_path + os.sep
+        lib_dir_paths.append(lib_dir_path)
     pass
   pass
 
-  return lib_dir_paths
+  return lib_dir_paths

ambari-server/src/main/resources/stacks/HDP/2.1/services/TEZ/configuration/tez-site.xml

@@ -21,7 +21,7 @@
 
   <property>
     <name>tez.lib.uris</name>
-    <value>hdfs:///apps/tez/,hdfs:///apps/tez/lib/</value>
+    <value>hdfs:///apps/tez/,hdfs:///apps/tez/lib/,hdfs:///apps/tez/tez.tar.gz</value>
     <description>The location of the Tez libraries which will be localized for DAGs</description>
   </property>
 

ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py

@@ -88,6 +88,16 @@ class TestHiveServer(RMFTestCase):
                               hdfs_user='hdfs'
     )
 
+    self.assertResourceCalled('CopyFromLocal', '/usr/lib/tez/tez*.tar.gz',
+                              mode=0755,
+                              owner='tez',
+                              dest_dir='/apps/tez/',
+                              dest_file='tez.tar.gz',
+                              kinnit_if_needed='',
+                              hadoop_conf_dir='/etc/hadoop/conf',
+                              hdfs_user='hdfs'
+    )
+
     self.assertResourceCalled('CopyFromLocal', '/usr/lib/tez/lib/*.jar',
                               mode=0755,
                               owner='tez',