AMBARI-4767. Move copyFromLocal to resource management package. (swagle)

ambari-agent/src/main/python/resource_management/libraries/providers/__init__.py

@@ -32,6 +32,7 @@ PROVIDERS = dict(
     PropertiesFile="resource_management.libraries.providers.properties_file.PropertiesFileProvider",
     MonitorWebserver="resource_management.libraries.providers.monitor_webserver.MonitorWebserverProvider",
     Repository="resource_management.libraries.providers.repository.RepositoryProvider",
-    HdfsDirectory="resource_management.libraries.providers.hdfs_directory.HdfsDirectoryProvider"
+    HdfsDirectory="resource_management.libraries.providers.hdfs_directory.HdfsDirectoryProvider",
+    CopyFromLocal="resource_management.libraries.providers.copy_from_local.CopyFromLocalProvider"
   ),
 )

ambari-agent/src/main/python/resource_management/libraries/providers/copy_from_local.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+from resource_management import *
+
+class CopyFromLocalProvider(Provider):
+  def action_run(self):
+    path = self.resource.path
+    dest_dir = self.resource.dest_dir
+    kinnit_if_needed = self.resource.kinnit_if_needed
+    stub_path = self.resource.stub_path
+    owner = self.resource.owner
+    group = self.resource.group
+    mode = self.resource.mode
+    hadoop_conf_path = self.resource.hadoop_conf_dir
+
+    copy_cmd = format("fs -copyFromLocal {path} {dest_dir}")
+    unless_cmd = format("{kinnit_if_needed} hadoop fs -ls {dest_dir} >/dev/null 2>&1")
+
+    ExecuteHadoop(copy_cmd,
+                  not_if=unless_cmd,
+                  user=owner,
+                  conf_dir=hadoop_conf_path,
+                  )
+
+    if not owner:
+      chown = None
+    else:
+      if not group:
+        chown = owner
+      else:
+        chown = format('{owner}:{group}')
+
+    if chown:
+      chown_cmd = format("fs -chown {chown} {dest_dir}")
+
+      ExecuteHadoop(chown_cmd,
+                    user=owner,
+                    conf_dir=hadoop_conf_path)
+    pass
+
+    if mode:
+      chmod_cmd = format('fs -chmod {mode} {dest_dir}')
+
+      ExecuteHadoop(chmod_cmd,
+                    user=owner,
+                    conf_dir=hadoop_conf_path)
+    pass

ambari-agent/src/main/python/resource_management/libraries/resources/__init__.py

@@ -26,4 +26,5 @@ from resource_management.libraries.resources.xml_config import *
 from resource_management.libraries.resources.properties_file import *
 from resource_management.libraries.resources.repository import *
 from resource_management.libraries.resources.monitor_webserver import *
-from resource_management.libraries.resources.hdfs_directory import *
+from resource_management.libraries.resources.hdfs_directory import *
+from resource_management.libraries.resources.copy_from_local import *

ambari-agent/src/main/python/resource_management/libraries/resources/copy_from_local.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+_all__ = ["CopyFromLocal"]
+from resource_management.core.base import Resource, ForcedListArgument, ResourceArgument, BooleanArgument
+
+class CopyFromLocal(Resource):
+  action = ForcedListArgument(default="run")
+
+  path = ResourceArgument(default=lambda obj: obj.name)
+  dest_dir = ResourceArgument()
+  owner = ResourceArgument()
+  group = ResourceArgument()
+  mode = ResourceArgument()
+  kinnit_if_needed = ResourceArgument(default='')
+  hadoop_conf_dir = ResourceArgument(default='/etc/hadoop/conf')
+
+  actions = Resource.actions + ["run"]

ambari-agent/src/test/python/resource_management/TestCopyFromLocal.py

@@ -0,0 +1,67 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+from unittest import TestCase
+from mock.mock import patch
+from resource_management import *
+
+@patch.object(System, "os_family", new = 'redhat')
+class TestCopyFromLocal(TestCase):
+
+  @patch("resource_management.libraries.providers.execute_hadoop.ExecuteHadoopProvider")
+  def test_run_default_args(self, execute_hadoop_mock):
+    with Environment() as env:
+      CopyFromLocal('/user/testdir/*.files',
+        owner='user1',
+        dest_dir='/apps/test/',
+        kinnit_if_needed='',
+        stub_path='/tmp/test_stub',
+        ignore_on_failure=True
+      )
+      self.assertEqual(execute_hadoop_mock.call_count, 2)
+      call_arg_list = execute_hadoop_mock.call_args_list
+      self.assertEqual('fs -copyFromLocal /user/testdir/*.files /apps/test/',
+                       call_arg_list[0][0][0].command)
+      self.assertEquals({'not_if': ' hadoop fs -ls /tmp/test_stub >/dev/null 2>&1', 'ignore_failures': False, 'user': 'user1', 'conf_dir': '/etc/hadoop/conf'},
+                        call_arg_list[0][0][0].arguments)
+      self.assertEquals('fs -chown user1 /apps/test/', call_arg_list[1][0][0].command)
+      self.assertEquals({'user': 'user1', 'conf_dir': '/etc/hadoop/conf'}, call_arg_list[1][0][0].arguments)
+
+
+  @patch("resource_management.libraries.providers.execute_hadoop.ExecuteHadoopProvider")
+  def test_run_with_chmod(self, execute_hadoop_mock):
+    with Environment() as env:
+      CopyFromLocal('/user/testdir/*.files',
+        mode=0655,
+        owner='user1',
+        group='hdfs',
+        dest_dir='/apps/test/',
+        kinnit_if_needed='',
+        stub_path='/tmp/test_stub',
+        ignore_on_failure=False
+      )
+      self.assertEqual(execute_hadoop_mock.call_count, 3)
+      call_arg_list = execute_hadoop_mock.call_args_list
+      self.assertEqual('fs -copyFromLocal /user/testdir/*.files /apps/test/',
+                       call_arg_list[0][0][0].command)
+      self.assertEquals({'not_if': ' hadoop fs -ls /tmp/test_stub >/dev/null 2>&1', 'ignore_failures': False, 'user': 'user1', 'conf_dir': '/etc/hadoop/conf'},
+                        call_arg_list[0][0][0].arguments)
+      self.assertEquals('fs -chown user1:hdfs /apps/test/', call_arg_list[1][0][0].command)
+      self.assertEquals({'user': 'user1', 'conf_dir': '/etc/hadoop/conf'}, call_arg_list[1][0][0].arguments)
+
+

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive_server.py

@@ -98,30 +98,23 @@ class HiveServer(Script):
       pass
 
       if app_dir_path:
-        copyFromLocal(path=params.tez_local_api_jars,
+        CopyFromLocal(params.tez_local_api_jars,
                       mode=0655,
                       owner=params.tez_user,
                       dest_dir=app_dir_path,
-                      kinnit_if_needed=kinit_if_needed,
-                      stub_path=params.tez_stub_path
+                      kinnit_if_needed=kinit_if_needed
         )
       pass
 
       if lib_dir_path:
-        copyFromLocal(path=params.tez_local_lib_jars,
+        CopyFromLocal(params.tez_local_lib_jars,
                       mode=0655,
                       owner=params.tez_user,
                       dest_dir=lib_dir_path,
                       kinnit_if_needed=kinit_if_needed,
-                      stub_path=params.tez_stub_path
         )
       pass
 
-      ExecuteHadoop(format('dfs -touchz {tez_stub_path}'),
-                    user = params.tez_user,
-                    conf_dir=params.hadoop_conf_dir
-      )
-
 
 def get_tez_hdfs_dir_paths(tez_lib_uris = None):
   hdfs_path_prefix = 'hdfs://'
@@ -138,47 +131,5 @@ def get_tez_hdfs_dir_paths(tez_lib_uris = None):
   return lib_dir_paths
 
 
-def copyFromLocal(path=None, owner=None, group=None, mode=None,
-                  dest_dir=None, kinnit_if_needed="", stub_path=None):
-  import params
-
-  if not stub_path:
-    stub_path = params.tez_stub_path
-  pass
-
-  copy_cmd = format("fs -copyFromLocal {path} {dest_dir}")
-  unless_cmd = format("{kinnit_if_needed} hadoop fs -ls {stub_path} >/dev/null 2>&1")
-
-  ExecuteHadoop(copy_cmd,
-                not_if=unless_cmd,
-                user=owner,
-                conf_dir=params.hadoop_conf_dir,
-                ignore_failures = True)
-
-  if not owner:
-    chown = None
-  else:
-    if not group:
-      chown = owner
-    else:
-      chown = format('{owner}:{group}')
-
-  if not chown:
-    chown_cmd = format("fs -chown {chown} {dest_dir}")
-
-    ExecuteHadoop(copy_cmd,
-                  user=owner,
-                  conf_dir=params.hadoop_conf_dir)
-  pass
-
-  if not mode:
-    chmod_cmd = format('fs -chmod {mode} {dest_dir}')
-
-    ExecuteHadoop(chmod_cmd,
-                  user=owner,
-                  conf_dir=params.hadoop_conf_dir)
-  pass
-
-
 if __name__ == "__main__":
   HiveServer().execute()

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/package/scripts/webhcat.py

@@ -84,55 +84,23 @@ def webhcat():
             path='/bin'
     )
 
-  copyFromLocal(path='/usr/lib/hadoop-mapreduce/hadoop-streaming-*.jar',
+  CopyFromLocal('/usr/lib/hadoop-mapreduce/hadoop-streaming-*.jar',
                 owner=params.webhcat_user,
                 mode=0755,
                 dest_dir=format("{webhcat_apps_dir}/hadoop-streaming.jar"),
                 kinnit_if_needed=kinit_if_needed
   )
 
-  copyFromLocal(path='/usr/share/HDP-webhcat/pig.tar.gz',
+  CopyFromLocal('/usr/share/HDP-webhcat/pig.tar.gz',
                 owner=params.webhcat_user,
                 mode=0755,
                 dest_dir=format("{webhcat_apps_dir}/pig.tar.gz"),
+                kinnit_if_needed=kinit_if_needed
   )
 
-  copyFromLocal(path='/usr/share/HDP-webhcat/hive.tar.gz',
+  CopyFromLocal('/usr/share/HDP-webhcat/hive.tar.gz',
                 owner=params.webhcat_user,
                 mode=0755,
-                dest_dir=format("{webhcat_apps_dir}/hive.tar.gz")
-  )
-
-
-def copyFromLocal(path=None, owner=None, group=None, mode=None, dest_dir=None, kinnit_if_needed=""):
-  import params
-
-  copy_cmd = format("fs -copyFromLocal {path} {dest_dir}")
-  unless_cmd = format("{kinnit_if_needed} hadoop fs -ls {dest_dir} >/dev/null 2>&1")
-
-  ExecuteHadoop(copy_cmd,
-                not_if=unless_cmd,
-                user=owner,
-                conf_dir=params.hadoop_conf_dir)
-
-  if not owner:
-    chown = None
-  else:
-    if not group:
-      chown = owner
-    else:
-      chown = format('{owner}:{group}')
-
-  if not chown:
-    chown_cmd = format("fs -chown {chown} {dest_dir}")
-
-    ExecuteHadoop(copy_cmd,
-                  user=owner,
-                  conf_dir=params.hadoop_conf_dir)
-
-  if not mode:
-    chmod_cmd = format('fs -chmod {mode} {dest_dir}')
-
-    ExecuteHadoop(chmod_cmd,
-                  user=owner,
-                  conf_dir=params.hadoop_conf_dir)
+                dest_dir=format("{webhcat_apps_dir}/hive.tar.gz"),
+                kinnit_if_needed=kinit_if_needed
+  )

ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py

@@ -98,23 +98,18 @@ class TestHiveServer(RMFTestCase):
                               action = ['create']
                               )
 
-    self.assertResourceCalled('ExecuteHadoop', 'fs -copyFromLocal /usr/lib/tez/tez*.jar /apps/tez/',
-                              not_if = ' hadoop fs -ls /tmp/tez_jars_copied >/dev/null 2>&1',
-                              user = 'tez',
-                              conf_dir = '/etc/hadoop/conf',
-                              ignore_failures=True
-    )
-
-    self.assertResourceCalled('ExecuteHadoop', 'fs -copyFromLocal /usr/lib/tez/lib/*.jar /apps/tez/lib/',
-                              not_if = ' hadoop fs -ls /tmp/tez_jars_copied >/dev/null 2>&1',
-                              user = 'tez',
-                              conf_dir = '/etc/hadoop/conf',
-                              ignore_failures=True
+    self.assertResourceCalled('CopyFromLocal', '/usr/lib/tez/tez*.jar',
+                              mode=0655,
+                              owner='tez',
+                              dest_dir='/apps/tez/',
+                              kinnit_if_needed=''
     )
 
-    self.assertResourceCalled('ExecuteHadoop', 'dfs -touchz /tmp/tez_jars_copied',
-                              user = 'tez',
-                              conf_dir = '/etc/hadoop/conf'
+    self.assertResourceCalled('CopyFromLocal', '/usr/lib/tez/lib/*.jar',
+                              mode=0655,
+                              owner='tez',
+                              dest_dir='/apps/tez/lib/',
+                              kinnit_if_needed=''
     )
 
     self.assertResourceCalled('Execute', 'env JAVA_HOME=/usr/jdk64/jdk1.7.0_45 /tmp/start_hiveserver2_script /var/log/hive/hive-server2.out /var/log/hive/hive-server2.log /var/run/hive/hive-server.pid /etc/hive/conf.server',

ambari-server/src/test/python/stacks/2.0.6/WEBHCAT/test_webhcat_server.py

@@ -154,20 +154,23 @@ class TestWebHCatServer(RMFTestCase):
       owner = 'hcat',
       group = 'hadoop',
     )
-    self.assertResourceCalled('ExecuteHadoop', 'fs -copyFromLocal /usr/lib/hadoop-mapreduce/hadoop-streaming-*.jar /apps/webhcat/hadoop-streaming.jar',
-      not_if = ' hadoop fs -ls /apps/webhcat/hadoop-streaming.jar >/dev/null 2>&1',
-      user = 'hcat',
-      conf_dir = '/etc/hadoop/conf',
+    self.assertResourceCalled('CopyFromLocal', '/usr/lib/hadoop-mapreduce/hadoop-streaming-*.jar',
+                              owner='hcat',
+                              mode=0755,
+                              dest_dir=format('/apps/webhcat/hadoop-streaming.jar'),
+                              kinnit_if_needed=''
     )
-    self.assertResourceCalled('ExecuteHadoop', 'fs -copyFromLocal /usr/share/HDP-webhcat/pig.tar.gz /apps/webhcat/pig.tar.gz',
-      not_if = ' hadoop fs -ls /apps/webhcat/pig.tar.gz >/dev/null 2>&1',
-      user = 'hcat',
-      conf_dir = '/etc/hadoop/conf',
+    self.assertResourceCalled('CopyFromLocal', '/usr/share/HDP-webhcat/pig.tar.gz',
+                              owner='hcat',
+                              mode=0755,
+                              dest_dir=format('/apps/webhcat/pig.tar.gz'),
+                              kinnit_if_needed=''
     )
-    self.assertResourceCalled('ExecuteHadoop', 'fs -copyFromLocal /usr/share/HDP-webhcat/hive.tar.gz /apps/webhcat/hive.tar.gz',
-      not_if = ' hadoop fs -ls /apps/webhcat/hive.tar.gz >/dev/null 2>&1',
-      user = 'hcat',
-      conf_dir = '/etc/hadoop/conf',
+    self.assertResourceCalled('CopyFromLocal', '/usr/share/HDP-webhcat/hive.tar.gz',
+                              owner='hcat',
+                              mode=0755,
+                              dest_dir=format('/apps/webhcat/hive.tar.gz'),
+                              kinnit_if_needed=''
     )
 
   def assert_configure_secured(self):
@@ -230,18 +233,21 @@ class TestWebHCatServer(RMFTestCase):
       path = ['/bin'],
       user = 'hcat',
     )
-    self.assertResourceCalled('ExecuteHadoop', 'fs -copyFromLocal /usr/lib/hadoop-mapreduce/hadoop-streaming-*.jar /apps/webhcat/hadoop-streaming.jar',
-      not_if = '/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa; hadoop fs -ls /apps/webhcat/hadoop-streaming.jar >/dev/null 2>&1',
-      user = 'hcat',
-      conf_dir = '/etc/hadoop/conf',
-    )
-    self.assertResourceCalled('ExecuteHadoop', 'fs -copyFromLocal /usr/share/HDP-webhcat/pig.tar.gz /apps/webhcat/pig.tar.gz',
-      not_if = ' hadoop fs -ls /apps/webhcat/pig.tar.gz >/dev/null 2>&1',
-      user = 'hcat',
-      conf_dir = '/etc/hadoop/conf',
-    )
-    self.assertResourceCalled('ExecuteHadoop', 'fs -copyFromLocal /usr/share/HDP-webhcat/hive.tar.gz /apps/webhcat/hive.tar.gz',
-      not_if = ' hadoop fs -ls /apps/webhcat/hive.tar.gz >/dev/null 2>&1',
-      user = 'hcat',
-      conf_dir = '/etc/hadoop/conf',
-    )
+    self.assertResourceCalled('CopyFromLocal', '/usr/lib/hadoop-mapreduce/hadoop-streaming-*.jar',
+                              owner='hcat',
+                              mode=0755,
+                              dest_dir=format('/apps/webhcat/hadoop-streaming.jar'),
+                              kinnit_if_needed='/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa;'
+    )
+    self.assertResourceCalled('CopyFromLocal', '/usr/share/HDP-webhcat/pig.tar.gz',
+                              owner='hcat',
+                              mode=0755,
+                              dest_dir=format('/apps/webhcat/pig.tar.gz'),
+                              kinnit_if_needed='/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa;'
+    )
+    self.assertResourceCalled('CopyFromLocal', '/usr/share/HDP-webhcat/hive.tar.gz',
+                              owner='hcat',
+                              mode=0755,
+                              dest_dir=format('/apps/webhcat/hive.tar.gz'),
+                              kinnit_if_needed='/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa;'
+    )