AMBARI-5447. WebHCat start Fail with authorization enabled. (swagle)

ambari-agent/src/main/python/resource_management/libraries/providers/copy_from_local.py

@@ -37,8 +37,9 @@ class CopyFromLocalProvider(Provider):
     copy_cmd = format("fs -copyFromLocal {path} {dest_dir}")
     dest_file_name = os.path.split(path)[1]
     dest_path = dest_dir + dest_file_name if dest_dir.endswith(os.sep) else dest_dir + os.sep + dest_file_name
-
-    unless_cmd = format("{kinnit_if_needed} hadoop fs -ls {dest_path} >/dev/null 2>&1")
+    # Need to run unless as resource user
+    su_cmd = 'su - {0} -c'.format(owner)
+    unless_cmd = format("{su_cmd} '{kinnit_if_needed} hadoop fs -ls {dest_path}' >/dev/null 2>&1")
 
     ExecuteHadoop(copy_cmd,
                   not_if=unless_cmd,

ambari-agent/src/test/python/resource_management/TestCopyFromLocal.py

@@ -36,7 +36,7 @@ class TestCopyFromLocal(TestCase):
       call_arg_list = execute_hadoop_mock.call_args_list
       self.assertEqual('fs -copyFromLocal /user/testdir/*.files /apps/test/',
                        call_arg_list[0][0][0].command)
-      self.assertEquals({'not_if': ' hadoop fs -ls /apps/test/*.files >/dev/null 2>&1', 'user': 'user1', 'conf_dir': '/etc/hadoop/conf'},
+      self.assertEquals({'not_if': "su - user1 -c ' hadoop fs -ls /apps/test/*.files' >/dev/null 2>&1", 'user': 'user1', 'conf_dir': '/etc/hadoop/conf'},
                         call_arg_list[0][0][0].arguments)
       self.assertEquals('fs -chown user1 /apps/test/*.files', call_arg_list[1][0][0].command)
       self.assertEquals({'user': 'hdfs', 'conf_dir': '/etc/hadoop/conf'}, call_arg_list[1][0][0].arguments)
@@ -57,7 +57,7 @@ class TestCopyFromLocal(TestCase):
       call_arg_list = execute_hadoop_mock.call_args_list
       self.assertEqual('fs -copyFromLocal /user/testdir/*.files /apps/test/',
                        call_arg_list[0][0][0].command)
-      self.assertEquals({'not_if': ' hadoop fs -ls /apps/test/*.files >/dev/null 2>&1', 'user': 'user1', 'conf_dir': '/etc/hadoop/conf'},
+      self.assertEquals({'not_if': "su - user1 -c ' hadoop fs -ls /apps/test/*.files' >/dev/null 2>&1", 'user': 'user1', 'conf_dir': '/etc/hadoop/conf'},
                         call_arg_list[0][0][0].arguments)
       self.assertEquals('fs -chown user1:hdfs /apps/test/*.files', call_arg_list[1][0][0].command)
       self.assertEquals({'user': 'hdfs', 'conf_dir': '/etc/hadoop/conf'}, call_arg_list[1][0][0].arguments)

ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/webhcat.py

@@ -20,7 +20,6 @@ Ambari Agent
 
 """
 from resource_management import *
-import sys
 
 
 def webhcat():
@@ -83,55 +82,26 @@ def webhcat():
             path='/bin'
     )
 
-  copyFromLocal(path='/usr/lib/hadoop/contrib/streaming/hadoop-streaming*.jar',
+  CopyFromLocal('/usr/lib/hadoop/contrib/streaming/hadoop-streaming*.jar',
                 owner=params.webhcat_user,
                 mode=0755,
-                dest_dir=format("{webhcat_apps_dir}/hadoop-streaming.jar"),
-                kinnit_if_needed=kinit_if_needed
+                dest_dir=params.webhcat_apps_dir,
+                kinnit_if_needed=kinit_if_needed,
+                hdfs_user=params.hdfs_user
   )
 
-  copyFromLocal(path='/usr/share/HDP-webhcat/pig.tar.gz',
+  CopyFromLocal('/usr/share/HDP-webhcat/pig.tar.gz',
                 owner=params.webhcat_user,
                 mode=0755,
-                dest_dir=format("{webhcat_apps_dir}/pig.tar.gz"),
+                dest_dir=params.webhcat_apps_dir,
+                kinnit_if_needed=kinit_if_needed,
+                hdfs_user=params.hdfs_user
   )
 
-  copyFromLocal(path='/usr/share/HDP-webhcat/hive.tar.gz',
+  CopyFromLocal('/usr/share/HDP-webhcat/hive.tar.gz',
                 owner=params.webhcat_user,
                 mode=0755,
-                dest_dir=format("{webhcat_apps_dir}/hive.tar.gz")
+                dest_dir=params.webhcat_apps_dir,
+                kinnit_if_needed=kinit_if_needed,
+                hdfs_user=params.hdfs_user
   )
-
-
-def copyFromLocal(path=None, owner=None, group=None, mode=None, dest_dir=None, kinnit_if_needed=""):
-  import params
-
-  copy_cmd = format("fs -copyFromLocal {path} {dest_dir}")
-  unless_cmd = format("{kinnit_if_needed} hadoop fs -ls {dest_dir} >/dev/null 2>&1")
-
-  ExecuteHadoop(copy_cmd,
-                not_if=unless_cmd,
-                user=owner,
-                conf_dir=params.hadoop_conf_dir)
-
-  if not owner:
-    chown = None
-  else:
-    if not group:
-      chown = owner
-    else:
-      chown = format('{owner}:{group}')
-
-  if not chown:
-    chown_cmd = format("fs -chown {chown} {dest_dir}")
-
-    ExecuteHadoop(copy_cmd,
-                  user=owner,
-                  conf_dir=params.hadoop_conf_dir)
-
-  if not mode:
-    chmod_cmd = format('fs -chmod {mode} {dest_dir}')
-
-    ExecuteHadoop(chmod_cmd,
-                  user=owner,
-                  conf_dir=params.hadoop_conf_dir)

ambari-server/src/test/python/stacks/1.3.2/WEBHCAT/test_webhcat_server.py

@@ -151,20 +151,26 @@ class TestWebHCatServer(RMFTestCase):
       owner = 'hcat',
       group = 'hadoop',
     )
-    self.assertResourceCalled('ExecuteHadoop', 'fs -copyFromLocal /usr/lib/hadoop/contrib/streaming/hadoop-streaming*.jar /apps/webhcat/hadoop-streaming.jar',
-      not_if = ' hadoop fs -ls /apps/webhcat/hadoop-streaming.jar >/dev/null 2>&1',
-      user = 'hcat',
-      conf_dir = '/etc/hadoop/conf',
+    self.assertResourceCalled('CopyFromLocal', '/usr/lib/hadoop/contrib/streaming/hadoop-streaming*.jar',
+                              owner='hcat',
+                              mode=0755,
+                              dest_dir='/apps/webhcat',
+                              kinnit_if_needed='',
+                              hdfs_user='hdfs'
     )
-    self.assertResourceCalled('ExecuteHadoop', 'fs -copyFromLocal /usr/share/HDP-webhcat/pig.tar.gz /apps/webhcat/pig.tar.gz',
-      not_if = ' hadoop fs -ls /apps/webhcat/pig.tar.gz >/dev/null 2>&1',
-      user = 'hcat',
-      conf_dir = '/etc/hadoop/conf',
+    self.assertResourceCalled('CopyFromLocal', '/usr/share/HDP-webhcat/pig.tar.gz',
+                              owner='hcat',
+                              mode=0755,
+                              dest_dir='/apps/webhcat',
+                              kinnit_if_needed='',
+                              hdfs_user='hdfs'
     )
-    self.assertResourceCalled('ExecuteHadoop', 'fs -copyFromLocal /usr/share/HDP-webhcat/hive.tar.gz /apps/webhcat/hive.tar.gz',
-      not_if = ' hadoop fs -ls /apps/webhcat/hive.tar.gz >/dev/null 2>&1',
-      user = 'hcat',
-      conf_dir = '/etc/hadoop/conf',
+    self.assertResourceCalled('CopyFromLocal', '/usr/share/HDP-webhcat/hive.tar.gz',
+                              owner='hcat',
+                              mode=0755,
+                              dest_dir='/apps/webhcat',
+                              kinnit_if_needed='',
+                              hdfs_user='hdfs'
     )
 
   def assert_configure_secured(self):
@@ -227,18 +233,24 @@ class TestWebHCatServer(RMFTestCase):
       path = ['/bin'],
       user = 'hcat',
     )
-    self.assertResourceCalled('ExecuteHadoop', 'fs -copyFromLocal /usr/lib/hadoop/contrib/streaming/hadoop-streaming*.jar /apps/webhcat/hadoop-streaming.jar',
-      not_if = '/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa; hadoop fs -ls /apps/webhcat/hadoop-streaming.jar >/dev/null 2>&1',
-      user = 'hcat',
-      conf_dir = '/etc/hadoop/conf',
+    self.assertResourceCalled('CopyFromLocal', '/usr/lib/hadoop/contrib/streaming/hadoop-streaming*.jar',
+                              owner='hcat',
+                              mode=0755,
+                              dest_dir='/apps/webhcat',
+                              kinnit_if_needed='/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa;',
+                              hdfs_user='hdfs'
     )
-    self.assertResourceCalled('ExecuteHadoop', 'fs -copyFromLocal /usr/share/HDP-webhcat/pig.tar.gz /apps/webhcat/pig.tar.gz',
-      not_if = ' hadoop fs -ls /apps/webhcat/pig.tar.gz >/dev/null 2>&1',
-      user = 'hcat',
-      conf_dir = '/etc/hadoop/conf',
-    )
-    self.assertResourceCalled('ExecuteHadoop', 'fs -copyFromLocal /usr/share/HDP-webhcat/hive.tar.gz /apps/webhcat/hive.tar.gz',
-      not_if = ' hadoop fs -ls /apps/webhcat/hive.tar.gz >/dev/null 2>&1',
-      user = 'hcat',
-      conf_dir = '/etc/hadoop/conf',
+    self.assertResourceCalled('CopyFromLocal', '/usr/share/HDP-webhcat/pig.tar.gz',
+                              owner='hcat',
+                              mode=0755,
+                              dest_dir='/apps/webhcat',
+                              kinnit_if_needed='/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa;',
+                              hdfs_user='hdfs'
     )
+    self.assertResourceCalled('CopyFromLocal', '/usr/share/HDP-webhcat/hive.tar.gz',
+                              owner='hcat',
+                              mode=0755,
+                              dest_dir='/apps/webhcat',
+                              kinnit_if_needed='/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa;',
+                              hdfs_user='hdfs'
+    )