AMBARI-4562. Pig service check fails on secure cluster. (aonishuk)

ambari-agent/src/main/python/resource_management/core/base.py

@@ -143,8 +143,6 @@ class Resource(object):
           self.arguments[key] = arg.validate(value)
         except InvalidArgument, exc:
           raise InvalidArgument("%s %s" % (self, exc))
-
-    Logger.debug("New resource %s: %s" % (self, self.arguments))
     
     if not self.env.test_mode:
       self.env.run()

ambari-agent/src/main/python/resource_management/core/logger.py

@@ -22,6 +22,7 @@ Ambari Agent
 
 __all__ = ["Logger"]
 import logging
+from resource_management.libraries.script.config_dictionary import UnknownConfiguration
 
 class Logger:
   logger = logging.getLogger("resource_management")
@@ -67,6 +68,9 @@ class Logger:
       # usually too long  
       elif isinstance(y, dict):
         val = "..."
+      # for configs which didn't come
+      elif isinstance(y, UnknownConfiguration):
+        val = "[EMPTY]"
       # correctly output 'mode' (as they are octal values like 0755)
       elif y and x == 'mode':
         val = oct(y)

ambari-agent/src/main/python/resource_management/libraries/providers/execute_hadoop.py

@@ -29,11 +29,7 @@ class ExecuteHadoopProvider(Provider):
     keytab = self.resource.keytab
     conf_dir = self.resource.conf_dir
     command = self.resource.command
-    
-    if self.resource.principal:
-      principal = self.resource.user
-    else:
-      principal = self.resource.principal
+    principal = self.resource.principal
     
     if isinstance(command, (list, tuple)):
       command = ' '.join(pipes.quote(x) for x in command)

ambari-agent/src/main/python/resource_management/libraries/resources/execute_hadoop.py

@@ -31,7 +31,7 @@ class ExecuteHadoop(Resource):
   try_sleep = ResourceArgument(default=0) # seconds
   user = ResourceArgument()
   logoutput = BooleanArgument(default=False)
-  principal = ResourceArgument()
+  principal = ResourceArgument(default=lambda obj: obj.user)
   
   conf_dir = ResourceArgument()
   

ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/package/scripts/params.py

@@ -30,6 +30,9 @@ hadoop_conf_dir = "/etc/hadoop/conf"
 hdfs_user = config['configurations']['global']['hdfs_user']
 smokeuser = config['configurations']['global']['smokeuser']
 user_group = config['configurations']['global']['user_group']
+security_enabled = config['configurations']['global']['security_enabled']
+smoke_user_keytab = config['configurations']['global']['smokeuser_keytab']
+kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
 
 # not supporting 32 bit jdk.
 java64_home = config['hostLevelParams']['java_home']

ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/package/scripts/service_check.py

@@ -39,7 +39,11 @@ class PigServiceCheck(Script):
       tries     = 3,
       try_sleep = 5,
       user      = params.smokeuser,
-      conf_dir = params.hadoop_conf_dir
+      conf_dir = params.hadoop_conf_dir,
+      # for kinit run
+      keytab = params.smoke_user_keytab,
+      security_enabled = params.security_enabled,
+      kinit_path_local = params.kinit_path_local
     )
   
     File( '/tmp/pigSmoke.sh',
@@ -61,11 +65,13 @@ class PigServiceCheck(Script):
     )
     
 def main():
+  command_type = "service_check"
+  print "Running "+command_type
+  command_data_file = '/var/lib/ambari-agent/data/command-2.json'
+  basedir = '/root/ambari/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package'
+  stroutfile = '/1.txt'
   import sys
-  command_type = 'service_check'
-  command_data_file = '/root/workspace/Pig/input.json'
-  basedir = '/root/workspace/Pig/main'
-  sys.argv = ["", command_type, command_data_file, basedir]
+  sys.argv = ["", command_type, command_data_file, basedir, stroutfile]
   
   PigServiceCheck().execute()
   

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py

@@ -30,6 +30,9 @@ hadoop_conf_dir = "/etc/hadoop/conf"
 hdfs_user = config['configurations']['global']['hdfs_user']
 smokeuser = config['configurations']['global']['smokeuser']
 user_group = config['configurations']['global']['user_group']
+security_enabled = config['configurations']['global']['security_enabled']
+smoke_user_keytab = config['configurations']['global']['smokeuser_keytab']
+kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
 
 # not supporting 32 bit jdk.
 java64_home = config['hostLevelParams']['java_home']

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/service_check.py

@@ -39,7 +39,11 @@ class PigServiceCheck(Script):
       tries     = 3,
       try_sleep = 5,
       user      = params.smokeuser,
-      conf_dir = params.hadoop_conf_dir
+      conf_dir = params.hadoop_conf_dir,
+      # for kinit run
+      keytab = params.smoke_user_keytab,
+      security_enabled = params.security_enabled,
+      kinit_path_local = params.kinit_path_local
     )
   
     File( '/tmp/pigSmoke.sh',
@@ -61,11 +65,13 @@ class PigServiceCheck(Script):
     )
     
 def main():
+  command_type = "service_check"
+  print "Running "+command_type
+  command_data_file = '/var/lib/ambari-agent/data/command-2.json'
+  basedir = '/root/ambari/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package'
+  stroutfile = '/1.txt'
   import sys
-  command_type = 'service_check'
-  command_data_file = '/root/workspace/Pig/input.json'
-  basedir = '/root/workspace/Pig/main'
-  sys.argv = ["", command_type, command_data_file, basedir]
+  sys.argv = ["", command_type, command_data_file, basedir, stroutfile]
   
   PigServiceCheck().execute()
   

ambari-server/src/test/python/stacks/1.3.2/PIG/test_pig_client.py

@@ -39,6 +39,11 @@ class TestHcatClient(RMFTestCase):
     self.assertResourceCalled('TemplateConfig', '/etc/pig/conf/pig.properties',
       owner = 'hdfs',
     )
+    self.assertResourceCalled('File', '/etc/pig/conf/log4j.properties',
+      owner = 'hdfs',
+      group = 'hadoop',
+      mode = 0644,
+    )
     self.assertNoMoreResources()
 
   def test_configure_secured(self):
@@ -58,4 +63,9 @@ class TestHcatClient(RMFTestCase):
     self.assertResourceCalled('TemplateConfig', '/etc/pig/conf/pig.properties',
       owner = 'hdfs',
     )
+    self.assertResourceCalled('File', '/etc/pig/conf/log4j.properties',
+      owner = 'hdfs',
+      group = 'hadoop',
+      mode = 0644,
+    )
     self.assertNoMoreResources()

ambari-server/src/test/python/stacks/1.3.2/PIG/test_pig_service_check.py

@@ -34,6 +34,9 @@ class TestServiceCheck(RMFTestCase):
       tries = 3,
       user = 'ambari-qa',
       conf_dir = '/etc/hadoop/conf',
+      security_enabled = False,
+      keytab = 'mockOfUnknownConfiguration',
+      kinit_path_local = '/usr/bin/kinit',
     )
        
     self.assertResourceCalled('File', '/tmp/pigSmoke.sh',
@@ -53,7 +56,7 @@ class TestServiceCheck(RMFTestCase):
       user = 'ambari-qa',
       conf_dir = '/etc/hadoop/conf',
     )
-    
+    self.assertNoMoreResources()
 
   def test_configure_secured(self):
     self.executeScript("1.3.2/services/PIG/package/scripts/service_check.py",
@@ -67,6 +70,9 @@ class TestServiceCheck(RMFTestCase):
       tries = 3,
       user = 'ambari-qa',
       conf_dir = '/etc/hadoop/conf',
+      security_enabled = True, 
+      keytab = '/etc/security/keytabs/smokeuser.headless.keytab',
+      kinit_path_local = '/usr/bin/kinit',
     )
        
     self.assertResourceCalled('File', '/tmp/pigSmoke.sh',
@@ -86,3 +92,4 @@ class TestServiceCheck(RMFTestCase):
       user = 'ambari-qa',
       conf_dir = '/etc/hadoop/conf',
     )
+    self.assertNoMoreResources()

ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_client.py

@@ -39,6 +39,11 @@ class TestHcatClient(RMFTestCase):
     self.assertResourceCalled('TemplateConfig', '/etc/pig/conf/pig.properties',
       owner = 'hdfs',
     )
+    self.assertResourceCalled('File', '/etc/pig/conf/log4j.properties',
+      owner = 'hdfs',
+      group = 'hadoop',
+      mode = 0644,
+    )
     self.assertNoMoreResources()
 
   def test_configure_secured(self):
@@ -58,4 +63,9 @@ class TestHcatClient(RMFTestCase):
     self.assertResourceCalled('TemplateConfig', '/etc/pig/conf/pig.properties',
       owner = 'hdfs',
     )
+    self.assertResourceCalled('File', '/etc/pig/conf/log4j.properties',
+      owner = 'hdfs',
+      group = 'hadoop',
+      mode = 0644,
+    )
     self.assertNoMoreResources()

ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_service_check.py

@@ -28,12 +28,14 @@ class TestServiceCheck(RMFTestCase):
                        command = "service_check",
                        config_file="default.json"
     )
-    
     self.assertResourceCalled('ExecuteHadoop', 'dfs -rmr pigsmoke.out passwd; hadoop dfs -put /etc/passwd passwd ',
       try_sleep = 5,
       tries = 3,
       user = 'ambari-qa',
       conf_dir = '/etc/hadoop/conf',
+      security_enabled = False,
+      keytab = 'mockOfUnknownConfiguration',
+      kinit_path_local = '/usr/bin/kinit'
     )
        
     self.assertResourceCalled('File', '/tmp/pigSmoke.sh',
@@ -53,7 +55,7 @@ class TestServiceCheck(RMFTestCase):
       user = 'ambari-qa',
       conf_dir = '/etc/hadoop/conf',
     )
-    
+    self.assertNoMoreResources()
 
   def test_configure_secured(self):
     self.executeScript("2.0.6/services/PIG/package/scripts/service_check.py",
@@ -67,6 +69,9 @@ class TestServiceCheck(RMFTestCase):
       tries = 3,
       user = 'ambari-qa',
       conf_dir = '/etc/hadoop/conf',
+      security_enabled = True, 
+      keytab = '/etc/security/keytabs/smokeuser.headless.keytab',
+      kinit_path_local = '/usr/bin/kinit'
     )
        
     self.assertResourceCalled('File', '/tmp/pigSmoke.sh',
@@ -86,3 +91,4 @@ class TestServiceCheck(RMFTestCase):
       user = 'ambari-qa',
       conf_dir = '/etc/hadoop/conf',
     )
+    self.assertNoMoreResources()

ambari-server/src/test/python/stacks/utils/RMFTestCase.py

@@ -29,6 +29,7 @@ from mock.mock import MagicMock, patch
 from resource_management.core.environment import Environment
 from resource_management.libraries.script.config_dictionary import ConfigDictionary
 from resource_management.libraries.script.script import Script
+from resource_management.libraries.script import config_dictionary 
 import platform
 
 PATH_TO_STACKS = os.path.normpath("main/resources/stacks/HDP")
@@ -81,7 +82,8 @@ class RMFTestCase(TestCase):
           with patch.object(Script, 'install_packages'):
             with patch('resource_management.libraries.functions.get_kinit_path', return_value=kinit_path_local):
               with patch.object(platform, 'linux_distribution', return_value=os_type):
-                method(RMFTestCase.env)
+                with patch.object(config_dictionary, 'UnknownConfiguration', return_value='mockOfUnknownConfiguration'):
+                  method(RMFTestCase.env)
   
   def getConfig(self):
     return self.config_dict
@@ -113,7 +115,14 @@ class RMFTestCase(TestCase):
     for resource in RMFTestCase.env.resource_list:
       print "'{0}', {1},".format(resource.__class__.__name__, self._ppformat(resource.name))
       for k,v in resource.arguments.iteritems():
-        print "  {0} = {1},".format(k, self._ppformat(v))
+
+        # correctly output octal mode numbers
+        if k == 'mode' and isinstance( v, int ):
+          val = oct(v)
+        else:
+          val = self._ppformat(v)
+          
+        print "  {0} = {1},".format(k, val)
       print
   
   def assertResourceCalled(self, resource_type, name, **kwargs):