AMBARI-16075. MR service check failed during EU (Intermittent) (ncole)

ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/mapred_service_check.py

@@ -23,6 +23,7 @@ import sys
 from resource_management import *
 from ambari_commons import OSConst
 from ambari_commons.os_family_impl import OsFamilyImpl
+from resource_management.libraries.functions.security_commons import cached_kinit_executor
 
 
 class MapReduce2ServiceCheck(Script):
@@ -133,27 +134,28 @@ class MapReduce2ServiceCheckDefault(MapReduce2ServiceCheck):
     )
     params.HdfsResource(None, action="execute")
 
+    # initialize the ticket
     if params.security_enabled:
-      kinit_cmd = format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal};")
+      cached_kinit_executor(params.kinit_path_local, params.smokeuser, params.smoke_user_keytab,
+                            params.smokeuser_principal, params.hostname, params.tmp_dir)
 
-      Execute(kinit_cmd,
-              user=params.smokeuser
-      )
-      
     ExecuteHadoop(run_wordcount_job,
                   tries=1,
                   try_sleep=5,
                   user=params.smokeuser,
                   bin_dir=params.execute_path,
                   conf_dir=params.hadoop_conf_dir,
-                  logoutput=True
-    )
+                  logoutput=True)
+
+    # the ticket may have expired, so re-initialize if needed
+    if params.security_enabled:
+      cached_kinit_executor(params.kinit_path_local, params.smokeuser, params.smoke_user_keytab,
+                            params.smokeuser_principal, params.hostname, params.tmp_dir)
 
     ExecuteHadoop(test_cmd,
                   user=params.smokeuser,
                   bin_dir=params.execute_path,
-                  conf_dir=params.hadoop_conf_dir
-    )
+                  conf_dir=params.hadoop_conf_dir)
 
 
 if __name__ == "__main__":

ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_service_check.py

@@ -91,7 +91,8 @@ class TestServiceCheck(RMFTestCase):
     )
     self.assertNoMoreResources()
 
-  def test_service_check_secured(self):
+  @patch("resource_management.libraries.functions.security_commons.cached_kinit_executor")
+  def test_service_check_secured(self, cached_kinit_mock):
 
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/mapred_service_check.py",
                       classname="MapReduce2ServiceCheck",
@@ -136,9 +137,12 @@ class TestServiceCheck(RMFTestCase):
         action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020',
         hadoop_conf_dir = '/etc/hadoop/conf',
     )
-    self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa@EXAMPLE.COM;',
-        user = 'ambari-qa',
-    )
+
+    self.assertTrue(2, cached_kinit_mock.call_count)
+    cached_kinit_mock.assert_called_with('/usr/bin/kinit', 'ambari-qa',
+                                         '/etc/security/keytabs/smokeuser.headless.keytab',
+                                         'ambari-qa@EXAMPLE.COM', 'c6401.ambari.apache.org', '/tmp')
+
     self.assertResourceCalled('ExecuteHadoop', 'jar /usr/lib/hadoop-mapreduce/hadoop-mapreduce-examples-2.*.jar wordcount /user/ambari-qa/mapredsmokeinput /user/ambari-qa/mapredsmokeoutput',
                       logoutput = True,
                       try_sleep = 5,