AMBARI-16956. Ensure smokeuser HDFS folder exists before running MR, YARN, PIG, OOZIE service checks (magyari_sandor)

ambari-server/src/main/resources/common-services/MAHOUT/1.0.0.2.3/package/scripts/params.py

@@ -63,6 +63,7 @@ hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_nam
 hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
 smokeuser = config['configurations']['cluster-env']['smokeuser']
 smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
+smoke_hdfs_user_mode = 0770
 user_group = config['configurations']['cluster-env']['user_group']
 security_enabled = config['configurations']['cluster-env']['security_enabled']
 smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']

ambari-server/src/main/resources/common-services/MAHOUT/1.0.0.2.3/package/scripts/service_check.py

@@ -34,6 +34,13 @@ class MahoutServiceCheck(Script):
         content = "Test text which will be converted to sequence file.",
         mode = 0755
     )
+
+    params.HdfsResource(format("/user/{smokeuser}"),
+                        type="directory",
+                        action="create_on_execute",
+                        owner=params.smokeuser,
+                        mode=params.smoke_hdfs_user_mode,
+                        )
     
     params.HdfsResource(format("/user/{smokeuser}/mahoutsmokeoutput"),
                        action="delete_on_execute",

ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py

@@ -112,6 +112,7 @@ execute_path = oozie_bin_dir + os.pathsep + hadoop_bin_dir
 oozie_user = config['configurations']['oozie-env']['oozie_user']
 smokeuser = config['configurations']['cluster-env']['smokeuser']
 smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
+smoke_hdfs_user_mode = 0770
 
 # This config actually contains {oozie_user}
 oozie_admin_users = format(config['configurations']['oozie-env']['oozie_admin_users'])

ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/service_check.py

@@ -82,6 +82,13 @@ class OozieServiceCheckDefault(OozieServiceCheck):
             logoutput=True
     )
 
+    params.HdfsResource(format("/user/{smokeuser}"),
+        type="directory",
+        action="create_on_execute",
+        owner=params.smokeuser,
+        mode=params.smoke_hdfs_user_mode,
+        )
+
     examples_dir = format('/user/{smokeuser}/examples')
     params.HdfsResource(examples_dir,
                         action = "delete_on_execute",

ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params_linux.py

@@ -62,6 +62,7 @@ hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_nam
 hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
 smokeuser = config['configurations']['cluster-env']['smokeuser']
 smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
+smoke_hdfs_user_mode = 0770
 user_group = config['configurations']['cluster-env']['user_group']
 security_enabled = config['configurations']['cluster-env']['security_enabled']
 smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']

ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/service_check.py

@@ -45,6 +45,13 @@ class PigServiceCheckLinux(PigServiceCheck):
     input_file = format('/user/{smokeuser}/passwd')
     output_dir = format('/user/{smokeuser}/pigsmoke.out')
 
+    params.HdfsResource(format("/user/{smokeuser}"),
+                        type="directory",
+                        action="create_on_execute",
+                        owner=params.smokeuser,
+                        mode=params.smoke_hdfs_user_mode,
+                        )
+
     params.HdfsResource(output_dir,
                         type="directory",
                         action="delete_on_execute",

ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/mapred_service_check.py

@@ -23,6 +23,7 @@ import sys
 from resource_management import *
 from ambari_commons import OSConst
 from ambari_commons.os_family_impl import OsFamilyImpl
+from resource_management.core.logger import Logger
 
 
 class MapReduce2ServiceCheck(Script):
@@ -120,6 +121,12 @@ class MapReduce2ServiceCheckDefault(MapReduce2ServiceCheck):
     test_cmd = format("fs -test -e {output_file}")
     run_wordcount_job = format("jar {jar_path} wordcount {input_file} {output_file}")
 
+    params.HdfsResource(format("/user/{smokeuser}"),
+                      type="directory",
+                      action="create_on_execute",
+                      owner=params.smokeuser,
+                      mode=params.smoke_hdfs_user_mode,
+    )
     params.HdfsResource(output_file,
                         action = "delete_on_execute",
                         type = "directory",

ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py

@@ -135,12 +135,14 @@ hdfs_tmp_dir = config['configurations']['hadoop-env']['hdfs_tmp_dir']
 
 smokeuser = config['configurations']['cluster-env']['smokeuser']
 smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
+smoke_hdfs_user_mode = 0770
 security_enabled = config['configurations']['cluster-env']['security_enabled']
 nm_security_marker_dir = "/var/lib/hadoop-yarn"
 nm_security_marker = format('{nm_security_marker_dir}/nm_security_enabled')
 current_nm_security_state = os.path.isfile(nm_security_marker)
 toggle_nm_security = (current_nm_security_state and not security_enabled) or (not current_nm_security_state and security_enabled)
 smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
+
 yarn_executor_container_group = config['configurations']['yarn-site']['yarn.nodemanager.linux-container-executor.group']
 yarn_nodemanager_container_executor_class =  config['configurations']['yarn-site']['yarn.nodemanager.container-executor.class']
 is_linux_container_executor = (yarn_nodemanager_container_executor_class == 'org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor')

ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service_check.py

@@ -86,6 +86,13 @@ class ServiceCheckDefault(ServiceCheck):
     import params
     env.set_params(params)
 
+    params.HdfsResource(format("/user/{smokeuser}"),
+                        type="directory",
+                        action="create_on_execute",
+                        owner=params.smokeuser,
+                        mode=params.smoke_hdfs_user_mode,
+                        )
+
     if params.stack_version_formatted_major and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.stack_version_formatted_major):
       path_to_distributed_shell_jar = format("{stack_root}/current/hadoop-yarn-client/hadoop-yarn-applications-distributedshell.jar")
     else:

ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_service_check.py

@@ -83,6 +83,20 @@ class TestServiceCheck(RMFTestCase):
         tries = 3,
         try_sleep = 5,
     )
+    self.assertResourceCalled('HdfsResource', '/user/ambari-qa',
+                              immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
+                              security_enabled = False,
+                              hadoop_bin_dir = '/usr/bin',
+                              keytab = UnknownConfigurationMock(),
+                              kinit_path_local = '/usr/bin/kinit',
+                              user = 'hdfs',
+                              dfs_type = '',
+                              mode = 0770,
+                              owner = 'ambari-qa',
+                              action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020',
+                              hadoop_conf_dir = '/etc/hadoop/conf',
+                              type = 'directory',
+                              )
     self.assertResourceCalled('HdfsResource', '/user/ambari-qa/examples',
         immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
         security_enabled = False,

ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_service_check.py

@@ -33,6 +33,20 @@ class TestPigServiceCheck(RMFTestCase):
                        stack_version = self.STACK_VERSION,
                        target = RMFTestCase.TARGET_COMMON_SERVICES
     )
+    self.assertResourceCalled('HdfsResource', '/user/ambari-qa',
+                              immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
+                              security_enabled = False,
+                              hadoop_bin_dir = '/usr/bin',
+                              keytab = UnknownConfigurationMock(),
+                              kinit_path_local = '/usr/bin/kinit',
+                              user = 'hdfs',
+                              dfs_type = '',
+                              mode = 0770,
+                              owner = 'ambari-qa',
+                              action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020',
+                              hadoop_conf_dir = '/etc/hadoop/conf',
+                              type = 'directory',
+                              )
     self.assertResourceCalled('HdfsResource', '/user/ambari-qa/pigsmoke.out',
         immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
         security_enabled = False,
@@ -109,7 +123,20 @@ class TestPigServiceCheck(RMFTestCase):
                        stack_version = self.STACK_VERSION,
                        target = RMFTestCase.TARGET_COMMON_SERVICES
     )
-    
+    self.assertResourceCalled('HdfsResource', '/user/ambari-qa',
+                              immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
+                              security_enabled = True,
+                              hadoop_bin_dir = '/usr/bin',
+                              keytab = '/etc/security/keytabs/hdfs.headless.keytab',
+                              kinit_path_local = '/usr/bin/kinit',
+                              user = 'hdfs',
+                              dfs_type = '',
+                              mode = 0770,
+                              owner = 'ambari-qa',
+                              action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020',
+                              hadoop_conf_dir = '/etc/hadoop/conf',
+                              type = 'directory',
+                              )
     self.assertResourceCalled('HdfsResource', '/user/ambari-qa/pigsmoke.out',
         immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
         security_enabled = True,

ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_service_check.py

@@ -40,6 +40,20 @@ class TestServiceCheck(RMFTestCase):
                       stack_version = self.STACK_VERSION,
                       target = RMFTestCase.TARGET_COMMON_SERVICES
     )
+    self.assertResourceCalled('HdfsResource', '/user/ambari-qa',
+                              immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
+                             security_enabled = False,
+                              hadoop_bin_dir = '/usr/bin',
+                              keytab = UnknownConfigurationMock(),
+                              kinit_path_local = '/usr/bin/kinit',
+                              user = 'hdfs',
+                              dfs_type = '',
+                              mode = 0770,
+                              owner = 'ambari-qa',
+                              action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020',
+                              hadoop_conf_dir = '/etc/hadoop/conf',
+                              type = 'directory',
+                              )
     self.assertResourceCalled('HdfsResource', '/user/ambari-qa/mapredsmokeoutput',
         immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
         security_enabled = False,
@@ -100,6 +114,20 @@ class TestServiceCheck(RMFTestCase):
                       stack_version = self.STACK_VERSION,
                       target = RMFTestCase.TARGET_COMMON_SERVICES
     )
+    self.assertResourceCalled('HdfsResource', '/user/ambari-qa',
+                              immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
+                              security_enabled = True,
+                              hadoop_bin_dir = '/usr/bin',
+                              keytab = '/etc/security/keytabs/hdfs.headless.keytab',
+                              kinit_path_local = '/usr/bin/kinit',
+                              user = 'hdfs',
+                              dfs_type = '',
+                              mode = 0770,
+                              owner = 'ambari-qa',
+                              action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020',
+                              hadoop_conf_dir = '/etc/hadoop/conf',
+                              type = 'directory',
+                              )
     self.assertResourceCalled('HdfsResource', '/user/ambari-qa/mapredsmokeoutput',
         immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
         security_enabled = True,

ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_service_check.py

@@ -28,6 +28,7 @@ from stacks.utils.RMFTestCase import *
 class TestServiceCheck(RMFTestCase):
   COMMON_SERVICES_PACKAGE_DIR = "YARN/2.1.0.2.0/package"
   STACK_VERSION = "2.0.6"
+  DEFAULT_IMMUTABLE_PATHS = ['/apps/hive/warehouse', '/apps/falcon', '/mr-history/done', '/app-logs', '/tmp']
 
   @patch("re.search")
   def test_service_check_default(self, re_search_mock):
@@ -44,6 +45,20 @@ class TestServiceCheck(RMFTestCase):
                           checked_call_mocks = [(0, "some test text, appTrackingUrl=http:"
                                 "//c6402.ambari.apache.org:8088/proxy/application_1429885383763_0001/, some test text")]
     )
+    self.assertResourceCalled('HdfsResource', '/user/ambari-qa',
+                              immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
+                              security_enabled = False,
+                              hadoop_bin_dir = '/usr/bin',
+                              keytab = UnknownConfigurationMock(),
+                              kinit_path_local = '/usr/bin/kinit',
+                              user = 'hdfs',
+                              dfs_type = '',
+                              mode = 0770,
+                              owner = 'ambari-qa',
+                              action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020',
+                              hadoop_conf_dir = '/etc/hadoop/conf',
+                              type = 'directory',
+                              )
     self.assertNoMoreResources()
 
 
@@ -62,5 +77,18 @@ class TestServiceCheck(RMFTestCase):
                           checked_call_mocks = [(0, "some test text, appTrackingUrl=http:"
                                "//c6402.ambari.apache.org:8088/proxy/application_1429885383763_0001/, some test text")]
     )
-
+    self.assertResourceCalled('HdfsResource', '/user/ambari-qa',
+                              immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
+                              security_enabled = True,
+                              hadoop_bin_dir = '/usr/bin',
+                              keytab = '/etc/security/keytabs/hdfs.headless.keytab',
+                              kinit_path_local = '/usr/bin/kinit',
+                              user = 'hdfs',
+                              dfs_type = '',
+                              mode = 0770,
+                              owner = 'ambari-qa',
+                              action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020',
+                              hadoop_conf_dir = '/etc/hadoop/conf',
+                              type = 'directory',
+                              )
     self.assertNoMoreResources()

ambari-server/src/test/python/stacks/2.2/PIG/test_pig_service_check.py

@@ -37,7 +37,24 @@ class TestPigServiceCheck(RMFTestCase):
                        stack_version=self.STACK_VERSION,
                        target=RMFTestCase.TARGET_COMMON_SERVICES
     )
-    
+    self.assertResourceCalled('HdfsResource', '/user/ambari-qa',
+                              immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
+                              dfs_type = '',
+                              security_enabled = True,
+                              hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+                              keytab = '/etc/security/keytabs/hdfs.headless.keytab',
+                              kinit_path_local = '/usr/bin/kinit',
+                              user = 'hdfs',
+                              mode = 0770,
+                              owner = 'ambari-qa',
+                              action = ['create_on_execute'],
+                              hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
+                              hdfs_site=self.getConfig()['configurations']['hdfs-site'],
+                              principal_name = 'hdfs@EXAMPLE.COM',
+                              default_fs='hdfs://c6401.ambari.apache.org:8020',
+                              hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+                              type = 'directory',
+                              )
     self.assertResourceCalled('HdfsResource', '/user/ambari-qa/pigsmoke.out',
         immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
         security_enabled = True,

ambari-server/src/test/python/stacks/2.3/MAHOUT/test_mahout_service_check.py

@@ -41,6 +41,21 @@ class TestMahoutClient(RMFTestCase):
         content = 'Test text which will be converted to sequence file.',
         mode = 0755,
     )
+    self.maxDiff=None
+    self.assertResourceCalled('HdfsResource', '/user/ambari-qa',
+                              immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
+                              security_enabled = False,
+                              hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+                              keytab = UnknownConfigurationMock(),
+                              kinit_path_local = '/usr/bin/kinit',
+                              user = 'hdfs',
+                              dfs_type = '',
+                              mode = 0770,
+                              owner = 'ambari-qa',
+                              action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020',
+                              hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+                              type = 'directory',
+                              )
     self.assertResourceCalled('HdfsResource', '/user/ambari-qa/mahoutsmokeoutput',
         immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
         security_enabled = False,