AMBARI-8612. Hard permissions should be handled on user defined data directories (Szilard Nemethy via ncole)

ambari-common/src/main/python/resource_management/core/providers/system.py

@@ -149,7 +149,11 @@ class DirectoryProvider(Provider):
     if not os.path.exists(path):
       Logger.info("Creating directory %s" % self.resource)
       if self.resource.recursive:
-        sudo.makedirs(path, self.resource.mode or 0755)
+        if self.resource.recursive_permission:
+          DirectoryProvider.makedirs_and_set_permission_recursively(path, self.resource.owner,
+                                                                    self.resource.group, self.resource.mode)
+        else:
+          sudo.makedirs(path, self.resource.mode or 0755)
       else:
         dirname = os.path.dirname(path)
         if not os.path.isdir(dirname):
@@ -163,6 +167,23 @@ class DirectoryProvider(Provider):
     _ensure_metadata(path, self.resource.owner, self.resource.group,
                         mode=self.resource.mode)
 
+  @staticmethod
+  def makedirs_and_set_permission_recursively(path, owner, group, mode):
+    folders=[]
+    path,folder=os.path.split(path)
+    while folder!="":
+      folders.append(folder)
+      path,folder=os.path.split(path)
+    if path!="":
+      folders.append(path)
+    folders.reverse()
+    dir_prefix=""
+    for folder in folders:
+      dir_prefix=os.path.join(dir_prefix, folder)
+      if not os.path.exists(dir_prefix):
+        sudo.makedir(dir_prefix, mode or 0755)
+        _ensure_metadata(dir_prefix, None, None, mode)
+
   def action_delete(self):
     path = self.resource.path
     if os.path.exists(path):

ambari-common/src/main/python/resource_management/core/resources/system.py

@@ -47,6 +47,7 @@ class Directory(Resource):
   owner = ResourceArgument()
   group = ResourceArgument()
   recursive = BooleanArgument(default=False) # this work for 'create', 'delete' is anyway recursive
+  recursive_permission = BooleanArgument(default=False) # sets given perms to all non-existent folders which are created recursively
 
   actions = Resource.actions + ["create", "delete"]
 

ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py

@@ -95,8 +95,6 @@ else:
   hadoop_home = "/usr/lib/hadoop"
   hadoop_secure_dn_user = hdfs_user
 
-hadoop_root_dir = "/hadoop"
-hadoop_data_dir = "/hadoop/hdfs"
 hadoop_dir = "/etc/hadoop"
 hadoop_conf_dir = "/etc/hadoop/conf"
 hadoop_conf_empty_dir = "/etc/hadoop/conf.empty"

ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py

@@ -83,6 +83,12 @@ def setup_users():
   set_uid(params.smoke_user, params.smoke_user_dirs)
 
   if params.has_hbase_masters:
+    Directory (params.hbase_tmp_dir,
+               owner = params.hbase_user,
+               mode=0775,
+               recursive = True,
+               recursive_permission = True
+    )
     set_uid(params.hbase_user, params.hbase_user_dirs)
     
 def set_uid(user, user_dirs):
@@ -106,16 +112,9 @@ def setup_hadoop_env():
     else:
       tc_owner = params.hdfs_user
 
-    Directory(params.hadoop_root_dir,
-              mode=0755
-    )
     Directory(params.hadoop_dir,
               mode=0755
     )
-    Directory(params.hadoop_data_dir,
-              owner=params.hdfs_user,
-              group=params.user_group
-    )
     if stackversion.find('Gluster') >= 0:
         Directory(params.hadoop_conf_empty_dir,
               recursive=True,

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/scripts/hbase.py

@@ -39,7 +39,8 @@ def hbase(name=None # 'master' or 'regionserver' or 'client'
   Directory (params.tmp_dir,
              owner = params.hbase_user,
              mode=0775,
-             recursive = True
+             recursive = True,
+             recursive_permission = True
   )
 
   Directory (params.local_dir,

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/hdfs_datanode.py

@@ -29,6 +29,7 @@ def create_dirs(data_dir, params):
   """
   Directory(data_dir,
             recursive=True,
+            recursive_permission=True,
             mode=0755,
             owner=params.hdfs_user,
             group=params.user_group,

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/hdfs_namenode.py

@@ -117,7 +117,8 @@ def create_name_dirs(directories):
             mode=0755,
             owner=params.hdfs_user,
             group=params.user_group,
-            recursive=True
+            recursive=True,
+            recursive_permission=True
   )
 
 

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/hdfs_snamenode.py

@@ -28,6 +28,7 @@ def snamenode(action=None, format=False):
   if action == "configure":
     Directory(params.fs_checkpoint_dir,
               recursive=True,
+              recursive_permission=True,
               mode=0755,
               owner=params.hdfs_user,
               group=params.user_group)

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/journalnode.py

@@ -71,6 +71,7 @@ class JournalNode(Script):
 
     Directory(params.jn_edits_dir,
               recursive=True,
+              recursive_permission=True,
               owner=params.hdfs_user,
               group=params.user_group
     )

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/OOZIE/package/scripts/oozie.py

@@ -128,12 +128,13 @@ def oozie_server_specific():
     not_if="ls {pid_file} >/dev/null 2>&1 && !(ps `cat {pid_file}` >/dev/null 2>&1)"
   )
   
-  oozie_server_directorties = [format("{oozie_home}/{oozie_tmp_dir}"), params.oozie_pid_dir, params.oozie_log_dir, params.oozie_tmp_dir, os.path.abspath(os.path.join(params.oozie_data_dir, "..")), params.oozie_data_dir, params.oozie_lib_dir, params.oozie_webapps_dir, params.oozie_webapps_conf_dir, params.oozie_server_dir]
+  oozie_server_directorties = [format("{oozie_home}/{oozie_tmp_dir}"), params.oozie_pid_dir, params.oozie_log_dir, params.oozie_tmp_dir, params.oozie_data_dir, params.oozie_lib_dir, params.oozie_webapps_dir, params.oozie_webapps_conf_dir, params.oozie_server_dir]
   Directory( oozie_server_directorties,
     owner = params.oozie_user,
     group = params.user_group,
     mode = 0755,
-    recursive = True
+    recursive = True,
+    recursive_permission=True
   )
   
   Directory(params.oozie_libext_dir,

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/params.py

@@ -117,8 +117,6 @@ hadoopMapredExamplesJarName = "hadoop-mapreduce-examples-2.*.jar"
 yarn_pid_dir = status_params.yarn_pid_dir
 mapred_pid_dir = status_params.mapred_pid_dir
 
-yarn_data_dir = "/hadoop/yarn"
-
 mapred_log_dir = format("{mapred_log_dir_prefix}/{mapred_user}")
 yarn_log_dir = format("{yarn_log_dir_prefix}/{yarn_user}")
 mapred_job_summary_log = format("{mapred_log_dir_prefix}/{mapred_user}/hadoop-mapreduce.jobsummary.log")

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/yarn.py

@@ -65,6 +65,7 @@ def yarn(name = None):
               owner=params.yarn_user,
               group=params.user_group,
               recursive=True,
+              recursive_permission=True,
               ignore_failures=True,
               mode=0775
               )
@@ -85,10 +86,6 @@ def yarn(name = None):
             recursive=True,
             ignore_failures=True,
   )
-  Directory(params.yarn_data_dir,
-            owner=params.yarn_user,
-            group=params.user_group
-  )
 
   XmlConfig("core-site.xml",
             conf_dir=params.hadoop_conf_dir,
@@ -142,7 +139,8 @@ def yarn(name = None):
     Directory(params.ats_leveldb_dir,
        owner=params.yarn_user,
        group=params.user_group,
-       recursive=True
+       recursive=True,
+       recursive_permission=True
     )
 
   File(params.rm_nodes_exclude_path,

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/ZOOKEEPER/package/scripts/zookeeper.py

@@ -58,6 +58,7 @@ def zookeeper(type = None):
   Directory(params.zk_data_dir,
             owner=params.zk_user,
             recursive=True,
+            recursive_permission=True,
             group=params.user_group
   )
 

ambari-server/src/main/resources/stacks/HDP/2.1/services/FALCON/package/scripts/falcon.py

@@ -68,7 +68,8 @@ def falcon(type, action = None):
                 owner=params.falcon_user,
                 group=params.user_group,
                 mode=0775,
-                recursive=True
+                recursive=True,
+                recursive_permission=True
       )
 
     if params.falcon_graph_serialize_path:
@@ -76,7 +77,8 @@ def falcon(type, action = None):
                 owner=params.falcon_user,
                 group=params.user_group,
                 mode=0775,
-                recursive=True
+                recursive=True,
+                recursive_permission=True
       )
 
   if type == 'server':
@@ -95,7 +97,8 @@ def falcon(type, action = None):
       params.HdfsDirectory(None, action="create")
       Directory(params.falcon_local_dir,
                 owner=params.falcon_user,
-                recursive=True
+                recursive=True,
+                recursive_permission=True
       )
       if params.falcon_embeddedmq_enabled == True:
         Directory(os.path.abspath(os.path.join(params.falcon_embeddedmq_data, "..")),

ambari-server/src/main/resources/stacks/HDP/2.1/services/STORM/package/scripts/storm.py

@@ -35,7 +35,8 @@ def storm():
   Directory([params.pid_dir, params.local_dir, params.conf_dir],
             owner=params.storm_user,
             group=params.user_group,
-            recursive=True
+            recursive=True,
+            recursive_permission=True
   )
 
   File(format("{conf_dir}/config.yaml"),

ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py

@@ -42,6 +42,7 @@ class TestHBaseClient(RMFTestCase):
       owner = 'hbase',
       mode=0775,
       recursive = True,
+      recursive_permission = True
     )
     self.assertResourceCalled('Directory', '/hadoop/hbase/local',
       owner = 'hbase',
@@ -123,6 +124,7 @@ class TestHBaseClient(RMFTestCase):
       owner = 'hbase',
       mode=0775,
       recursive = True,
+      recursive_permission = True
     )
     self.assertResourceCalled('Directory', '/hadoop/hbase/local',
       owner = 'hbase',

ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py

@@ -184,6 +184,7 @@ class TestHBaseMaster(RMFTestCase):
       owner = 'hbase',
       mode = 0775,
       recursive = True,
+      recursive_permission = True
     )
     self.assertResourceCalled('Directory', '/hadoop/hbase/local',
       owner = 'hbase',
@@ -293,6 +294,7 @@ class TestHBaseMaster(RMFTestCase):
       owner = 'hbase',
       mode = 0775,
       recursive = True,
+      recursive_permission = True
     )
     self.assertResourceCalled('Directory', '/hadoop/hbase/local',
       owner = 'hbase',

ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py

@@ -117,6 +117,7 @@ class TestHbaseRegionServer(RMFTestCase):
       owner = 'hbase',
       mode=0775,
       recursive = True,
+      recursive_permission = True
     )
     self.assertResourceCalled('Directory', '/hadoop/hbase/local',
       owner = 'hbase',
@@ -226,6 +227,7 @@ class TestHbaseRegionServer(RMFTestCase):
       owner = 'hbase',
       mode=0775,
       recursive = True,
+      recursive_permission = True
     )
     self.assertResourceCalled('Directory', '/hadoop/hbase/local',
       owner = 'hbase',

ambari-server/src/test/python/stacks/2.0.6/HDFS/test_datanode.py

@@ -368,6 +368,7 @@ class TestDatanode(RMFTestCase):
                               group = 'hadoop',
                               mode = 0755,
                               recursive = True,
+                              recursive_permission = True
                               )
 
   def assert_configure_secured(self):
@@ -413,4 +414,5 @@ class TestDatanode(RMFTestCase):
                               group = 'hadoop',
                               mode = 0755,
                               recursive = True,
+                              recursive_permission = True
                               )

ambari-server/src/test/python/stacks/2.0.6/HDFS/test_journalnode.py

@@ -160,6 +160,7 @@ class TestJournalnode(RMFTestCase):
                               owner = 'hdfs',
                               group = 'hadoop',
                               recursive = True,
+                              recursive_permission = True
                               )
     self.assertResourceCalled('Directory', '/etc/security/limits.d',
                               owner = 'root',
@@ -197,6 +198,7 @@ class TestJournalnode(RMFTestCase):
                               owner = 'hdfs',
                               group = 'hadoop',
                               recursive = True,
+                              recursive_permission = True
                               )
     self.assertResourceCalled('Directory', '/etc/security/limits.d',
                               owner = 'root',

ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py

@@ -528,6 +528,7 @@ class TestNamenode(RMFTestCase):
                               group = 'hadoop',
                               recursive = True,
                               mode = 0755,
+                              recursive_permission = True
                               )
 
   def assert_configure_secured(self):
@@ -566,6 +567,7 @@ class TestNamenode(RMFTestCase):
                               group = 'hadoop',
                               recursive = True,
                               mode = 0755,
+                              recursive_permission = True
                               )
 
   @patch("resource_management.libraries.script.Script.put_structured_out")

ambari-server/src/test/python/stacks/2.0.6/HDFS/test_snamenode.py

@@ -221,6 +221,7 @@ class TestSNamenode(RMFTestCase):
                               group = 'hadoop',
                               mode = 0755,
                               recursive = True,
+                              recursive_permission = True
                               )
 
   def assert_configure_secured(self):
@@ -259,4 +260,5 @@ class TestSNamenode(RMFTestCase):
                               group = 'hadoop',
                               mode = 0755,
                               recursive = True,
+                              recursive_permission = True
                               )

ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py

@@ -187,60 +187,63 @@ class TestOozieServer(RMFTestCase):
         group = 'hadoop',
         recursive = True,
         mode = 0755,
+        recursive_permission = True
     )
     self.assertResourceCalled('Directory', '/var/run/oozie',
                               owner = 'oozie',
                               group = 'hadoop',
                               recursive = True,
                               mode = 0755,
+                              recursive_permission = True
                               )
     self.assertResourceCalled('Directory', '/var/log/oozie',
                               owner = 'oozie',
                               group = 'hadoop',
                               recursive = True,
                               mode = 0755,
+                              recursive_permission = True
                               )
     self.assertResourceCalled('Directory', '/var/tmp/oozie',
                               owner = 'oozie',
                               group = 'hadoop',
                               recursive = True,
                               mode = 0755,
-                              )
-    self.assertResourceCalled('Directory', '/hadoop/oozie',
-                              owner = 'oozie',
-                              group = 'hadoop',
-                              recursive = True,
-                              mode = 0755,
+                              recursive_permission = True
                               )
     self.assertResourceCalled('Directory', '/hadoop/oozie/data',
                               owner = 'oozie',
                               group = 'hadoop',
                               recursive = True,
                               mode = 0755,
+                              recursive_permission = True
                               )
     self.assertResourceCalled('Directory', '/var/lib/oozie/',
                               owner = 'oozie',
                               group = 'hadoop',
                               recursive = True,
                               mode = 0755,
+                              recursive_permission = True
                               )
     self.assertResourceCalled('Directory', '/var/lib/oozie/oozie-server/webapps/',
                               owner = 'oozie',
                               group = 'hadoop',
                               recursive = True,
                               mode = 0755,
+                              recursive_permission = True
                               )
     self.assertResourceCalled('Directory', '/var/lib/oozie/oozie-server/conf',
                               owner = 'oozie',
                               group = 'hadoop',
                               recursive = True,
                               mode = 0755,
+                              recursive_permission = True
                               )
     self.assertResourceCalled('Directory', '/var/lib/oozie/oozie-server',
                               owner = 'oozie',
                               group = 'hadoop',
                               recursive = True,
                               mode = 0755,
+                              recursive_permission = True
                               )
     self.assertResourceCalled('Directory', '/usr/lib/oozie/libext',
         recursive = True,
@@ -337,60 +340,63 @@ class TestOozieServer(RMFTestCase):
         group = 'hadoop',
         recursive = True,
         mode = 0755,
+        recursive_permission = True
     )
     self.assertResourceCalled('Directory', '/var/run/oozie',
                               owner = 'oozie',
                               group = 'hadoop',
                               recursive = True,
                               mode = 0755,
+                              recursive_permission = True
                               )
     self.assertResourceCalled('Directory', '/var/log/oozie',
                               owner = 'oozie',
                               group = 'hadoop',
                               recursive = True,
                               mode = 0755,
+                              recursive_permission = True
                               )
     self.assertResourceCalled('Directory', '/var/tmp/oozie',
                               owner = 'oozie',
                               group = 'hadoop',
                               recursive = True,
                               mode = 0755,
-                              )
-    self.assertResourceCalled('Directory', '/hadoop/oozie',
-                              owner = 'oozie',
-                              group = 'hadoop',
-                              recursive = True,
-                              mode = 0755,
+                              recursive_permission = True
                               )
     self.assertResourceCalled('Directory', '/hadoop/oozie/data',
                               owner = 'oozie',
                               group = 'hadoop',
                               recursive = True,
                               mode = 0755,
+                              recursive_permission = True
                               )
     self.assertResourceCalled('Directory', '/var/lib/oozie/',
                               owner = 'oozie',
                               group = 'hadoop',
                               recursive = True,
                               mode = 0755,
+                              recursive_permission = True
                               )
     self.assertResourceCalled('Directory', '/var/lib/oozie/oozie-server/webapps/',
                               owner = 'oozie',
                               group = 'hadoop',
                               recursive = True,
                               mode = 0755,
+                              recursive_permission = True
                               )
     self.assertResourceCalled('Directory', '/var/lib/oozie/oozie-server/conf',
                               owner = 'oozie',
                               group = 'hadoop',
                               recursive = True,
                               mode = 0755,
+                              recursive_permission = True
                               )
     self.assertResourceCalled('Directory', '/var/lib/oozie/oozie-server',
                               owner = 'oozie',
                               group = 'hadoop',
                               recursive = True,
                               mode = 0755,
+                              recursive_permission = True
                               )
     self.assertResourceCalled('Directory', '/usr/lib/oozie/libext',
         recursive = True,

ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py

@@ -219,10 +219,6 @@ class TestHistoryServer(RMFTestCase):
       recursive = True,
       ignore_failures = True,
     )
-    self.assertResourceCalled('Directory', '/hadoop/yarn',
-      owner = 'yarn',
-      group = 'hadoop'
-    )
     self.assertResourceCalled('XmlConfig', 'core-site.xml',
       owner = 'hdfs',
       group = 'hadoop',
@@ -415,10 +411,6 @@ class TestHistoryServer(RMFTestCase):
       recursive = True,
       ignore_failures = True,
     )
-    self.assertResourceCalled('Directory', '/hadoop/yarn',
-      owner = 'yarn',
-      group = 'hadoop'
-    )
     self.assertResourceCalled('XmlConfig', 'core-site.xml',
       owner = 'hdfs',
       group = 'hadoop',

ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_client.py

@@ -74,10 +74,6 @@ class TestMapReduce2Client(RMFTestCase):
       recursive = True,
       ignore_failures = True,
     )
-    self.assertResourceCalled('Directory', '/hadoop/yarn',
-      owner = 'yarn',
-      group = 'hadoop'
-    )
     self.assertResourceCalled('XmlConfig', 'core-site.xml',
       owner = 'hdfs',
       group = 'hadoop',
@@ -211,10 +207,6 @@ class TestMapReduce2Client(RMFTestCase):
       recursive = True,
       ignore_failures = True,
     )
-    self.assertResourceCalled('Directory', '/hadoop/yarn',
-      owner = 'yarn',
-      group = 'hadoop'
-    )
     self.assertResourceCalled('XmlConfig', 'core-site.xml',
       owner = 'hdfs',
       group = 'hadoop',

ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py

@@ -182,27 +182,31 @@ class TestNodeManager(RMFTestCase):
                               mode = 0775,
                               recursive = True,
                               ignore_failures = True,
+                              recursive_permission=True
                               )
     self.assertResourceCalled('Directory', '/hadoop/yarn/local1',
                               owner = 'yarn',
                               recursive = True,
                               group = 'hadoop',
                               ignore_failures = True,
-                              mode = 0775
+                              mode = 0775,
+                              recursive_permission=True
                               )
     self.assertResourceCalled('Directory', '/hadoop/yarn/log',
                               owner = 'yarn',
                               group = 'hadoop',
                               recursive = True,
                               ignore_failures = True,
-                              mode = 0775
+                              mode = 0775,
+                              recursive_permission=True
                               )
     self.assertResourceCalled('Directory', '/hadoop/yarn/log1',
                               owner = 'yarn',
                               group = 'hadoop',
                               recursive = True,
                               ignore_failures = True,
-                              mode = 0775
+                              mode = 0775,
+                              recursive_permission=True
                               )
     self.assertResourceCalled('Directory', '/var/run/hadoop-yarn',
       owner = 'yarn',
@@ -244,10 +248,6 @@ class TestNodeManager(RMFTestCase):
       recursive = True,
       ignore_failures = True,
     )
-    self.assertResourceCalled('Directory', '/hadoop/yarn',
-      owner = 'yarn',
-      group = 'hadoop'
-    )
     self.assertResourceCalled('XmlConfig', 'core-site.xml',
       owner = 'hdfs',
       group = 'hadoop',
@@ -405,14 +405,16 @@ class TestNodeManager(RMFTestCase):
                               group = 'hadoop',
                               recursive = True,
                               ignore_failures = True,
-                              mode = 0775
+                              mode = 0775,
+                              recursive_permission=True
                               )
     self.assertResourceCalled('Directory', '/hadoop/yarn/log',
                               owner = 'yarn',
                               group = 'hadoop',
                               recursive = True,
                               ignore_failures = True,
-                              mode = 0775
+                              mode = 0775,
+                              recursive_permission=True
                               )
     self.assertResourceCalled('Directory', '/var/run/hadoop-yarn',
       owner = 'yarn',
@@ -454,10 +456,6 @@ class TestNodeManager(RMFTestCase):
       recursive = True,
       ignore_failures = True,
     )
-    self.assertResourceCalled('Directory', '/hadoop/yarn',
-      owner = 'yarn',
-      group = 'hadoop'
-    )
     self.assertResourceCalled('XmlConfig', 'core-site.xml',
       owner = 'hdfs',
       group = 'hadoop',

ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py

@@ -189,10 +189,6 @@ class TestResourceManager(RMFTestCase):
       recursive = True,
       ignore_failures = True,
     )
-    self.assertResourceCalled('Directory', '/hadoop/yarn',
-      owner = 'yarn',
-      group = 'hadoop'
-    )
     self.assertResourceCalled('XmlConfig', 'core-site.xml',
       owner = 'hdfs',
       group = 'hadoop',
@@ -324,10 +320,6 @@ class TestResourceManager(RMFTestCase):
       recursive = True,
       ignore_failures = True,
     )
-    self.assertResourceCalled('Directory', '/hadoop/yarn',
-      owner = 'yarn',
-      group = 'hadoop'
-    )
     self.assertResourceCalled('XmlConfig', 'core-site.xml',
       owner = 'hdfs',
       group = 'hadoop',

ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_client.py

@@ -75,10 +75,6 @@ class TestYarnClient(RMFTestCase):
       recursive = True,
       ignore_failures = True,
     )
-    self.assertResourceCalled('Directory', '/hadoop/yarn',
-      owner = 'yarn',
-      group = 'hadoop'
-    )
     self.assertResourceCalled('XmlConfig', 'core-site.xml',
       owner = 'hdfs',
       group = 'hadoop',
@@ -212,10 +208,6 @@ class TestYarnClient(RMFTestCase):
       recursive = True,
       ignore_failures = True,
     )
-    self.assertResourceCalled('Directory', '/hadoop/yarn',
-      owner = 'yarn',
-      group = 'hadoop'
-    )
     self.assertResourceCalled('XmlConfig', 'core-site.xml',
       owner = 'hdfs',
       group = 'hadoop',
@@ -366,10 +358,6 @@ class TestYarnClient(RMFTestCase):
       recursive = True,
       ignore_failures = True,
     )
-    self.assertResourceCalled('Directory', '/hadoop/yarn',
-      owner = 'yarn',
-      group = 'hadoop'
-    )
     self.assertResourceCalled('XmlConfig', 'core-site.xml',
       owner = 'hdfs',
       group = 'hadoop',

ambari-server/src/test/python/stacks/2.0.6/ZOOKEEPER/test_zookeeper_client.py

@@ -64,6 +64,7 @@ class TestZookeeperClient(RMFTestCase):
       owner = 'zookeeper',
       group = 'hadoop',
       recursive = True,
+      recursive_permission = True
     )
     self.assertResourceCalled('File',
                               '/etc/zookeeper/conf/log4j.properties',
@@ -120,6 +121,7 @@ class TestZookeeperClient(RMFTestCase):
       owner = 'zookeeper',
       group = 'hadoop',
       recursive = True,
+      recursive_permission = True
     )
     self.assertResourceCalled('File',
                               '/etc/zookeeper/conf/log4j.properties',

ambari-server/src/test/python/stacks/2.0.6/ZOOKEEPER/test_zookeeper_server.py

@@ -133,6 +133,7 @@ class TestZookeeperServer(RMFTestCase):
       owner = 'zookeeper',
       group = 'hadoop',
       recursive = True,
+      recursive_permission = True
     )
     self.assertResourceCalled('File', '/hadoop/zookeeper/myid',
       content = '1',
@@ -185,6 +186,7 @@ class TestZookeeperServer(RMFTestCase):
       owner = 'zookeeper',
       group = 'hadoop',
       recursive = True,
+      recursive_permission = True
     )
     self.assertResourceCalled('File', '/hadoop/zookeeper/myid',
       content = '1',

ambari-server/src/test/python/stacks/2.0.6/hooks/before-ANY/test_before_any.py

@@ -127,6 +127,12 @@ class TestHookBeforeInstall(RMFTestCase):
     self.assertResourceCalled('Execute', '/tmp/changeUid.sh ambari-qa /tmp/hadoop-ambari-qa,/tmp/hsperfdata_ambari-qa,/home/ambari-qa,/tmp/ambari-qa,/tmp/sqoop-ambari-qa',
         not_if = 'test $(id -u ambari-qa) -gt 1000',
     )
+    self.assertResourceCalled('Directory', '/hadoop/hbase',
+        owner = 'hbase',
+        mode = 0775,
+        recursive = True,
+        recursive_permission = True
+    )
     self.assertResourceCalled('File', '/tmp/changeUid.sh',
         content = StaticFile('changeToSecureUid.sh'),
         mode = 0555,
@@ -134,16 +140,9 @@ class TestHookBeforeInstall(RMFTestCase):
     self.assertResourceCalled('Execute', '/tmp/changeUid.sh hbase /home/hbase,/tmp/hbase,/usr/bin/hbase,/var/log/hbase,/hadoop/hbase',
         not_if = 'test $(id -u hbase) -gt 1000',
     )
-    self.assertResourceCalled('Directory', '/hadoop',
-        mode = 0755
-    )
     self.assertResourceCalled('Directory', '/etc/hadoop',
         mode = 0755
     )
-    self.assertResourceCalled('Directory', '/hadoop/hdfs',
-        owner = 'hdfs',
-        group = 'hadoop'
-    )
     self.assertResourceCalled('Directory', '/etc/hadoop/conf.empty',
         owner = 'hdfs',
         group = 'hadoop',

ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py

@@ -123,6 +123,7 @@ class TestFalconServer(RMFTestCase):
     self.assertResourceCalled('Directory', '/hadoop/falcon',
                               owner = 'falcon',
                               recursive = True,
+                              recursive_permission = True
                               )
     self.assertResourceCalled('Directory', '/hadoop/falcon/embeddedmq',
                               owner = 'falcon'

ambari-server/src/test/python/stacks/2.1/STORM/test_storm_base.py

@@ -37,16 +37,19 @@ class TestStormBase(RMFTestCase):
       owner = 'storm',
       group = 'hadoop',
       recursive = True,
+      recursive_permission = True
     )
     self.assertResourceCalled('Directory', '/hadoop/storm',
       owner = 'storm',
       group = 'hadoop',
       recursive = True,
+      recursive_permission = True
     )
     self.assertResourceCalled('Directory', '/etc/storm/conf',
       owner = 'storm',
       group = 'hadoop',
       recursive = True,
+      recursive_permission = True
     )
     self.assertResourceCalled('File', '/etc/storm/conf/config.yaml',
       owner = 'storm',
@@ -76,16 +79,19 @@ class TestStormBase(RMFTestCase):
       owner = 'storm',
       group = 'hadoop',
       recursive = True,
+      recursive_permission = True
     )
     self.assertResourceCalled('Directory', '/hadoop/storm',
       owner = 'storm',
       group = 'hadoop',
       recursive = True,
+      recursive_permission = True
     )
     self.assertResourceCalled('Directory', '/etc/storm/conf',
       owner = 'storm',
       group = 'hadoop',
       recursive = True,
+      recursive_permission = True
     )
     self.assertResourceCalled('File', '/etc/storm/conf/config.yaml',
       owner = 'storm',

ambari-server/src/test/python/stacks/2.1/YARN/test_apptimelineserver.py

@@ -115,10 +115,6 @@ class TestAppTimelineServer(RMFTestCase):
                               recursive = True,
                               ignore_failures = True,
                               )
-    self.assertResourceCalled('Directory', '/hadoop/yarn',
-                              owner = 'yarn',
-                              group = 'hadoop'
-                              )
     self.assertResourceCalled('XmlConfig', 'core-site.xml',
                               owner = 'hdfs',
                               group = 'hadoop',
@@ -154,7 +150,8 @@ class TestAppTimelineServer(RMFTestCase):
     self.assertResourceCalled('Directory', '/var/log/hadoop-yarn/timeline',
                               owner = 'yarn',
                               group = 'hadoop',
-                              recursive = True
+                              recursive = True,
+                              recursive_permission = True
                               )
     self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude',
                               owner = 'yarn',