AMBARI-15776. Not all log and PID dirs have "hadoop" group (aonishuk)

ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume.py

@@ -94,13 +94,16 @@ def flume(action = None):
       )
       
     Directory(params.flume_run_dir,
+              group=params.user_group,
+              owner=params.flume_user,
     )
 
     Directory(params.flume_conf_dir,
               create_parents = True,
               owner=params.flume_user,
               )
-    Directory(params.flume_log_dir, 
+    Directory(params.flume_log_dir,
+              group=params.user_group,
               owner=params.flume_user,
               cd_access="a",
               mode=0755,

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/utils.py

@@ -208,6 +208,7 @@ def service(action=None, name=None, user=None, options="", create_pid_dir=False,
     if create_pid_dir:
       Directory(pid_dir,
                 owner=user,
+                group=params.user_group,
                 create_parents = True)
     if create_log_dir:
       if name == "nfs3":
@@ -218,6 +219,7 @@ def service(action=None, name=None, user=None, options="", create_pid_dir=False,
       else:
         Directory(log_dir,
                   owner=user,
+                  group=params.user_group,
                   create_parents = True)
 
   if params.security_enabled and name == "datanode":

ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py

@@ -181,6 +181,7 @@ def yarn(name = None):
   )
   Directory([params.yarn_log_dir_prefix],
             owner=params.yarn_user,
+            group=params.user_group,
             create_parents = True,
             ignore_failures=True,
             cd_access = 'a',

ambari-server/src/test/python/stacks/2.0.6/FLUME/test_flume.py

@@ -227,7 +227,9 @@ class TestFlumeHandler(RMFTestCase):
     self.assertNoMoreResources()
 
   def assert_configure_default(self, check_mc=True):
-    self.assertResourceCalled('Directory', '/var/run/flume',)
+    self.assertResourceCalled('Directory', '/var/run/flume',
+                              owner = 'flume',
+                              group = 'hadoop')
     self.assertResourceCalled('Directory',
                               '/etc/flume/conf',
                               owner='flume',
@@ -236,6 +238,7 @@ class TestFlumeHandler(RMFTestCase):
     self.assertResourceCalled('Directory',
                               '/var/log/flume',
                               owner = 'flume',
+                              group = 'hadoop',
                               cd_access = 'a', 
                               mode=0755
     )
@@ -275,13 +278,22 @@ class TestFlumeHandler(RMFTestCase):
       )
 
   def assert_configure_many(self):
-    self.assertResourceCalled('Directory', '/var/run/flume')
+    self.assertResourceCalled('Directory',
+                              '/var/run/flume',
+                              owner='flume',
+                              group = 'hadoop'
+                              )
     self.assertResourceCalled('Directory',
                               '/etc/flume/conf',
                               owner='flume',
                               create_parents = True)
 
-    self.assertResourceCalled('Directory', '/var/log/flume', owner = 'flume', cd_access = 'a', mode=0755)
+    self.assertResourceCalled('Directory',
+                              '/var/log/flume',
+                              owner = 'flume',
+                              group = 'hadoop',
+                              cd_access = 'a',
+                              mode=0755)
 
     top = build_flume(self.getConfig()['configurations']['flume-conf']['content'])
 
@@ -438,7 +450,9 @@ class TestFlumeHandler(RMFTestCase):
                        stack_version = self.STACK_VERSION,
                        target = RMFTestCase.TARGET_COMMON_SERVICES)
 
-    self.assertResourceCalled('Directory', '/var/run/flume',)
+    self.assertResourceCalled('Directory', '/var/run/flume',
+                              group = 'hadoop',
+                              owner='flume',)
     self.assertResourceCalled('Directory',
                               '/etc/flume/conf',
                               owner='flume',
@@ -448,6 +462,7 @@ class TestFlumeHandler(RMFTestCase):
                               '/var/log/flume',
                               owner = 'flume',
                               cd_access = 'a',
+                              group = 'hadoop',
                               mode=0755)
 
     self.assertResourceCalled('Directory',
@@ -489,7 +504,9 @@ class TestFlumeHandler(RMFTestCase):
                        stack_version = self.STACK_VERSION,
                        target = RMFTestCase.TARGET_COMMON_SERVICES)
 
-    self.assertResourceCalled('Directory', '/var/run/flume',)
+    self.assertResourceCalled('Directory', '/var/run/flume',
+                              group = 'hadoop',
+                              owner='flume')
     self.assertResourceCalled('Directory',
                               '/usr/hdp/current/flume-server/conf',
                               owner='flume',
@@ -497,7 +514,8 @@ class TestFlumeHandler(RMFTestCase):
 
     self.assertResourceCalled('Directory',
                               '/var/log/flume',
-                              owner = 'flume', 
+                              owner = 'flume',
+                              group = 'hadoop',
                               cd_access = 'a', 
                               mode=0755)
 

ambari-server/src/test/python/stacks/2.0.6/HDFS/test_datanode.py

@@ -57,10 +57,12 @@ class TestDatanode(RMFTestCase):
                               )
     self.assertResourceCalled('Directory', '/var/run/hadoop/hdfs',
                               owner = 'hdfs',
+                              group = 'hadoop',
                               create_parents = True,
                               )
     self.assertResourceCalled('Directory', '/var/log/hadoop/hdfs',
                               owner = 'hdfs',
+                              group = 'hadoop',
                               create_parents = True,
                               )
     self.assertResourceCalled('File', '/var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid',
@@ -117,10 +119,12 @@ class TestDatanode(RMFTestCase):
                               )
     self.assertResourceCalled('Directory', '/var/run/hadoop/hdfs',
                               owner = 'hdfs',
+                              group = 'hadoop',
                               create_parents = True,
                               )
     self.assertResourceCalled('Directory', '/var/log/hadoop/hdfs',
                               owner = 'hdfs',
+                              group = 'hadoop',
                               create_parents = True,
                               )
     self.assertResourceCalled('File', '/var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid',
@@ -155,10 +159,12 @@ class TestDatanode(RMFTestCase):
                               )
     self.assertResourceCalled('Directory', '/var/run/hadoop/hdfs',
                               owner = 'hdfs',
+                              group = 'hadoop',
                               create_parents = True,
                               )
     self.assertResourceCalled('Directory', '/var/log/hadoop/hdfs',
                               owner = 'hdfs',
+                              group = 'hadoop',
                               create_parents = True,
                               )
     self.assertResourceCalled('File', '/var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid',
@@ -196,10 +202,12 @@ class TestDatanode(RMFTestCase):
                               )
     self.assertResourceCalled('Directory', '/var/run/hadoop/hdfs',
                               owner = 'hdfs',
+                              group = 'hadoop',
                               create_parents = True,
                               )
     self.assertResourceCalled('Directory', '/var/log/hadoop/hdfs',
                               owner = 'hdfs',
+                              group = 'hadoop',
                               create_parents = True,
                               )
     self.assertResourceCalled('File', '/var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid',

ambari-server/src/test/python/stacks/2.0.6/HDFS/test_journalnode.py

@@ -55,10 +55,12 @@ class TestJournalnode(RMFTestCase):
                               )
     self.assertResourceCalled('Directory', '/var/run/hadoop/hdfs',
                               owner = 'hdfs',
+                              group = 'hadoop',
                               create_parents = True,
                               )
     self.assertResourceCalled('Directory', '/var/log/hadoop/hdfs',
                               owner = 'hdfs',
+                              group = 'hadoop',
                               create_parents = True,
                               )
     self.assertResourceCalled('File', '/var/run/hadoop/hdfs/hadoop-hdfs-journalnode.pid',
@@ -113,10 +115,12 @@ class TestJournalnode(RMFTestCase):
                               )
     self.assertResourceCalled('Directory', '/var/run/hadoop/hdfs',
                               owner = 'hdfs',
+                              group = 'hadoop',
                               create_parents = True,
                               )
     self.assertResourceCalled('Directory', '/var/log/hadoop/hdfs',
                               owner = 'hdfs',
+                              group = 'hadoop',
                               create_parents = True,
                               )
     self.assertResourceCalled('File', '/var/run/hadoop/hdfs/hadoop-hdfs-journalnode.pid',

ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py

@@ -75,10 +75,12 @@ class TestNamenode(RMFTestCase):
     )
     self.assertResourceCalled('Directory', '/var/run/hadoop/hdfs',
                               owner = 'hdfs',
+                              group = 'hadoop',
                               create_parents = True,
                               )
     self.assertResourceCalled('Directory', '/var/log/hadoop/hdfs',
                               owner = 'hdfs',
+                              group = 'hadoop',
                               create_parents = True,
                               )
     self.assertResourceCalled('File', '/var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid',
@@ -192,10 +194,12 @@ class TestNamenode(RMFTestCase):
     self.assertResourceCalled('Directory', '/var/run/hadoop/hdfs',
                               owner = 'hdfs',
                               create_parents = True,
+                              group = 'hadoop'
                               )
     self.assertResourceCalled('Directory', '/var/log/hadoop/hdfs',
                               owner = 'hdfs',
                               create_parents = True,
+                              group = 'hadoop'
                               )
     self.assertResourceCalled('File', '/var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid',
         action = ['delete'],
@@ -320,10 +324,12 @@ class TestNamenode(RMFTestCase):
                               )
     self.assertResourceCalled('Directory', '/var/run/hadoop/hdfs',
                               owner = 'hdfs',
+                              group = 'hadoop',
                               create_parents = True,
                               )
     self.assertResourceCalled('Directory', '/var/log/hadoop/hdfs',
                               owner = 'hdfs',
+                              group = 'hadoop',
                               create_parents = True,
                               )
     self.assertResourceCalled('File', '/var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid',
@@ -422,10 +428,12 @@ class TestNamenode(RMFTestCase):
                               )
     self.assertResourceCalled('Directory', '/var/run/hadoop/hdfs',
                               owner = 'hdfs',
+                              group = 'hadoop',
                               create_parents = True,
                               )
     self.assertResourceCalled('Directory', '/var/log/hadoop/hdfs',
                               owner = 'hdfs',
+                              group = 'hadoop',
                               create_parents = True,
                               )
     self.assertResourceCalled('File', '/var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid',
@@ -522,10 +530,12 @@ class TestNamenode(RMFTestCase):
                               )
     self.assertResourceCalled('Directory', '/var/run/hadoop/hdfs',
                               owner = 'hdfs',
+                              group = 'hadoop',
                               create_parents = True,
                               )
     self.assertResourceCalled('Directory', '/var/log/hadoop/hdfs',
                               owner = 'hdfs',
+                              group = 'hadoop',
                               create_parents = True,
                               )
     self.assertResourceCalled('File', '/var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid',
@@ -625,10 +635,12 @@ class TestNamenode(RMFTestCase):
                               )
     self.assertResourceCalled('Directory', '/var/run/hadoop/hdfs',
                               owner = 'hdfs',
+                              group = 'hadoop',
                               create_parents = True,
                               )
     self.assertResourceCalled('Directory', '/var/log/hadoop/hdfs',
                               owner = 'hdfs',
+                              group = 'hadoop',
                               create_parents = True,
                               )
     self.assertResourceCalled('File', '/var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid',
@@ -734,10 +746,12 @@ class TestNamenode(RMFTestCase):
     )
     self.assertResourceCalled('Directory', '/var/run/hadoop/hdfs',
                               owner = 'hdfs',
+                              group = 'hadoop',
                               create_parents = True,
                               )
     self.assertResourceCalled('Directory', '/var/log/hadoop/hdfs',
                               owner = 'hdfs',
+                              group = 'hadoop',
                               create_parents = True,
                               )
     self.assertResourceCalled('File', '/var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid',
@@ -839,10 +853,12 @@ class TestNamenode(RMFTestCase):
     # command is run before the namenode launches
     self.assertResourceCalled('Directory', '/var/run/hadoop/hdfs',
                               owner = 'hdfs',
+                              group = 'hadoop',
                               create_parents = True,
                               )
     self.assertResourceCalled('Directory', '/var/log/hadoop/hdfs',
                               owner = 'hdfs',
+                              group = 'hadoop',
                               create_parents = True,
                               )
     self.assertResourceCalled('File', '/var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid',
@@ -952,10 +968,12 @@ class TestNamenode(RMFTestCase):
     # command is run before the namenode launches
     self.assertResourceCalled('Directory', '/var/run/hadoop/hdfs',
                               owner = 'hdfs',
+                              group = 'hadoop',
                               create_parents = True,
                               )
     self.assertResourceCalled('Directory', '/var/log/hadoop/hdfs',
                               owner = 'hdfs',
+                              group = 'hadoop',
                               create_parents = True,
                               )
     self.assertResourceCalled('File', '/var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid',

ambari-server/src/test/python/stacks/2.0.6/HDFS/test_nfsgateway.py

@@ -61,6 +61,7 @@ class TestNFSGateway(RMFTestCase):
                               )
     self.assertResourceCalled('Directory', '/var/run/hadoop/root',
                               owner = 'root',
+                              group = 'hadoop',
                               create_parents = True,
                               )
     self.assertResourceCalled('Directory', '/var/log/hadoop/root',
@@ -128,6 +129,7 @@ class TestNFSGateway(RMFTestCase):
                               )
     self.assertResourceCalled('Directory', '/var/run/hadoop/root',
                               owner = 'root',
+                              group = 'hadoop',
                               create_parents = True,
                               )
     self.assertResourceCalled('Directory', '/var/log/hadoop/root',

ambari-server/src/test/python/stacks/2.0.6/HDFS/test_snamenode.py

@@ -62,10 +62,12 @@ class TestSNamenode(RMFTestCase):
                               )
     self.assertResourceCalled('Directory', '/var/run/hadoop/hdfs',
                               owner = 'hdfs',
+                              group = 'hadoop',
                               create_parents = True,
                               )
     self.assertResourceCalled('Directory', '/var/log/hadoop/hdfs',
                               owner = 'hdfs',
+                              group = 'hadoop',
                               create_parents = True,
                               )
     self.assertResourceCalled('File', '/var/run/hadoop/hdfs/hadoop-hdfs-secondarynamenode.pid',
@@ -130,10 +132,12 @@ class TestSNamenode(RMFTestCase):
                               )
     self.assertResourceCalled('Directory', '/var/run/hadoop/hdfs',
                               owner = 'hdfs',
+                              group = 'hadoop',
                               create_parents = True,
                               )
     self.assertResourceCalled('Directory', '/var/log/hadoop/hdfs',
                               owner = 'hdfs',
+                              group = 'hadoop',
                               create_parents = True,
                               )
     self.assertResourceCalled('File', '/var/run/hadoop/hdfs/hadoop-hdfs-secondarynamenode.pid',

ambari-server/src/test/python/stacks/2.0.6/HDFS/test_zkfc.py

@@ -90,10 +90,12 @@ class TestZkfc(RMFTestCase):
     self.assertResourceCalled('Directory', '/var/run/hadoop/hdfs',
                               owner = 'hdfs',
                               create_parents = True,
+                              group = 'hadoop'
                               )
     self.assertResourceCalled('Directory', '/var/log/hadoop/hdfs',
                               owner = 'hdfs',
                               create_parents = True,
+                              group = 'hadoop'
                               )
     self.assertResourceCalled('File', '/var/run/hadoop/hdfs/hadoop-hdfs-zkfc.pid',
         action = ['delete'],
@@ -185,10 +187,12 @@ class TestZkfc(RMFTestCase):
     self.assertResourceCalled('Directory', '/var/run/hadoop/hdfs',
                               owner = 'hdfs',
                               create_parents = True,
+                              group = 'hadoop'
                               )
     self.assertResourceCalled('Directory', '/var/log/hadoop/hdfs',
                               owner = 'hdfs',
                               create_parents = True,
+                              group = 'hadoop'
                               )
     self.assertResourceCalled('File', '/var/run/hadoop/hdfs/hadoop-hdfs-zkfc.pid',
         action = ['delete'],
@@ -280,10 +284,12 @@ class TestZkfc(RMFTestCase):
     self.assertResourceCalled('Directory', '/var/run/hadoop/hdfs',
                               owner = 'hdfs',
                               create_parents = True,
+                              group = 'hadoop'
                               )
     self.assertResourceCalled('Directory', '/var/log/hadoop/hdfs',
                               owner = 'hdfs',
                               create_parents = True,
+                              group = 'hadoop'
                               )
     self.assertResourceCalled('File', '/var/run/hadoop/hdfs/hadoop-hdfs-zkfc.pid',
         action = ['delete'],
@@ -360,9 +366,11 @@ class TestZkfc(RMFTestCase):
     self.assertResourceCalled('Directory', '/var/run/hadoop/hdfs',
                               owner = 'hdfs',
                               create_parents = True,
+                              group = 'hadoop'
                               )
     self.assertResourceCalled('Directory', '/var/log/hadoop/hdfs',
                               owner = 'hdfs',
+                              group = 'hadoop',
                               create_parents = True,
                               )
     self.assertResourceCalled('File', '/var/run/hadoop/hdfs/hadoop-hdfs-zkfc.pid',

ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py

@@ -356,6 +356,7 @@ class TestHistoryServer(RMFTestCase):
     )
     self.assertResourceCalled('Directory', '/var/log/hadoop-yarn',
       owner = 'yarn',
+      group = 'hadoop',
       create_parents = True,
       ignore_failures = True,
       cd_access = 'a',
@@ -627,6 +628,7 @@ class TestHistoryServer(RMFTestCase):
     )
     self.assertResourceCalled('Directory', '/var/log/hadoop-yarn',
       owner = 'yarn',
+      group = 'hadoop',
       create_parents = True,
       ignore_failures = True,
       cd_access = 'a',

ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_client.py

@@ -87,6 +87,7 @@ class TestMapReduce2Client(RMFTestCase):
     )
     self.assertResourceCalled('Directory', '/var/log/hadoop-yarn',
       owner = 'yarn',
+      group = 'hadoop',
       create_parents = True,
       ignore_failures = True,
       cd_access = 'a',
@@ -254,6 +255,7 @@ class TestMapReduce2Client(RMFTestCase):
     )
     self.assertResourceCalled('Directory', '/var/log/hadoop-yarn',
       owner = 'yarn',
+      group = 'hadoop',
       create_parents = True,
       ignore_failures = True,
       cd_access = 'a',

ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py

@@ -215,6 +215,7 @@ class TestNodeManager(RMFTestCase):
     )
     self.assertResourceCalled('Directory', '/var/log/hadoop-yarn',
       owner = 'yarn',
+      group = 'hadoop',
       create_parents = True,
       ignore_failures = True,
       cd_access = 'a',
@@ -401,6 +402,7 @@ class TestNodeManager(RMFTestCase):
     )
     self.assertResourceCalled('Directory', '/var/log/hadoop-yarn',
       owner = 'yarn',
+      group = 'hadoop',
       create_parents = True,
       ignore_failures = True,
       cd_access = 'a',

ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py

@@ -224,6 +224,7 @@ class TestResourceManager(RMFTestCase):
     )
     self.assertResourceCalled('Directory', '/var/log/hadoop-yarn',
       owner = 'yarn',
+      group = 'hadoop',
       create_parents = True,
       ignore_failures = True,
       cd_access = 'a',
@@ -387,6 +388,7 @@ class TestResourceManager(RMFTestCase):
     )
     self.assertResourceCalled('Directory', '/var/log/hadoop-yarn',
       owner = 'yarn',
+      group = 'hadoop',
       create_parents = True,
       ignore_failures = True,
       cd_access = 'a',

ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_client.py

@@ -87,6 +87,7 @@ class TestYarnClient(RMFTestCase):
     )
     self.assertResourceCalled('Directory', '/var/log/hadoop-yarn',
       owner = 'yarn',
+      group = 'hadoop',
       create_parents = True,
       ignore_failures = True,
       cd_access = 'a',
@@ -254,6 +255,7 @@ class TestYarnClient(RMFTestCase):
     )
     self.assertResourceCalled('Directory', '/var/log/hadoop-yarn',
       owner = 'yarn',
+      group = 'hadoop',
       create_parents = True,
       ignore_failures = True,
       cd_access = 'a',
@@ -429,6 +431,7 @@ class TestYarnClient(RMFTestCase):
     )
     self.assertResourceCalled('Directory', '/var/log/hadoop-yarn',
       owner = 'yarn',
+      group = 'hadoop',
       create_parents = True,
       ignore_failures = True,
       cd_access = 'a',

ambari-server/src/test/python/stacks/2.0.6/configs/flume_only.json

@@ -52,6 +52,14 @@
         },
         "flume-log4j": {
           "content": "log4jproperties\nline2"
+        },
+        "cluster-env": {
+          "managed_hdfs_resource_property_names": "",
+          "security_enabled": "false",
+          "ignore_groupsusers_create": "false",
+          "smokeuser": "ambari-qa",
+          "kerberos_domain": "EXAMPLE.COM",
+          "user_group": "hadoop"
         }
     },
     "configuration_attributes": {

ambari-server/src/test/python/stacks/2.0.6/configs/flume_target.json

@@ -104,7 +104,15 @@
             "oozie.authentication.type": "simple", 
             "oozie.service.JPAService.jdbc.driver": "org.apache.derby.jdbc.EmbeddedDriver", 
             "oozie.system.id": "oozie-${user.name}"
-        }, 
+        },
+        "cluster-env": {
+            "managed_hdfs_resource_property_names": "",
+            "security_enabled": "false",
+            "ignore_groupsusers_create": "false",
+            "smokeuser": "ambari-qa",
+            "kerberos_domain": "EXAMPLE.COM",
+            "user_group": "hadoop"
+        },
         "storm-site": {
             "topology.tuple.serializer": "backtype.storm.serialization.types.ListDelegateSerializer", 
             "topology.workers": "1", 

ambari-server/src/test/python/stacks/2.1/YARN/test_apptimelineserver.py

@@ -138,6 +138,7 @@ class TestAppTimelineServer(RMFTestCase):
                               )
     self.assertResourceCalled('Directory', '/var/log/hadoop-yarn',
                               owner = 'yarn',
+                              group = 'hadoop',
                               create_parents = True,
                               ignore_failures = True,
                               cd_access = 'a',

ambari-server/src/test/python/stacks/2.3/YARN/test_ats_1_5.py

@@ -94,6 +94,7 @@ class TestAts(RMFTestCase):
                               )
     self.assertResourceCalled('Directory', '/var/log/hadoop-yarn',
                               owner = 'yarn',
+                              group = 'hadoop',
                               ignore_failures = True,
                               create_parents = True,
                               cd_access = 'a',