AMBARI-19328. Fix permissions for Druid config files and data directories (Nishant Bangarwa via smohanty)

ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-historical.xml

@@ -61,7 +61,7 @@
   </property>
   <property>
     <name>druid.segmentCache.locations</name>
-    <value>[{"path":"/apps/druid/segmentCache","maxSize"\:300000000000}]</value>
+    <value>[{"path":"/apps/druid/segmentCache","maxSize":300000000000}]</value>
     <description>Segments assigned to a Historical node are first stored on the local file system (in a disk cache) and
       then served by the Historical node. These locations define where that local cache resides.
     </description>
@@ -78,7 +78,7 @@
   </property>
   <property>
     <name>druid.segmentCache.infoDir</name>
-    <value>/apps/druid/segmentCache</value>
+    <value>/apps/druid/segmentCache/info_dir</value>
     <description>Historical nodes keep track of the segments they are serving so that when the process is restarted they
       can reload the same segments without waiting for the Coordinator to reassign. This path defines where this
       metadata is kept. Directory will be created if needed.

ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-middlemanager.xml

@@ -55,7 +55,7 @@
   </property>
   <property>
     <name>druid.indexer.task.baseTaskDir</name>
-    <value>/tmp/persistent/tasks</value>
+    <value>/apps/druid/tasks</value>
     <description>
       Base temporary working directory for druid tasks.
     </description>

ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid.py

@@ -16,13 +16,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 
 """
+import json
 import os
 from resource_management.libraries.resources.properties_file import PropertiesFile
 from resource_management.core.resources.system import Directory, Execute, File
 from resource_management.core.source import DownloadSource
 from resource_management.core.source import InlineTemplate
 from resource_management.libraries.functions import format
-from resource_management.libraries.resources import XmlConfig
 from resource_management.libraries.functions.show_logs import show_logs
 from resource_management.core.logger import Logger
 
@@ -34,7 +34,8 @@ def druid(upgrade_type=None, nodeType=None):
   # Environment Variables
   File(format("{params.druid_conf_dir}/druid-env.sh"),
        owner=params.druid_user,
-       content=InlineTemplate(params.druid_env_sh_template)
+       content=InlineTemplate(params.druid_env_sh_template),
+       mode = 0700
        )
 
   # common config
@@ -60,6 +61,7 @@ def druid(upgrade_type=None, nodeType=None):
                  properties=druid_common_config,
                  owner=params.druid_user,
                  group=params.user_group,
+                 mode = 0600
                  )
   Logger.info("Created common.runtime.properties")
 
@@ -92,6 +94,7 @@ def druid(upgrade_type=None, nodeType=None):
                    properties=node_config,
                    owner=params.druid_user,
                    group=params.user_group,
+                   mode = 0600
                    )
     Logger.info(format("Created druid-{node_type_lowercase} runtime.properties"))
 
@@ -177,17 +180,38 @@ def create_hadoop_directory(hadoop_dir):
 def ensure_base_directories():
   import params
   Directory(
-    [params.druid_log_dir, params.druid_pid_dir, params.druid_common_conf_dir, params.druid_coordinator_conf_dir,
-     params.druid_broker_conf_dir, params.druid_middlemanager_conf_dir, params.druid_historical_conf_dir,
-     params.druid_overlord_conf_dir, params.druid_router_conf_dir, params.druid_segment_infoDir],
+    [params.druid_log_dir, params.druid_pid_dir],
     mode=0755,
-    cd_access='a',
     owner=params.druid_user,
     group=params.user_group,
     create_parents=True,
     recursive_ownership=True,
   )
 
+  Directory(
+    [params.druid_conf_dir, params.druid_common_conf_dir, params.druid_coordinator_conf_dir,
+     params.druid_broker_conf_dir, params.druid_middlemanager_conf_dir, params.druid_historical_conf_dir,
+     params.druid_overlord_conf_dir, params.druid_router_conf_dir, params.druid_segment_infoDir,
+     params.druid_tasks_dir],
+    mode=0700,
+    owner=params.druid_user,
+    group=params.user_group,
+    create_parents=True,
+    recursive_ownership=True,
+  )
+
+  segment_cache_locations = json.loads(params.druid_segment_cache_locations)
+  for segment_cache_location in segment_cache_locations:
+    Directory(
+      segment_cache_location["path"],
+      mode=0700,
+      owner=params.druid_user,
+      group=params.user_group,
+      create_parents=True,
+      recursive_ownership=True,
+    )
+
+
 
 def get_daemon_cmd(params=None, node_type=None, command=None):
   return format('source {params.druid_conf_dir}/druid-env.sh ; {params.druid_home}/bin/node.sh {node_type} {command}')

ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py

@@ -67,6 +67,8 @@ druid_router_conf_dir = druid_conf_dir + "/router"
 druid_extensions_dir = druid_home + "/extensions"
 druid_hadoop_dependencies_dir = druid_home + "/hadoop-dependencies"
 druid_segment_infoDir = config['configurations']['druid-historical']['druid.segmentCache.infoDir']
+druid_segment_cache_locations = config['configurations']['druid-historical']['druid.segmentCache.locations']
+druid_tasks_dir = config['configurations']['druid-middlemanager']['druid.indexer.task.baseTaskDir']
 druid_user = config['configurations']['druid-env']['druid_user']
 druid_log_dir = config['configurations']['druid-env']['druid_log_dir']
 druid_classpath = config['configurations']['druid-env']['druid_classpath']

ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py

@@ -321,7 +321,6 @@ class TestDruid(RMFTestCase):
 
     self.assertResourceCalled('Directory', '/var/log/druid',
                               mode=0755,
-                              cd_access='a',
                               owner='druid',
                               group='hadoop',
                               create_parents=True,
@@ -330,7 +329,14 @@ class TestDruid(RMFTestCase):
 
     self.assertResourceCalled('Directory', '/var/run/druid',
                               mode=0755,
-                              cd_access='a',
+                              owner='druid',
+                              group='hadoop',
+                              create_parents=True,
+                              recursive_ownership=True
+                              )
+
+    self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf'),
+                              mode=0700,
                               owner='druid',
                               group='hadoop',
                               create_parents=True,
@@ -338,8 +344,7 @@ class TestDruid(RMFTestCase):
                               )
 
     self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/_common'),
-                              mode=0755,
-                              cd_access='a',
+                              mode=0700,
                               owner='druid',
                               group='hadoop',
                               create_parents=True,
@@ -347,8 +352,7 @@ class TestDruid(RMFTestCase):
                               )
 
     self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/coordinator'),
-                              mode=0755,
-                              cd_access='a',
+                              mode=0700,
                               owner='druid',
                               group='hadoop',
                               create_parents=True,
@@ -356,8 +360,7 @@ class TestDruid(RMFTestCase):
                               )
 
     self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/broker'),
-                              mode=0755,
-                              cd_access='a',
+                              mode=0700,
                               owner='druid',
                               group='hadoop',
                               create_parents=True,
@@ -365,8 +368,7 @@ class TestDruid(RMFTestCase):
                               )
 
     self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/middleManager'),
-                              mode=0755,
-                              cd_access='a',
+                              mode=0700,
                               owner='druid',
                               group='hadoop',
                               create_parents=True,
@@ -374,8 +376,7 @@ class TestDruid(RMFTestCase):
                               )
 
     self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/historical'),
-                              mode=0755,
-                              cd_access='a',
+                              mode=0700,
                               owner='druid',
                               group='hadoop',
                               create_parents=True,
@@ -383,8 +384,7 @@ class TestDruid(RMFTestCase):
                               )
 
     self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/overlord'),
-                              mode=0755,
-                              cd_access='a',
+                              mode=0700,
                               owner='druid',
                               group='hadoop',
                               create_parents=True,
@@ -392,8 +392,23 @@ class TestDruid(RMFTestCase):
                               )
 
     self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/router'),
-                              mode=0755,
-                              cd_access='a',
+                              mode=0700,
+                              owner='druid',
+                              group='hadoop',
+                              create_parents=True,
+                              recursive_ownership=True
+                              )
+
+    self.assertResourceCalled('Directory', '/apps/druid/segmentCache/info_dir',
+                              mode=0700,
+                              owner='druid',
+                              group='hadoop',
+                              create_parents=True,
+                              recursive_ownership=True
+                              )
+
+    self.assertResourceCalled('Directory', '/apps/druid/tasks',
+                              mode=0700,
                               owner='druid',
                               group='hadoop',
                               create_parents=True,
@@ -401,8 +416,7 @@ class TestDruid(RMFTestCase):
                               )
 
     self.assertResourceCalled('Directory', '/apps/druid/segmentCache',
-                              mode=0755,
-                              cd_access='a',
+                              mode=0700,
                               owner='druid',
                               group='hadoop',
                               create_parents=True,
@@ -411,7 +425,8 @@ class TestDruid(RMFTestCase):
 
     self.assertResourceCalled('File', format('/usr/hdp/current/{role}/conf/druid-env.sh'),
                               owner = 'druid',
-                              content = InlineTemplate(self.getConfig()['configurations']['druid-env']['content'])
+                              content = InlineTemplate(self.getConfig()['configurations']['druid-env']['content']),
+                              mode = 0700
                               )
     druid_common_config = mutable_config_dict(self.getConfig()['configurations']['druid-common'])
     druid_common_config['druid.host'] = 'c6401.ambari.apache.org'
@@ -424,7 +439,8 @@ class TestDruid(RMFTestCase):
                               dir=format("/usr/hdp/current/{role}/conf/_common"),
                               properties=druid_common_config,
                               owner='druid',
-                              group='hadoop'
+                              group='hadoop',
+                              mode = 0600
                               )
 
     self.assertResourceCalled('File', format('/usr/hdp/current/{role}/conf/_common/druid-log4j.xml'),
@@ -445,7 +461,8 @@ class TestDruid(RMFTestCase):
                               dir=format('/usr/hdp/current/{role}/conf/coordinator'),
                               properties=self.getConfig()['configurations']['druid-coordinator'],
                               owner='druid',
-                              group='hadoop'
+                              group='hadoop',
+                              mode = 0600
                               )
 
     self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/coordinator/jvm.config"),
@@ -463,7 +480,8 @@ class TestDruid(RMFTestCase):
                               dir=format('/usr/hdp/current/{role}/conf/overlord'),
                               properties=self.getConfig()['configurations']['druid-overlord'],
                               owner='druid',
-                              group='hadoop'
+                              group='hadoop',
+                              mode = 0600
                               )
 
     self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/overlord/jvm.config"),
@@ -481,7 +499,8 @@ class TestDruid(RMFTestCase):
                               dir=format('/usr/hdp/current/{role}/conf/historical'),
                               properties=self.getConfig()['configurations']['druid-historical'],
                               owner='druid',
-                              group='hadoop'
+                              group='hadoop',
+                              mode = 0600
                               )
 
     self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/historical/jvm.config"),
@@ -500,7 +519,8 @@ class TestDruid(RMFTestCase):
                           dir=format('/usr/hdp/current/{role}/conf/broker'),
                           properties=self.getConfig()['configurations']['druid-broker'],
                           owner='druid',
-                          group='hadoop'
+                          group='hadoop',
+                          mode = 0600
                           )
 
     self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/broker/jvm.config"),
@@ -519,7 +539,8 @@ class TestDruid(RMFTestCase):
                           dir=format('/usr/hdp/current/{role}/conf/middleManager'),
                           properties=self.getConfig()['configurations']['druid-middlemanager'],
                           owner='druid',
-                          group='hadoop'
+                          group='hadoop',
+                          mode = 0600
                           )
 
     self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/middleManager/jvm.config"),
@@ -537,7 +558,8 @@ class TestDruid(RMFTestCase):
                               dir=format('/usr/hdp/current/{role}/conf/router'),
                               properties=self.getConfig()['configurations']['druid-router'],
                               owner='druid',
-                              group='hadoop'
+                              group='hadoop',
+                              mode = 0600
                               )
 
     self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/router/jvm.config"),

ambari-server/src/test/python/stacks/2.6/configs/default.json

@@ -405,7 +405,8 @@
       "druid.extensions.loadList": "[\"mysql-metadata-storage\", \"druid-datasketches\"]"
     },
     "druid-historical" : {
-      "druid.segmentCache.infoDir" : "/apps/druid/segmentCache"
+      "druid.segmentCache.infoDir" : "/apps/druid/segmentCache/info_dir",
+      "druid.segmentCache.locations" :"[{\"path\":\"/apps/druid/segmentCache\",\"maxSize\":300000000000}]"
     },
     "druid-coordinator" : {
       "druid.service" : "druid/coordinator"
@@ -418,7 +419,8 @@
     },
     "druid-middlemanager" : {
       "druid.service" : "druid/middlemanager",
-      "druid.indexer.task.hadoopWorkingPath" : "/tmp/druid-indexing"
+      "druid.indexer.task.hadoopWorkingPath" : "/tmp/druid-indexing",
+      "druid.indexer.task.baseTaskDir" : "/apps/druid/tasks"
     },
     "druid-router" : {
       "druid.service" : "druid/router"