AMBARI-20779 Create Ranger KMS HDFS audit folder as part of install (mugdha)

ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/metainfo.xml

@@ -45,6 +45,15 @@
               <primary>true</primary>
             </log>
           </logs>
+          <dependencies>
+            <dependency>
+              <name>HDFS/HDFS_CLIENT</name>
+              <scope>host</scope>
+              <auto-deploy>
+                <enabled>true</enabled>
+              </auto-deploy>
+            </dependency>
+          </dependencies>
         </component>
 
       </components>

ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py

@@ -506,6 +506,26 @@ def enable_kms_plugin():
       mode = 0640
       )
 
+    # create ranger kms audit directory
+    if params.xa_audit_hdfs_is_enabled and params.has_namenode and params.has_hdfs_client_on_node:
+      params.HdfsResource("/ranger/audit",
+                        type="directory",
+                        action="create_on_execute",
+                        owner=params.hdfs_user,
+                        group=params.hdfs_user,
+                        mode=0755,
+                        recursive_chmod=True
+      )
+      params.HdfsResource("/ranger/audit/kms",
+                        type="directory",
+                        action="create_on_execute",
+                        owner=params.kms_user,
+                        group=params.kms_group,
+                        mode=0750,
+                        recursive_chmod=True
+      )
+      params.HdfsResource(None, action="execute")
+
     if params.xa_audit_hdfs_is_enabled and len(params.namenode_host) > 1:
       Logger.info('Audit to Hdfs enabled in NameNode HA environment, creating hdfs-site.xml')
       XmlConfig("hdfs-site.xml",

ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py

@@ -29,6 +29,9 @@ from resource_management.libraries.functions import StackFeature
 from resource_management.libraries.functions.get_bare_principal import get_bare_principal
 from resource_management.libraries.functions.is_empty import is_empty
 from resource_management.libraries.functions.setup_ranger_plugin_xml import generate_ranger_service_config
+from resource_management.libraries.resources.hdfs_resource import HdfsResource
+from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions import get_kinit_path
 
 config  = Script.get_config()
 tmp_dir = Script.get_tmp_dir()
@@ -296,4 +299,32 @@ xa_audit_hdfs_is_enabled = default("/configurations/ranger-kms-audit/xasecure.au
 namenode_host = default("/clusterHostInfo/namenode_host", [])
 
 # need this to capture cluster name from where ranger kms plugin is enabled
-cluster_name = config['clusterName']
+cluster_name = config['clusterName']
+
+has_namenode = len(namenode_host) > 0
+hdfs_user = default("/configurations/hadoop-env/hdfs_user", None)
+hdfs_user_keytab = default("/configurations/hadoop-env/hdfs_user_keytab", None)
+hdfs_principal_name = default("/configurations/hadoop-env/hdfs_principal_name", None)
+default_fs = default("/configurations/core-site/fs.defaultFS", None)
+hdfs_site = config['configurations']['hdfs-site'] if has_namenode else None
+hadoop_bin_dir = stack_select.get_hadoop_dir("bin") if has_namenode else None
+kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
+
+import functools
+# create partial functions with common arguments for every HdfsResource call
+# to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code
+HdfsResource = functools.partial(
+  HdfsResource,
+  user=hdfs_user,
+  security_enabled = security_enabled,
+  keytab = hdfs_user_keytab,
+  kinit_path_local = kinit_path_local,
+  hadoop_bin_dir = hadoop_bin_dir,
+  hadoop_conf_dir = hadoop_conf_dir,
+  principal_name = hdfs_principal_name,
+  hdfs_site = hdfs_site,
+  default_fs = default_fs
+)
+
+local_component_list = default("/localComponents", [])
+has_hdfs_client_on_node = 'HDFS_CLIENT' in local_component_list

ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json

@@ -4,8 +4,8 @@
   "general_deps" : {
     "_comment" : "dependencies for all cases",
     "MAHOUT_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", "RESOURCEMANAGER-START"],
-    "RANGER_USERSYNC-START" : ["RANGER_ADMIN-START", "RANGER_KMS_SERVER-START"],
-    "RANGER_KMS_SERVER-START" : ["RANGER_ADMIN-START"],
+    "RANGER_USERSYNC-START" : ["RANGER_ADMIN-START"],
+    "RANGER_KMS_SERVER-START" : ["RANGER_ADMIN-START", "NAMENODE-START"],
     "RANGER_KMS_SERVICE_CHECK-SERVICE_CHECK" : ["RANGER_KMS_SERVER-START"],
     "PHOENIX_QUERY_SERVER-START": ["HBASE_MASTER-START"],
     "ATLAS_SERVICE_CHECK-SERVICE_CHECK": ["ATLAS_SERVER-START"],

ambari-server/src/test/python/stacks/2.5/RANGER_KMS/test_kms_server.py

@@ -151,6 +151,55 @@ class TestRangerKMS(RMFTestCase):
       mode = 0640
     )
 
+    self.assertResourceCalled('HdfsResource', '/ranger/audit',
+                        type = 'directory',
+                        action = ['create_on_execute'],
+                        owner = 'hdfs',
+                        group = 'hdfs',
+                        mode = 0755,
+                        recursive_chmod = True,
+                        user = 'hdfs',
+                        security_enabled = False,
+                        keytab = None,
+                        kinit_path_local = '/usr/bin/kinit',
+                        hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+                        hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+                        principal_name = None,
+                        hdfs_site = self.getConfig()['configurations']['hdfs-site'],
+                        default_fs = 'hdfs://c6401.ambari.apache.org:8020'
+    )
+
+    self.assertResourceCalled('HdfsResource', '/ranger/audit/kms',
+                        type = 'directory',
+                        action = ['create_on_execute'],
+                        owner = 'kms',
+                        group = 'kms',
+                        mode = 0750,
+                        recursive_chmod = True,
+                        user = 'hdfs',
+                        security_enabled = False,
+                        keytab = None,
+                        kinit_path_local = '/usr/bin/kinit',
+                        hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+                        hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+                        principal_name = None,
+                        hdfs_site = self.getConfig()['configurations']['hdfs-site'],
+                        default_fs = 'hdfs://c6401.ambari.apache.org:8020'
+    )
+
+    self.assertResourceCalled('HdfsResource', None,
+                        action = ['execute'],
+                        user = 'hdfs',
+                        security_enabled = False,
+                        keytab = None,
+                        kinit_path_local = '/usr/bin/kinit',
+                        hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+                        hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+                        principal_name = None,
+                        hdfs_site = self.getConfig()['configurations']['hdfs-site'],
+                        default_fs = 'hdfs://c6401.ambari.apache.org:8020'
+    )
+
     self.assertResourceCalled('File', '/usr/hdp/current/ranger-kms/conf/hdfs-site.xml',
       action = ['delete'],
     )
@@ -537,6 +586,55 @@ class TestRangerKMS(RMFTestCase):
       mode = 0640
     )
 
+    self.assertResourceCalled('HdfsResource', '/ranger/audit',
+                        type = 'directory',
+                        action = ['create_on_execute'],
+                        owner = 'hdfs',
+                        group = 'hdfs',
+                        mode = 0755,
+                        recursive_chmod = True,
+                        user = 'hdfs',
+                        security_enabled = True,
+                        keytab = '/etc/security/keytabs/hdfs.headless.keytab',
+                        kinit_path_local = '/usr/bin/kinit',
+                        hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+                        hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+                        principal_name = 'hdfs-cl1@EXAMPLE.COM',
+                        hdfs_site = self.getConfig()['configurations']['hdfs-site'],
+                        default_fs = 'hdfs://c6401.ambari.apache.org:8020'
+    )
+
+    self.assertResourceCalled('HdfsResource', '/ranger/audit/kms',
+                        type = 'directory',
+                        action = ['create_on_execute'],
+                        owner = 'kms',
+                        group = 'kms',
+                        mode = 0750,
+                        recursive_chmod = True,
+                        user = 'hdfs',
+                        security_enabled = True,
+                        keytab = '/etc/security/keytabs/hdfs.headless.keytab',
+                        kinit_path_local = '/usr/bin/kinit',
+                        hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+                        hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+                        principal_name = 'hdfs-cl1@EXAMPLE.COM',
+                        hdfs_site = self.getConfig()['configurations']['hdfs-site'],
+                        default_fs = 'hdfs://c6401.ambari.apache.org:8020'
+    )
+
+    self.assertResourceCalled('HdfsResource', None,
+                        action = ['execute'],
+                        user = 'hdfs',
+                        security_enabled = True,
+                        keytab = '/etc/security/keytabs/hdfs.headless.keytab',
+                        kinit_path_local = '/usr/bin/kinit',
+                        hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+                        hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+                        principal_name = 'hdfs-cl1@EXAMPLE.COM',
+                        hdfs_site = self.getConfig()['configurations']['hdfs-site'],
+                        default_fs = 'hdfs://c6401.ambari.apache.org:8020'
+    )
+
     self.assertResourceCalled('File', '/usr/hdp/current/ranger-kms/conf/hdfs-site.xml',
       action = ['delete'],
     )

ambari-server/src/test/python/stacks/2.5/configs/ranger-kms-secured.json

@@ -751,7 +751,7 @@
             "hdfs_log_dir_prefix": "/var/log/hadoop", 
             "hdfs_user_nofile_limit": "128000", 
             "hdfs_user": "hdfs", 
-            "hdfs_principal_name": "hdfs-test_cluster01@EXAMPLE.COM", 
+            "hdfs_principal_name": "hdfs-cl1@EXAMPLE.COM",
             "keyserver_host": " ", 
             "namenode_opt_maxnewsize": "128m", 
             "hdfs_user_keytab": "/etc/security/keytabs/hdfs.headless.keytab",