AMBARI-16446. Ranger audit properties should be configured to use Ambari installed Solr by LogSearch (Mugdha Varadkar via gautam)

ambari-common/src/main/python/resource_management/libraries/functions/constants.py

@@ -82,3 +82,4 @@ class StackFeature:
   RANGER_KERBEROS_SUPPORT = "ranger_kerberos_support"
   HIVE_METASTORE_SITE_SUPPORT = "hive_metastore_site_support"
   RANGER_USERSYNC_PASSWORD_JCEKS = "ranger_usersync_password_jceks"
+  RANGER_LOGSEARCH_DEPENDENT = "ranger_logsearch_dependent"

ambari-common/src/main/python/resource_management/libraries/functions/solr_cloud_util.py

@@ -66,7 +66,7 @@ def create_collection(zookeeper_quorum, solr_znode, collection, config_set, java
     max_shards = replication_factor * shards
 
   create_collection_cmd = format('{solr_cli_prefix} --create-collection -c {collection} -cs {config_set} -s {shards} -r {replication_factor} '\
-    '-m {max_shards} -rt {retry} -i {interval}')
+    '-m {max_shards} -rt {retry} -i {interval} -ns')
 
   if router_name is not None and router_field is not None:
     create_collection_cmd += format(' -rn {router_name} -rf {router_field}')

ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py

@@ -238,6 +238,11 @@ _DEFAULT_STACK_FEATURES = {
       "name": "ranger_usersync_password_jceks",
       "description": "Saving Ranger Usersync credentials in jceks",
       "min_version": "2.5.0.0"
+    },
+    {
+      "name": "ranger_logsearch_dependent",
+      "description": "Ranger audit properties should be configured to use Ambari installed LogSearch (AMBARI-16446)",
+      "min_version": "2.5.0.0"
     }
   ]
 }

ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py

@@ -63,6 +63,7 @@ stack_supports_ranger_audit_db = stack_version_formatted and check_stack_feature
 stack_supports_ranger_log4j =  stack_version_formatted and check_stack_feature(StackFeature.RANGER_LOG4J_SUPPORT, stack_version_formatted)
 stack_supports_ranger_kerberos = stack_version_formatted and check_stack_feature(StackFeature.RANGER_KERBEROS_SUPPORT, stack_version_formatted)
 stack_supports_usersync_passwd = stack_version_formatted and check_stack_feature(StackFeature.RANGER_USERSYNC_PASSWORD_JCEKS, stack_version_formatted)
+stack_supports_logsearch_dependent = stack_version_formatted and check_stack_feature(StackFeature.RANGER_LOGSEARCH_DEPENDENT, stack_version_formatted)
 
 downgrade_from_version = default("/commandParams/downgrade_from_version", None)
 upgrade_direction = default("/commandParams/upgrade_direction", None)
@@ -252,3 +253,16 @@ has_namenode = len(namenode_hosts) > 0
 
 ugsync_policymgr_alias = config["configurations"]["ranger-ugsync-site"]["ranger.usersync.policymgr.alias"]
 ugsync_policymgr_keystore = config["configurations"]["ranger-ugsync-site"]["ranger.usersync.policymgr.keystore"]
+
+# ranger solr
+ranger_solr_config_set = config['configurations']['ranger-env']['ranger_solr_config_set']
+ranger_solr_collection_name = config['configurations']['ranger-env']['ranger_solr_collection_name']
+ranger_solr_shards = config['configurations']['ranger-env']['ranger_solr_shards']
+zookeeper_hosts_list = config['clusterHostInfo']['zookeeper_hosts']
+zookeeper_hosts_list.sort()
+zookeeper_hosts = ",".join(zookeeper_hosts_list)
+logsearch_solr_znode = config['configurations']['logsearch-solr-env']['logsearch_solr_znode']
+ranger_solr_conf = format('{ranger_home}/contrib/solr_for_audit_setup/conf')
+logsearch_solr_hosts = default("/clusterHostInfo/logsearch_solr_hosts", [])
+replication_factor = 2 if len(logsearch_solr_hosts) > 1 else 1
+has_logsearch = len(logsearch_solr_hosts) > 0

ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py

@@ -25,6 +25,7 @@ from resource_management.libraries.functions.format import format
 from resource_management.core.logger import Logger
 from resource_management.core import shell
 from ranger_service import ranger_service
+from setup_ranger_xml import setup_ranger_audit_solr
 import upgrade
 import os, errno
 
@@ -73,6 +74,9 @@ class RangerAdmin(Script):
     import params
     env.set_params(params)
     self.configure(env, upgrade_type=upgrade_type)
+
+    if params.stack_supports_logsearch_dependent and params.has_logsearch:
+      setup_ranger_audit_solr()
     ranger_service('ranger_admin')
 
 

ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py

@@ -19,6 +19,7 @@ limitations under the License.
 """
 import os
 import re
+import random
 from resource_management.core.logger import Logger
 from resource_management.core.resources.system import File, Directory, Execute, Link
 from resource_management.core.source import DownloadSource, InlineTemplate
@@ -30,6 +31,7 @@ from resource_management.libraries.functions.format import format
 from resource_management.libraries.functions.is_empty import is_empty
 from resource_management.core.utils import PasswordString
 from resource_management.core.shell import as_sudo
+from resource_management.libraries.functions import solr_cloud_util
 
 # This file contains functions used for setup/configure of Ranger Admin and Ranger Usersync.
 # The design is to mimic what is done by the setup.sh script bundled by Ranger component currently.
@@ -523,4 +525,31 @@ def create_core_site_xml(conf_dir):
            owner = params.unix_user,
            group = params.unix_group,
            mode=0644
-      )
+      )
+
+def setup_ranger_audit_solr():
+  import params
+
+  random_num = random.random()
+  tmp_config_set_folder = format('{tmp_dir}/ranger_config_{ranger_solr_config_set}_{random_num}')
+
+  solr_cloud_util.upload_configuration_to_zk(
+    zookeeper_quorum = params.zookeeper_hosts,
+    solr_znode = params.logsearch_solr_znode,
+    config_set = params.ranger_solr_config_set,
+    config_set_dir = params.ranger_solr_conf,
+    tmp_config_set_dir = tmp_config_set_folder,
+    java64_home = params.java_home,
+    user = params.unix_user,
+    group = params.unix_group)
+
+  solr_cloud_util.create_collection(
+    zookeeper_quorum = params.zookeeper_hosts,
+    solr_znode = params.logsearch_solr_znode,
+    collection = params.ranger_solr_collection_name,
+    config_set = params.ranger_solr_config_set,
+    java64_home = params.java_home,
+    user = params.unix_user,
+    group = params.unix_group,
+    shards = params.ranger_solr_shards,
+    replication_factor = params.replication_factor)

ambari-server/src/main/resources/common-services/RANGER/0.6.0/configuration/ranger-admin-site.xml

@@ -137,4 +137,20 @@
     <description>Ranger trust-store password</description>
   </property>
 
+  <property>
+    <name>ranger.audit.solr.zookeepers</name>
+    <value>NONE</value>
+    <description>Solr Zookeeper string</description>
+    <depends-on>
+      <property>
+        <type>logsearch-solr-env</type>
+        <name>logsearch_solr_znode</name>
+      </property>
+      <property>
+        <type>ranger-env</type>
+        <name>is_solrCloud_enabled</name>
+      </property>
+    </depends-on>
+  </property>
+
 </configuration>

ambari-server/src/main/resources/common-services/RANGER/0.6.0/configuration/ranger-env.xml

@@ -26,4 +26,19 @@
     <deleted>true</deleted>
   </property>
 
+  <property>
+   <name>ranger_solr_config_set</name>
+    <value>ranger_audits</value>
+  </property>
+
+  <property>
+    <name>ranger_solr_collection_name</name>
+    <value>ranger_audits</value>
+  </property>
+
+  <property>
+    <name>ranger_solr_shards</name>
+    <value>1</value>
+  </property>
+
 </configuration>

ambari-server/src/main/resources/common-services/RANGER/0.6.0/metainfo.xml

@@ -27,6 +27,19 @@
       <version>0.6.0</version>
 
       <components>
+        <component>
+          <name>RANGER_ADMIN</name>
+          <dependencies>
+            <dependency>
+              <name>LOGSEARCH/LOGSEARCH_SOLR_CLIENT</name>
+              <scope>host</scope>
+              <auto-deploy>
+                <enabled>true</enabled>
+              </auto-deploy>
+            </dependency>
+          </dependencies>
+        </component>
+
         <component>
           <name>RANGER_TAGSYNC</name>
           <displayName>Ranger Tagsync</displayName>
@@ -52,6 +65,15 @@
         </theme>
       </themes>
 
+      <requiredServices>
+        <service>LOGSEARCH</service>
+      </requiredServices>
+
+      <configuration-dependencies>
+        <config-type>admin-log4j</config-type>
+        <config-type>usersync-log4j.xml</config-type>
+      </configuration-dependencies>
+
     </service>
   </services>
 </metainfo>

ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json

@@ -220,6 +220,11 @@
       "name": "ranger_usersync_password_jceks",
       "description": "Saving Ranger Usersync credentials in jceks",
       "min_version": "2.5.0.0"
+    },
+    {
+      "name": "ranger_logsearch_dependent",
+      "description": "Ranger audit properties should be configured to use Ambari installed LogSearch (AMBARI-16446)",
+      "min_version": "2.5.0.0"
     }
   ]
 }

ambari-server/src/main/resources/stacks/HDP/2.5/role_command_order.json

@@ -6,6 +6,7 @@
     "ZEPPELIN_MASTER-START" : ["NAMENODE-START"],
     "HIVE_SERVER_INTERACTIVE-START": ["NODEMANAGER-START", "MYSQL_SERVER-START"],
     "HIVE_SERVER_INTERACTIVE-RESTART": ["NODEMANAGER-RESTART", "MYSQL_SERVER-RESTART"],
-    "HIVE_SERVICE_CHECK-SERVICE_CHECK": ["HIVE_SERVER-START", "HIVE_METASTORE-START", "WEBHCAT_SERVER-START", "HIVE_SERVER_INTERACTIVE-START"]
+    "HIVE_SERVICE_CHECK-SERVICE_CHECK": ["HIVE_SERVER-START", "HIVE_METASTORE-START", "WEBHCAT_SERVER-START", "HIVE_SERVER_INTERACTIVE-START"],
+    "RANGER_ADMIN-START": ["ZOOKEEPER_SERVER-START", "LOGSEARCH_SOLR-START"]
   }
 }

ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER_KMS/configuration/ranger-kms-audit.xml

@@ -55,4 +55,31 @@
     <deleted>true</deleted>
   </property>
 
+  <property>
+    <name>xasecure.audit.destination.solr.urls</name>
+    <value>{{ranger_audit_solr_urls}}</value>
+    <description>Solr URL</description>
+    <value-attributes>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+    <depends-on>
+      <property>
+        <type>ranger-admin-site</type>
+        <name>ranger.audit.solr.urls</name>
+      </property>
+    </depends-on>
+  </property>
+
+  <property>
+    <name>xasecure.audit.destination.solr.zookeepers</name>
+    <value>none</value>
+    <description>Solr Zookeeper string</description>
+    <depends-on>
+      <property>
+        <type>ranger-admin-site</type>
+        <name>ranger.audit.solr.zookeepers</name>
+      </property>
+    </depends-on>
+  </property>
+
 </configuration>

ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py

@@ -930,13 +930,15 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
 
     putTagsyncAppProperty = self.putProperty(configurations, "tagsync-application-properties", services)
     putTagsyncSiteProperty = self.putProperty(configurations, "ranger-tagsync-site", services)
+    putRangerAdminProperty = self.putProperty(configurations, "ranger-admin-site", services)
+    putRangerEnvProperty = self.putProperty(configurations, "ranger-env", services)
 
     has_ranger_tagsync = False
     if 'RANGER' in servicesList:
       ranger_tagsync_host = self.__getHostsForComponent(services, "RANGER", "RANGER_TAGSYNC")
       has_ranger_tagsync = len(ranger_tagsync_host) > 0
 
-    if 'ATLAS' in servicesList:
+    if 'ATLAS' in servicesList and has_ranger_tagsync:
       putTagsyncSiteProperty('ranger.tagsync.source.atlas', 'true')
     else:
       putTagsyncSiteProperty('ranger.tagsync.source.atlas', 'false')
@@ -963,6 +965,60 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
     else:
       putTagsyncAppProperty('atlas.kafka.bootstrap.servers', 'localhost:6667')
 
+    if 'LOGSEARCH' in servicesList and zookeeper_host_port:
+      putRangerEnvProperty('is_solrCloud_enabled', 'true')
+      zookeeper_host_port = zookeeper_host_port.split(',')
+      zookeeper_host_port.sort()
+      zookeeper_host_port = ",".join(zookeeper_host_port)
+      logsearch_solr_znode = '/logsearch'
+      ranger_audit_zk_port = ''
+      if 'logsearch-solr-env' in services['configurations'] and \
+        ('logsearch_solr_znode' in services['configurations']['logsearch-solr-env']['properties']):
+        logsearch_solr_znode = services['configurations']['logsearch-solr-env']['properties']['logsearch_solr_znode']
+        ranger_audit_zk_port = '{0}{1}'.format(zookeeper_host_port, logsearch_solr_znode)
+      putRangerAdminProperty('ranger.audit.solr.zookeepers', ranger_audit_zk_port)
+    else:
+      putRangerEnvProperty('is_solrCloud_enabled', 'false')
+
+    if 'ranger-env' in configurations and configurations["ranger-env"]["properties"]["is_solrCloud_enabled"]:
+      isSolrCloudEnabled = configurations and configurations["ranger-env"]["properties"]["is_solrCloud_enabled"] == "true"
+    elif 'ranger-env' in services['configurations'] and 'is_solrCloud_enabled' in services['configurations']["ranger-env"]["properties"]:
+      isSolrCloudEnabled = services['configurations']["ranger-env"]["properties"]["is_solrCloud_enabled"]  == "true"
+    else:
+      isSolrCloudEnabled = False
+
+    if not isSolrCloudEnabled:
+      putRangerAdminProperty('ranger.audit.solr.zookeepers', 'NONE')
+
+    ranger_services = [
+      {'service_name': 'HDFS', 'audit_file': 'ranger-hdfs-audit'},
+      {'service_name': 'YARN', 'audit_file': 'ranger-yarn-audit'},
+      {'service_name': 'HBASE', 'audit_file': 'ranger-hbase-audit'},
+      {'service_name': 'HIVE', 'audit_file': 'ranger-hive-audit'},
+      {'service_name': 'KNOX', 'audit_file': 'ranger-knox-audit'},
+      {'service_name': 'KAFKA', 'audit_file': 'ranger-kafka-audit'},
+      {'service_name': 'STORM', 'audit_file': 'ranger-storm-audit'},
+      {'service_name': 'RANGER_KMS', 'audit_file': 'ranger-kms-site'}
+    ]
+
+    for item in range(len(ranger_services)):
+      if ranger_services[item]['service_name'] in servicesList:
+        component_audit_file =  ranger_services[item]['audit_file']
+        if component_audit_file in services["configurations"]:
+          ranger_audit_dict = [
+            {'filename': 'ranger-admin-site', 'configname': 'ranger.audit.solr.urls', 'target_configname': 'xasecure.audit.destination.solr.urls'},
+            {'filename': 'ranger-admin-site', 'configname': 'ranger.audit.solr.zookeepers', 'target_configname': 'xasecure.audit.destination.solr.zookeepers'}
+          ]
+          putRangerAuditProperty = self.putProperty(configurations, component_audit_file, services)
+
+          for item in ranger_audit_dict:
+            if item['filename'] in services["configurations"] and item['configname'] in  services["configurations"][item['filename']]["properties"]:
+              if item['filename'] in configurations and item['configname'] in  configurations[item['filename']]["properties"]:
+                rangerAuditProperty = configurations[item['filename']]["properties"][item['configname']]
+              else:
+                rangerAuditProperty = services["configurations"][item['filename']]["properties"][item['configname']]
+              putRangerAuditProperty(item['target_configname'], rangerAuditProperty)
+
   def validateRangerTagsyncConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
     ranger_tagsync_properties = getSiteProperties(configurations, "ranger-tagsync-site")
     validationItems = []

ambari-server/src/test/python/stacks/2.2/configs/ranger-admin-secured.json

@@ -185,6 +185,9 @@
         "all_hosts": [
             "c6401.ambari.apache.org",
             "c6402.ambari.apache.org"
+        ],
+        "zookeeper_hosts": [
+         "c6401.ambari.apache.org"
         ]
     }
 }