AMBARI-17462. Install Log search client package with Ranger Service (Part-2) (Mugdha Varadkar via gautam)

ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py

@@ -269,15 +269,17 @@ logsearch_solr_hosts = default("/clusterHostInfo/logsearch_solr_hosts", [])
 has_logsearch = len(logsearch_solr_hosts) > 0
 is_solrCloud_enabled = default('/configurations/ranger-env/is_solrCloud_enabled', False)
 solr_znode = '/ranger_audits'
-if is_solrCloud_enabled:
-  solr_znode = config['configurations']['ranger-admin-site']['ranger.audit.solr.zookeepers']
+if stack_supports_logsearch_client and is_solrCloud_enabled:
+  solr_znode = default('/configurations/ranger-admin-site/ranger.audit.solr.zookeepers', 'NONE')
   if solr_znode != '' and solr_znode.upper() != 'NONE':
-    solr_znode = solr_znode.split('/')[1]
-    solr_znode = format('/{solr_znode}')
+    solr_znode = solr_znode.split('/')
+    if len(solr_znode) > 1 and len(solr_znode) == 2:
+      solr_znode = solr_znode[1]
+      solr_znode = format('/{solr_znode}')
   if has_logsearch:
     solr_znode = config['configurations']['logsearch-solr-env']['logsearch_solr_znode']
 solr_user = default('/configurations/logsearch-solr-env/logsearch_solr_user', unix_user)
-custom_log4j = True if has_logsearch else False
+custom_log4j = has_logsearch
 
 # get comma separated list of zookeeper hosts
 zookeeper_port = default('/configurations/zoo.cfg/clientPort', None)

ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py

@@ -30,6 +30,7 @@ from resource_management.core.logger import Logger
 from resource_management.core import shell
 from ranger_service import ranger_service
 from setup_ranger_xml import setup_ranger_audit_solr
+from resource_management.libraries.functions import solr_cloud_util
 import upgrade
 import os, errno
 
@@ -82,6 +83,10 @@ class RangerAdmin(Script):
     env.set_params(params)
     self.configure(env, upgrade_type=upgrade_type)
 
+    if params.stack_supports_logsearch_client and params.is_solrCloud_enabled:
+      solr_cloud_util.setup_solr_client(params.config, user = params.solr_user, custom_log4j = params.custom_log4j)
+      setup_ranger_audit_solr()
+
     ranger_service('ranger_admin')
 
 

ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py

@@ -191,10 +191,6 @@ def setup_ranger_admin(upgrade_type=None):
 
   create_core_site_xml(ranger_conf)
 
-  if params.stack_supports_logsearch_client and params.is_solrCloud_enabled:
-    solr_cloud_util.setup_solr_client(params.config, user = params.solr_user, custom_log4j = params.custom_log4j)
-    setup_ranger_audit_solr()
-
 def setup_ranger_db(stack_version=None):
   import params
   

ambari-server/src/test/python/stacks/2.5/RANGER/test_ranger_admin.py

@@ -50,6 +50,44 @@ class TestRangerAdmin(RMFTestCase):
       target = RMFTestCase.TARGET_COMMON_SERVICES
     )
     self.assert_configure_default()
+
+    self.assertResourceCalled('Directory', '/var/log/ambari-logsearch-solr-client',
+        owner = 'logsearch-solr',
+        group = 'hadoop',
+        create_parents = True,
+        mode = 0755,
+        cd_access = 'a',
+    )
+    self.assertResourceCalled('Directory', '/usr/lib/ambari-logsearch-solr-client',
+        group = 'hadoop',
+        cd_access = 'a',
+        create_parents = True,
+        mode = 0755,
+        owner = 'logsearch-solr',
+        recursive_ownership = True,
+    )
+    self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh',
+        content = StaticFile('/usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh'),
+        owner = 'logsearch-solr',
+        group = 'hadoop',
+        mode = 0755,
+    )
+    self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-solr-client/log4j.properties',
+        owner = 'logsearch-solr',
+        content = InlineTemplate(self.getConfig()['configurations']['logsearch-solr-client-log4j']['content']),
+        group = 'hadoop',
+        mode = 0644,
+    )
+    self.assertResourceCalled('File', '/var/log/ambari-logsearch-solr-client/solr-client.log',
+        content = '',
+        owner = 'logsearch-solr',
+        group = 'hadoop',
+        mode = 0664,
+    )
+    self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/ambari-solr --download-config --config-dir /tmp/ranger_config_ranger_audits_0.[0-9]* --config-set ranger_audits --retry 30 --interval 5')
+    self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/ambari-solr --upload-config --config-dir /usr/hdp/current/ranger-admin/contrib/solr_for_audit_setup/conf --config-set ranger_audits --retry 30 --interval 5')
+    self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/ambari-solr --create-collection --collection ranger_audits --config-set ranger_audits --shards 1 --replication 1 --max-shards 1 --retry 5 --interval 10')
+
     self.assertResourceCalled('Execute', '/usr/bin/ranger-admin-start',
       environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
       not_if = 'ps -ef | grep proc_rangeradmin | grep -v grep',
@@ -99,6 +137,48 @@ class TestRangerAdmin(RMFTestCase):
       target = RMFTestCase.TARGET_COMMON_SERVICES
     )
     self.assert_configure_secured()
+
+    self.assertResourceCalled('Directory', '/var/log/ambari-logsearch-solr-client',
+        owner = 'logsearch-solr',
+        group = 'hadoop',
+        create_parents = True,
+        mode = 0755,
+        cd_access = 'a',
+    )
+    self.assertResourceCalled('Directory', '/usr/lib/ambari-logsearch-solr-client',
+        group = 'hadoop',
+        cd_access = 'a',
+        create_parents = True,
+        mode = 0755,
+        owner = 'logsearch-solr',
+        recursive_ownership = True,
+    )
+    self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh',
+        content = StaticFile('/usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh'),
+        owner = 'logsearch-solr',
+        group = 'hadoop',
+        mode = 0755,
+    )
+    self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-solr-client/log4j.properties',
+        owner = 'logsearch-solr',
+        content = InlineTemplate(self.getConfig()['configurations']['logsearch-solr-client-log4j']['content']),
+        group = 'hadoop',
+        mode = 0644,
+    )
+    self.assertResourceCalled('File', '/var/log/ambari-logsearch-solr-client/solr-client.log',
+        content = '',
+        owner = 'logsearch-solr',
+        group = 'hadoop',
+        mode = 0664,
+    )
+    self.assertResourceCalled('File', '/usr/hdp/current/ranger-admin/conf/ranger_solr_jass.conf',
+      content = Template('ranger_solr_jass_conf.j2'),
+      owner = 'ranger',
+    )
+    self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/ambari-solr --download-config --config-dir /tmp/ranger_config_ranger_audits_0.[0-9]* --config-set ranger_audits --retry 30 --interval 5')
+    self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/ambari-solr --upload-config --config-dir /usr/hdp/current/ranger-admin/contrib/solr_for_audit_setup/conf --config-set ranger_audits --retry 30 --interval 5')
+    self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/ambari-solr --create-collection --collection ranger_audits --config-set ranger_audits --shards 1 --replication 1 --max-shards 1 --retry 5 --interval 10')    
+
     self.assertResourceCalled('Execute', '/usr/bin/ranger-admin-start',
       environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
       not_if = 'ps -ef | grep proc_rangeradmin | grep -v grep',
@@ -249,42 +329,6 @@ class TestRangerAdmin(RMFTestCase):
       configuration_attributes = self.getConfig()['configuration_attributes']['core-site'],
       mode = 0644
     )
-    self.assertResourceCalled('Directory', '/var/log/ambari-logsearch-solr-client',
-        owner = 'logsearch-solr',
-        group = 'hadoop',
-        create_parents = True,
-        mode = 0755,
-        cd_access = 'a',
-    )
-    self.assertResourceCalled('Directory', '/usr/lib/ambari-logsearch-solr-client',
-        group = 'hadoop',
-        cd_access = 'a',
-        create_parents = True,
-        mode = 0755,
-        owner = 'logsearch-solr',
-        recursive_ownership = True,
-    )
-    self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh',
-        content = StaticFile('/usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh'),
-        owner = 'logsearch-solr',
-        group = 'hadoop',
-        mode = 0755,
-    )
-    self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-solr-client/log4j.properties',
-        owner = 'logsearch-solr',
-        content = InlineTemplate(self.getConfig()['configurations']['logsearch-solr-client-log4j']['content']),
-        group = 'hadoop',
-        mode = 0644,
-    )
-    self.assertResourceCalled('File', '/var/log/ambari-logsearch-solr-client/solr-client.log',
-        content = '',
-        owner = 'logsearch-solr',
-        group = 'hadoop',
-        mode = 0664,
-    )
-    self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/ambari-solr --download-config --config-dir /tmp/ranger_config_ranger_audits_0.[0-9]* --config-set ranger_audits --retry 30 --interval 5')
-    self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/ambari-solr --upload-config --config-dir /usr/hdp/current/ranger-admin/contrib/solr_for_audit_setup/conf --config-set ranger_audits --retry 30 --interval 5')
-    self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/ambari-solr --create-collection --collection ranger_audits --config-set ranger_audits --shards 1 --replication 1 --max-shards 1 --retry 5 --interval 10')
 
   def assert_configure_secured(self):
 
@@ -427,44 +471,4 @@ class TestRangerAdmin(RMFTestCase):
       configurations = self.getConfig()['configurations']['core-site'],
       configuration_attributes = self.getConfig()['configuration_attributes']['core-site'],
       mode = 0644
-    )
-    self.assertResourceCalled('Directory', '/var/log/ambari-logsearch-solr-client',
-        owner = 'logsearch-solr',
-        group = 'hadoop',
-        create_parents = True,
-        mode = 0755,
-        cd_access = 'a',
-    )
-    self.assertResourceCalled('Directory', '/usr/lib/ambari-logsearch-solr-client',
-        group = 'hadoop',
-        cd_access = 'a',
-        create_parents = True,
-        mode = 0755,
-        owner = 'logsearch-solr',
-        recursive_ownership = True,
-    )
-    self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh',
-        content = StaticFile('/usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh'),
-        owner = 'logsearch-solr',
-        group = 'hadoop',
-        mode = 0755,
-    )
-    self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-solr-client/log4j.properties',
-        owner = 'logsearch-solr',
-        content = InlineTemplate(self.getConfig()['configurations']['logsearch-solr-client-log4j']['content']),
-        group = 'hadoop',
-        mode = 0644,
-    )
-    self.assertResourceCalled('File', '/var/log/ambari-logsearch-solr-client/solr-client.log',
-        content = '',
-        owner = 'logsearch-solr',
-        group = 'hadoop',
-        mode = 0664,
-    )
-    self.assertResourceCalled('File', '/usr/hdp/current/ranger-admin/conf/ranger_solr_jass.conf',
-      content = Template('ranger_solr_jass_conf.j2'),
-      owner = 'ranger',
-    )
-    self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/ambari-solr --download-config --config-dir /tmp/ranger_config_ranger_audits_0.[0-9]* --config-set ranger_audits --retry 30 --interval 5')
-    self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/ambari-solr --upload-config --config-dir /usr/hdp/current/ranger-admin/contrib/solr_for_audit_setup/conf --config-set ranger_audits --retry 30 --interval 5')
-    self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh --zookeeper-connect-string c6401.ambari.apache.org:2181/ambari-solr --create-collection --collection ranger_audits --config-set ranger_audits --shards 1 --replication 1 --max-shards 1 --retry 5 --interval 10')    
+    )