AMBARI-17833: EU fails during restart of Ranger Admin at setup_ranger_audit_solr (Mugdha Varadkar via jluniya)

ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py

@@ -265,8 +265,8 @@ ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
 policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url']
 if 'admin-properties' in config['configurations'] and 'policymgr_external_url' in config['configurations']['admin-properties'] and policymgr_mgr_url.endswith('/'):
   policymgr_mgr_url = policymgr_mgr_url.rstrip('/')
-xa_audit_db_name = config['configurations']['admin-properties']['audit_db_name']
-xa_audit_db_user = config['configurations']['admin-properties']['audit_db_user']
+xa_audit_db_name = default('/configurations/admin-properties/audit_db_name', 'ranger_audits')
+xa_audit_db_user = default('/configurations/admin-properties/audit_db_user', 'rangerlogger')
 xa_db_host = config['configurations']['admin-properties']['db_host']
 repo_name = str(config['clusterName']) + '_hbase'
 
@@ -290,7 +290,9 @@ java_share_dir = '/usr/share/java'
 enable_ranger_hbase = False
 if has_ranger_admin:
   enable_ranger_hbase = (config['configurations']['ranger-hbase-plugin-properties']['ranger-hbase-plugin-enabled'].lower() == 'yes')
-  xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password']) if stack_supports_ranger_audit_db else None
+  xa_audit_db_password = ''
+  if not is_empty(config['configurations']['admin-properties']['audit_db_password']) and stack_supports_ranger_audit_db:
+    xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password'])
   repo_config_password = unicode(config['configurations']['ranger-hbase-plugin-properties']['REPOSITORY_CONFIG_PASSWORD'])
   xa_audit_db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower()
   previous_jdbc_jar_name = None

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py

@@ -42,7 +42,7 @@ from resource_management.libraries.resources.hdfs_resource import HdfsResource
 from resource_management.libraries.functions.format_jvm_option import format_jvm_option
 from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages
 from resource_management.libraries.functions.hdfs_utils import is_https_enabled_in_hdfs
-
+from resource_management.libraries.functions import is_empty
 
 
 config = Script.get_config()
@@ -399,8 +399,8 @@ ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
 policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url']
 if 'admin-properties' in config['configurations'] and 'policymgr_external_url' in config['configurations']['admin-properties'] and policymgr_mgr_url.endswith('/'):
   policymgr_mgr_url = policymgr_mgr_url.rstrip('/')
-xa_audit_db_name = config['configurations']['admin-properties']['audit_db_name']
-xa_audit_db_user = config['configurations']['admin-properties']['audit_db_user']
+xa_audit_db_name = default('/configurations/admin-properties/audit_db_name', 'ranger_audits')
+xa_audit_db_user = default('/configurations/admin-properties/audit_db_user', 'rangerlogger')
 xa_db_host = config['configurations']['admin-properties']['db_host']
 repo_name = str(config['clusterName']) + '_hadoop'
 
@@ -430,7 +430,9 @@ is_https_enabled = is_https_enabled_in_hdfs(config['configurations']['hdfs-site'
 
 if has_ranger_admin:
   enable_ranger_hdfs = (config['configurations']['ranger-hdfs-plugin-properties']['ranger-hdfs-plugin-enabled'].lower() == 'yes')
-  xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password']) if stack_supports_ranger_audit_db else None
+  xa_audit_db_password = ''
+  if not is_empty(config['configurations']['admin-properties']['audit_db_password']) and stack_supports_ranger_audit_db:
+    xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password'])
   repo_config_password = unicode(config['configurations']['ranger-hdfs-plugin-properties']['REPOSITORY_CONFIG_PASSWORD'])
   xa_audit_db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower()
   previous_jdbc_jar_name = None

ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py

@@ -619,8 +619,8 @@ xml_configurations_supported = config['configurations']['ranger-env']['xml_confi
 policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url']
 if 'admin-properties' in config['configurations'] and 'policymgr_external_url' in config['configurations']['admin-properties'] and policymgr_mgr_url.endswith('/'):
   policymgr_mgr_url = policymgr_mgr_url.rstrip('/')
-xa_audit_db_name = config['configurations']['admin-properties']['audit_db_name']
-xa_audit_db_user = config['configurations']['admin-properties']['audit_db_user']
+xa_audit_db_name = default('/configurations/admin-properties/audit_db_name', 'ranger_audits')
+xa_audit_db_user = default('/configurations/admin-properties/audit_db_user', 'rangerlogger')
 xa_db_host = config['configurations']['admin-properties']['db_host']
 repo_name = str(config['clusterName']) + '_hive'
 
@@ -716,7 +716,9 @@ if has_ranger_admin:
 
 
   xa_audit_db_is_enabled = False
-  xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password']) if stack_supports_ranger_audit_db else None
+  xa_audit_db_password = ''
+  if not is_empty(config['configurations']['admin-properties']['audit_db_password']) and stack_supports_ranger_audit_db:
+    xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password'])
   ranger_audit_solr_urls = config['configurations']['ranger-admin-site']['ranger.audit.solr.urls']
   if xml_configurations_supported and stack_supports_ranger_audit_db:
     xa_audit_db_is_enabled = config['configurations']['ranger-hive-audit']['xasecure.audit.destination.db']

ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py

@@ -184,9 +184,11 @@ if has_ranger_admin and is_supported_kafka_ranger:
     policymgr_mgr_url = policymgr_mgr_url.rstrip('/')
   xa_audit_db_flavor = config['configurations']['admin-properties']['DB_FLAVOR']
   xa_audit_db_flavor = xa_audit_db_flavor.lower() if xa_audit_db_flavor else None
-  xa_audit_db_name = config['configurations']['admin-properties']['audit_db_name']
-  xa_audit_db_user = config['configurations']['admin-properties']['audit_db_user']
-  xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password']) if stack_supports_ranger_audit_db else None
+  xa_audit_db_name = default('/configurations/admin-properties/audit_db_name', 'ranger_audits')
+  xa_audit_db_user = default('/configurations/admin-properties/audit_db_user', 'rangerlogger')
+  xa_audit_db_password = ''
+  if not is_empty(config['configurations']['admin-properties']['audit_db_password']) and stack_supports_ranger_audit_db:
+    xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password'])
   xa_db_host = config['configurations']['admin-properties']['db_host']
   repo_name = str(config['clusterName']) + '_kafka'
 

ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py

@@ -38,6 +38,7 @@ from resource_management.libraries.functions.get_not_managed_resources import ge
 from resource_management.libraries.functions.stack_features import check_stack_feature
 from resource_management.libraries.functions.stack_features import get_stack_feature_version
 from resource_management.libraries.functions.constants import StackFeature
+from resource_management.libraries.functions import is_empty
 
 # server configurations
 config = Script.get_config()
@@ -264,8 +265,8 @@ ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
 policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url']
 if 'admin-properties' in config['configurations'] and 'policymgr_external_url' in config['configurations']['admin-properties'] and policymgr_mgr_url.endswith('/'):
   policymgr_mgr_url = policymgr_mgr_url.rstrip('/')
-xa_audit_db_name = config['configurations']['admin-properties']['audit_db_name']
-xa_audit_db_user = config['configurations']['admin-properties']['audit_db_user']
+xa_audit_db_name = default('/configurations/admin-properties/audit_db_name', 'ranger_audits')
+xa_audit_db_user = default('/configurations/admin-properties/audit_db_user', 'rangerlogger')
 xa_db_host = config['configurations']['admin-properties']['db_host']
 repo_name = str(config['clusterName']) + '_knox'
 
@@ -283,7 +284,9 @@ jdk_location = config['hostLevelParams']['jdk_location']
 java_share_dir = '/usr/share/java'
 if has_ranger_admin:
   enable_ranger_knox = (config['configurations']['ranger-knox-plugin-properties']['ranger-knox-plugin-enabled'].lower() == 'yes')
-  xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password']) if stack_supports_ranger_audit_db else None
+  xa_audit_db_password = ''
+  if not is_empty(config['configurations']['admin-properties']['audit_db_password']) and stack_supports_ranger_audit_db:
+    xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password'])
   repo_config_password = unicode(config['configurations']['ranger-knox-plugin-properties']['REPOSITORY_CONFIG_PASSWORD'])
   xa_audit_db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower()
   previous_jdbc_jar_name= None

ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py

@@ -298,7 +298,10 @@ for host in zookeeper_hosts:
   index += 1
   if index < len(zookeeper_hosts):
     zookeeper_quorum += ","
+
+# solr kerberised
 solr_jaas_file = None
+is_solr_kerberos_enabled = default('/configurations/ranger-admin-site/ranger.is.solr.kerberised', False)
 
 if security_enabled:
   if has_ranger_tagsync:
@@ -308,18 +311,16 @@ if security_enabled:
     tagsync_keytab_path = config['configurations']['ranger-tagsync-site']['ranger.tagsync.kerberos.keytab']
 
   if stack_supports_ranger_kerberos:
+    ranger_admin_keytab = config['configurations']['ranger-admin-site']['ranger.admin.kerberos.keytab']
     ranger_admin_principal = config['configurations']['ranger-admin-site']['ranger.admin.kerberos.principal']
     if not is_empty(ranger_admin_principal) and ranger_admin_principal != '':
       ranger_admin_jaas_principal = ranger_admin_principal.replace('_HOST', ranger_host.lower())
-    ranger_admin_keytab = config['configurations']['ranger-admin-site']['ranger.admin.kerberos.keytab']
-
-    if not is_empty(ranger_admin_principal) and ranger_admin_principal != '':
-      if stack_supports_logsearch_client and is_solrCloud_enabled:
+      if stack_supports_logsearch_client and is_solrCloud_enabled and is_solr_kerberos_enabled:
         solr_jaas_file = format('{ranger_home}/conf/ranger_solr_jass.conf')
         solr_kerberos_principal = ranger_admin_jaas_principal
         solr_kerberos_keytab = ranger_admin_keytab
-    else:
-      solr_jaas_file = None
+      else:
+        solr_jaas_file = None
 
 # logic to create core-site.xml if hdfs not installed
 if stack_supports_ranger_kerberos and not has_namenode:
@@ -327,12 +328,17 @@ if stack_supports_ranger_kerberos and not has_namenode:
     'hadoop.security.authentication': 'kerberos' if security_enabled else 'simple'
   }
 
-  realm = 'EXAMPLE.COM'
   if security_enabled:
-    ranger_admin_principal = config['configurations']['ranger-admin-site']['ranger.admin.kerberos.principal']
+    realm = 'EXAMPLE.COM'
+    ranger_admin_bare_principal = 'rangeradmin'
+    ranger_usersync_bare_principal = 'rangerusersync'
+    ranger_tagsync_bare_principal = 'rangertagsync'
+
     ranger_usersync_principal = config['configurations']['ranger-ugsync-site']['ranger.usersync.kerberos.principal']
-    ranger_admin_bare_principal = get_bare_principal(ranger_admin_principal)
-    ranger_usersync_bare_principal = get_bare_principal(ranger_usersync_principal)
+    if not is_empty(ranger_admin_principal) and ranger_admin_principal != '':
+      ranger_admin_bare_principal = get_bare_principal(ranger_admin_principal)
+    if not is_empty(ranger_usersync_principal) and ranger_usersync_principal != '':
+      ranger_usersync_bare_principal = get_bare_principal(ranger_usersync_principal)
     realm = config['configurations']['kerberos-env']['realm']
 
     rule_dict = [
@@ -341,7 +347,8 @@ if stack_supports_ranger_kerberos and not has_namenode:
     ]
 
     if has_ranger_tagsync:
-      ranger_tagsync_bare_principal = get_bare_principal(ranger_tagsync_principal)
+      if not is_empty(ranger_tagsync_principal) and ranger_tagsync_principal != '':
+        ranger_tagsync_bare_principal = get_bare_principal(ranger_tagsync_principal)
       rule_dict.append({'principal': ranger_tagsync_bare_principal, 'user': 'rangertagsync'})
 
     core_site_auth_to_local_property = ''

ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py

@@ -557,12 +557,14 @@ def create_core_site_xml(conf_dir):
 
 def setup_ranger_audit_solr():
   import params
-  jaas_file = params.solr_jaas_file if params.security_enabled else None
-  if params.security_enabled and params.stack_supports_ranger_kerberos:
-    File(format("{solr_jaas_file}"),
-      content=Template("ranger_solr_jass_conf.j2"),
-      owner=params.unix_user
-    )
+
+  if params.security_enabled and params.stack_supports_ranger_kerberos and params.is_solr_kerberos_enabled:
+    if params.solr_jaas_file is not None:
+      File(format("{solr_jaas_file}"),
+        content=Template("ranger_solr_jass_conf.j2"),
+        owner=params.unix_user
+      )
+
   check_znode()
 
   solr_cloud_util.upload_configuration_to_zk(
@@ -573,7 +575,7 @@ def setup_ranger_audit_solr():
     tmp_dir = params.tmp_dir,
     java64_home = params.java_home,
     user = params.unix_user,
-    jaas_file=jaas_file,
+    jaas_file=params.solr_jaas_file,
     retry=30, interval=5)
 
   solr_cloud_util.create_collection(

ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py

@@ -91,9 +91,11 @@ if has_ranger_admin:
   if 'admin-properties' in config['configurations'] and 'policymgr_external_url' in config['configurations']['admin-properties'] and policymgr_mgr_url.endswith('/'):
     policymgr_mgr_url = policymgr_mgr_url.rstrip('/')
   xa_audit_db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower()
-  xa_audit_db_name = config['configurations']['admin-properties']['audit_db_name']
-  xa_audit_db_user = config['configurations']['admin-properties']['audit_db_user']
-  xa_audit_db_password = config['configurations']['admin-properties']['audit_db_password']
+  xa_audit_db_name = default('/configurations/admin-properties/audit_db_name', 'ranger_audits')
+  xa_audit_db_user = default('/configurations/admin-properties/audit_db_user', 'rangerlogger')
+  xa_audit_db_password = ''
+  if not is_empty(config['configurations']['admin-properties']['audit_db_password']) and stack_supports_ranger_audit_db:
+    xa_audit_db_password = config['configurations']['admin-properties']['audit_db_password']
   xa_db_host = config['configurations']['admin-properties']['db_host']
 
   admin_uname = config['configurations']['ranger-env']['admin_username']

ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py

@@ -38,6 +38,7 @@ from resource_management.libraries.functions.stack_features import get_stack_fea
 from resource_management.libraries.functions import StackFeature
 from resource_management.libraries.functions.expect import expect
 from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster
+from resource_management.libraries.functions import is_empty
 
 # server configurations
 config = Script.get_config()
@@ -225,8 +226,8 @@ ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
 policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url']
 if 'admin-properties' in config['configurations'] and 'policymgr_external_url' in config['configurations']['admin-properties'] and policymgr_mgr_url.endswith('/'):
   policymgr_mgr_url = policymgr_mgr_url.rstrip('/')
-xa_audit_db_name = config['configurations']['admin-properties']['audit_db_name']
-xa_audit_db_user = config['configurations']['admin-properties']['audit_db_user']
+xa_audit_db_name = default('/configurations/admin-properties/audit_db_name', 'ranger_audits')
+xa_audit_db_user = default('/configurations/admin-properties/audit_db_user', 'rangerlogger')
 xa_db_host = config['configurations']['admin-properties']['db_host']
 repo_name = str(config['clusterName']) + '_storm'
 
@@ -250,7 +251,9 @@ java_share_dir = '/usr/share/java'
 
 if has_ranger_admin:
   enable_ranger_storm = (config['configurations']['ranger-storm-plugin-properties']['ranger-storm-plugin-enabled'].lower() == 'yes')
-  xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password']) if stack_supports_ranger_audit_db else None
+  xa_audit_db_password = ''
+  if not is_empty(config['configurations']['admin-properties']['audit_db_password']) and stack_supports_ranger_audit_db:
+    xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password'])
   repo_config_password = unicode(config['configurations']['ranger-storm-plugin-properties']['REPOSITORY_CONFIG_PASSWORD'])
   xa_audit_db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower()
   previous_jdbc_jar_name = None

ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py

@@ -34,6 +34,7 @@ from resource_management.libraries.functions.get_not_managed_resources import ge
 from resource_management.libraries.functions.version import format_stack_version
 from resource_management.libraries.functions.default import default
 from resource_management.libraries import functions
+from resource_management.libraries.functions import is_empty
 
 import status_params
 
@@ -404,9 +405,11 @@ if has_ranger_admin:
     if 'admin-properties' in config['configurations'] and 'policymgr_external_url' in config['configurations']['admin-properties'] and policymgr_mgr_url.endswith('/'):
       policymgr_mgr_url = policymgr_mgr_url.rstrip('/')
     xa_audit_db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower()
-    xa_audit_db_name = config['configurations']['admin-properties']['audit_db_name']
-    xa_audit_db_user = config['configurations']['admin-properties']['audit_db_user']
-    xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password']) if stack_supports_ranger_audit_db else None
+    xa_audit_db_name = default('/configurations/admin-properties/audit_db_name', 'ranger_audits')
+    xa_audit_db_user = default('/configurations/admin-properties/audit_db_user', 'rangerlogger')
+    xa_audit_db_password = ''
+    if not is_empty(config['configurations']['admin-properties']['audit_db_password']) and stack_supports_ranger_audit_db:
+      xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password'])
     xa_db_host = config['configurations']['admin-properties']['db_host']
     repo_name = str(config['clusterName']) + '_yarn'
 

ambari-server/src/test/python/stacks/2.5/configs/ranger-admin-secured.json

@@ -281,6 +281,7 @@
             "ssl.client.keystore.type": "jks"
         }, 
         "ranger-admin-site": {
+            "ranger.is.solr.kerberised": "true",
             "ranger.admin.kerberos.cookie.domain": "{{ranger_host}}", 
             "ranger.kms.service.user.hdfs": "hdfs", 
             "ranger.spnego.kerberos.principal": "HTTP/_HOST@EXAMPLE.COM",