AMBARI-13349. Create all necessary keytabs and principals for Ranger Service(Mugdha Varadkar via gautam)

ambari-common/src/main/python/resource_management/libraries/functions/constants.py

@@ -79,3 +79,4 @@ class StackFeature:
   HIVE_ENV_HEAPSIZE = "hive_env_heapsize"
   RANGER_KMS_HSM_SUPPORT = "ranger_kms_hsm_support"
   RANGER_LOG4J_SUPPORT = "ranger_log4j_support"
+  RANGER_KERBEROS_SUPPORT = "ranger_kerberos_support"

ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py

@@ -228,6 +228,11 @@ _DEFAULT_STACK_FEATURES = {
       "name": "ranger_log4j_support",
       "description": "Ranger supporting log-4j properties (AMBARI-15681)",
       "min_version": "2.5.0.0"
+    },
+    {
+      "name": "ranger_kerberos_support",
+      "description": "Ranger Kerberos support",
+      "min_version": "2.5.0.0"
     }
   ]
 }

ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/alerts/alert_ranger_admin_passwd_check.py

@@ -23,6 +23,7 @@ import urllib2
 import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set.
 import logging
 from resource_management.core.environment import Environment
+from resource_management.libraries.script import Script
 
 logger = logging.getLogger()
 RANGER_ADMIN_URL = '{{admin-properties/policymgr_external_url}}'
@@ -30,6 +31,7 @@ ADMIN_USERNAME = '{{ranger-env/admin_username}}'
 ADMIN_PASSWORD = '{{ranger-env/admin_password}}'
 RANGER_ADMIN_USERNAME = '{{ranger-env/ranger_admin_username}}'
 RANGER_ADMIN_PASSWORD = '{{ranger-env/ranger_admin_password}}'
+SECURITY_ENABLED = '{{cluster-env/security_enabled}}'
 
 def get_tokens():
   """
@@ -38,7 +40,7 @@ def get_tokens():
 
   :return tuple
   """
-  return (RANGER_ADMIN_URL, ADMIN_USERNAME, ADMIN_PASSWORD, RANGER_ADMIN_USERNAME, RANGER_ADMIN_PASSWORD)
+  return (RANGER_ADMIN_URL, ADMIN_USERNAME, ADMIN_PASSWORD, RANGER_ADMIN_USERNAME, RANGER_ADMIN_PASSWORD, SECURITY_ENABLED)
 
 
 def execute(configurations={}, parameters={}, host_name=None):
@@ -61,6 +63,9 @@ def execute(configurations={}, parameters={}, host_name=None):
   admin_password = None
   ranger_admin_username = None
   ranger_admin_password = None
+  security_enabled = False
+
+  stack_is_hdp25_or_further = Script.is_stack_greater_or_equal("2.5")
 
   if RANGER_ADMIN_URL in configurations:
     ranger_link = configurations[RANGER_ADMIN_URL]
@@ -81,33 +86,40 @@ def execute(configurations={}, parameters={}, host_name=None):
   if RANGER_ADMIN_PASSWORD in configurations:
     ranger_admin_password = configurations[RANGER_ADMIN_PASSWORD]
 
+  if SECURITY_ENABLED in configurations:
+    security_enabled = str(configurations[SECURITY_ENABLED]).upper() == 'TRUE'
+
   label = None
   result_code = 'OK'
 
   try:
-    admin_http_code = check_ranger_login(ranger_auth_link, admin_username, admin_password)
-    if admin_http_code == 200:
-      get_user_code = get_ranger_user(ranger_get_user, admin_username, admin_password, ranger_admin_username)
-      if get_user_code:
-        user_http_code = check_ranger_login(ranger_auth_link, ranger_admin_username, ranger_admin_password)
-        if user_http_code == 200:
-          result_code = 'OK'
-          label = 'Login Successful for users {0} and {1}'.format(admin_username, ranger_admin_username)
-        elif user_http_code == 401:
-          result_code = 'CRITICAL'
-          label = 'User:{0} credentials on Ambari UI are not in sync with Ranger'.format(ranger_admin_username)
+    if security_enabled and stack_is_hdp25_or_further:
+      result_code = 'UNKNOWN'
+      label = 'This alert will get skipped for Ranger Admin on kerberos env'
+    else:
+      admin_http_code = check_ranger_login(ranger_auth_link, admin_username, admin_password)
+      if admin_http_code == 200:
+        get_user_code = get_ranger_user(ranger_get_user, admin_username, admin_password, ranger_admin_username)
+        if get_user_code:
+          user_http_code = check_ranger_login(ranger_auth_link, ranger_admin_username, ranger_admin_password)
+          if user_http_code == 200:
+            result_code = 'OK'
+            label = 'Login Successful for users {0} and {1}'.format(admin_username, ranger_admin_username)
+          elif user_http_code == 401:
+            result_code = 'CRITICAL'
+            label = 'User:{0} credentials on Ambari UI are not in sync with Ranger'.format(ranger_admin_username)
+          else:
+            result_code = 'WARNING'
+            label = 'Ranger Admin service is not reachable, please restart the service'
         else:
-          result_code = 'WARNING'
-          label = 'Ranger Admin service is not reachable, please restart the service'
+          result_code = 'OK'
+          label = 'Login Successful for user: {0}. User:{1} user not yet synced with Ranger'.format(admin_username, ranger_admin_username)
+      elif admin_http_code == 401:
+        result_code = 'CRITICAL'
+        label = 'User:{0} credentials on Ambari UI are not in sync with Ranger'.format(admin_username)
       else:
-        result_code = 'OK'
-        label = 'Login Successful for user: {0}. User:{1} user not yet synced with Ranger'.format(admin_username, ranger_admin_username)
-    elif admin_http_code == 401:
-      result_code = 'CRITICAL'
-      label = 'User:{0} credentials on Ambari UI are not in sync with Ranger'.format(admin_username)
-    else:
-      result_code = 'WARNING'
-      label = 'Ranger Admin service is not reachable, please restart the service'
+        result_code = 'WARNING'
+        label = 'Ranger Admin service is not reachable, please restart the service'
 
   except Exception, e:
     label = str(e)

ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py

@@ -61,6 +61,7 @@ stack_supports_usersync_non_root =  stack_version_formatted and check_stack_feat
 stack_supports_ranger_tagsync =  stack_version_formatted and check_stack_feature(StackFeature.RANGER_TAGSYNC_COMPONENT, stack_version_formatted)
 stack_supports_ranger_audit_db = stack_version_formatted and check_stack_feature(StackFeature.RANGER_AUDIT_DB_SUPPORT, stack_version_formatted)
 stack_supports_ranger_log4j =  stack_version_formatted and check_stack_feature(StackFeature.RANGER_LOG4J_SUPPORT, stack_version_formatted)
+stack_supports_ranger_kerberos = stack_version_formatted and check_stack_feature(StackFeature.RANGER_KERBEROS_SUPPORT, stack_version_formatted)
 
 downgrade_from_version = default("/commandParams/downgrade_from_version", None)
 upgrade_direction = default("/commandParams/upgrade_direction", None)
@@ -233,3 +234,8 @@ tagsync_pid_file = format('{ranger_pid_dir}/tagsync.pid')
 admin_log4j = config['configurations']['admin-log4j']['content']
 usersync_log4j = config['configurations']['usersync-log4j']['content']
 tagsync_log4j = config['configurations']['tagsync-log4j']['content']
+
+# ranger kerberos
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+namenode_hosts = default("/clusterHostInfo/namenode_host", [])
+has_namenode = len(namenode_hosts) > 0

ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py

@@ -173,6 +173,7 @@ def setup_ranger_admin(upgrade_type=None):
 
   do_keystore_setup(upgrade_type=upgrade_type)
 
+  create_core_site_xml(ranger_conf)
 
 def setup_ranger_db(stack_version=None):
   import params
@@ -425,6 +426,8 @@ def setup_usersync(upgrade_type=None):
         mode = 0640
     )
 
+  create_core_site_xml(ranger_ugsync_conf)
+
 def setup_tagsync(upgrade_type=None):
   import params
 
@@ -517,10 +520,25 @@ def setup_tagsync(upgrade_type=None):
     only_if=format("ls {tagsync_services_file}"),
     sudo=True)
 
+  create_core_site_xml(ranger_tagsync_conf)
+
 def ranger_credential_helper(lib_path, alias_key, alias_value, file_path):
   import params
 
   java_bin = format('{java_home}/bin/java')
   file_path = format('jceks://file{file_path}')
   cmd = (java_bin, '-cp', lib_path, 'org.apache.ranger.credentialapi.buildks', 'create', alias_key, '-value', PasswordString(alias_value), '-provider', file_path)
-  Execute(cmd, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True)
+  Execute(cmd, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True)
+
+def create_core_site_xml(conf_dir):
+  import params
+
+  if params.stack_supports_ranger_kerberos and params.security_enabled and params.has_namenode:
+    XmlConfig("core-site.xml",
+      conf_dir=conf_dir,
+      configurations=params.config['configurations']['core-site'],
+      configuration_attributes=params.config['configuration_attributes']['core-site'],
+      owner=params.unix_user,
+      group=params.unix_group,
+      mode=0644
+    )

ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json

@@ -205,6 +205,11 @@
       "name": "ranger_log4j_support",
       "description": "Ranger supporting log-4j properties (AMBARI-15681)",
       "min_version": "2.5.0.0"
+    },
+    {
+      "name": "ranger_kerberos_support",
+      "description": "Ranger Kerberos support",
+      "min_version": "2.5.0.0"
     }
   ]
 }

ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/alerts.json

@@ -0,0 +1,76 @@
+{
+  "RANGER": {
+    "service": [],
+    "RANGER_ADMIN": [
+      {
+        "name": "ranger_admin_process",
+        "label": "Ranger Admin Process",
+        "description": "This host-level alert is triggered if the Ranger Admin Web UI is unreachable.",
+        "interval": 1,
+        "scope": "ANY",
+        "source": {
+          "type": "WEB",
+          "uri": {
+              "http": "{{admin-properties/policymgr_external_url}}",
+              "https": "{{admin-properties/policymgr_external_url}}",
+              "kerberos_keytab": "{{ranger-admin-site/ranger.spnego.kerberos.keytab}}",
+              "kerberos_principal": "{{ranger-admin-site/ranger.spnego.kerberos.principal}}",
+              "https_property": "{{ranger-admin-site/ranger.service.https.attrib.ssl.enabled}}",
+              "https_property_value": "true",
+              "connection_timeout": 5.0
+            },
+          "reporting": {
+            "ok": {
+              "text": "HTTP {0} response in {2:.3f}s"
+            },
+            "warning": {
+              "text": "HTTP {0} response from {1} in {2:.3f}s ({3})"
+            },
+            "critical": {
+              "text": "Connection failed to {1} ({3})"
+            }
+          }
+        }
+      },
+      {
+        "name": "ranger_admin_password_check",
+        "label": "Ranger Admin password check",
+        "description": "This alert is used to ensure that the Ranger Admin password in Ambari is correct.",
+        "interval": 30,
+        "scope": "ANY",
+        "source": {
+          "type": "SCRIPT",
+          "path": "RANGER/0.4.0/package/alerts/alert_ranger_admin_passwd_check.py",
+          "parameters": []
+        }
+      }
+    ],
+    "RANGER_USERSYNC": [
+      {
+        "name": "ranger_usersync_process",
+        "label": "Ranger Usersync Process",
+        "description": "This host-level alert is triggered if the Ranger Usersync cannot be determined to be up.",
+        "interval": 1,
+        "scope": "HOST",
+        "source": {
+          "type": "PORT",
+          "uri": "{{ranger-ugsync-site/ranger.usersync.port}}",
+          "default_port": 5151,
+          "reporting": {
+            "ok": {
+              "text": "TCP OK - {0:.3f}s response on port {1}"
+            },
+            "warning": {
+              "text": "TCP OK - {0:.3f}s response on port {1}",
+              "value": 1.5
+            },
+            "critical": {
+              "text": "Connection failed: {0} to {1}:{2}",
+              "value": 5.0
+            }
+          }
+        }
+      }
+    ]
+  }
+}

ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/configuration/ranger-admin-site.xml

@@ -48,4 +48,79 @@
     <deleted>true</deleted>
   </property>
 
-</configuration>
+  <property>
+    <name>ranger.admin.kerberos.token.valid</name>
+    <value>30</value>
+    <description></description>
+  </property>
+
+  <property>
+    <name>ranger.admin.kerberos.cookie.domain</name>
+    <value></value>
+    <description></description>
+    <value-attributes>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+  </property>
+
+  <property>
+    <name>ranger.admin.kerberos.cookie.path</name>
+    <value>/</value>
+    <description></description>
+  </property>
+
+  <property>
+    <name>ranger.spnego.kerberos.principal</name>
+    <value></value>
+    <description></description>
+    <value-attributes>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+  </property>
+
+  <property>
+    <name>ranger.spnego.kerberos.keytab</name>
+    <value></value>
+    <description></description>
+    <value-attributes>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+  </property>
+
+  <property>
+    <name>ranger.admin.kerberos.principal</name>
+    <value></value>
+    <description></description>
+    <value-attributes>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+  </property>
+
+  <property>
+    <name>ranger.admin.kerberos.keytab</name>
+    <value></value>
+    <description></description>
+    <value-attributes>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+  </property>
+
+  <property>
+    <name>ranger.lookup.kerberos.principal</name>
+    <value></value>
+    <description></description>
+    <value-attributes>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+  </property>
+
+  <property>
+    <name>ranger.lookup.kerberos.keytab</name>
+    <value></value>
+    <description></description>
+    <value-attributes>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+  </property>
+
+</configuration>

ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/configuration/ranger-tagsync-site.xml

@@ -184,4 +184,22 @@
     </value-attributes>
   </property>
 
+  <property>
+    <name>ranger.tagsync.kerberos.principal</name>
+    <value></value>
+    <description></description>
+    <value-attributes>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+  </property>
+
+  <property>
+    <name>ranger.tagsync.kerberos.keytab</name>
+    <value></value>
+    <description></description>
+    <value-attributes>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+  </property>
+
 </configuration>

ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/configuration/ranger-ugsync-site.xml

@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+<configuration>
+
+  <property>
+    <name>ranger.usersync.kerberos.principal</name>
+    <value></value>
+    <description></description>
+    <value-attributes>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+  </property>
+
+  <property>
+    <name>ranger.usersync.kerberos.keytab</name>
+    <value></value>
+    <description></description>
+    <value-attributes>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+  </property>
+
+</configuration>

ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/kerberos.json

@@ -0,0 +1,115 @@
+{
+  "services": [
+    {
+      "name": "RANGER",
+      "identities": [
+        {
+          "name": "/spnego"
+        },
+        {
+          "name": "/smokeuser"
+        }
+      ],
+      "configurations": [
+        {
+          "ranger-admin-site": {
+            "ranger.admin.kerberos.cookie.domain": "{{ranger_host}}"
+          }
+        }
+      ],
+      "components": [
+        {
+          "name": "RANGER_ADMIN",
+          "identities": [
+            {
+              "name": "rangeradmin",
+              "principal": {
+                "value": "rangeradmin/_HOST@${realm}",
+                "type" : "service",
+                "configuration": "ranger-admin-site/ranger.admin.kerberos.principal",
+                "local_username" : "${ranger-env/ranger_user}"
+              },
+              "keytab": {
+                "file": "${keytab_dir}/rangeradmin.service.keytab",
+                "owner": {
+                  "name": "${ranger-env/ranger_user}",
+                  "access": "r"
+                },
+                "configuration": "ranger-admin-site/ranger.admin.kerberos.keytab"
+              }
+            },
+            {
+              "name": "rangerlookup",
+              "principal": {
+                "value": "rangerlookup/_HOST@${realm}",
+                "configuration": "ranger-admin-site/ranger.lookup.kerberos.principal",
+                "type" : "service"
+              },
+              "keytab": {
+                "file": "${keytab_dir}/rangerlookup.service.keytab",
+                "owner": {
+                  "name": "${ranger-env/ranger_user}",
+                  "access": "r"
+                },
+                "configuration": "ranger-admin-site/ranger.lookup.kerberos.keytab"
+              }
+            },
+            {
+              "name": "/spnego",
+              "principal": {
+                "configuration": "ranger-admin-site/ranger.spnego.kerberos.principal"
+              },
+              "keytab": {
+                "configuration": "ranger-admin-site/ranger.spnego.kerberos.keytab"
+              }
+            }
+          ]
+        },
+        {
+          "name": "RANGER_USERSYNC",
+          "identities": [
+            {
+              "name": "rangerusersync",
+              "principal": {
+                "value": "rangerusersync/_HOST@${realm}",
+                "type" : "service",
+                "configuration" : "ranger-ugsync-site/ranger.usersync.kerberos.principal",
+                "local_username" : "rangerusersync"
+              },
+              "keytab": {
+                "file": "${keytab_dir}/rangerusersync.service.keytab",
+                "owner": {
+                  "name": "${ranger-env/ranger_user}",
+                  "access": "r"
+                },
+                "configuration": "ranger-ugsync-site/ranger.usersync.kerberos.keytab"
+              }
+            }
+          ]
+        },
+        {
+          "name": "RANGER_TAGSYNC",
+          "identities": [
+            {
+              "name": "rangertagsync",
+              "principal": {
+                "value": "rangertagsync/_HOST@${realm}",
+                "type" : "service",
+                "configuration": "ranger-tagsync-site/ranger.tagsync.kerberos.principal",
+                "local_username" : "rangertagsync"
+              },
+              "keytab": {
+                "file": "${keytab_dir}/rangertagsync.service.keytab",
+                "owner": {
+                  "name": "${ranger-env/ranger_user}",
+                  "access": "r"
+                },
+                "configuration": "ranger-tagsync-site/ranger.tagsync.kerberos.keytab"
+              }
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}