AMBARI-16182. Add new component level identity for RANGER_KMS in kerberos.json for stack 2.5(Mugdha Varadkar via gautam)

ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions_v2.py

@@ -371,7 +371,7 @@ class RangeradminV2:
     response,error_message,time_in_millis = self.call_curl_request(component_user,component_user_keytab,component_user_principal,search_repo_url,False,method,data,header)
     if response and len(response) > 0:
       response_json = json.loads(response)
-      if response_json['name'].lower() == name.lower():
+      if 'name' in response_json and response_json['name'].lower() == name.lower():
         Logger.info('Repository created Successfully')
         service_name = response_json['name']
         service_type = response_json['type']

ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py

@@ -33,6 +33,8 @@ from resource_management.core.exceptions import Fail
 from resource_management.core.logger import Logger
 from resource_management.libraries.functions.format import format
 from resource_management.libraries.functions.ranger_functions import Rangeradmin
+from resource_management.libraries.functions.ranger_functions_v2 import RangeradminV2
+from resource_management.libraries.functions.decorator import safe_retry
 from resource_management.core.utils import PasswordString
 from resource_management.core.shell import as_sudo
 import re
@@ -343,16 +345,14 @@ def enable_kms_plugin():
   import params
 
   if params.has_ranger_admin:
-    count = 0
-    while count < 5:
-      ranger_flag = check_ranger_service()
-      if ranger_flag:
-        break
-      else:
-        time.sleep(5) # delay for 5 seconds
-        count = count + 1
+
+    if params.stack_supports_ranger_kerberos and params.security_enabled:
+      ranger_flag = check_ranger_service_support_kerberos()
     else:
-      Logger.error("Ranger service is not reachable after {0} tries".format(count))
+      ranger_flag = check_ranger_service()
+
+    if not ranger_flag:
+      Logger.error('Error in Get/Create service for Ranger Kms.')
 
     current_datetime = datetime.now()
 
@@ -458,20 +458,16 @@ def check_ranger_service():
     if user_resp_code is not None and user_resp_code == 200:
       get_repo_flag = get_repo(params.policymgr_mgr_url, params.repo_name, ambari_username_password_for_ranger)
       if not get_repo_flag:
-        create_repo_flag = create_repo(params.policymgr_mgr_url, json.dumps(params.kms_ranger_plugin_repo), ambari_username_password_for_ranger)
-        if create_repo_flag:
-          return True
-        else:
-          return False
+        return create_repo(params.policymgr_mgr_url, json.dumps(params.kms_ranger_plugin_repo), ambari_username_password_for_ranger)
       else:
         return True
     else:
-      Logger.error('Ambari admin user creation failed')
       return False
   else:
-    Logger.error('Ranger service is not reachable host')
+    Logger.error('Ranger service is not reachable')
     return False
 
+@safe_retry(times=5, sleep_time=8, backoff_factor=1.5, err_class=Fail, return_on_fail=False)
 def create_repo(url, data, usernamepassword):
   try:
     base_url = url + '/service/public/v2/api/service'
@@ -493,15 +489,13 @@ def create_repo(url, data, usernamepassword):
       return False
   except urllib2.URLError, e:
     if isinstance(e, urllib2.HTTPError):
-      Logger.error("Error creating service. Http status code - {0}. \n {1}".format(e.code, e.read()))
-      return False
+      raise Fail("Error creating service. Http status code - {0}. \n {1}".format(e.code, e.read()))
     else:
-      Logger.error("Error creating service. Reason - {0}.".format(e.reason))
-      return False
+      raise Fail("Error creating service. Reason - {0}.".format(e.reason))
   except socket.timeout as e:
-    Logger.error("Error creating service. Reason - {0}".format(e))
-    return False
+    raise Fail("Error creating service. Reason - {0}".format(e))
 
+@safe_retry(times=5, sleep_time=8, backoff_factor=1.5, err_class=Fail, return_on_fail=False)
 def get_repo(url, name, usernamepassword):
   try:
     base_url = url + '/service/public/v2/api/service?serviceName=' + name + '&serviceType=kms&isEnabled=true'
@@ -526,11 +520,29 @@ def get_repo(url, name, usernamepassword):
       return False
   except urllib2.URLError, e:
     if isinstance(e, urllib2.HTTPError):
-      Logger.error("Error getting {0} service. Http status code - {1}. \n {2}".format(name, e.code, e.read()))
-      return False
+      raise Fail("Error getting {0} service. Http status code - {1}. \n {2}".format(name, e.code, e.read()))
     else:
-      Logger.error("Error getting {0} service. Reason - {1}.".format(name, e.reason))
-      return False
+      raise Fail("Error getting {0} service. Reason - {1}.".format(name, e.reason))
   except socket.timeout as e:
-    Logger.error("Error creating service. Reason - {0}".format(e))
+    raise Fail("Error creating service. Reason - {0}".format(e))
+
+def check_ranger_service_support_kerberos():
+  import params
+
+  ranger_adm_obj = RangeradminV2(url=params.policymgr_mgr_url)
+  response_code = ranger_adm_obj.check_ranger_login_curl(params.kms_user, params.rangerkms_keytab, params.rangerkms_principal, params.policymgr_mgr_url, True)
+
+  if response_code is not None and response_code[0] == 200:
+    get_repo_name_response = ranger_adm_obj.get_repository_by_name_curl(params.kms_user, params.rangerkms_keytab, params.rangerkms_principal, params.repo_name, 'kms', 'true')
+    if get_repo_name_response is not None:
+      Logger.info('KMS repository {0} exist'.format(get_repo_name_response['name']))
+      return True
+    else:
+      create_repo_response = ranger_adm_obj.create_repository_curl(params.kms_user, params.rangerkms_keytab, params.rangerkms_principal, params.repo_name, json.dumps(params.kms_ranger_plugin_repo), None)
+      if create_repo_response is not None and len(create_repo_response) > 0:
+        return True
+      else:
+        return False
+  else:
+    Logger.error('Ranger service is not reachable')
     return False

ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py

@@ -38,6 +38,7 @@ stack_version_formatted = format_stack_version(stack_version_unformatted)
 
 stack_supports_config_versioning =  stack_version_formatted and check_stack_feature(StackFeature.CONFIG_VERSIONING, stack_version_formatted)
 stack_support_kms_hsm = stack_version_formatted and check_stack_feature(StackFeature.RANGER_KMS_HSM_SUPPORT, stack_version_formatted)
+stack_supports_ranger_kerberos = stack_version_formatted and check_stack_feature(StackFeature.RANGER_KERBEROS_SUPPORT, stack_version_formatted)
 hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
 security_enabled = config['configurations']['cluster-env']['security_enabled']
 
@@ -177,6 +178,9 @@ kms_plugin_config = {
   'provider' : format('kms://http@{kms_host}:{kms_port}/kms') 
 }
 
+if stack_supports_ranger_kerberos:
+  kms_plugin_config['policy.download.auth.users'] = 'keyadmin'
+
 kms_ranger_plugin_repo = {
   'isEnabled' : 'true',
   'configs' : kms_plugin_config,
@@ -212,3 +216,10 @@ jce_source_dir = format('{tmp_dir}/jce_dir')
 enable_kms_hsm = default("/configurations/dbks-site/ranger.ks.hsm.enabled", False)
 hms_partition_alias = default("/configurations/dbks-site/ranger.ks.hsm.partition.password.alias", "ranger.kms.hsm.partition.password")
 hms_partition_passwd = default("/configurations/kms-env/hsm_partition_password", None)
+
+# kms kerberos from stack 2.5 onward
+rangerkms_keytab = config['configurations']['dbks-site']['ranger.ks.kerberos.keytab']
+if stack_supports_ranger_kerberos and security_enabled:
+  rangerkms_principal = default("/configurations/dbks-site/ranger.ks.kerberos.principal", None)
+  if rangerkms_principal is not None:
+    rangerkms_principal = rangerkms_principal.replace('_HOST', kms_host.lower())

ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/configuration/ranger-admin-site.xml

@@ -49,7 +49,7 @@
   </property>
 
   <property>
-    <name>ranger.admin.kerberos.token.valid</name>
+    <name>ranger.admin.kerberos.token.valid.seconds</name>
     <value>30</value>
     <description></description>
   </property>

ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER_KMS/configuration/dbks-site.xml

@@ -75,4 +75,22 @@
     <description>HSM partition password alias</description>
   </property>
 
+  <property>
+    <name>ranger.ks.kerberos.principal</name>
+    <value></value>
+    <description></description>
+    <value-attributes>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+  </property>
+
+  <property>
+    <name>ranger.ks.kerberos.keytab</name>
+    <value></value>
+    <description></description>
+    <value-attributes>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+  </property>
+
 </configuration>

ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER_KMS/kerberos.json

@@ -0,0 +1,65 @@
+{
+  "services": [
+    {
+      "name": "RANGER_KMS",
+      "identities": [
+        {
+          "name": "/spnego",
+          "keytab": {
+            "configuration": "kms-site/hadoop.kms.authentication.kerberos.keytab"
+          }
+        },
+        {
+          "name": "/smokeuser"
+        }
+      ],
+      "auth_to_local_properties" : [
+        "kms-site/hadoop.kms.authentication.kerberos.name.rules"
+      ],
+      "configurations": [
+        {
+          "kms-site": {
+            "hadoop.kms.authentication.type": "kerberos",
+            "hadoop.kms.authentication.kerberos.principal": "*"
+          }
+        }
+      ],
+      "components": [
+        {
+          "name": "RANGER_KMS_SERVER",
+          "identities": [
+            {
+              "name": "/spnego",
+              "principal": {
+                "configuration": "kms-site/hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.principal"
+              },
+              "keytab": {
+                "configuration": "kms-site/hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.keytab"
+              }
+            },
+            {
+              "name": "/smokeuser"
+            },
+            {
+              "name": "rangerkms",
+              "principal": {
+                "value": "rangerkms/_HOST@${realm}",
+                "type" : "service",
+                "configuration": "dbks-site/ranger.ks.kerberos.principal",
+                "local_username" : "keyadmin"
+              },
+              "keytab": {
+                "file": "${keytab_dir}/rangerkms.service.keytab",
+                "owner": {
+                  "name": "${kms-env/kms_user}",
+                  "access": "r"
+                },
+                "configuration": "dbks-site/ranger.ks.kerberos.keytab"
+              }
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}