AMBARI-16290: Handle repository creation for Hive in Ranger for kerberised environments(gautam)

ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions_v2.py

@@ -98,7 +98,8 @@ class RangeradminV2:
 
   def create_ranger_repository(self, component, repo_name, repo_properties,
                                ambari_ranger_admin, ambari_ranger_password,
-                               admin_uname, admin_password, policy_user, is_security_enabled, component_user, component_user_principal, component_user_keytab):
+                               admin_uname, admin_password, policy_user, is_security_enabled = False, component_user = None,
+                               component_user_principal = None, component_user_keytab = None):
     if not is_security_enabled :
       response_code = self.check_ranger_login_urllib2(self.base_url)
       repo_data = json.dumps(repo_properties)
@@ -345,10 +346,12 @@ class RangeradminV2:
       response_stripped = response[1:len(response) - 1]
       if response_stripped and len(response_stripped) > 0:
         response_json = json.loads(response_stripped)
-        if response_json['name'].lower() == name.lower():
+        if 'name' in response_json and response_json['name'].lower() == name.lower():
           return response_json
         else:
           return None
+      else:
+        return None
     except Fail, fail:
       raise Fail(str(fail))
 
@@ -364,120 +367,24 @@ class RangeradminV2:
     :param data: service definition of the repository
     :return:
     """
-    search_repo_url = self.url_repos_pub
-    header = 'Content-Type: application/json'
-    method = 'POST'
-
-    response,error_message,time_in_millis = self.call_curl_request(component_user,component_user_keytab,component_user_principal,search_repo_url,False,method,data,header)
-    if response and len(response) > 0:
-      response_json = json.loads(response)
-      if 'name' in response_json and response_json['name'].lower() == name.lower():
-        Logger.info('Repository created Successfully')
-        service_name = response_json['name']
-        service_type = response_json['type']
-        if service_type in ['hdfs','hive','hbase','knox','storm']:
-          policy_list = self.get_policy_by_repo_name(component_user,component_user_keytab,component_user_principal,service_name,service_type,'true')
-          if policy_list is not None and len(policy_list) > 0:
-            policy_update_count = 0
-            for policy in policy_list:
-              updated_policy_object = self.get_policy_params(service_type,policy,policy_user=policy_user)
-              response,error_message,time_in_millis = self.update_ranger_policy(component_user,component_user_keytab,component_user_principal,updated_policy_object['id'],json.dumps(updated_policy_object))
-              if response and len(response) > 0:
-                policy_update_count += 1
-              else:
-                Logger.info("Policy updated failed")
-            if len(policy_list) == policy_update_count:
-              Logger.info("Ranger Repository created successfully and policies updated successfully providing ambari-qa user all permissions")
-              return response_json
-        else:
-          return response_json
-      else:
-        Logger.info('Repository creation failed')
-        return None
-    else:
-      Logger.info('Repository creation failed')
-      return None
-
-
-
-  @safe_retry(times=5, sleep_time=8, backoff_factor=1.5, err_class=Fail, return_on_fail=None)
-  def get_policy_by_repo_name(self, component_user,component_user_keytab,component_user_principal,name, component, status):
-    """
-    :param name: repository name
-    :param component: component name for which policy needs to be searched
-    :param status: true or false
-    :param usernamepassword: user credentials using which policy needs to be searched
-    :return Returns successful response else None
-    """
     try:
-      # time.sleep(5)
-      search_policy_url = self.url_policies_get+ '?serviceType=' + component + '&isEnabled=' + status
-
-      search_policy_url = search_policy_url.format(servicename=name)
-      method = 'GET'
-      response,error_message,time_in_millis = self.call_curl_request(component_user,component_user_keytab,component_user_principal,search_policy_url,False,request_method=method)
-      if response and len(response) > 0:
-        response = json.loads(response)
-        return response
-      else:
-        return None
-    except Fail, fail:
-      raise Fail(str(fail))
-
-  @safe_retry(times=5, sleep_time=8, backoff_factor=1.5, err_class=Fail, return_on_fail=None)
-  def update_ranger_policy(self,component_user,component_user_keytab,component_user_principal, policyId, data):
-    """
-    :param policyId: policy id which needs to be updated
-    :param data: policy data that needs to be updated
-    :param usernamepassword: user credentials using which policy needs to be updated
-    :return Returns successful response and response code else None
-    """
-    try:
-      update_url = self.url_policies + '/' + str(policyId)
+      search_repo_url = self.url_repos_pub
       header = 'Content-Type: application/json'
-      method = 'PUT'
+      method = 'POST'
 
-      response,error_message,time_in_millis = self.call_curl_request(component_user,component_user_keytab,component_user_principal,update_url,False,method,data,header=header)
+      response,error_message,time_in_millis = self.call_curl_request(component_user,component_user_keytab,component_user_principal,search_repo_url,False,method,data,header)
       if response and len(response) > 0:
-        Logger.info('Policy updated Successfully')
-        
         response_json = json.loads(response)
-        return response_json,error_message,time_in_millis
+        if 'name' in response_json and response_json['name'].lower() == name.lower():
+          Logger.info('Repository created Successfully')
+          return response_json
+        elif 'exists'.lower() in response_json.lower():
+          Logger.info('Repository {name} already exists'.format(name=name))
+        else:
+          Logger.info('Repository creation failed')
+          return None
       else:
-        Logger.error('Update Policy failed')
-        return None, None,None
+        Logger.info('Repository creation failed')
+        return None
     except Fail, fail:
       raise Fail(str(fail))
-
-  def get_policy_params(self, typeOfPolicy, policyObj, policy_user):
-    """
-    :param typeOfPolicy: component name for which policy has to be get
-    :param policyObj: policy dict
-    :param policy_user: policy user that needs to be updated
-    :returns Returns updated policy dict
-    """
-    typeOfPolicy = typeOfPolicy.lower()
-    policy_record = ''
-    if typeOfPolicy == "hdfs":
-      policy_record  = {'users': [policy_user], 'accesses': [{'isAllowed': True,'type': 'read' }, {'isAllowed': True,'type': 'write' },{'isAllowed': True,'type': 'execute' }],'delegateAdmin': True}
-    elif typeOfPolicy == "hive":
-      policy_record = {'users': [policy_user],
-                                   'accesses': [{'isAllowed': True,'type': 'select' }, {'isAllowed': True,'type': 'update' }, {'isAllowed': True,'type': 'create' },
-                                                {'isAllowed': True,'type': 'drop' }, {'isAllowed': True,'type': 'alter' }, {'isAllowed': True,'type': 'index' },
-                                                {'isAllowed': True,'type': 'lock' }, {'isAllowed': True,'type': 'all' }],'delegateAdmin':True }
-    elif typeOfPolicy == "hbase":
-      policy_record = {'users': [policy_user], 'accesses': [{'isAllowed': True,'type': 'read' }, {'isAllowed': True,'type': 'write' },
-                                                             {'isAllowed': True,'type': 'create' }],'delegateAdmin':True }
-    elif typeOfPolicy == "knox":
-      policy_record = {'users': [policy_user], 'accesses': [{'isAllowed': True,'type': 'allow' }],'delegateAdmin':True }
-    elif typeOfPolicy == "storm":
-      policy_record = {'users': [policy_user],
-                                   'accesses': [{'isAllowed': True,'type': 'submitTopology' }, {'isAllowed': True,'type': 'fileUpload' },{'isAllowed': True,'type': 'getNimbusConf' },
-                                                {'isAllowed': True,'type': 'getClusterInfo' },{'isAllowed': True,'type': 'fileDownload' } , {'isAllowed': True,'type': 'killTopology' },
-                                                {'isAllowed': True,'type': 'rebalance' }, {'isAllowed': True,'type': 'activate' }, {'isAllowed': True,'type': 'deactivate' },
-                                                {'isAllowed': True,'type': 'getTopologyConf' }, {'isAllowed': True,'type': 'getTopology' }, {'isAllowed': True,'type': 'getUserTopology' },
-                                                {'isAllowed': True,'type': 'getTopologyInfo' }, {'isAllowed': True,'type': 'uploadNewCredential' }],'delegateAdmin':True}
-
-    if policy_record != '':
-      policyObj['policyItems'].append(policy_record)
-    return policyObj

ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py

@@ -70,19 +70,12 @@ def setup_ranger_plugin(component_select_name, service_name,
 
   if plugin_enabled:
     if api_version is not None and api_version == 'v2':
-
       ranger_adm_obj = RangeradminV2(url=policymgr_mgr_url, skip_if_rangeradmin_down=skip_if_rangeradmin_down)
-      if is_security_enabled and is_stack_supports_ranger_kerberos:
-        ranger_adm_obj.create_ranger_repository(service_name, repo_name, plugin_repo_dict,
+      ranger_adm_obj.create_ranger_repository(service_name, repo_name, plugin_repo_dict,
                                               ranger_env_properties['ranger_admin_username'], ranger_env_properties['ranger_admin_password'],
                                               ranger_env_properties['admin_username'], ranger_env_properties['admin_password'],
                                               policy_user,is_security_enabled,component_user,component_user_principal,component_user_keytab)
 
-      else:
-        ranger_adm_obj.create_ranger_repository(service_name, repo_name, plugin_repo_dict,
-                                                ranger_env_properties['ranger_admin_username'], ranger_env_properties['ranger_admin_password'],
-                                                ranger_env_properties['admin_username'], ranger_env_properties['admin_password'],
-                                                policy_user)
     else:
       ranger_adm_obj = Rangeradmin(url=policymgr_mgr_url, skip_if_rangeradmin_down=skip_if_rangeradmin_down)
       ranger_adm_obj.create_ranger_repository(service_name, repo_name, plugin_repo_dict,

ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py

@@ -342,6 +342,8 @@ if has_ranger_admin:
     hbase_ranger_plugin_config['tag.download.auth.users'] = hbase_user
     hbase_ranger_plugin_config['policy.grant.revoke.auth.users'] = hbase_user
 
+  if stack_supports_ranger_kerberos:
+    hbase_ranger_plugin_config['ambari.service.check.user'] = policy_user
 
     hbase_ranger_plugin_repo = {
       'isEnabled': 'true',

ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/setup_ranger_hbase.py

@@ -64,7 +64,7 @@ def setup_ranger_hbase(upgrade_type=None):
 
     if params.xml_configurations_supported:
       api_version=None
-      if params.stack_supports_ranger_kerberos and params.security_enabled:
+      if params.stack_supports_ranger_kerberos:
         api_version='v2'
       from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
       setup_ranger_plugin('hbase-client', 'hbase', params.downloaded_custom_connector, params.driver_curl_source,

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py

@@ -489,6 +489,9 @@ if has_ranger_admin:
     hdfs_ranger_plugin_config['policy.download.auth.users'] = hdfs_user
     hdfs_ranger_plugin_config['tag.download.auth.users'] = hdfs_user
 
+  if stack_supports_ranger_kerberos:
+    hdfs_ranger_plugin_config['ambari.service.check.user'] = policy_user
+
     hdfs_ranger_plugin_repo = {
       'isEnabled': 'true',
       'configs': hdfs_ranger_plugin_config,

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py

@@ -46,7 +46,7 @@ def setup_ranger_hdfs(upgrade_type=None):
     if params.xml_configurations_supported:
         from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
         api_version=None
-        if params.stack_supports_ranger_kerberos and params.security_enabled:
+        if params.stack_supports_ranger_kerberos:
           api_version='v2'
         setup_ranger_plugin('hadoop-client', 'hdfs',
                              params.downloaded_custom_connector, params.driver_curl_source,

ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py

@@ -662,6 +662,9 @@ if has_ranger_admin:
     hive_ranger_plugin_config['tag.download.auth.users'] = hive_user
     hive_ranger_plugin_config['policy.grant.revoke.auth.users'] = hive_user
 
+  if stack_supports_ranger_kerberos:
+    hive_ranger_plugin_config['ambari.service.check.user'] = policy_user
+
     hive_ranger_plugin_repo = {
       'isEnabled': 'true',
       'configs': hive_ranger_plugin_config,

ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py

@@ -55,7 +55,7 @@ def setup_ranger_hive(upgrade_type = None):
 
     if params.xml_configurations_supported:
       api_version=None
-      if params.stack_supports_ranger_kerberos and params.security_enabled:
+      if params.stack_supports_ranger_kerberos:
         api_version='v2'
       from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
       setup_ranger_plugin('hive-server2', 'hive',
@@ -75,7 +75,7 @@ def setup_ranger_hive(upgrade_type = None):
                           stack_version_override = stack_version, skip_if_rangeradmin_down= not params.retryAble, api_version=api_version,
                           is_security_enabled = params.security_enabled,
                           is_stack_supports_ranger_kerberos = params.stack_supports_ranger_kerberos,
-                          component_user_principal=params.hive_server_principal if params.security_enabled else None,
+                          component_user_principal=params.hive_principal if params.security_enabled else None,
                           component_user_keytab=params.hive_server2_keytab if params.security_enabled else None)
     else:
       from resource_management.libraries.functions.setup_ranger_plugin import setup_ranger_plugin

ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py

@@ -214,7 +214,7 @@ if has_ranger_admin and is_supported_kafka_ranger:
   if stack_supports_ranger_kerberos and security_enabled:
     ranger_plugin_config['policy.download.auth.users'] = kafka_user
     ranger_plugin_config['tag.download.auth.users'] = kafka_user
-
+    ranger_plugin_config['ambari.service.check.user'] = policy_user
 
   #For curl command in ranger plugin to get db connector
   jdk_location = config['hostLevelParams']['jdk_location']

ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py

@@ -324,6 +324,9 @@ if has_ranger_admin:
     knox_ranger_plugin_config['policy.download.auth.users'] = knox_user
     knox_ranger_plugin_config['tag.download.auth.users'] = knox_user
 
+  if stack_supports_ranger_kerberos:
+    knox_ranger_plugin_config['ambari.service.check.user'] = policy_user
+
     knox_ranger_plugin_repo = {
       'isEnabled': 'true',
       'configs': knox_ranger_plugin_config,

ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/setup_ranger_knox.py

@@ -56,7 +56,7 @@ def setup_ranger_knox(upgrade_type=None):
 
     if params.xml_configurations_supported:
       api_version=None
-      if params.stack_supports_ranger_kerberos and params.security_enabled:
+      if params.stack_supports_ranger_kerberos:
         api_version='v2'
       from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
       setup_ranger_plugin('knox-server', 'knox',

ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py

@@ -291,6 +291,7 @@ if has_ranger_admin:
   if stack_supports_ranger_kerberos and security_enabled:
     storm_ranger_plugin_config['policy.download.auth.users'] = storm_user
     storm_ranger_plugin_config['tag.download.auth.users'] = storm_user
+    storm_ranger_plugin_config['ambari.service.check.user'] = policy_user
 
     storm_ranger_plugin_repo = {
       'isEnabled': 'true',

ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/setup_ranger_storm.py

@@ -57,8 +57,7 @@ def setup_ranger_storm(upgrade_type=None):
 
     if params.xml_configurations_supported:
       api_version=None
-      if params.stack_supports_ranger_kerberos and params.security_enabled:
-        Logger.info('setting stack_version as v2')
+      if params.stack_supports_ranger_kerberos:
         api_version='v2'
       from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
       setup_ranger_plugin('storm-nimbus', 'storm',

ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py

@@ -415,6 +415,9 @@ if has_ranger_admin:
       'assetType': '1'
     }
 
+    if stack_supports_ranger_kerberos:
+      ranger_plugin_config['ambari.service.check.user'] = policy_user
+
     if stack_supports_ranger_kerberos and security_enabled:
       ranger_plugin_config['policy.download.auth.users'] = yarn_user
       ranger_plugin_config['tag.download.auth.users'] = yarn_user