AMBARI-15587. improve ranger kms install integeration (alexantonenko)

ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py

@@ -256,6 +256,16 @@ def kms(upgrade_type=None):
       content=params.kms_log4j,
       mode=0644
     )
+    if params.stack_is_hdp23_or_further and params.security_enabled:
+      # core-site.xml linking required by setup for HDFS encryption
+      XmlConfig("core-site.xml",
+        conf_dir=params.kms_conf_dir,
+        configurations=params.config['configurations']['core-site'],
+        configuration_attributes=params.config['configuration_attributes']['core-site'],
+        owner=params.kms_user,
+        group=params.kms_group,
+        mode=0644
+      )
 
 def copy_jdbc_connector(stack_version=None):
   import params

ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py

@@ -18,6 +18,7 @@ limitations under the License.
 
 """
 import os
+from resource_management.libraries.functions import conf_select
 from resource_management.libraries.script import Script
 from resource_management.libraries.functions.version import format_stack_version, compare_versions
 from resource_management.libraries.functions.format import format
@@ -33,11 +34,13 @@ stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
 stack_version_formatted = format_stack_version(stack_version_unformatted)
 
 stack_is_hdp23_or_further = Script.is_stack_greater_or_equal("2.3")
+hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
+security_enabled = config['configurations']['cluster-env']['security_enabled']
 
 if stack_is_hdp23_or_further:
   kms_home = '/usr/hdp/current/ranger-kms'
   kms_conf_dir = '/usr/hdp/current/ranger-kms/conf'
-  
+
 kms_log_dir = default("/configurations/kms-env/kms_log_dir", "/var/log/ranger/kms")
 java_home = config['hostLevelParams']['java_home']
 kms_user  = default("/configurations/kms-env/kms_user", "kms")
@@ -203,4 +206,4 @@ if current_host in ranger_kms_hosts:
 check_db_connection_jar_name = "DBConnectionVerification.jar"
 check_db_connection_jar = format("/usr/lib/ambari-agent/{check_db_connection_jar_name}")
 ranger_kms_jdbc_connection_url = config['configurations']['dbks-site']['ranger.ks.jpa.jdbc.url']
-ranger_kms_jdbc_driver = config['configurations']['dbks-site']['ranger.ks.jpa.jdbc.driver']
+ranger_kms_jdbc_driver = config['configurations']['dbks-site']['ranger.ks.jpa.jdbc.driver']

ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py

@@ -460,6 +460,9 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
     servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
     putRangerKmsDbksProperty = self.putProperty(configurations, "dbks-site", services)
     putRangerKmsProperty = self.putProperty(configurations, "kms-properties", services)
+    kmsEnvProperties = getSiteProperties(services['configurations'], 'kms-env')
+    putCoreSiteProperty = self.putProperty(configurations, "core-site", services)
+    putCoreSitePropertyAttribute = self.putPropertyAttribute(configurations, "core-site")
 
     if 'kms-properties' in services['configurations'] and ('DB_FLAVOR' in services['configurations']['kms-properties']['properties']):
 
@@ -492,6 +495,15 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
         for key in rangerKmsDbProperties:
           putRangerKmsDbksProperty(key, rangerKmsDbProperties.get(key))
 
+    if kmsEnvProperties and self.checkSiteProperties(kmsEnvProperties, 'kms_user') and 'KERBEROS' in servicesList:
+      kmsUser = kmsEnvProperties['kms_user']
+      kmsUserOld = getOldValue(self, services, 'kms-env', 'kms_user')
+      putCoreSiteProperty('hadoop.proxyuser.{0}.groups'.format(kmsUser), '*')
+      if kmsUserOld is not None and kmsUser != kmsUserOld:
+        putCoreSitePropertyAttribute("hadoop.proxyuser.{0}.groups".format(kmsUserOld), 'delete', 'true')
+        services["forced-configurations"].append({"type" : "core-site", "name" : "hadoop.proxyuser.{0}.groups".format(kmsUserOld)})
+        services["forced-configurations"].append({"type" : "core-site", "name" : "hadoop.proxyuser.{0}.groups".format(kmsUser)})
+
   def recommendRangerConfigurations(self, configurations, clusterData, services, hosts):
     super(HDP23StackAdvisor, self).recommendRangerConfigurations(configurations, clusterData, services, hosts)
     servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
@@ -1025,7 +1037,7 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
         validationItems.append({"config-name": PROP_NAME, "item": self.getWarnItem(message.format(PROP_NAME, str(limit)))})
 
     return self.toConfigurationValidationProblems(validationItems, "hdfs-client")
-  
+
   def isComponentUsingCardinalityForLayout(self, componentName):
     return componentName in ['NFS_GATEWAY', 'PHOENIX_QUERY_SERVER', 'SPARK_THRIFTSERVER']
 

ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py

@@ -2215,3 +2215,107 @@ class TestHDP23StackAdvisor(TestCase):
     problems = self.stackAdvisor.validateHAWQHdfsClientConfigurations(properties, defaults, configurations, services, hosts)
     self.assertEqual(len(problems), 1)
     self.assertEqual(problems[0], expected)
+
+  def test_recommendRangerKMSConfigurations(self):
+    clusterData = {}
+    services = {
+      "Versions": {
+        "stack_version" : "2.3",
+        },
+      "services": [
+        {
+          "StackServices": {
+            "service_name": "RANGER",
+            "service_version": "0.5.0.2.3"
+          },
+          "components": [
+            {
+              "StackServiceComponents": {
+                "component_name": "RANGER_ADMIN",
+                "hostnames": ["host1"]
+              }
+            }
+          ]
+        }
+        ],
+      "configurations": {
+        "kms-env": {
+          "properties": {
+            "kms_user": "kmsname"
+            }
+        },
+        "core-site": {
+          "properties": {
+          }
+        }
+      },
+      "forced-configurations": []
+    }
+    expected = {
+      'kms-properties': {
+        'properties': {}
+      },
+      'dbks-site': {
+        'properties': {}
+      },
+      'core-site': {
+        'properties': {
+        }
+      }
+    }
+
+    # non kerberized cluster. There should be no proxyuser configs
+    recommendedConfigurations = {}
+    self.stackAdvisor.recommendRangerKMSConfigurations(recommendedConfigurations, clusterData, services, None)
+    self.assertEquals(recommendedConfigurations, expected)
+
+    # kerberized cluster
+    services['services'].append({
+      "StackServices": {
+        "service_name": "KERBEROS"
+      }
+    })
+
+    expected = {
+      'kms-properties': {
+        'properties': {}
+      },
+      'dbks-site': {
+        'properties': {}
+      },
+      'core-site': {
+        'properties': {
+          'hadoop.proxyuser.kmsname.groups': '*'
+        }
+      }
+    }
+
+    # on kerberized cluster property should be recommended
+    recommendedConfigurations = {}
+    self.stackAdvisor.recommendRangerKMSConfigurations(recommendedConfigurations, clusterData, services, None)
+    self.assertEquals(recommendedConfigurations, expected)
+
+    recommendedConfigurations = {}
+    services['changed-configurations'] = [
+      {
+        'type': 'kms-env',
+        'name': 'kms_user',
+        'old_value': 'kmsname'
+      }
+    ]
+    services['configurations']['kms-env']['properties']['kms_user'] = 'kmsnew'
+
+    expected['core-site'] = {
+      'properties': {
+        'hadoop.proxyuser.kmsnew.groups': '*'
+      },
+      'property_attributes': {
+        'hadoop.proxyuser.kmsname.groups': {
+          'delete': 'true'
+        }
+      }
+    }
+
+    # kms_user was changed, old property should be removed
+    self.stackAdvisor.recommendRangerKMSConfigurations(recommendedConfigurations, clusterData, services, None)
+    self.assertEquals(recommendedConfigurations, expected)