AMBARI-19915 Add Ranger KMS SSL properties in ambari stack (mugdha)

ambari-common/src/main/python/resource_management/libraries/functions/constants.py

@@ -117,3 +117,4 @@ class StackFeature:
   ATLAS_HDFS_SITE_ON_NAMENODE_HA='atlas_hdfs_site_on_namenode_ha'
   HIVE_INTERACTIVE_GA_SUPPORT='hive_interactive_ga'
   SECURE_RANGER_SSL_PASSWORD = "secure_ranger_ssl_password"
+  RANGER_KMS_SSL = "ranger_kms_ssl"

ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/kms-env.xml

@@ -58,6 +58,16 @@
     <value>9292</value>
     <description/>
     <on-ambari-upgrade add="true"/>
+    <depends-on>
+      <property>
+        <type>ranger-kms-site</type>
+        <name>ranger.service.https.port</name>
+      </property>
+      <property>
+        <type>ranger-kms-site</type>
+        <name>ranger.service.https.attrib.ssl.enabled</name>
+      </property>
+    </depends-on>
   </property>
   <property>
     <name>create_db_user</name>

ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py

@@ -140,6 +140,16 @@ def kms(upgrade_type=None):
       create_parents = True
     )
 
+    Directory("/etc/security/serverKeys",
+      create_parents = True,
+      cd_access = "a"
+    )
+
+    Directory("/etc/ranger/kms",
+      create_parents = True,
+      cd_access = "a"
+    )
+
     copy_jdbc_connector()
 
     File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
@@ -270,6 +280,8 @@ def kms(upgrade_type=None):
     do_keystore_setup(params.credential_provider_path, params.masterkey_alias, params.kms_master_key_password)
     if params.stack_support_kms_hsm and params.enable_kms_hsm:
       do_keystore_setup(params.credential_provider_path, params.hms_partition_alias, unicode(params.hms_partition_passwd))
+    if params.stack_supports_ranger_kms_ssl and params.ranger_kms_ssl_enabled:
+      do_keystore_setup(params.ranger_kms_cred_ssl_path, params.ranger_kms_ssl_keystore_alias, params.ranger_kms_ssl_passwd)
 
     # remove plain-text password from xml configs
     dbks_site_copy = {}
@@ -288,9 +300,17 @@ def kms(upgrade_type=None):
       mode=0644
     )
 
+    ranger_kms_site_copy = {}
+    ranger_kms_site_copy.update(params.config['configurations']['ranger-kms-site'])
+    if params.stack_supports_ranger_kms_ssl:
+      # remove plain-text password from xml configs
+      for prop in params.ranger_kms_site_password_properties:
+        if prop in ranger_kms_site_copy:
+          ranger_kms_site_copy[prop] = "_"
+
     XmlConfig("ranger-kms-site.xml",
       conf_dir=params.kms_conf_dir,
-      configurations=params.config['configurations']['ranger-kms-site'],
+      configurations=ranger_kms_site_copy,
       configuration_attributes=params.config['configuration_attributes']['ranger-kms-site'],
       owner=params.kms_user,
       group=params.kms_group,

ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/params.py

@@ -48,6 +48,7 @@ stack_support_kms_hsm = check_stack_feature(StackFeature.RANGER_KMS_HSM_SUPPORT,
 stack_supports_ranger_kerberos = check_stack_feature(StackFeature.RANGER_KERBEROS_SUPPORT, version_for_stack_feature_checks)
 stack_supports_pid = check_stack_feature(StackFeature.RANGER_KMS_PID_SUPPORT, version_for_stack_feature_checks)
 stack_supports_ranger_audit_db = check_stack_feature(StackFeature.RANGER_AUDIT_DB_SUPPORT, version_for_stack_feature_checks)
+stack_supports_ranger_kms_ssl = check_stack_feature(StackFeature.RANGER_KMS_SSL, version_for_stack_feature_checks)
 
 hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
 security_enabled = config['configurations']['cluster-env']['security_enabled']
@@ -279,4 +280,9 @@ if security_enabled:
 
 plugin_audit_password_property = 'xasecure.audit.destination.db.password'
 kms_plugin_password_properties = ['xasecure.policymgr.clientssl.keystore.password', 'xasecure.policymgr.clientssl.truststore.password']
-dbks_site_password_properties = ['ranger.db.encrypt.key.password', 'ranger.ks.jpa.jdbc.password', 'ranger.ks.hsm.partition.password']
+dbks_site_password_properties = ['ranger.db.encrypt.key.password', 'ranger.ks.jpa.jdbc.password', 'ranger.ks.hsm.partition.password']
+ranger_kms_site_password_properties = ['ranger.service.https.attrib.keystore.pass']
+ranger_kms_cred_ssl_path = config['configurations']['ranger-kms-site']['ranger.credential.provider.path']
+ranger_kms_ssl_keystore_alias = config['configurations']['ranger-kms-site']['ranger.service.https.attrib.keystore.credential.alias']
+ranger_kms_ssl_passwd = config['configurations']['ranger-kms-site']['ranger.service.https.attrib.keystore.pass']
+ranger_kms_ssl_enabled = config['configurations']['ranger-kms-site']['ranger.service.https.attrib.ssl.enabled']

ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json

@@ -387,6 +387,11 @@
       "name": "secure_ranger_ssl_password",
       "description": "Securing Ranger Admin and Usersync SSL and Trustore related passwords in jceks",
       "min_version": "2.6.0.0"
+    },
+    {
+      "name": "ranger_kms_ssl",
+      "description": "Ranger KMS SSL properties in ambari stack",
+      "min_version": "2.6.0.0"
     }
   ]
 }

ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml

@@ -338,6 +338,13 @@
             <replace key="content" find="log4j.appender.kms-audit=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.kms-audit=org.apache.log4j.DailyRollingFileAppender&#xA;log4j.appender.kms-audit.MaxFileSize = {{ranger_kms_audit_log_maxfilesize}}MB"/>
             <replace key="content" find="log4j.appender.kms-audit=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.kms-audit=org.apache.log4j.DailyRollingFileAppender&#xA;log4j.appender.kms-audit.MaxBackupIndex = {{ranger_kms_audit_log_maxbackupindex}}"/>
           </definition>
+          <definition xsi:type="configure" id="hdp_2_6_0_0_remove_ranger_kms_duplicate_ssl">
+            <type>ranger-kms-site</type>
+            <transfer operation="delete" delete-key="ranger.https.attrib.keystore.file"
+              if-type="ranger-kms-site" if-key="ranger.service.https.attrib.keystore.file" if-key-state="present"/>
+            <transfer operation="delete" delete-key="ranger.service.https.attrib.clientAuth"
+              if-type="ranger-kms-site" if-key="ranger.service.https.attrib.client.auth" if-key-state="present"/>
+          </definition>
         </changes>
       </component>
     </service>

ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml

@@ -556,6 +556,10 @@
         </task>
       </execute-stage>
 
+      <execute-stage service="RANGER_KMS" component="RANGER_KMS_SERVER" title="Apply config changes for Ranger KMS">
+        <task xsi:type="configure" id="hdp_2_6_0_0_remove_ranger_kms_duplicate_ssl"/>
+      </execute-stage>
+
       <!-- KNOX -->
       <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Apply config changes for Knox Gateway">
         <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_knox_audit_db"/>

ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml

@@ -654,6 +654,7 @@
         <pre-upgrade>
           <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_kms_audit_db" />
           <task xsi:type="configure" id="kms_log4j_parameterize" />
+          <task xsi:type="configure" id="hdp_2_6_0_0_remove_ranger_kms_duplicate_ssl" />
           <task xsi:type="execute" hosts="any" sequential="true">
             <summary>Upgrading Ranger KMS database schema</summary>
             <script>scripts/kms_server.py</script>

ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml

@@ -229,6 +229,13 @@
             <replace key="content" find="log4j.appender.kms-audit=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.kms-audit=org.apache.log4j.DailyRollingFileAppender&#xA;log4j.appender.kms-audit.MaxFileSize = {{ranger_kms_audit_log_maxfilesize}}MB"/>
             <replace key="content" find="log4j.appender.kms-audit=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.kms-audit=org.apache.log4j.DailyRollingFileAppender&#xA;log4j.appender.kms-audit.MaxBackupIndex = {{ranger_kms_audit_log_maxbackupindex}}"/>
           </definition>
+          <definition xsi:type="configure" id="hdp_2_6_0_0_remove_ranger_kms_duplicate_ssl">
+            <type>ranger-kms-site</type>
+            <transfer operation="delete" delete-key="ranger.https.attrib.keystore.file"
+              if-type="ranger-kms-site" if-key="ranger.service.https.attrib.keystore.file" if-key-state="present"/>
+            <transfer operation="delete" delete-key="ranger.service.https.attrib.clientAuth"
+              if-type="ranger-kms-site" if-key="ranger.service.https.attrib.client.auth" if-key-state="present"/>
+          </definition>
         </changes>
       </component>
     </service>

ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml

@@ -563,6 +563,10 @@
         </task>
       </execute-stage>
 
+      <execute-stage service="RANGER_KMS" component="RANGER_KMS_SERVER" title="Apply config changes for Ranger KMS">
+        <task xsi:type="configure" id="hdp_2_6_0_0_remove_ranger_kms_duplicate_ssl"/>
+      </execute-stage>
+
       <!--ATLAS-->
       <execute-stage service="ATLAS" component="ATLAS_SERVER" title="Parameterizing Atlas Log4J Properties">
         <task xsi:type="configure" id="atlas_log4j_parameterize">

ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml

@@ -643,6 +643,7 @@
         <pre-upgrade>
           <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_kms_audit_db" />
           <task xsi:type="configure" id="kms_log4j_parameterize" />
+          <task xsi:type="configure" id="hdp_2_6_0_0_remove_ranger_kms_duplicate_ssl" />
           <task xsi:type="execute" hosts="any" sequential="true">
             <summary>Upgrading Ranger KMS database schema</summary>
             <script>scripts/kms_server.py</script>

ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml

@@ -263,6 +263,13 @@
         <replace key="content" find="log4j.appender.kms-audit=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.kms-audit=org.apache.log4j.DailyRollingFileAppender&#xA;log4j.appender.kms-audit.MaxFileSize = {{ranger_kms_audit_log_maxfilesize}}MB"/>
         <replace key="content" find="log4j.appender.kms-audit=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.kms-audit=org.apache.log4j.DailyRollingFileAppender&#xA;log4j.appender.kms-audit.MaxBackupIndex = {{ranger_kms_audit_log_maxbackupindex}}"/>
       </definition>
+      <definition xsi:type="configure" id="hdp_2_6_0_0_remove_ranger_kms_duplicate_ssl">
+        <type>ranger-kms-site</type>
+        <transfer operation="delete" delete-key="ranger.https.attrib.keystore.file"
+          if-type="ranger-kms-site" if-key="ranger.service.https.attrib.keystore.file" if-key-state="present"/>
+        <transfer operation="delete" delete-key="ranger.service.https.attrib.clientAuth"
+          if-type="ranger-kms-site" if-key="ranger.service.https.attrib.client.auth" if-key-state="present"/>
+      </definition>
     </changes>
     </component>
     </service>

ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml

@@ -399,6 +399,10 @@
         </task>
       </execute-stage>
 
+      <execute-stage service="RANGER_KMS" component="RANGER_KMS_SERVER" title="Apply config changes for Ranger KMS">
+        <task xsi:type="configure" id="hdp_2_6_0_0_remove_ranger_kms_duplicate_ssl"/>
+      </execute-stage>
+
       <!--ATLAS-->
       <execute-stage service="ATLAS" component="ATLAS_SERVER" title="Parameterizing Atlas Log4J Properties">
         <task xsi:type="configure" id="atlas_log4j_parameterize">

ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml

@@ -553,6 +553,7 @@
       <component name="RANGER_KMS_SERVER">
         <pre-upgrade>
           <task xsi:type="configure" id="kms_log4j_parameterize" />
+          <task xsi:type="configure" id="hdp_2_6_0_0_remove_ranger_kms_duplicate_ssl" />
           <task xsi:type="execute" hosts="any" sequential="true">
             <summary>Upgrading Ranger KMS database schema</summary>
             <script>scripts/kms_server.py</script>

ambari-server/src/main/resources/stacks/HDP/2.6/services/RANGER_KMS/configuration/ranger-kms-site.xml

@@ -0,0 +1,68 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+  <property>
+    <name>ranger.service.https.attrib.keystore.file</name>
+    <value>/etc/security/serverKeys/ranger-kms-keystore.jks</value>
+    <on-ambari-upgrade add="false"/>
+    <description/>
+  </property>
+  <property>
+    <name>ranger.service.https.attrib.client.auth</name>
+    <value>want</value>
+    <on-ambari-upgrade add="false"/>
+    <description/>
+  </property>
+  <property>
+    <name>ranger.service.https.attrib.keystore.keyalias</name>
+    <value>rangerkms</value>
+    <on-ambari-upgrade add="false"/>
+    <description/>
+  </property>
+  <property>
+    <name>ranger.service.https.attrib.keystore.pass</name>
+    <value>rangerkms</value>
+    <property-type>PASSWORD</property-type>
+    <value-attributes>
+      <type>password</type>
+    </value-attributes>
+    <on-ambari-upgrade add="false"/>
+    <description/>
+  </property>
+  <property>
+    <name>ranger.credential.provider.path</name>
+    <value>/etc/ranger/kms/rangerkms.jceks</value>
+    <on-ambari-upgrade add="false"/>
+    <description/>
+  </property>
+  <property>
+    <name>ranger.service.https.attrib.keystore.credential.alias</name>
+    <value>keyStoreCredentialAlias</value>
+    <on-ambari-upgrade add="false"/>
+    <description/>
+  </property>
+  <property>
+    <name>ajp.enabled</name>
+    <value>false</value>
+    <on-ambari-upgrade add="false"/>
+    <description/>
+  </property>
+</configuration>

ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py

@@ -33,7 +33,8 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
           "DRUID": self.recommendDruidConfigurations,
           "ATLAS": self.recommendAtlasConfigurations,
           "TEZ": self.recommendTezConfigurations,
-          "RANGER": self.recommendRangerConfigurations
+          "RANGER": self.recommendRangerConfigurations,
+          "RANGER_KMS": self.recommendRangerKMSConfigurations
       }
       parentRecommendConfDict.update(childRecommendConfDict)
       return parentRecommendConfDict
@@ -301,3 +302,20 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
                             "Need to set ranger.usersync.group.searchenabled as true, as ranger.usersync.ldap.deltasync is enabled")})
 
     return self.toConfigurationValidationProblems(validationItems, "ranger-ugsync-site")
+
+  def recommendRangerKMSConfigurations(self, configurations, clusterData, services, hosts):
+    super(HDP26StackAdvisor, self).recommendRangerKMSConfigurations(configurations, clusterData, services, hosts)
+    putRangerKmsEnvProperty = self.putProperty(configurations, "kms-env", services)
+
+    ranger_kms_ssl_enabled = False
+    ranger_kms_ssl_port = "9393"
+    if 'ranger-kms-site' in services['configurations'] and 'ranger.service.https.attrib.ssl.enabled' in services['configurations']['ranger-kms-site']['properties']:
+      ranger_kms_ssl_enabled = services['configurations']['ranger-kms-site']['properties']['ranger.service.https.attrib.ssl.enabled'].lower() == "true"
+
+    if 'ranger-kms-site' in services['configurations'] and 'ranger.service.https.port' in services['configurations']['ranger-kms-site']['properties']:
+      ranger_kms_ssl_port = services['configurations']['ranger-kms-site']['properties']['ranger.service.https.port']
+
+    if ranger_kms_ssl_enabled:
+      putRangerKmsEnvProperty("kms_port", ranger_kms_ssl_port)
+    else:
+      putRangerKmsEnvProperty("kms_port", "9292")

ambari-server/src/test/python/stacks/2.5/RANGER_KMS/test_kms_server.py

@@ -208,6 +208,16 @@ class TestRangerKMS(RMFTestCase):
       create_parents = True
     )
 
+    self.assertResourceCalled('Directory', '/etc/security/serverKeys',
+      create_parents = True,
+      cd_access = "a",
+    )
+
+    self.assertResourceCalled('Directory', '/etc/ranger/kms',
+      create_parents = True,
+      cd_access = "a",
+    )
+
     self.assertResourceCalled('File', '/usr/hdp/current/ranger-kms/ews/webapp/lib/mysql-connector-java-old.jar',
         action = ['delete'],
     )
@@ -559,6 +569,16 @@ class TestRangerKMS(RMFTestCase):
       create_parents = True
     )
 
+    self.assertResourceCalled('Directory', '/etc/security/serverKeys',
+      create_parents = True,
+      cd_access = "a",
+    )
+
+    self.assertResourceCalled('Directory', '/etc/ranger/kms',
+      create_parents = True,
+      cd_access = "a",
+    )
+
     self.assertResourceCalled('File', '/usr/hdp/current/ranger-kms/ews/webapp/lib/mysql-connector-java-old.jar',
         action = ['delete'],
     )

ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py

@@ -780,6 +780,75 @@ class TestHDP26StackAdvisor(TestCase):
     self.stackAdvisor.recommendRangerConfigurations(recommendedConfigurations, clusterData, services, None)
     self.assertEquals(recommendedConfigurations, expected)
 
+  def test_recommendRangerKMSConfigurations(self):
+    clusterData = {}
+    services = {
+      "ambari-server-properties": {
+        "ambari-server.user": "root"
+        },
+      "Versions": {
+        "stack_version" : "2.6",
+        },
+      "services": [
+        {
+          "StackServices": {
+            "service_name": "RANGER_KMS",
+            "service_version": "0.7.0.2.6"
+          },
+          "components": [
+            {
+              "StackServiceComponents": {
+                "component_name": "RANGER_KMS_SERVER",
+                "hostnames": ["host1"]
+              }
+            }
+          ]
+        }
+      ],
+      "configurations": {
+        'ranger-kms-site': {
+          'properties': {
+            "ranger.service.https.attrib.ssl.enabled": "true",
+            "ranger.service.https.port": "9393"
+          }
+        }
+      }
+    }
+
+    expected = {
+      'kms-site': {
+        'properties': {},
+        'property_attributes': {
+          'hadoop.kms.proxyuser.HTTP.users': {'delete': 'true'},
+          'hadoop.kms.proxyuser.root.hosts': {'delete': 'true'},
+          'hadoop.kms.proxyuser.root.users': {'delete': 'true'},
+          'hadoop.kms.proxyuser.HTTP.hosts': {'delete': 'true'}
+        }
+      },
+      'core-site': {
+        'properties': {}
+      },
+      'kms-properties': {
+        'properties': {}
+      },
+      'ranger-kms-audit': {
+        'properties': {}
+      },
+      'kms-env': {
+        'properties': {
+          'kms_port': '9393'
+        }
+      },
+      'dbks-site': {
+        'properties': {}
+      }
+    }
+
+    recommendedConfigurations = {}
+
+    self.stackAdvisor.recommendRangerKMSConfigurations(recommendedConfigurations, clusterData, services, None)
+    self.assertEquals(recommendedConfigurations, expected)
+
 def load_json(self, filename):
   file = os.path.join(self.testDirectory, filename)
   with open(file, 'rb') as f: