AMBARI-17415: Ambari configuration for ranger-tagsync needs to support property for atlas keystore filename (Mugdha Varadkar via jluniya)

ambari-server/src/main/resources/common-services/RANGER/0.6.0/configuration/ranger-tagsync-site.xml

@@ -88,7 +88,7 @@
   <property>
     <name>ranger.tagsync.source.atlasrest.download.interval.millis</name>
     <display-name>AtlasREST Source: Atlas source download interval</display-name>
-    <value/>
+    <value>60000</value>
     <value-attributes>
       <empty-value-valid>true</empty-value-valid>
     </value-attributes>
@@ -137,6 +137,20 @@
       <empty-value-valid>true</empty-value-valid>
     </value-attributes>
     <on-ambari-upgrade add="true"/>
+    <depends-on>
+      <property>
+        <type>application-properties</type>
+        <name>atlas.server.http.port</name>
+      </property>
+      <property>
+        <type>application-properties</type>
+        <name>atlas.server.https.port</name>
+      </property>
+      <property>
+        <type>application-properties</type>
+        <name>atlas.enableTLS</name>
+      </property>
+    </depends-on>
   </property>
   <property>
     <name>ranger.tagsync.kerberos.principal</name>
@@ -162,4 +176,19 @@
     <description/>
     <on-ambari-upgrade add="true"/>
   </property>
+  <property>
+    <name>ranger.tagsync.source.atlasrest.keystore.filename</name>
+    <value>/etc/ranger/tagsync/conf/atlasuser.jceks</value>
+    <description>Tagsync atlasrest keystore file</description>
+    <value-attributes>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>ranger.tagsync.source.atlasrest.username</name>
+    <value>admin</value>
+    <description/>
+    <on-ambari-upgrade add="true"/>
+  </property>
 </configuration>

ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py

@@ -229,7 +229,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
       keyserverHostsString = services["configurations"]["hadoop-env"]["properties"]["keyserver_host"]
       keyserverPortString = services["configurations"]["hadoop-env"]["properties"]["keyserver_port"]
 
-    # Irrespective of what hadoop-env has, if Ranger-KMS is installed, we use its values. 
+    # Irrespective of what hadoop-env has, if Ranger-KMS is installed, we use its values.
     rangerKMSServerHosts = self.getHostsWithComponent("RANGER_KMS", "RANGER_KMS_SERVER", services, hosts)
     if rangerKMSServerHosts is not None and len(rangerKMSServerHosts) > 0:
       rangerKMSServerHostsArray = []
@@ -1134,7 +1134,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
           validationItems.append({"config-name" : address_property, "item" :
             self.getErrorItem(address_property + " does not contain a valid host:port authority: " + value)})
 
-    #Adding Ranger Plugin logic here 
+    #Adding Ranger Plugin logic here
     ranger_plugin_properties = getSiteProperties(configurations, "ranger-hdfs-plugin-properties")
     ranger_plugin_enabled = ranger_plugin_properties['ranger-hdfs-plugin-enabled'] if ranger_plugin_properties else 'No'
     servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
@@ -1237,8 +1237,8 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
 
   def validateHiveServer2Configurations(self, properties, recommendedDefaults, configurations, services, hosts):
     hive_server2 = properties
-    validationItems = [] 
-    #Adding Ranger Plugin logic here 
+    validationItems = []
+    #Adding Ranger Plugin logic here
     ranger_plugin_properties = getSiteProperties(configurations, "ranger-hive-plugin-properties")
     hive_env_properties = getSiteProperties(configurations, "hive-env")
     ranger_plugin_enabled = 'hive_security_authorization' in hive_env_properties and hive_env_properties['hive_security_authorization'].lower() == 'ranger'
@@ -1376,7 +1376,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
                               "item": self.getWarnItem(
                               "{0} and {1} sum should not exceed {2}".format(prop_name1, prop_name2, props_max_sum))})
 
-    #Adding Ranger Plugin logic here 
+    #Adding Ranger Plugin logic here
     ranger_plugin_properties = getSiteProperties(configurations, "ranger-hbase-plugin-properties")
     ranger_plugin_enabled = ranger_plugin_properties['ranger-hbase-plugin-enabled'] if ranger_plugin_properties else 'No'
     prop_name = 'hbase.security.authorization'
@@ -1430,7 +1430,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
                               "item": self.getWarnItem(
                                 "If bucketcache ioengine is enabled, {0} should be set".format(prop_name3))})
 
-    # Validate hbase.security.authentication. 
+    # Validate hbase.security.authentication.
     # Kerberos works only when security enabled.
     if "hbase.security.authentication" in properties:
       hbase_security_kerberos = properties["hbase.security.authentication"].lower() == "kerberos"
@@ -1505,6 +1505,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
     validationItems = []
     ranger_plugin_properties = getSiteProperties(configurations, "ranger-storm-plugin-properties")
     ranger_plugin_enabled = ranger_plugin_properties['ranger-storm-plugin-enabled'] if ranger_plugin_properties else 'No'
+    servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
     if ranger_plugin_enabled.lower() == 'yes':
       # ranger-hdfs-plugin must be enabled in ranger-env
       ranger_env = getServicesSiteProperties(services, 'ranger-env')
@@ -1513,6 +1514,11 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
         validationItems.append({"config-name": 'ranger-storm-plugin-enabled',
                                 "item": self.getWarnItem(
                                   "ranger-storm-plugin-properties/ranger-storm-plugin-enabled must correspond ranger-env/ranger-storm-plugin-enabled")})
+    if ("RANGER" in servicesList) and (ranger_plugin_enabled.lower() == 'Yes'.lower()) and not 'KERBEROS' in servicesList:
+      validationItems.append({"config-name": "ranger-storm-plugin-enabled",
+                              "item": self.getWarnItem(
+                                "Ranger Storm plugin should not be enabled in non-kerberos environment.")})
+
     return self.toConfigurationValidationProblems(validationItems, "ranger-storm-plugin-properties")
 
   def validateYARNEnvConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
@@ -1546,13 +1552,12 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
     return self.toConfigurationValidationProblems(validationItems, "ranger-yarn-plugin-properties")
 
   def validateRangerConfigurationsEnv(self, properties, recommendedDefaults, configurations, services, hosts):
+    ranger_env_properties = properties
     validationItems = []
-    if "ranger-storm-plugin-enabled" in properties and "ranger-storm-plugin-enabled" in recommendedDefaults and \
-      properties["ranger-storm-plugin-enabled"] != recommendedDefaults["ranger-storm-plugin-enabled"]:
-        validationItems.append({"config-name": "ranger-storm-plugin-enabled",
-                                "item": self.getWarnItem(
-                                  "Ranger Storm plugin should not be enabled in non-kerberos environment.")})
-
+    servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+    if "ranger-storm-plugin-enabled" in ranger_env_properties and ranger_env_properties['ranger-storm-plugin-enabled'].lower() == 'yes' and not 'KERBEROS' in servicesList:
+      validationItems.append({"config-name": "ranger-storm-plugin-enabled",
+                              "item": self.getWarnItem("Ranger Storm plugin should not be enabled in non-kerberos environment.")})
     return self.toConfigurationValidationProblems(validationItems, "ranger-env")
 
   def getMastersWithMultipleInstances(self):
@@ -1573,7 +1578,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
   def getAffectedConfigs(self, services):
     affectedConfigs = super(HDP22StackAdvisor, self).getAffectedConfigs(services)
 
-    # There are configs that are not defined in the stack but added/removed by 
+    # There are configs that are not defined in the stack but added/removed by
     # stack-advisor. Here we add such configs in order to clear the config
     # filtering down in base class
     configsList = [affectedConfig["type"] + "/" + affectedConfig["name"] for affectedConfig in affectedConfigs]

ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py

@@ -724,7 +724,8 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
       "HBASE": {"hbase-site": self.validateHBASEConfigurations},
       "KAKFA": {"kafka-broker": self.validateKAFKAConfigurations},
       "YARN": {"yarn-site": self.validateYARNConfigurations},
-      "RANGER": {"admin-properties": self.validateRangerAdminConfigurations}
+      "RANGER": {"admin-properties": self.validateRangerAdminConfigurations,
+                 "ranger-env": self.validateRangerConfigurationsEnv}
     }
     self.mergeValidators(parentValidators, childValidators)
     return parentValidators
@@ -907,6 +908,11 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
                                 "If Ranger Kafka Plugin is enabled."\
                                 "{0} needs to be set to {1}".format(prop_name,prop_val))})
 
+    if ("RANGER" in servicesList) and (ranger_plugin_enabled.lower() == 'Yes'.lower()) and not 'KERBEROS' in servicesList:
+      validationItems.append({"config-name": "ranger-kafka-plugin-enabled",
+                              "item": self.getWarnItem(
+                                "Ranger Kafka plugin should not be enabled in non-kerberos environment.")})
+
     return self.toConfigurationValidationProblems(validationItems, "kafka-broker")
 
   def validateYARNConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
@@ -917,7 +923,7 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
       yarn_resource_proxy_enabled = yarn_site['yarn.resourcemanager.proxy-user-privileges.enabled']
       if yarn_resource_proxy_enabled.lower() == 'true':
         validationItems.append({"config-name": 'yarn.resourcemanager.proxy-user-privileges.enabled',
-          "item": self.getWarnItem("If Ranger KMS service is installed set yarn.resourcemanager.proxy-user-privileges.enabled "\
+          "item": self.getWarnItem("If Ranger KMS service is installed set yarn.resourcemanager.proxy-user-privileges.enabled " \
           "property value as false under yarn-site"
         )})
 
@@ -937,3 +943,22 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
                                'item':self.getWarnItem('Ranger External URL should not contain trailing slash "/"')})
     return self.toConfigurationValidationProblems(validationItems,'admin-properties')
 
+  def validateRangerConfigurationsEnv(self, properties, recommendedDefaults, configurations, services, hosts):
+    parentValidationProblems = super(HDP23StackAdvisor, self).validateRangerConfigurationsEnv(properties, recommendedDefaults, configurations, services, hosts)
+    ranger_env_properties = properties
+    validationItems = []
+    security_enabled = False
+
+    servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+    if 'KERBEROS' in servicesList:
+      security_enabled = True
+
+    if "ranger-kafka-plugin-enabled" in ranger_env_properties and ranger_env_properties["ranger-kafka-plugin-enabled"].lower() == 'yes' and not security_enabled:
+      validationItems.append({"config-name": "ranger-kafka-plugin-enabled",
+                              "item": self.getWarnItem(
+                                "Ranger Kafka plugin should not be enabled in non-kerberos environment.")})
+
+    validationProblems = self.toConfigurationValidationProblems(validationItems, "ranger-env")
+    validationProblems.extend(parentValidationProblems)
+    return validationProblems
+

ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml

@@ -62,6 +62,13 @@
         </osSpecific>
       </osSpecifics>
 
+      <configuration-dependencies>
+        <config-type>ranger-atlas-audit</config-type>
+        <config-type>ranger-atlas-plugin-properties</config-type>
+        <config-type>ranger-atlas-policymgr-ssl</config-type>
+        <config-type>ranger-atlas-security</config-type>
+      </configuration-dependencies>
+
       <requiredServices>
         <service>KAFKA</service>
       </requiredServices>

ambari-server/src/main/resources/stacks/HDP/2.5/services/RANGER/configuration/ranger-tagsync-site.xml

@@ -34,4 +34,13 @@
     </value-attributes>
     <on-ambari-upgrade add="true"/>
   </property>
+  <property>
+    <name>ranger.tagsync.source.atlasrest.keystore.filename</name>
+    <value>/usr/hdp/current/ranger-tagsync/conf/atlasuser.jceks</value>
+    <description>Tagsync atlasrest keystore file</description>
+    <value-attributes>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
 </configuration>

ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py

@@ -1399,12 +1399,29 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
       has_ranger_tagsync = len(ranger_tagsync_host) > 0
 
     if 'ATLAS' in servicesList and has_ranger_tagsync:
+      atlas_hosts = self.getHostNamesWithComponent("ATLAS", "ATLAS_SERVER", services)
+      atlas_host = 'localhost' if len(atlas_hosts) == 0 else atlas_hosts[0]
+      protocol = 'http'
+      atlas_port = '21000'
+
+      if 'application-properties' in services['configurations'] and 'atlas.enableTLS' in services['configurations']['application-properties']['properties'] \
+        and services['configurations']['application-properties']['properties']['atlas.enableTLS'].lower() == 'true':
+        protocol = 'https'
+        if 'application-properties' in services['configurations'] and 'atlas.server.https.port' in services['configurations']['application-properties']['properties']:
+          atlas_port = services['configurations']['application-properties']['properties']['atlas.server.https.port']
+      else:
+        protocol = 'http'
+        if 'application-properties' in services['configurations'] and 'atlas.server.http.port' in services['configurations']['application-properties']['properties']:
+          atlas_port = services['configurations']['application-properties']['properties']['atlas.server.http.port']
+
+      atlas_rest_endpoint = '{0}://{1}:{2}'.format(protocol, atlas_host, atlas_port)
+
       putTagsyncSiteProperty('ranger.tagsync.source.atlas', 'true')
+      putTagsyncSiteProperty('ranger.tagsync.source.atlasrest.endpoint', atlas_rest_endpoint)
 
     zookeeper_host_port = self.getZKHostPortString(services)
     if zookeeper_host_port and has_ranger_tagsync:
-      zookeeper_host_list = zookeeper_host_port.split(',')
-      putTagsyncAppProperty('atlas.kafka.zookeeper.connect', zookeeper_host_list[0])
+      putTagsyncAppProperty('atlas.kafka.zookeeper.connect', zookeeper_host_port)
 
     if 'KAFKA' in servicesList and has_ranger_tagsync:
       kafka_hosts = self.getHostNamesWithComponent("KAFKA", "KAFKA_BROKER", services)

ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py

@@ -3912,6 +3912,14 @@ class TestHDP22StackAdvisor(TestCase):
       }
     }
     services = {
+      "services":
+        [
+          {
+            "StackServices": {
+              "service_name" : "STORM"
+            }
+          }
+        ],
       "configurations": configurations
     }
     res_expected = []
@@ -3978,10 +3986,26 @@ class TestHDP22StackAdvisor(TestCase):
     recommendedDefaults = {
       "ranger-storm-plugin-enabled": "No",
     }
-    configurations = {}
-    services = {}
+    configurations = {
+      "cluster-env": {
+        "properties": {
+          "security_enabled": "false",
+          }
+      }
+    }
+    services = {
+      "services":
+        [
+          {
+            "StackServices": {
+              "service_name" : "STORM"
+            }
+          }
+        ]
+    }
 
     # Test with ranger plugin enabled, validation fails
     res_expected = [{'config-type': 'ranger-env', 'message': 'Ranger Storm plugin should not be enabled in non-kerberos environment.', 'type': 'configuration', 'config-name': 'ranger-storm-plugin-enabled', 'level': 'WARN'}]
+
     res = self.stackAdvisor.validateRangerConfigurationsEnv(properties, recommendedDefaults, configurations, services, {})
     self.assertEquals(res, res_expected)

ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py

@@ -2110,3 +2110,35 @@ class TestHDP23StackAdvisor(TestCase):
     self.stackAdvisor.getComponentHostNames = return_c6401_hostname
     self.stackAdvisor.recommendLogsearchConfigurations(configurations, clusterData, services, hosts)
     self.assertEquals(configurations, expected)
+
+  def test_validateRangerConfigurationsEnv(self):
+    properties = {
+      "ranger-kafka-plugin-enabled": "Yes",
+      }
+    recommendedDefaults = {
+      "ranger-kafka-plugin-enabled": "No",
+      }
+
+    configurations = {
+      "cluster-env": {
+        "properties": {
+          "security_enabled": "false",
+          }
+      }
+    }
+    services = {
+      "services":
+        [
+          {
+            "StackServices": {
+              "service_name" : "KAFKA"
+            }
+          }
+        ]
+      }
+
+    # Test with ranger plugin enabled, validation fails
+    res_expected = [{'config-type': 'ranger-env', 'message': 'Ranger Kafka plugin should not be enabled in non-kerberos environment.', 'type': 'configuration', 'config-name': 'ranger-kafka-plugin-enabled', 'level': 'WARN'}]
+
+    res = self.stackAdvisor.validateRangerConfigurationsEnv(properties, recommendedDefaults, configurations, services, {})
+    self.assertEquals(res, res_expected)