AMBARI-14911: Populate hawq_site properties when YARN is installed after HAWQ. (adenissov via jaoki)

ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml

@@ -153,6 +153,12 @@
     <description>
       The address of YARN resource manager server.
     </description>
+    <depends-on>
+      <property>
+        <type>yarn-site</type>
+        <name>yarn.resourcemanager.address</name>
+      </property>
+    </depends-on>
   </property>
 
   <property>
@@ -161,6 +167,12 @@
     <description>
       The address of YARN scheduler server.
     </description>
+    <depends-on>
+      <property>
+        <type>yarn-site</type>
+        <name>yarn.resourcemanager.scheduler.address</name>
+      </property>
+    </depends-on>
   </property>
 
   <property>

ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py

@@ -678,19 +678,32 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
 
 
   def recommendHAWQConfigurations(self, configurations, clusterData, services, hosts):
+    if "hawq-site" not in services["configurations"]:
+      return
+    hawq_site = services["configurations"]["hawq-site"]["properties"]
     putHawqSiteProperty = self.putProperty(configurations, "hawq-site", services)
-    if self.isHawqMasterComponentOnAmbariServer(services):
-      if "hawq-site" in services["configurations"] and "hawq_master_address_port" in services["configurations"]["hawq-site"]["properties"]:
-        putHawqSiteProperty('hawq_master_address_port', '')
-    # calculate optimal number of virtual segments
     componentsListList = [service["components"] for service in services["services"]]
     componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist]
+    servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+
+    # remove master port when master is colocated with Ambari server
+    if self.isHawqMasterComponentOnAmbariServer(services) and "hawq_master_address_port" in hawq_site:
+        putHawqSiteProperty('hawq_master_address_port', '')
+
+    # calculate optimal number of virtual segments
     numSegments = len(self.__getHosts(componentsList, "HAWQSEGMENT"))
     # update default if segments are deployed
-    if numSegments and "hawq-site" in services["configurations"] and "default_segment_num" in services["configurations"]["hawq-site"]["properties"]:
+    if numSegments and "default_segment_num" in hawq_site:
       factor = 6 if numSegments < 50 else 4
       putHawqSiteProperty('default_segment_num', numSegments * factor)
-          
+
+    # update YARN RM urls with the values from yarn-site if YARN is installed
+    if "YARN" in servicesList and "yarn-site" in services["configurations"]:
+      yarn_site = services["configurations"]["yarn-site"]["properties"]
+      for hs_prop, ys_prop in self.getHAWQYARNPropertyMapping().items():
+        if hs_prop in hawq_site and ys_prop in yarn_site:
+          putHawqSiteProperty(hs_prop, yarn_site[ys_prop])
+
   def getServiceConfigurationValidators(self):
     parentValidators = super(HDP23StackAdvisor, self).getServiceConfigurationValidators()
     childValidators = {
@@ -946,8 +959,20 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
     display_name = 'HAWQ Segment temp directory'
     self.validateIfRootDir (properties, validationItems, prop_name, display_name)
 
+    # 3. Check YARN RM address properties
+    servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+    if "YARN" in servicesList and "yarn-site" in configurations:
+      yarn_site = getSiteProperties(configurations, "yarn-site")
+      for hs_prop, ys_prop in self.getHAWQYARNPropertyMapping().items():
+        if hs_prop in hawq_site and ys_prop in yarn_site and hawq_site[hs_prop] != yarn_site[ys_prop]:
+          message = "Expected value: {0} (this property should have the same value as the property {1} in yarn-site)".format(yarn_site[ys_prop], ys_prop)
+          validationItems.append({"config-name": hs_prop, "item": self.getWarnItem(message)})
+
     return self.toConfigurationValidationProblems(validationItems, "hawq-site")
   
   
   def isComponentUsingCardinalityForLayout(self, componentName):
     return componentName in ['NFS_GATEWAY', 'PHOENIX_QUERY_SERVER', 'SPARK_THRIFTSERVER']
+
+  def getHAWQYARNPropertyMapping(self):
+    return { "hawq_rm_yarn_address": "yarn.resourcemanager.address", "hawq_rm_yarn_scheduler_address": "yarn.resourcemanager.scheduler.address" }

ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py

@@ -1323,8 +1323,14 @@ class TestHDP23StackAdvisor(TestCase):
     componentsListList = [service["components"] for service in services["services"]]
     componentsList = [item for sublist in componentsListList for item in sublist]
     hawqSegmentComponent = [component["StackServiceComponents"] for component in componentsList if component["StackServiceComponents"]["component_name"] == "HAWQSEGMENT"][0]
-    services["configurations"]["hawq-site"] = {"properties": {"default_segment_num": "24"}}
 
+    # setup default configuration values
+    services["configurations"]["hawq-site"] = {"properties": {"default_segment_num": "24",
+                                                              "hawq_rm_yarn_address": "localhost:8032",
+                                                              "hawq_rm_yarn_scheduler_address": "localhost:8030"}}
+    services["configurations"]["yarn-site"] = {"properties": {"yarn.resourcemanager.address": "host1:8050",
+                                                              "yarn.resourcemanager.scheduler.address": "host1:8030"}}
+    services["services"].append({"StackServices" : {"service_name" : "YARN"}, "components":[]})
     configurations = {}
     clusterData = {}
 
@@ -1333,6 +1339,10 @@ class TestHDP23StackAdvisor(TestCase):
     self.stackAdvisor.recommendHAWQConfigurations(configurations, clusterData, services, None)
     self.assertEquals(configurations["hawq-site"]["properties"]["default_segment_num"], str(3 * 6))
 
+    # check derived properties
+    self.assertEquals(configurations["hawq-site"]["properties"]["hawq_rm_yarn_address"], "host1:8050")
+    self.assertEquals(configurations["hawq-site"]["properties"]["hawq_rm_yarn_scheduler_address"], "host1:8030")
+
     # Test 2 - with 49 segments
     hawqSegmentComponent["hostnames"] = ["host" + str(i) for i in range(49)]
     self.stackAdvisor.recommendHAWQConfigurations(configurations, clusterData, services, None)
@@ -1345,6 +1355,7 @@ class TestHDP23StackAdvisor(TestCase):
 
     # Test 4 - with no segments
     configurations = {}
+    services["configurations"]["hawq-site"] = {"properties":{'hawq-site': {'properties': {}}}}
     hawqSegmentComponent["hostnames"] = []
     self.stackAdvisor.recommendHAWQConfigurations(configurations, clusterData, services, None)
     self.assertEquals(configurations, {'hawq-site': {'properties': {}}})
@@ -1616,3 +1627,41 @@ class TestHDP23StackAdvisor(TestCase):
     services["configurations"]["hdfs-site"]["properties"]["dfs.allow.truncate"] = "false"
     problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, hosts)
     self.assertEqual(len(problems), 0)
+
+  def test_validateHAWQConfigurations(self):
+    services = self.load_json("services-hawq-3-hosts.json")
+    # setup default configuration values
+    configurations = services["configurations"]
+    configurations["hawq-site"] = {"properties": {"hawq_rm_yarn_address": "localhost:8032",
+                                                  "hawq_rm_yarn_scheduler_address": "localhost:8030"}}
+    configurations["yarn-site"] = {"properties": {"yarn.resourcemanager.address": "host1:8050",
+                                                  "yarn.resourcemanager.scheduler.address": "host1:8030"}}
+    services["services"].append({"StackServices" : {"service_name" : "YARN"}, "components":[]})
+    properties = configurations["hawq-site"]["properties"]
+    defaults = {}
+    hosts = {}
+
+    expected_warnings = {
+      'hawq_rm_yarn_address': {
+        'config-type': 'hawq-site',
+        'message': 'Expected value: host1:8050 (this property should have the same value as the property yarn.resourcemanager.address in yarn-site)',
+        'type': 'configuration',
+        'config-name': 'hawq_rm_yarn_address',
+        'level': 'WARN'
+      },
+      'hawq_rm_yarn_scheduler_address': {
+        'config-type': 'hawq-site',
+        'message': 'Expected value: host1:8030 (this property should have the same value as the property yarn.resourcemanager.scheduler.address in yarn-site)',
+        'type': 'configuration',
+        'config-name': 'hawq_rm_yarn_scheduler_address',
+        'level': 'WARN'
+      }
+    }
+
+    problems = self.stackAdvisor.validateHAWQConfigurations(properties, defaults, configurations, services, hosts)
+    problems_dict = {}
+    for problem in problems:
+      problems_dict[problem['config-name']] = problem
+    self.assertEqual(len(problems), 2)
+    self.assertEqual(problems_dict, expected_warnings)
+