AMBARI-16992: Recommend hawq_rm_nvcore_limit_perseg in HAWQ service advisor (mithmatt via bhuvnesh2703)

ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py

@@ -142,6 +142,7 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor):
       hawqHosts = hawqMasterHosts.union(hawqSegmentHosts)
       numSegments = len(hawqSegmentHosts)
       minHawqHostsMemory = min([host['Hosts']['total_mem'] for host in hosts['items'] if host['Hosts']['host_name'] in hawqHosts])
+      minHawqHostsCoreCount = min([host['Hosts']['cpu_count'] for host in hosts['items'] if host['Hosts']['host_name'] in hawqHosts])
 
     if "hawq-site" in services["configurations"]:
       hawq_site = services["configurations"]["hawq-site"]["properties"]
@@ -167,6 +168,9 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor):
           buckets = factor * numSegments
         putHawqSiteProperty('default_hash_table_bucket_number', buckets)
 
+      if "hawq_rm_nvcore_limit_perseg" in hawq_site:
+        putHawqSiteProperty('hawq_rm_nvcore_limit_perseg', minHawqHostsCoreCount)
+
       # update YARN RM urls with the values from yarn-site if YARN is installed
       if "YARN" in servicesList and "yarn-site" in services["configurations"]:
         yarn_site = services["configurations"]["yarn-site"]["properties"]

ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py

@@ -141,7 +141,8 @@ class TestHAWQ200ServiceAdvisor(TestCase):
       },
       "hawq-site": {
         "properties": {
-          "hawq_rm_memory_limit_perseg": "67108864KB"
+          "hawq_rm_memory_limit_perseg": "67108864KB",
+          "hawq_rm_nvcore_limit_perseg": "16"
         }
       }
     }
@@ -184,30 +185,52 @@ class TestHAWQ200ServiceAdvisor(TestCase):
         {
           "Hosts": {
             "host_name": "c6401.ambari.apache.org",
+            "cpu_count" : 2,
             "total_mem": 33554432
           }
         },
         {
           "Hosts": {
             "host_name": "c6402.ambari.apache.org",
+            "cpu_count" : 4,
             "total_mem": 33554433
           }
         },
         {
           "Hosts": {
             "host_name": "c6403.ambari.apache.org",
+            "cpu_count" : 1,
             "total_mem": 33554434
           }
         },
         {
           "Hosts": {
             "host_name": "c6404.ambari.apache.org",
+            "cpu_count" : 2,
             "total_mem": 33554435
           }
         }
       ]
     }
 
+    ## Test if hawq_rm_nvcore_limit_perseg is set correctly
+
+    # Case 1:
+    # HAWQ Hosts Core Count: c6401.ambari.apache.org - 2, c6402.ambari.apache.org - 4, c6404.ambari.apache.org - 2
+    # Non HAWQ Hosts Core Count: c6401.ambari.apache.org - 1
+    # Recommend hawq_rm_nvcore_limit_perseg as 2
+    self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts)
+    self.assertEquals(configurations["hawq-site"]["properties"]["hawq_rm_nvcore_limit_perseg"], "2")
+
+    # Case 2:
+    # HAWQ Hosts Core Count: c6401.ambari.apache.org - 2, c6402.ambari.apache.org - 2, c6404.ambari.apache.org - 2
+    # Non HAWQ Hosts Core Count: c6401.ambari.apache.org - 1
+    # Recommend hawq_rm_nvcore_limit_perseg as 2
+    hosts["items"][1]["Hosts"]["cpu_count"] = 2
+    self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts)
+    self.assertEquals(configurations["hawq-site"]["properties"]["hawq_rm_nvcore_limit_perseg"], "2")
+
+
     ## Test if vm.overcommit_memory is set correctly
 
     # Case 1: All machines have total_mem above 32GB (total_mem >= 33554432)
@@ -221,6 +244,7 @@ class TestHAWQ200ServiceAdvisor(TestCase):
     self.assertEquals(configurations["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"], "1")
 
     ## Test if hawq_rm_memory_limit_perseg is set correctly
+
     # Case 1: Minimum host memory is ~ 2 GB (2048MB), recommended val must be .75% of 2GB as vm.overcommit_memory = 1 and in MB
     hosts["items"][0]["Hosts"]["total_mem"] = 2097152
     self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts)

ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py

@@ -1994,25 +1994,29 @@ class TestHDP23StackAdvisor(TestCase):
         {
           "Hosts": {
             "host_name": "c6401.ambari.apache.org",
-            "total_mem": 12345678
+            "cpu_count" : 2,
+            "total_mem": 33554432
           }
         },
         {
           "Hosts": {
             "host_name": "c6402.ambari.apache.org",
-            "total_mem": 12345678
+            "cpu_count" : 4,
+            "total_mem": 33554433
           }
         },
         {
           "Hosts": {
             "host_name": "c6403.ambari.apache.org",
-            "total_mem": 12345678
+            "cpu_count" : 1,
+            "total_mem": 33554434
           }
         },
         {
           "Hosts": {
             "host_name": "c6404.ambari.apache.org",
-            "total_mem": 12345678
+            "cpu_count" : 2,
+            "total_mem": 33554435
           }
         }
       ]