AMBARI-16970: Update derivation of hawq_rm_memory_limit_perseg (bhuvnesh2703)

ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml

@@ -180,10 +180,10 @@
   <property>
     <name>hawq_rm_memory_limit_perseg</name>
     <display-name>Segment Memory Usage Limit</display-name>
-    <value>64GB</value>
+    <value>67108864KB</value>
     <description>
       The maximum memory that can be used by a HAWQ segment when Resource Manager ('hawq_global_rm_type') is set to Standalone ('none').
-      The default is 64GB.
+      Its value depends on vm.overcommit_memory, vm.overcommit_ratio and system memory on HAWQ hosts.
     </description>
     <depends-on>
       <property>

ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-sysctl-env.xml

@@ -347,7 +347,6 @@
       1: Always overcommit
       2: Don't overcommit.
       For production environments, the value of 2 is recommended and means that processes will not be killed but will receive errors upon memory allocation.
-      The default value is 1.
     </description>
     <value-attributes>
       <type>int</type>
@@ -361,7 +360,7 @@
   <property>
     <name>vm.overcommit_ratio</name>
     <display-name>VM Overcommit Ratio</display-name>
-    <value/>
+    <value>50</value>
     <description>
       When overcommit_memory is set to 2, the committed address space is not permitted to exceed swap plus this percentage of physical RAM.
       Default is 50.

ambari-server/src/main/resources/common-services/HAWQ/2.0.0/service_advisor.py

@@ -174,18 +174,20 @@ class HAWQ200ServiceAdvisor(service_advisor.ServiceAdvisor):
           if hs_prop in hawq_site and ys_prop in yarn_site:
             putHawqSiteProperty(hs_prop, yarn_site[ys_prop])
 
-    # set vm.overcommit_memory to 2 if the minimum memory among all hawqHosts is greater than 32GB
     if "hawq-sysctl-env" in services["configurations"] and "vm.overcommit_memory" in services["configurations"]["hawq-sysctl-env"]["properties"]:
       MEM_THRESHOLD = 33554432 # 32GB, minHawqHostsMemory is represented in kB
       # Set the value for hawq_rm_memory_limit_perseg based on vm.overcommit value and RAM available on HAWQ Hosts
-      # Available RAM Formula = SWAP + RAM * vm.overcommit_ratio / 100. (SWAP not considered for recommendation here)
-      # Default value of vm.overcommit_ratio from configuration is '', so first time during install it will be blank, for subsequent calls it will have the recommended value.
-      if "vm.overcommit_ratio" in services["configurations"]["hawq-sysctl-env"]["properties"] and len(str(services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_ratio"])) > 0:
+      # If value of hawq_rm_memory_limit_perseg is 67108864KB, it indicates hawq is being added and recommendation
+      # has not be made yet, since after recommendation it will be in GB in case its 67108864KB.
+      if "vm.overcommit_ratio" in services["configurations"]["hawq-sysctl-env"]["properties"]:
         vm_overcommit_ratio = int(services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_ratio"])
-        vm_overcommit_mem_value = int(services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"])
       else:
         vm_overcommit_ratio = 50
+      if "hawq_rm_memory_limit_perseg" in services["configurations"]["hawq-site"]["properties"] and services["configurations"]["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"] == "67108864KB":
         vm_overcommit_mem_value = 2 if minHawqHostsMemory >= MEM_THRESHOLD else 1
+      else:
+        # set vm.overcommit_memory to 2 if the minimum memory among all hawqHosts is greater than 32GB
+        vm_overcommit_mem_value = int(services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"])
       host_ram_kb = minHawqHostsMemory * vm_overcommit_ratio / 100 if vm_overcommit_mem_value == 2 else minHawqHostsMemory
       host_ram_gb = float(host_ram_kb) / (1024 * 1024)
       recommended_mem_percentage = {

ambari-server/src/test/python/stacks/2.3/HAWQ/test_service_advisor.py

@@ -141,7 +141,7 @@ class TestHAWQ200ServiceAdvisor(TestCase):
       },
       "hawq-site": {
         "properties": {
-          "hawq_rm_memory_limit_perseg": "64GB"
+          "hawq_rm_memory_limit_perseg": "67108864KB"
         }
       }
     }
@@ -211,13 +211,12 @@ class TestHAWQ200ServiceAdvisor(TestCase):
     ## Test if vm.overcommit_memory is set correctly
 
     # Case 1: All machines have total_mem above 32GB (total_mem >= 33554432)
-    services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_ratio"] = ''
     self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts)
     self.assertEquals(configurations["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"], "2")
 
     # Case 2: One machine has total_mem below 32GB
     hosts["items"][0]["Hosts"]["total_mem"] = 33554431
-    services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_ratio"] = ''
+    services["configurations"]["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"] = "67108864KB"
     self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts)
     self.assertEquals(configurations["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"], "1")
 
@@ -238,8 +237,7 @@ class TestHAWQ200ServiceAdvisor(TestCase):
     hosts["items"][0]["Hosts"]["total_mem"] = 67108864
     hosts["items"][1]["Hosts"]["total_mem"] = 77108864
     hosts["items"][3]["Hosts"]["total_mem"] = 87108864
-    services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_ratio"] = 50
-    services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"] = 2
+    services["configurations"]["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"] = "67108864KB"
     self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts)
     self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "24GB")
 
@@ -257,13 +255,11 @@ class TestHAWQ200ServiceAdvisor(TestCase):
     self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts)
     self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "436GB")
 
-
     # Case 6: Minimum host memory is ~ 1024 GB, vm.overcommit_ratio = 75, vm.overcommit_memory = 2
     # recommended val must be .95% of (1024*75)/100 and in GB
     hosts["items"][0]["Hosts"]["total_mem"] = 1073741824
     hosts["items"][1]["Hosts"]["total_mem"] = 2073741824
     hosts["items"][3]["Hosts"]["total_mem"] = 3073741824
     services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_ratio"] = 75
-    services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"] = 2
     self.serviceAdvisor.getServiceConfigurationRecommendations(self.stackAdvisor, configurations, None, services, hosts)
-    self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "730GB")
+    self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "730GB")