Browse Source

AMBARI-8020. /recommendations API mising "tez.am.java.opts" (aonishuk)

Andrew Onishuk 10 years ago
parent
commit
6a29afb0ff

+ 4 - 0
ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py

@@ -36,6 +36,9 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
   def recommendTezConfigurations(self, configurations, clusterData):
   def recommendTezConfigurations(self, configurations, clusterData):
     putTezProperty = self.putProperty(configurations, "tez-site")
     putTezProperty = self.putProperty(configurations, "tez-site")
     putTezProperty("tez.am.resource.memory.mb", int(clusterData['amMemory']) * 2 if int(clusterData['amMemory']) < 3072 else int(clusterData['amMemory']))
     putTezProperty("tez.am.resource.memory.mb", int(clusterData['amMemory']) * 2 if int(clusterData['amMemory']) < 3072 else int(clusterData['amMemory']))
+    putTezProperty("tez.am.java.opts",
+                   "-server -Xmx" + str(int(0.8 * clusterData["amMemory"]))
+                   + "m -Djava.net.preferIPv4Stack=true -XX:+UseNUMA -XX:+UseParallelGC")
 
 
     taskResourceMemory = clusterData['mapMemory'] if clusterData['mapMemory'] > 2048 else int(clusterData['reduceMemory'])
     taskResourceMemory = clusterData['mapMemory'] if clusterData['mapMemory'] > 2048 else int(clusterData['reduceMemory'])
     taskResourceMemory = min(clusterData['containers'] * clusterData['ramPerContainer'], taskResourceMemory)
     taskResourceMemory = min(clusterData['containers'] * clusterData['ramPerContainer'], taskResourceMemory)
@@ -55,6 +58,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
 
 
   def validateTezConfigurations(self, properties, recommendedDefaults, configurations):
   def validateTezConfigurations(self, properties, recommendedDefaults, configurations):
     validationItems = [ {"config-name": 'tez.am.resource.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.am.resource.memory.mb')},
     validationItems = [ {"config-name": 'tez.am.resource.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.am.resource.memory.mb')},
+                        {"config-name": 'tez.am.java.opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'tez.am.java.opts')},
                         {"config-name": 'tez.task.resource.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.task.resource.memory.mb')},
                         {"config-name": 'tez.task.resource.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.task.resource.memory.mb')},
                         {"config-name": 'tez.runtime.io.sort.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.runtime.io.sort.mb')},
                         {"config-name": 'tez.runtime.io.sort.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.runtime.io.sort.mb')},
                         {"config-name": 'tez.runtime.unordered.output.buffer.size-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.runtime.unordered.output.buffer.size-mb')},]
                         {"config-name": 'tez.runtime.unordered.output.buffer.size-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.runtime.unordered.output.buffer.size-mb')},]

+ 3 - 0
ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py

@@ -55,6 +55,7 @@ class TestHDP22StackAdvisor(TestCase):
       "tez-site": {
       "tez-site": {
         "properties": {
         "properties": {
           "tez.am.resource.memory.mb": "4000",
           "tez.am.resource.memory.mb": "4000",
+          "tez.am.java.opts": "-server -Xmx1600m -Djava.net.preferIPv4Stack=true -XX:+UseNUMA -XX:+UseParallelGC",
           "tez.task.resource.memory.mb": "768",
           "tez.task.resource.memory.mb": "768",
           "tez.runtime.io.sort.mb": "307",
           "tez.runtime.io.sort.mb": "307",
           "tez.runtime.unordered.output.buffer.size-mb": "57"
           "tez.runtime.unordered.output.buffer.size-mb": "57"
@@ -77,6 +78,7 @@ class TestHDP22StackAdvisor(TestCase):
       "tez-site": {
       "tez-site": {
         "properties": {
         "properties": {
           "tez.am.resource.memory.mb": "3100",
           "tez.am.resource.memory.mb": "3100",
+          "tez.am.java.opts": "-server -Xmx2480m -Djava.net.preferIPv4Stack=true -XX:+UseNUMA -XX:+UseParallelGC",
           "tez.task.resource.memory.mb": "768",
           "tez.task.resource.memory.mb": "768",
           "tez.runtime.io.sort.mb": "307",
           "tez.runtime.io.sort.mb": "307",
           "tez.runtime.unordered.output.buffer.size-mb": "57"
           "tez.runtime.unordered.output.buffer.size-mb": "57"
@@ -99,6 +101,7 @@ class TestHDP22StackAdvisor(TestCase):
       "tez-site": {
       "tez-site": {
         "properties": {
         "properties": {
           "tez.am.resource.memory.mb": "4000",
           "tez.am.resource.memory.mb": "4000",
+          "tez.am.java.opts": "-server -Xmx1600m -Djava.net.preferIPv4Stack=true -XX:+UseNUMA -XX:+UseParallelGC",
           "tez.task.resource.memory.mb": "760",
           "tez.task.resource.memory.mb": "760",
           "tez.runtime.io.sort.mb": "304",
           "tez.runtime.io.sort.mb": "304",
           "tez.runtime.unordered.output.buffer.size-mb": "57"
           "tez.runtime.unordered.output.buffer.size-mb": "57"