AMBARI-11538. Ambari does not warn if hive tez container size exceeds yarn container max size. (Andrew Onischuk via mpapirkovskyy)

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py

@@ -727,6 +727,11 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
 
     return uid_min
 
+  def mergeValidators(self, parentValidators, childValidators):
+    for service, configsDict in childValidators.iteritems():
+      if service not in parentValidators:
+        parentValidators[service] = {}
+      parentValidators[service].update(configsDict)
 
 # Validation helper methods
 def getSiteProperties(configurations, siteName):

ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py

@@ -74,13 +74,19 @@ class HDP21StackAdvisor(HDP206StackAdvisor):
       "HIVE": {"hive-site": self.validateHiveConfigurations},
       "TEZ": {"tez-site": self.validateTezConfigurations}
     }
-    parentValidators.update(childValidators)
+    self.mergeValidators(parentValidators, childValidators)
     return parentValidators
 
   def validateHiveConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
     validationItems = [ {"config-name": 'hive.tez.container.size', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'hive.tez.container.size')},
                         {"config-name": 'hive.tez.java.opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'hive.tez.java.opts')},
                         {"config-name": 'hive.auto.convert.join.noconditionaltask.size', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'hive.auto.convert.join.noconditionaltask.size')} ]
+    yarnSiteProperties = getSiteProperties(configurations, "yarn-site")
+    if yarnSiteProperties:
+      yarnSchedulerMaximumAllocationMb = to_number(yarnSiteProperties["yarn.scheduler.maximum-allocation-mb"])
+      hiveTezContainerSize = to_number(properties['hive.tez.container.size'])
+      if hiveTezContainerSize is not None and yarnSchedulerMaximumAllocationMb is not None and hiveTezContainerSize > yarnSchedulerMaximumAllocationMb:
+        validationItems.append({"config-name": 'hive.tez.container.size', "item": self.getWarnItem("yarn.scheduler.maximum-allocation-mb is less than hive.tez.container.size")})
     return self.toConfigurationValidationProblems(validationItems, "hive-site")
 
   def validateTezConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):

ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py

@@ -573,7 +573,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
       "MAPREDUCE2": {"mapred-site": self.validateMapReduce2Configurations},
       "TEZ": {"tez-site": self.validateTezConfigurations}
     }
-    parentValidators.update(childValidators)
+    self.mergeValidators(parentValidators, childValidators)
     return parentValidators
 
   def validateTezConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
@@ -770,7 +770,6 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
     return self.toConfigurationValidationProblems(validationItems, "hdfs-site")
 
   def validateHiveServer2Configurations(self, properties, recommendedDefaults, configurations, services, hosts):
-    super(HDP22StackAdvisor, self).validateHiveConfigurations(properties, recommendedDefaults, configurations, services, hosts)
     hive_server2 = properties
     validationItems = [] 
     #Adding Ranger Plugin logic here 
@@ -826,7 +825,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
     return self.toConfigurationValidationProblems(validationItems, "hive-env")
 
   def validateHiveConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
-    super(HDP22StackAdvisor, self).validateHiveConfigurations(properties, recommendedDefaults, configurations, services, hosts)
+    parentValidationProblems = super(HDP22StackAdvisor, self).validateHiveConfigurations(properties, recommendedDefaults, configurations, services, hosts)
     hive_site = properties
     validationItems = []
     #Adding Ranger Plugin logic here
@@ -866,7 +865,9 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
                               "item": self.getWarnItem("Correct values are {0}".format(stripe_size_values))
                              }
       )
-    return self.toConfigurationValidationProblems(validationItems, "hive-site")
+    configurationValidationProblems = self.toConfigurationValidationProblems(validationItems, "hive-site")
+    configurationValidationProblems.extend(parentValidationProblems)
+    return configurationValidationProblems
 
   def validateHBASEConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
     hbase_site = properties

ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py

@@ -93,7 +93,7 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
         "HIVE": {"hiveserver2-site": self.validateHiveServer2Configurations},
         "HBASE": {"hbase-site": self.validateHBASEConfigurations}
       }
-      parentValidators.update(childValidators)
+      self.mergeValidators(parentValidators, childValidators)
       return parentValidators
 
   def validateHDFSConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):

ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py

@@ -779,3 +779,41 @@ class TestHDP206StackAdvisor(TestCase):
     # unknown service
     unknown_component = self.stackAdvisor.getHostWithComponent("UNKNOWN", "NODEMANAGER", services, hosts)
     self.assertEquals(nodemanager, None)
+
+  def test_mergeValidators(self):
+    childValidators = {
+      "HDFS": {"hdfs-site": "validateHDFSConfigurations2.3"},
+      "HIVE": {"hiveserver2-site": "validateHiveServer2Configurations2.3"},
+      "HBASE": {"hbase-site": "validateHBASEConfigurations2.3",
+                "newconf": "new2.3"},
+      "NEWSERVICE" : {"newserviceconf": "abc2.3"}
+    }
+    parentValidators = {
+      "HDFS": {"hdfs-site": "validateHDFSConfigurations2.2",
+               "hadoop-env": "validateHDFSConfigurationsEnv2.2"},
+      "YARN": {"yarn-env": "validateYARNEnvConfigurations2.2"},
+      "HIVE": {"hiveserver2-site": "validateHiveServer2Configurations2.2",
+               "hive-site": "validateHiveConfigurations2.2",
+               "hive-env": "validateHiveConfigurationsEnv2.2"},
+      "HBASE": {"hbase-site": "validateHBASEConfigurations2.2",
+                "hbase-env": "validateHBASEEnvConfigurations2.2"},
+      "MAPREDUCE2": {"mapred-site": "validateMapReduce2Configurations2.2"},
+      "TEZ": {"tez-site": "validateTezConfigurations2.2"}
+    }
+    expected = {
+      "HDFS": {"hdfs-site": "validateHDFSConfigurations2.3",
+               "hadoop-env": "validateHDFSConfigurationsEnv2.2"},
+      "YARN": {"yarn-env": "validateYARNEnvConfigurations2.2"},
+      "HIVE": {"hiveserver2-site": "validateHiveServer2Configurations2.3",
+               "hive-site": "validateHiveConfigurations2.2",
+               "hive-env": "validateHiveConfigurationsEnv2.2"},
+      "HBASE": {"hbase-site": "validateHBASEConfigurations2.3",
+                "hbase-env": "validateHBASEEnvConfigurations2.2",
+                "newconf": "new2.3"},
+      "MAPREDUCE2": {"mapred-site": "validateMapReduce2Configurations2.2"},
+      "TEZ": {"tez-site": "validateTezConfigurations2.2"},
+      "NEWSERVICE" : {"newserviceconf": "abc2.3"}
+    }
+
+    self.stackAdvisor.mergeValidators(parentValidators, childValidators)
+    self.assertEquals(expected, parentValidators)

ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py

@@ -246,3 +246,30 @@ class TestHDP21StackAdvisor(TestCase):
 
     res = self.stackAdvisor.validateHDFSConfigurationsEnv(properties, recommendedDefaults, configurations, '', '')
     self.assertEquals(res, res_expected)
+
+  def test_validateHiveConfigurations(self):
+    configurations = {'yarn-site': {'properties': {'yarn.scheduler.maximum-allocation-mb': '4096'}}}
+
+    # 1) ok: hive.tez.container.size > recommended
+    recommendedDefaults = {'hive.tez.container.size': '1024',
+                           'hive.tez.java.opts': '-Xmx256m',
+                           'hive.auto.convert.join.noconditionaltask.size': '1000000000'}
+    properties = {'hive.tez.container.size': '2048',
+                  'hive.tez.java.opts': '-Xmx300m',
+                  'hive.auto.convert.join.noconditionaltask.size': '1100000000'}
+    res_expected = []
+
+    res = self.stackAdvisor.validateHiveConfigurations(properties, recommendedDefaults, configurations, '', '')
+    self.assertEquals(res, res_expected)
+
+    # 2) fail: yarn.scheduler.maximum-allocation-mb < hive.tez.container.size
+    configurations = {'yarn-site': {'properties': {'yarn.scheduler.maximum-allocation-mb': '256'}}}
+    res_expected = [{'config-type': 'hive-site',
+                     'message': 'yarn.scheduler.maximum-allocation-mb is less than hive.tez.container.size',
+                     'type': 'configuration',
+                     'config-name': 'hive.tez.container.size',
+                     'level': 'WARN'},
+                    ]
+
+    res = self.stackAdvisor.validateHiveConfigurations(properties, recommendedDefaults, configurations, '', '')
+    self.assertEquals(res, res_expected)

ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py

@@ -2323,7 +2323,14 @@ class TestHDP22StackAdvisor(TestCase):
     pass
 
   def test_validateHiveConfigurations(self):
-    properties = {"hive_security_authorization": "None", "hive.exec.orc.default.stripe.size": "8388608"}
+    properties = {"hive_security_authorization": "None",
+                  "hive.exec.orc.default.stripe.size": "8388608",
+                  'hive.tez.container.size': '2048',
+                  'hive.tez.java.opts': '-Xmx300m',
+                  'hive.auto.convert.join.noconditionaltask.size': '1100000000'}
+    recommendedDefaults = {'hive.tez.container.size': '1024',
+                           'hive.tez.java.opts': '-Xmx256m',
+                           'hive.auto.convert.join.noconditionaltask.size': '1000000000'}
     configurations = {
       "hive-site": {
         "properties": {"hive.security.authorization.enabled": "true"}
@@ -2338,7 +2345,7 @@ class TestHDP22StackAdvisor(TestCase):
 
     # Test for 'ranger-hive-plugin-properties' not being in configs
     res_expected = []
-    res = self.stackAdvisor.validateHiveConfigurations(properties, {}, configurations, services, {})
+    res = self.stackAdvisor.validateHiveConfigurations(properties, recommendedDefaults, configurations, services, {})
     self.assertEquals(res, res_expected)
 
     pass