AMBARI-17835 Maximum validation failure for 'yarn.scheduler.maximum-allocation-mb' after dependency change (dsen)

ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java

@@ -98,6 +98,7 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend
   private static final String CONFIGURATIONS_PROPERTY = "configurations";
   private static final String CHANGED_CONFIGURATIONS_PROPERTY = "changed-configurations";
   private static final String AMBARI_SERVER_CONFIGURATIONS_PROPERTY = "ambari-server-properties";
+  private static final String STACK_ADVISOR_COMMAND_TYPE_PROPERTY = "stack-advisor-command-type";
 
   private File recommendationsDir;
   private String recommendationsArtifactsLifetime;
@@ -163,6 +164,7 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend
       populateConfigurations(root, request);
       populateConfigGroups(root, request);
       populateAmbariServerInfo(root);
+      populateCommandType(root);
       data.servicesJSON = mapper.writeValueAsString(root);
     } catch (Exception e) {
       // should not happen
@@ -183,6 +185,10 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend
     }
   }
 
+  protected void populateCommandType(ObjectNode root) throws StackAdvisorException {
+    root.put(STACK_ADVISOR_COMMAND_TYPE_PROPERTY, getCommandType().toString());
+  }
+
   private void populateConfigurations(ObjectNode root,
                                       StackAdvisorRequest request) {
     Map<String, Map<String, Map<String, String>>> configurations =

ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py

@@ -26,6 +26,8 @@ import socket
 import re
 import xml.etree.ElementTree as ET
 
+from stack_advisor import ActionType
+
 class HDP22StackAdvisor(HDP21StackAdvisor):
 
   def getServiceConfigurationRecommenderDict(self):
@@ -70,14 +72,11 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
       putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-mb', 'maximum', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])
 
       # Above is the default calculated 'maximum' values derived purely from hosts.
-      # However, there are 'maximum' and other attributes that actually change based on the values
-      #  of other configs. We need to update those values.
+      # However, there are 'maximum' and other attributes that actually should come from services["configurations"] during validations.
 
       # On a first stackadvisor invocation from the UI we have stack defaults in services["configurations"].
       # So services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"] is always 5120 during first recommendation.
-      # Get a value from recommendations instead of services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]
-      # in this case (if 'changed-configurations' are empty).
-      if ('changed-configurations' in services.keys() and services['changed-configurations'] and "yarn-site" in services["configurations"]):
+      if (self.getActionType(services) == ActionType.VALIDATE_CONFIGURATIONS and "yarn-site" in services["configurations"]):
         if ("yarn.nodemanager.resource.memory-mb" in services["configurations"]["yarn-site"]["properties"]):
           putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-mb', 'maximum', services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])
           putYarnPropertyAttribute('yarn.scheduler.minimum-allocation-mb', 'maximum', services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])
@@ -355,8 +354,19 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
     container_size = min(clusterData['containers'] * clusterData['ramPerContainer'], container_size, yarnMaxAllocationSize)
 
     putHiveSiteProperty("hive.tez.container.size", min(int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]), container_size))
-    putHiveSitePropertyAttribute("hive.tez.container.size", "minimum", int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]))
-    putHiveSitePropertyAttribute("hive.tez.container.size", "maximum", int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]))
+
+    # "maximum" and "minimum" attributes should come from services["configurations"] during validations.
+    # If one of dependencies are in changed-confogurations another can be absent. In this case we need to take both values from 'services'.
+    if self.getActionType(services) == ActionType.VALIDATE_CONFIGURATIONS or ('changed-configurations' in services and services['changed-configurations'] \
+        and (self.isPropertyInChangedConfigs('yarn-site', 'yarn.scheduler.minimum-allocation-mb', services['changed-configurations']) or self.isPropertyInChangedConfigs('yarn-site', 'yarn.scheduler.maximum-allocation-mb', services['changed-configurations']))):
+      if "yarn-site" in services["configurations"]:
+        if "yarn.scheduler.minimum-allocation-mb" in services["configurations"]["yarn-site"]["properties"]:
+          putHiveSitePropertyAttribute("hive.tez.container.size", "minimum", int(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]))
+        if "yarn.scheduler.maximum-allocation-mb" in services["configurations"]["yarn-site"]["properties"]:
+          putHiveSitePropertyAttribute("hive.tez.container.size", "maximum", int(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]))
+    else:
+      putHiveSitePropertyAttribute("hive.tez.container.size", "minimum", int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]))
+      putHiveSitePropertyAttribute("hive.tez.container.size", "maximum", int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]))
 
     putHiveSiteProperty("hive.prewarm.enabled", "false")
     putHiveSiteProperty("hive.prewarm.numcontainers", "3")
@@ -1037,6 +1047,16 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
     putMapredPropertyAttribute = self.putPropertyAttribute(configurations, "mapred-site")
     yarnMinAllocationSize = int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])
     yarnMaxAllocationSize = min(30 * int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]), int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]))
+
+    # "maximum" and "minimum" attributes should come from services["configurations"] during validations.
+    # If one of dependencies are in changed-confogurations another can be absent. In this case we need to take both values from 'services'.
+    if self.getActionType(services) == ActionType.VALIDATE_CONFIGURATIONS or ('changed-configurations' in services and services['changed-configurations'] \
+        and (self.isPropertyInChangedConfigs('yarn-site', 'yarn.scheduler.minimum-allocation-mb', services['changed-configurations']) or self.isPropertyInChangedConfigs('yarn-site', 'yarn.scheduler.maximum-allocation-mb', services['changed-configurations']))):
+      if "yarn-site" in services["configurations"] and "yarn.scheduler.minimum-allocation-mb" in services["configurations"]["yarn-site"]["properties"] \
+          and "yarn.scheduler.maximum-allocation-mb" in services["configurations"]["yarn-site"]["properties"]:
+        yarnMinAllocationSize = int(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])
+        yarnMaxAllocationSize = min(30 * int(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]), int(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]))
+
     putMapredPropertyAttribute("mapreduce.map.memory.mb", "maximum", yarnMaxAllocationSize)
     putMapredPropertyAttribute("mapreduce.map.memory.mb", "minimum", yarnMinAllocationSize)
     putMapredPropertyAttribute("mapreduce.reduce.memory.mb", "maximum", yarnMaxAllocationSize)

ambari-server/src/main/resources/stacks/stack_advisor.py

@@ -298,6 +298,12 @@ class StackAdvisor(object):
     """
     pass
 
+class ActionType:
+  RECOMMEND_COMPONENT_LAYOUT_ACTION = 'recommend-component-layout'
+  VALIDATE_COMPONENT_LAYOUT_ACTION = 'validate-component-layout'
+  RECOMMEND_CONFIGURATIONS = 'recommend-configurations'
+  RECOMMEND_CONFIGURATION_DEPENDENCIES = 'recommend-configuration-dependencies'
+  VALIDATE_CONFIGURATIONS = 'validate-configurations'
 
 class DefaultStackAdvisor(StackAdvisor):
   """
@@ -313,7 +319,6 @@ class DefaultStackAdvisor(StackAdvisor):
     # Dictionary that maps serviceName or componentName to serviceAdvisor
     self.serviceAdvisorsDict = {}
 
-
   def getActiveHosts(self, hosts):
     """ Filters the list of specified hosts object and returns
         a list of hosts which are not in maintenance mode. """
@@ -987,13 +992,13 @@ class DefaultStackAdvisor(StackAdvisor):
       config[configType]["properties"] = {}
     def appendProperty(key, value):
       # If property exists in changedConfigs, do not override, use user defined property
-      if self.__isPropertyInChangedConfigs(configType, key, changedConfigs):
+      if self.isPropertyInChangedConfigs(configType, key, changedConfigs):
         config[configType]["properties"][key] = userConfigs[configType]['properties'][key]
       else:
         config[configType]["properties"][key] = str(value)
     return appendProperty
 
-  def __isPropertyInChangedConfigs(self, configType, propertyName, changedConfigs):
+  def isPropertyInChangedConfigs(self, configType, propertyName, changedConfigs):
     for changedConfig in changedConfigs:
       if changedConfig['type']==configType and changedConfig['name']==propertyName:
         return True
@@ -1016,7 +1021,7 @@ class DefaultStackAdvisor(StackAdvisor):
 
     def updatePropertyWithCallback(key, value, callback):
       # If property exists in changedConfigs, do not override, use user defined property
-      if self.__isPropertyInChangedConfigs(configType, key, changedConfigs):
+      if self.isPropertyInChangedConfigs(configType, key, changedConfigs):
         config[configType]["properties"][key] = userConfigs[configType]['properties'][key]
       else:
         # Give the callback an empty string if the mapping doesn't exist
@@ -1083,3 +1088,6 @@ class DefaultStackAdvisor(StackAdvisor):
 
   def getServiceNames(self, services):
     return [service["StackServices"]["service_name"] for service in services["services"]]
+
+  def getActionType(self, services):
+    return services.get("stack-advisor-command-type", ActionType.RECOMMEND_CONFIGURATIONS)

ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommandTest.java

@@ -240,6 +240,24 @@ public class StackAdvisorCommandTest {
     assertEquals("b", serverProperties.iterator().next().getTextValue());
   }
 
+  @Test
+  public void testPopulateCommandType() throws Exception {
+    File file = mock(File.class);
+    String recommendationsArtifactsLifetime = "1w";
+    StackAdvisorRunner stackAdvisorRunner = mock(StackAdvisorRunner.class);
+    AmbariMetaInfo ambariMetaInfo = mock(AmbariMetaInfo.class);
+    StackAdvisorCommand<TestResource> cmd = new TestStackAdvisorCommand(file, recommendationsArtifactsLifetime, "test", 1,
+      stackAdvisorRunner, ambariMetaInfo);
+    ObjectNode objectNode = (ObjectNode) cmd.mapper.readTree("{\"Versions\": " +
+      "{\"stack_name\": \"stack\", \"stack_version\":\"1.0.0\"}}");
+
+    cmd.populateCommandType(objectNode);
+
+    JsonNode commandType = objectNode.get("stack-advisor-command-type");
+    assertNotNull(commandType);
+    assertEquals(StackAdvisorCommandType.RECOMMEND_COMPONENT_LAYOUT.toString(), commandType.getTextValue());
+  }
+
   @Test
   public void testPopulateStackHierarchy_noParents() throws Exception {
     File file = mock(File.class);

ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py

@@ -1136,24 +1136,19 @@ class TestHDP22StackAdvisor(TestCase):
     self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, hosts)
     self.assertEquals(configurations, expected)
 
-    # With no 'changed-configurations', we should get updated 'maximum's from recommendations
-    # Else update from services.
+    # During a recommendation request, we should get updated 'maximum's from recommendations
+    # During a validation request, update from services.
     #
     # On a first stackadvisor invocation from the UI we have stack defaults in services["configurations"].
     # So services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"] is always 5120 during first recommendation.
-    # Get a value from recommendations instead of services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]
-    # in this case (if 'changed-configurations' are empty).
-    #
-    # Test not empty 'changed-configurations':
-    services["changed-configurations"] = [
-      {
-        "type": "yarn-env",
-        "name": "min_user_id"
-      }
-    ]
-    services.pop("configurations", None)
+
+    # Test validate-configurations:
+    services.pop("changed-configurations", None)
+    services["stack-advisor-command-type"] = 'validate-configurations'
     services["configurations"] = {"yarn-site": {"properties": {"yarn.nodemanager.resource.memory-mb": '4321', "yarn.nodemanager.resource.cpu-vcores": '9'}},
                                   "yarn-env": {"properties": {"min_user_id": "500"}}}
+
+    # Expected recommendations from 'servicess'
     expected["yarn-site"]["property_attributes"]["yarn.scheduler.minimum-allocation-vcores"]["maximum"] = '9'
     expected["yarn-site"]["property_attributes"]["yarn.scheduler.maximum-allocation-vcores"]["maximum"] = '9'
     expected["yarn-site"]["property_attributes"]["yarn.scheduler.maximum-allocation-mb"]["maximum"] = '4321'