AMBARI-10589. BE: Stack advisor endpoints should support config-groups specific calls - fixes (dsen via srimanth)

ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java

@@ -26,6 +26,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse;
 import org.apache.ambari.server.state.PropertyDependencyInfo;
 import org.apache.commons.lang.StringUtils;
 
@@ -44,6 +45,7 @@ public class StackAdvisorRequest {
   private Map<String, Set<String>> hostGroupBindings = new HashMap<String, Set<String>>();
   private Map<String, Map<String, Map<String, String>>> configurations = new HashMap<String, Map<String, Map<String, String>>>();
   private List<PropertyDependencyInfo> changedConfigurations = new LinkedList<PropertyDependencyInfo>();
+  private Set<RecommendationResponse.ConfigGroup> configGroups;
 
   public String getStackName() {
     return stackName;
@@ -97,6 +99,14 @@ public class StackAdvisorRequest {
     this.changedConfigurations = changedConfigurations;
   }
 
+  public Set<RecommendationResponse.ConfigGroup> getConfigGroups() {
+    return configGroups;
+  }
+
+  public void setConfigGroups(Set<RecommendationResponse.ConfigGroup> configGroups) {
+    this.configGroups = configGroups;
+  }
+
   private StackAdvisorRequest(String stackName, String stackVersion) {
     this.stackName = stackName;
     this.stackVersion = stackVersion;
@@ -157,6 +167,12 @@ public class StackAdvisorRequest {
       return this;
     }
 
+    public StackAdvisorRequestBuilder withConfigGroups(
+      Set<RecommendationResponse.ConfigGroup> configGroups) {
+      this.instance.configGroups = configGroups;
+      return this;
+    }
+
     public StackAdvisorRequest build() {
       return this.instance;
     }

ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java

@@ -44,7 +44,6 @@ import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest;
 import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorResponse;
 import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRunner;
 import org.apache.ambari.server.controller.spi.Resource;
-import org.apache.ambari.server.state.PropertyDependencyInfo;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.logging.Log;
@@ -81,6 +80,7 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend
       + "&services/StackServices/service_name.in(%s)";
   private static final String SERVICES_PROPERTY = "services";
   private static final String SERVICES_COMPONENTS_PROPERTY = "components";
+  private static final String CONFIG_GROUPS_PROPERTY = "config-groups";
   private static final String COMPONENT_INFO_PROPERTY = "StackServiceComponents";
   private static final String COMPONENT_NAME_PROPERTY = "component_name";
   private static final String COMPONENT_HOSTNAMES_PROPERTY = "hostnames";
@@ -146,6 +146,7 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend
       populateStackHierarchy(root);
       populateComponentHostsMap(root, request.getComponentHostsMap());
       populateConfigurations(root, request);
+      populateConfigGroups(root, request);
       data.servicesJSON = mapper.writeValueAsString(root);
     } catch (Exception e) {
       // should not happen
@@ -181,6 +182,15 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend
     root.put(CHANGED_CONFIGURATIONS_PROPERTY, changedConfigs);
   }
 
+  private void populateConfigGroups(ObjectNode root,
+                                    StackAdvisorRequest request) {
+    if (request.getConfigGroups() != null &&
+      !request.getConfigGroups().isEmpty()) {
+      JsonNode configGroups = mapper.valueToTree(request.getConfigGroups());
+      root.put(CONFIG_GROUPS_PROPERTY, configGroups);
+    }
+  }
+
   protected void populateStackHierarchy(ObjectNode root) {
     ObjectNode version = (ObjectNode) root.get("Versions");
     TextNode stackName = (TextNode) version.get("stack_name");

ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/recommendations/RecommendationResponse.java

@@ -18,6 +18,8 @@
 
 package org.apache.ambari.server.api.services.stackadvisor.recommendations;
 
+import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
@@ -71,6 +73,10 @@ public class RecommendationResponse extends StackAdvisorResponse {
     @JsonProperty("blueprint_cluster_binding")
     private BlueprintClusterBinding blueprintClusterBinding;
 
+    @JsonProperty("config-groups")
+    @JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
+    private Set<ConfigGroup> configGroups;
+
     public Blueprint getBlueprint() {
       return blueprint;
     }
@@ -86,6 +92,14 @@ public class RecommendationResponse extends StackAdvisorResponse {
     public void setBlueprintClusterBinding(BlueprintClusterBinding blueprintClusterBinding) {
       this.blueprintClusterBinding = blueprintClusterBinding;
     }
+
+    public Set<ConfigGroup> getConfigGroups() {
+      return configGroups;
+    }
+
+    public void setConfigGroups(Set<ConfigGroup> configGroups) {
+      this.configGroups = configGroups;
+    }
   }
 
   public static class Blueprint {
@@ -121,7 +135,7 @@ public class RecommendationResponse extends StackAdvisorResponse {
     private Map<String, ValueAttributesInfo> propertyAttributes;
 
     public BlueprintConfigurations() {
-      System.out.println(this);
+
     }
 
     public Map<String, String> getProperties() {
@@ -202,4 +216,48 @@ public class RecommendationResponse extends StackAdvisorResponse {
     }
   }
 
+  public static class ConfigGroup {
+
+    @JsonProperty
+    private List<String> hosts;
+
+    @JsonProperty
+    @JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
+    private Map<String, BlueprintConfigurations> configurations =
+      new HashMap<String, BlueprintConfigurations>();
+
+    @JsonProperty("dependent_configurations")
+    @JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
+    private Map<String, BlueprintConfigurations> dependentConfigurations =
+      new HashMap<String, BlueprintConfigurations>();
+
+    public ConfigGroup() {
+
+    }
+
+    public List<String> getHosts() {
+      return hosts;
+    }
+
+    public void setHosts(List<String> hosts) {
+      this.hosts = hosts;
+    }
+
+    public Map<String, BlueprintConfigurations> getConfigurations() {
+      return configurations;
+    }
+
+    public void setConfigurations(Map<String, BlueprintConfigurations> configurations) {
+      this.configurations = configurations;
+    }
+
+    public Map<String, BlueprintConfigurations> getDependentConfigurations() {
+      return dependentConfigurations;
+    }
+
+    public void setDependentConfigurations(Map<String, BlueprintConfigurations> dependentConfigurations) {
+      this.dependentConfigurations = dependentConfigurations;
+    }
+  }
+
 }

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RecommendationResourceProvider.java

@@ -26,10 +26,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-import javax.ws.rs.WebApplicationException;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.Response.Status;
-
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorException;
 import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest;
@@ -57,6 +53,9 @@ public class RecommendationResourceProvider extends StackAdvisorResourceProvider
   protected static final String SERVICES_PROPERTY_ID = "services";
   protected static final String RECOMMEND_PROPERTY_ID = "recommend";
 
+  protected static final String CONFIG_GROUPS_PROPERTY_ID = PropertyHelper
+      .getPropertyId("recommendations", "config-groups");
+
   protected static final String BLUEPRINT_CONFIGURATIONS_PROPERTY_ID = PropertyHelper
       .getPropertyId("recommendations/blueprint", "configurations");
 
@@ -92,10 +91,10 @@ public class RecommendationResourceProvider extends StackAdvisorResourceProvider
     try {
       response = saHelper.recommend(recommendationRequest);
     } catch (StackAdvisorRequestException e) {
-      LOG.warn("Error occured during recommnedation", e);
+      LOG.warn("Error occured during recommendation", e);
       throw new IllegalArgumentException(e.getMessage(), e);
     } catch (StackAdvisorException e) {
-      LOG.warn("Error occured during recommnedation", e);
+      LOG.warn("Error occured during recommendation", e);
       throw new SystemException(e.getMessage(), e);
     }
 
@@ -112,6 +111,8 @@ public class RecommendationResourceProvider extends StackAdvisorResourceProvider
         setResourceProperty(resource, HOSTS_PROPERTY_ID, response.getHosts(), getPropertyIds());
         setResourceProperty(resource, SERVICES_PROPERTY_ID, response.getServices(),
             getPropertyIds());
+        setResourceProperty(resource, CONFIG_GROUPS_PROPERTY_ID,
+          response.getRecommendations().getConfigGroups(), getPropertyIds());
         setResourceProperty(resource, BLUEPRINT_CONFIGURATIONS_PROPERTY_ID, response
             .getRecommendations().getBlueprint().getConfigurations(), getPropertyIds());
 

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java

@@ -34,6 +34,7 @@ import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorHelper;
 import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest;
 import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest.StackAdvisorRequestBuilder;
 import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest.StackAdvisorRequestType;
+import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse;
 import org.apache.ambari.server.controller.AmbariManagementController;
 import org.apache.ambari.server.controller.spi.Request;
 import org.apache.ambari.server.controller.spi.Resource.Type;
@@ -68,6 +69,10 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi
   private static final String BINDING_HOST_GROUPS_HOSTS_PROPERTY = "hosts";
   private static final String BINDING_HOST_GROUPS_HOSTS_NAME_PROPERTY = "fqdn";
 
+  private static final String CONFIG_GROUPS_PROPERTY = "recommendations/config_groups";
+  private static final String CONFIG_GROUPS_CONFIGURATIONS_PROPERTY = "configurations";
+  private static final String CONFIG_GROUPS_HOSTS_PROPERTY = "hosts";
+
   protected static StackAdvisorHelper saHelper;
 
   @Inject
@@ -108,12 +113,14 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi
         requestType == StackAdvisorRequestType.CONFIGURATION_DEPENDENCIES ?
           calculateChangedConfigurations(request) : Collections.<PropertyDependencyInfo>emptyList();
 
+      Set<RecommendationResponse.ConfigGroup> configGroups = calculateConfigGroups(request);
       return StackAdvisorRequestBuilder.
         forStack(stackName, stackVersion).ofType(requestType).forHosts(hosts).
         forServices(services).forHostComponents(hgComponentsMap).
         forHostsGroupBindings(hgHostsMap).
         withComponentHostsMap(componentHostsMap).
         withConfigurations(configurations).
+        withConfigGroups(configGroups).
         withChangedConfigurations(changedConfigurations).build();
     } catch (Exception e) {
       LOG.warn("Error occured during preparation of stack advisor request", e);
@@ -198,6 +205,40 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi
     return configs;
   }
 
+  protected Set<RecommendationResponse.ConfigGroup> calculateConfigGroups(Request request) {
+
+    Set<RecommendationResponse.ConfigGroup> configGroups =
+      new HashSet<RecommendationResponse.ConfigGroup>();
+
+    Set<HashMap<String, Object>> configGroupsProperties =
+      (HashSet<HashMap<String, Object>>) getRequestProperty(request, CONFIG_GROUPS_PROPERTY);
+    if (configGroupsProperties != null) {
+      for (HashMap<String, Object> props : configGroupsProperties) {
+        RecommendationResponse.ConfigGroup configGroup = new RecommendationResponse.ConfigGroup();
+        configGroup.setHosts((List<String>) props.get(CONFIG_GROUPS_HOSTS_PROPERTY));
+
+        for (Map<String, String> property : (Set<Map<String, String>>) props.get(CONFIG_GROUPS_CONFIGURATIONS_PROPERTY)) {
+          for (Map.Entry<String, String> entry : property.entrySet()) {
+            String[] propertyPath = entry.getKey().split("/"); // length == 3
+            String siteName = propertyPath[0];
+            String propertyName = propertyPath[2];
+
+            if (!configGroup.getConfigurations().containsKey(siteName)) {
+              RecommendationResponse.BlueprintConfigurations configurations =
+                new RecommendationResponse.BlueprintConfigurations();
+              configGroup.getConfigurations().put(siteName, configurations);
+              configGroup.getConfigurations().get(siteName).setProperties(new HashMap<String, String>());
+            }
+            configGroup.getConfigurations().get(siteName).getProperties().put(propertyName, entry.getValue());
+          }
+        }
+        configGroups.add(configGroup);
+      }
+    }
+
+    return configGroups;
+  }
+
   protected static final String CONFIGURATIONS_PROPERTY_ID = "recommendations/blueprint/configurations/";
 
   protected Map<String, Map<String, Map<String, String>>> calculateConfigurations(Request request) {

ambari-server/src/main/resources/properties.json

@@ -390,7 +390,8 @@
         "recommendations/blueprint_cluster_binding",
         "recommendations/blueprint_cluster_binding/host_groups",
         "recommendations/blueprint_cluster_binding/host_groups/name",
-        "recommendations/blueprint_cluster_binding/host_groups/hosts"
+        "recommendations/blueprint_cluster_binding/host_groups/hosts",
+        "recommendations/config_groups"
     ],
     "Validation":[
         "Validation/id",

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py

@@ -435,7 +435,6 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
                   collectorHostName, str(", ".join(hostMasterComponents[collectorHostName]))))
 
             # Not enough physical memory
-            # TODO Add AMBARI_METRICS Collector Xmx property to ams-env
             requiredMemory = getMemorySizeRequired(hostComponents[collectorHostName], configurations)
             if host["Hosts"]["total_mem"] * 1024 < requiredMemory:  # in bytes
               message = "Not enough total RAM on the host {0}, " \

ambari-server/src/main/resources/stacks/stack_advisor.py

@@ -473,6 +473,64 @@ class DefaultStackAdvisor(StackAdvisor):
   def getConfigurationsValidationItems(self, services, hosts):
     return []
 
+  def recommendConfigGroupsConfigurations(self, recommendations, services, components, hosts,
+                            servicesList):
+    recommendations["recommendations"]["config-groups"] = []
+    for configGroup in services["config-groups"]:
+
+      # Override configuration with the config group values
+      cgServices = services.copy()
+      for configName in configGroup["configurations"].keys():
+        if configName in cgServices["configurations"]:
+          cgServices["configurations"][configName]["properties"].update(
+            configGroup["configurations"][configName]['properties'])
+        else:
+          cgServices["configurations"][configName] = \
+          configGroup["configurations"][configName]
+
+      # Override hosts with the config group hosts
+      cgHosts = {"items": [host for host in hosts["items"] if
+                           host["Hosts"]["host_name"] in configGroup["hosts"]]}
+
+      # Override clusterSummary
+      cgClusterSummary = self.getConfigurationClusterSummary(servicesList,
+                                                             cgHosts,
+                                                             components,
+                                                             cgServices)
+
+      configurations = {}
+
+      for service in servicesList:
+        calculation = self.getServiceConfigurationRecommender(service)
+        if calculation is not None:
+          calculation(configurations, cgClusterSummary, cgServices, cgHosts)
+
+      cgRecommendation = {
+        "configurations": {},
+        "dependent_configurations": {},
+        "hosts": configGroup["hosts"]
+      }
+
+      recommendations["recommendations"]["config-groups"].append(
+        cgRecommendation)
+
+      # Parse results.
+      for config in configurations.keys():
+        cgRecommendation["configurations"][config] = {}
+        cgRecommendation["dependent_configurations"][config] = {}
+        # property + property_attributes
+        for configElement in configurations[config].keys():
+          cgRecommendation["configurations"][config][configElement] = {}
+          cgRecommendation["dependent_configurations"][config][
+            configElement] = {}
+          for property, value in configurations[config][configElement].items():
+            if config in configGroup["configurations"]:
+              cgRecommendation["configurations"][config][configElement][
+                property] = value
+            else:
+              cgRecommendation["dependent_configurations"][config][
+                configElement][property] = value
+
   def recommendConfigurations(self, services, hosts):
     stackName = services["Versions"]["stack_name"]
     stackVersion = services["Versions"]["stack_version"]
@@ -499,12 +557,17 @@ class DefaultStackAdvisor(StackAdvisor):
       }
     }
 
-    configurations = recommendations["recommendations"]["blueprint"]["configurations"]
+    # If recommendation for config groups
+    if "config-groups" in services:
+      self.recommendConfigGroupsConfigurations(recommendations, services, components, hosts,
+                                 servicesList)
+    else:
+      configurations = recommendations["recommendations"]["blueprint"]["configurations"]
 
-    for service in servicesList:
-      calculation = self.getServiceConfigurationRecommender(service)
-      if calculation is not None:
-        calculation(configurations, clusterSummary, services, hosts)
+      for service in servicesList:
+        calculation = self.getServiceConfigurationRecommender(service)
+        if calculation is not None:
+          calculation(configurations, clusterSummary, services, hosts)
 
     return recommendations
 
@@ -624,30 +687,36 @@ class DefaultStackAdvisor(StackAdvisor):
   # returns recommendations only for changed and depended properties
   def filterResult(self, result, services):
     allRequestedProperties = self.getAllRequestedProperties(services)
+    self.filterConfigs(result['recommendations']['blueprint']['configurations'], allRequestedProperties)
+    if "config-groups" in services:
+      for configGroup in result['recommendations']["config-groups"]:
+        self.filterConfigs(configGroup["configurations"], allRequestedProperties)
+        self.filterConfigs(configGroup["dependent_configurations"], allRequestedProperties)
+    return result
+
+  def filterConfigs(self, configs, requestedProperties):
 
-    configs = result['recommendations']['blueprint']['configurations']
     filteredConfigs = {}
     for type, names in configs.items():
       for name in names['properties']:
-        if type in allRequestedProperties.keys() and \
-                name in allRequestedProperties[type]:
+        if type in requestedProperties.keys() and \
+                name in requestedProperties[type]:
           if type not in filteredConfigs.keys():
             filteredConfigs[type] = {'properties': {}}
           filteredConfigs[type]['properties'][name] = \
             configs[type]['properties'][name]
       if 'property_attributes' in names.keys():
         for name in names['property_attributes']:
-          if type in allRequestedProperties.keys() and \
-                  name in allRequestedProperties[type]:
+          if type in requestedProperties.keys() and \
+                  name in requestedProperties[type]:
             if type not in filteredConfigs.keys():
               filteredConfigs[type] = {'property_Attributes': {}}
             elif 'property_attributes' not in filteredConfigs[type].keys():
               filteredConfigs[type]['property_attributes'] = {}
             filteredConfigs[type]['property_attributes'][name] = \
               configs[type]['property_attributes'][name]
-
-    result['recommendations']['blueprint']['configurations'] = filteredConfigs
-    return result
+    configs.clear()
+    configs.update(filteredConfigs)
 
   def getAllRequestedProperties(self, services):
     affectedConfigs = self.getAffectedConfigs(services)

ambari-server/src/test/python/TestStackAdvisor.py

@@ -339,3 +339,39 @@ class TestStackAdvisorInitialization(TestCase):
       }
     }
     self.assertEquals(actualRecommendLayoutResponse, expectedRecommendLayoutResponse)
+
+    # Config groups support by default
+    services["config-groups"] = [{
+      "configurations": {
+      },
+      "hosts": [
+        'host2'
+      ]
+    }]
+
+    actualConfigGroupRecommendConfigResponse = \
+      default_stack_advisor.recommendConfigurations(services, hosts)
+    expectedConfigGroupRecommendConfigResponse = {
+      "Versions": {"stack_name": "HDP1", "stack_version": "2.0.6"},
+      "hosts": ["host1", "host2"],
+      "services": ['GANGLIA', 'HBASE', 'HDFS', 'PIG', 'TEZ', 'ZOOKEEPER'],
+      "recommendations": {
+        'config-groups': [
+          {
+            'configurations': {},
+            'dependent_configurations': {},
+            'hosts': [
+              'host2'
+            ]
+          }
+        ],
+        "blueprint": {
+          "configurations": {},
+          "host_groups": []
+        },
+        "blueprint_cluster_binding": {
+          "host_groups": []
+        }
+      }
+    }
+    self.assertEquals(actualConfigGroupRecommendConfigResponse, expectedConfigGroupRecommendConfigResponse)

ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py

@@ -1686,3 +1686,4 @@ class TestHDP22StackAdvisor(TestCase):
 
     res = self.stackAdvisor.validateMapReduce2Configurations(properties, recommendedDefaults, {}, '', '')
     self.assertEquals(res, res_expected)
+

ambari-web/app/config.js

@@ -73,7 +73,7 @@ App.supports = {
   opsDuringRollingUpgrade: false,
   customizedWidgets: false,
   customizedWidgetLayout: false,
-  enhancedConfigs: false
+  enhancedConfigs: true
 };
 
 if (App.enableExperimental) {