AMBARI-10316 Stack-advisor mechanism when recommending configuration-dependencies has incorrect services.json content (dsen)

ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java

@@ -1224,42 +1224,4 @@ public class AmbariMetaInfo {
     return kerberosServiceDescriptors;
   }
 
-  /**
-   * Get set of all the depended-by properties down to directed acyclic graph(DAG)
-   * of dependencies between configuration properties
-   * @param stackName the stack name
-   * @param stackVersion the stack version
-   * @param changedConfigs the list of changed configurations
-   * @return set of all depended-by properties including all changedConfigs
-   */
-  public Set<PropertyDependencyInfo> getDependedByProperties(String stackName,
-                                                             String stackVersion,
-                                                             List<PropertyDependencyInfo> changedConfigs) {
-    StackInfo stack = getStackManager().getStack(stackName, stackVersion);
-
-    if (changedConfigs == null) {
-      return Collections.emptySet();
-    }
-    int size = 0;
-    Set<PropertyDependencyInfo> configs =
-      new HashSet<PropertyDependencyInfo>();
-
-    configs.addAll(changedConfigs);
-
-    while (size != configs.size()) {
-      size = configs.size();
-      for (ServiceInfo service: stack.getServices()) {
-        for (PropertyInfo pi: service.getProperties()) {
-          String type = ConfigHelper.fileNameToConfigType(pi.getFilename());
-          String name = pi.getName();
-          PropertyDependencyInfo dep =
-            new PropertyDependencyInfo(type, name);
-          if (configs.contains(dep) && !configs.containsAll(pi.getDependedByProperties())) {
-            configs.addAll(pi.getDependedByProperties());
-          }
-        }
-      }
-    }
-    return configs;
-  }
 }

ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java

@@ -71,10 +71,13 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend
 
   private static final String GET_HOSTS_INFO_URI = "/api/v1/hosts"
       + "?fields=Hosts&Hosts/host_name.in(%s)";
-  private static final String GET_SERVICES_INFO_URI = "/api/v1/stacks/%s/versions/%s"
+  private static final String GET_SERVICES_INFO_URI = "/api/v1/stacks/%s/versions/%s/"
       + "?fields=Versions/stack_name,Versions/stack_version,Versions/parent_stack_version"
       + ",services/StackServices/service_name,services/StackServices/service_version"
       + ",services/components/StackServiceComponents,services/components/dependencies,services/components/auto_deploy"
+      + ",services/configurations/StackConfigurations/property_depends_on"
+      + ",services/configurations/StackConfigurations/property_depended_by"
+      + ",services/configurations/StackConfigurations/type"
       + "&services/StackServices/service_name.in(%s)";
   private static final String SERVICES_PROPERTY = "services";
   private static final String SERVICES_COMPONENTS_PROPERTY = "components";
@@ -83,7 +86,6 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend
   private static final String COMPONENT_HOSTNAMES_PROPERTY = "hostnames";
   private static final String CONFIGURATIONS_PROPERTY = "configurations";
   private static final String CHANGED_CONFIGURATIONS_PROPERTY = "changed-configurations";
-  private static final String DEPENDED_CONFIGURATIONS_PROPERTY = "depended-configurations";
 
   private File recommendationsDir;
   private String stackAdvisorScript;
@@ -174,15 +176,9 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend
         }
       }
     }
-    // Populating changed configs info and depended by configs info
-    Set<PropertyDependencyInfo> dependedProperties =
-      metaInfo.getDependedByProperties(request.getStackName(),
-      request.getStackVersion(), request.getChangedConfigurations());
 
     JsonNode changedConfigs = mapper.valueToTree(request.getChangedConfigurations());
-    JsonNode dependendConfigs = mapper.valueToTree(dependedProperties);
     root.put(CHANGED_CONFIGURATIONS_PROPERTY, changedConfigs);
-    root.put(DEPENDED_CONFIGURATIONS_PROPERTY, dependendConfigs);
   }
 
   protected void populateStackHierarchy(ObjectNode root) {

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackLevelConfigurationResourceProvider.java

@@ -55,13 +55,13 @@ public class StackLevelConfigurationResourceProvider extends
       .getPropertyId("StackLevelConfigurations", "property_value");
 
   public static final String PROPERTY_VALUE_ATTRIBUTES_PROPERTY_ID = PropertyHelper
-      .getPropertyId("StackConfigurations", "property_value_attributes");
+      .getPropertyId("StackLevelConfigurations", "property_value_attributes");
 
   public static final String DEPENDS_ON_PROPERTY_ID = PropertyHelper
-      .getPropertyId("StackConfigurations", "property_depends_on");
+      .getPropertyId("StackLevelConfigurations", "property_depends_on");
 
   public static final String DEPENDED_BY_PROPERTY_ID = PropertyHelper
-      .getPropertyId("StackConfigurations", "property_depended_by");
+      .getPropertyId("StackLevelConfigurations", "property_depended_by");
 
   public static final String PROPERTY_DESCRIPTION_PROPERTY_ID = PropertyHelper
       .getPropertyId("StackLevelConfigurations", "property_description");

ambari-server/src/main/resources/scripts/stack_advisor.py

@@ -51,7 +51,7 @@ def loadJson(path):
     with open(path, 'r') as f:
       return json.load(f)
   except Exception, err:
-    raise StackAdvisorException("File not found at: {0}".format(hostsFile))
+    raise StackAdvisorException("File not found at: {0}".format(path))
 
 
 def dumpJson(json_object, dump_file):
@@ -107,9 +107,8 @@ def main(argv=None):
     result = stackAdvisor.recommendConfigurations(services, hosts)
     result_file = os.path.join(actionDir, "configurations.json")
   elif action == RECOMMEND_CONFIGURATION_DEPENDENCIES:
-    result = stackAdvisor.recommendConfigurations(services, hosts)
+    result = stackAdvisor.recommendConfigurationDependencies(services, hosts)
     result_file = os.path.join(actionDir, "configurations.json")
-    result = filterResult(result, services)
   else: # action == VALIDATE_CONFIGURATIONS
     result = stackAdvisor.validateConfigurations(services, hosts)
     result_file = os.path.join(actionDir, "configurations-validation.json")
@@ -117,45 +116,6 @@ def main(argv=None):
   dumpJson(result, result_file)
   pass
 
-# returns recommendations only for changed and depended properties
-def filterResult(result, services):
-  allRequestedProperties = getAllRequestedProperties(services)
-
-  configs = result['recommendations']['blueprint']['configurations']
-  filteredConfigs = {}
-  for type, names in configs.items():
-    for name in names['properties']:
-      if type in allRequestedProperties.keys() and \
-              name in allRequestedProperties[type]:
-        if type not in filteredConfigs.keys():
-          filteredConfigs[type] = {'properties': {}}
-        filteredConfigs[type]['properties'][name] = \
-          configs[type]['properties'][name]
-    if 'property_attributes' in names.keys():
-      for name in names['property_attributes']:
-        if type in allRequestedProperties.keys() and \
-                name in allRequestedProperties[type]:
-          if type not in filteredConfigs.keys():
-            filteredConfigs[type] = {'property_Attributes': {}}
-          elif 'property_attributes' not in filteredConfigs[type].keys():
-            filteredConfigs[type]['property_attributes'] = {}
-          filteredConfigs[type]['property_attributes'][name] = \
-            configs[type]['property_attributes'][name]
-
-  result['recommendations']['blueprint']['configurations'] = filteredConfigs
-  return result
-
-def getAllRequestedProperties(services):
-  changedConfigs = []
-  changedConfigs.extend(services['changed-configurations'])
-  changedConfigs.extend(services['depended-configurations'])
-  allRequestedProperties = {}
-  for config in changedConfigs:
-    if config['type'] in allRequestedProperties:
-      allRequestedProperties[config['type']].append(config['name'])
-    else:
-      allRequestedProperties[config['type']] = [config['name']]
-  return allRequestedProperties
 
 def instantiateStackAdvisor(stackName, stackVersion, parentVersions):
   """Instantiates StackAdvisor implementation for the specified Stack"""

ambari-server/src/main/resources/stacks/stack_advisor.py

@@ -18,6 +18,8 @@ limitations under the License.
 """
 
 import socket
+import re
+
 
 class StackAdvisor(object):
   """
@@ -613,4 +615,71 @@ class DefaultStackAdvisor(StackAdvisor):
         for component in service['components']:
           if component["StackServiceComponents"]["component_name"] == componentName:
             return component["StackServiceComponents"]["hostnames"]
-  pass
+  pass
+
+  def recommendConfigurationDependencies(self, services, hosts):
+    result = self.recommendConfigurations(services, hosts)
+    return self.filterResult(result, services)
+
+  # returns recommendations only for changed and depended properties
+  def filterResult(self, result, services):
+    allRequestedProperties = self.getAllRequestedProperties(services)
+
+    configs = result['recommendations']['blueprint']['configurations']
+    filteredConfigs = {}
+    for type, names in configs.items():
+      for name in names['properties']:
+        if type in allRequestedProperties.keys() and \
+                name in allRequestedProperties[type]:
+          if type not in filteredConfigs.keys():
+            filteredConfigs[type] = {'properties': {}}
+          filteredConfigs[type]['properties'][name] = \
+            configs[type]['properties'][name]
+      if 'property_attributes' in names.keys():
+        for name in names['property_attributes']:
+          if type in allRequestedProperties.keys() and \
+                  name in allRequestedProperties[type]:
+            if type not in filteredConfigs.keys():
+              filteredConfigs[type] = {'property_Attributes': {}}
+            elif 'property_attributes' not in filteredConfigs[type].keys():
+              filteredConfigs[type]['property_attributes'] = {}
+            filteredConfigs[type]['property_attributes'][name] = \
+              configs[type]['property_attributes'][name]
+
+    result['recommendations']['blueprint']['configurations'] = filteredConfigs
+    return result
+
+  def getAllRequestedProperties(self, services):
+    affectedConfigs = self.getAffectedConfigs(services)
+    allRequestedProperties = {}
+    for config in affectedConfigs:
+      if config['type'] in allRequestedProperties:
+        allRequestedProperties[config['type']].append(config['name'])
+      else:
+        allRequestedProperties[config['type']] = [config['name']]
+    return allRequestedProperties
+
+  def getAffectedConfigs(self, services):
+    """returns properties dict including changed-configurations and depended-by configs"""
+    changedConfigs = services['changed-configurations']
+    allDependencies = []
+
+    for item in services['services']:
+      allDependencies.extend(item['configurations'])
+
+    dependencies = []
+    dependencies.extend(changedConfigs)
+
+    size = 0
+    while size != len(dependencies):
+      size = len(dependencies)
+      for config in allDependencies:
+        type = re.sub('\.xml$', '', config['StackConfigurations']['type'])
+        name = config['StackConfigurations']['property_name']
+
+        if {"type": type, "name": name} in dependencies:
+          for dependedConfig in config['StackConfigurations']['property_depended_by']:
+            if dependedConfig not in dependencies:
+              dependencies.append(dependedConfig)
+
+    return  dependencies

ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java

@@ -818,18 +818,6 @@ public class AmbariMetaInfoTest {
       originalProperty.getDescription());
     Assert.assertEquals(6, redefinedService.getConfigDependencies().size());
     Assert.assertEquals(7, redefinedService.getConfigDependenciesWithComponents().size());
-
-    // Test directed-acyclic-graph (DAG) of dependencies between configurations
-    List<PropertyDependencyInfo> changedConfigs = new LinkedList<PropertyDependencyInfo>();
-    String type = ConfigHelper.fileNameToConfigType(newProperty.getFilename());
-    String name = newProperty.getName();
-    changedConfigs.add(new PropertyDependencyInfo(type, name));
-    Set<PropertyDependencyInfo> dependedByProperties = metaInfo.getDependedByProperties(stackInfo.getName(), stackInfo.getVersion(), changedConfigs);
-    Assert.assertEquals(3, dependedByProperties.size());
-    Assert.assertTrue(dependedByProperties.contains(new PropertyDependencyInfo("yarn-site", "new-enhanced-yarn-property2")));
-    Assert.assertTrue(dependedByProperties.contains(new PropertyDependencyInfo("yarn-site", "new-enhanced-yarn-property")));
-    Assert.assertTrue(dependedByProperties.contains(new PropertyDependencyInfo("yarn-site", "new-yarn-property")));
-
   }
 
   @Test

ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py

@@ -778,45 +778,9 @@ class TestHDP22StackAdvisor(TestCase):
           "type": "yarn-env",
           "name": "min_user_id"
         },
-      ],
-      "depended-configurations": [
-        {
-          "type" : "mapred-site",
-          "name" : "yarn.app.mapreduce.am.admin-command-opts"
-        }, {
-          "type" : "yarn-site",
-          "name" : "yarn.scheduler.maximum-allocation-mb"
-        }, {
-          "type" : "yarn-site",
-          "name" : "yarn.scheduler.minimum-allocation-mb"
-        }, {
-          "type" : "mapred-site",
-          "name" : "mapreduce.reduce.java.opts"
-        }, {
-          "type" : "mapred-site",
-          "name" : "mapreduce.map.java.opts"
-        }, {
-          "type" : "mapred-site",
-          "name" : "yarn.app.mapreduce.am.command-opts"
-        }, {
-          "type" : "mapred-site",
-          "name" : "yarn.app.mapreduce.am.resource.mb"
-        }, {
-          "type" : "yarn-site",
-          "name" : "yarn.nodemanager.resource.memory-mb"
-        }, {
-          "type" : "mapred-site",
-          "name" : "mapreduce.task.io.sort.mb"
-        }, {
-          "type" : "mapred-site",
-          "name" : "mapreduce.reduce.memory.mb"
-        }, {
-          "type" : "mapred-site",
-          "name" : "mapreduce.map.memory.mb"
-        }
       ]
 
-      }
+    }
     hosts = {
       "items" : [
         {