AMBARI-14745. Ambari server throws error when unused configs are present in blueprint. (Oliver Szabo via rnettleton)

ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java

@@ -30,6 +30,7 @@ import org.apache.ambari.server.serveraction.kerberos.KerberosInvalidConfigurati
 import org.apache.ambari.server.state.Cluster;
 import org.apache.ambari.server.state.SecurityType;
 import org.apache.ambari.server.utils.StageUtils;
+import org.apache.commons.collections.MapUtils;
 import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -74,11 +75,38 @@ public class ClusterConfigurationRequest {
     // set initial configuration (not topology resolved)
     this.configurationProcessor = new BlueprintConfigurationProcessor(clusterTopology);
     this.stackAdvisorBlueprintProcessor = stackAdvisorBlueprintProcessor;
+    removeOrphanConfigTypes(clusterTopology);
     if (setInitial) {
       setConfigurationsOnCluster(clusterTopology, TopologyManager.INITIAL_CONFIG_TAG, Collections.<String>emptySet());
     }
   }
 
+  /**
+   * Remove config-types, if there is no any services related to them (except cluster-env and global).
+   */
+  private void removeOrphanConfigTypes(ClusterTopology clusterTopology) {
+    Configuration configuration = clusterTopology.getConfiguration();
+    Collection<String> configTypes = configuration.getAllConfigTypes();
+    for (String configType : configTypes) {
+      if (!configType.equals("cluster-env") && !configType.equals("global")) {
+        String service = clusterTopology.getBlueprint().getStack().getServiceForConfigType(configType);
+        if (!clusterTopology.getBlueprint().getServices().contains(service)) {
+          configuration.removeConfigType(configType);
+          LOG.info("Not found any service for config type '{}'. It will be removed from configuration.", configType);
+          Map<String, HostGroupInfo> hostGroupInfoMap = clusterTopology.getHostGroupInfo();
+          if (MapUtils.isNotEmpty(hostGroupInfoMap)) {
+            for (Map.Entry<String, HostGroupInfo> hostGroupInfo : hostGroupInfoMap.entrySet()) {
+              if (hostGroupInfo.getValue().getConfiguration() != null) {
+                hostGroupInfo.getValue().getConfiguration().removeConfigType(configType);
+                LOG.info("Not found any service for config type '{}'. It will be removed from host group scoped configuration.", configType);
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+
   public ClusterConfigurationRequest(AmbariContext ambariContext, ClusterTopology topology, boolean setInitial, StackAdvisorBlueprintProcessor stackAdvisorBlueprintProcessor, boolean configureSecurity) {
     this(ambariContext, topology, setInitial, stackAdvisorBlueprintProcessor);
     this.configureSecurity = configureSecurity;

ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java

@@ -344,4 +344,19 @@ public class Configuration {
   public void setParentConfiguration(Configuration parent) {
     parentConfiguration = parent;
   }
+
+  /**
+   * Remove all occurrences of a config type
+   */
+  public void removeConfigType(String configType) {
+    if (properties != null && properties.containsKey(configType)) {
+      properties.remove(configType);
+    }
+    if (attributes != null && attributes.containsKey(configType)) {
+      attributes.remove(configType);
+    }
+    if (parentConfiguration != null) {
+      parentConfiguration.removeConfigType(configType);
+    }
+  }
 }

ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java

@@ -28,6 +28,17 @@ import org.apache.ambari.server.controller.internal.Stack;
 import org.apache.ambari.server.serveraction.kerberos.KerberosInvalidConfigurationException;
 import org.apache.ambari.server.state.Cluster;
 import org.apache.ambari.server.state.Clusters;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import com.google.common.collect.Maps;
+import org.easymock.EasyMock;
 import org.easymock.Capture;
 import org.easymock.CaptureType;
 import org.easymock.EasyMockRule;
@@ -40,14 +51,6 @@ import org.powermock.api.easymock.PowerMock;
 import org.powermock.core.classloader.annotations.PrepareForTest;
 import org.powermock.modules.junit4.PowerMockRunner;
 
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
 import static org.easymock.EasyMock.anyBoolean;
 import static org.easymock.EasyMock.anyObject;
 import static org.easymock.EasyMock.anyString;
@@ -56,6 +59,8 @@ import static org.easymock.EasyMock.expectLastCall;
 import static org.easymock.EasyMock.eq;
 import static org.easymock.EasyMock.newCapture;
 import static org.easymock.EasyMock.verify;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 import static org.easymock.EasyMock.capture;
 import static org.junit.Assert.assertEquals;
 
@@ -184,6 +189,7 @@ public class ClusterConfigurationRequestTest {
     expect(clusters.getCluster("testCluster")).andReturn(cluster).anyTimes();
 
     expect(blueprint.getStack()).andReturn(stack).anyTimes();
+    expect(stack.getServiceForConfigType(anyString())).andReturn("KERBEROS").anyTimes();
     expect(stack.getAllConfigurationTypes(anyString())).andReturn(Collections.<String>singletonList("testConfigType")
     ).anyTimes();
     expect(stack.getExcludedConfigurationTypes(anyString())).andReturn(Collections.<String>emptySet()).anyTimes();
@@ -308,4 +314,57 @@ public class ClusterConfigurationRequestTest {
 
   }
 
+  @Test
+  public void testProcessClusterConfigRequestRemoveUnusedConfigTypes() {
+    // GIVEN
+    Configuration configuration = createConfigurations();
+    Set<String> services = new HashSet<String>();
+    services.add("HDFS");
+    services.add("RANGER");
+    Map<String, HostGroupInfo> hostGroupInfoMap = Maps.newHashMap();
+    HostGroupInfo hg1 = new HostGroupInfo("hg1");
+    hg1.setConfiguration(createConfigurations());
+    hostGroupInfoMap.put("hg1", hg1);
+
+    expect(topology.getConfiguration()).andReturn(configuration).anyTimes();
+    expect(topology.getBlueprint()).andReturn(blueprint).anyTimes();
+    expect(topology.getHostGroupInfo()).andReturn(hostGroupInfoMap);
+    expect(blueprint.getStack()).andReturn(stack).anyTimes();
+    expect(blueprint.getServices()).andReturn(services).anyTimes();
+    expect(stack.getServiceForConfigType("hdfs-site")).andReturn("HDFS").anyTimes();
+    expect(stack.getServiceForConfigType("admin-properties")).andReturn("RANGER").anyTimes();
+    expect(stack.getServiceForConfigType("yarn-site")).andReturn("YARN").anyTimes();
+
+    EasyMock.replay(stack, blueprint, topology);
+    // WHEN
+    new ClusterConfigurationRequest(ambariContext, topology, false, stackAdvisorBlueprintProcessor);
+    // THEN
+    assertFalse(configuration.getFullProperties().containsKey("yarn-site"));
+    assertFalse(configuration.getFullAttributes().containsKey("yarn-site"));
+    assertTrue(configuration.getFullAttributes().containsKey("hdfs-site"));
+    assertTrue(configuration.getFullProperties().containsKey("cluster-env"));
+    assertTrue(configuration.getFullProperties().containsKey("global"));
+    assertFalse(hg1.getConfiguration().getFullAttributes().containsKey("yarn-site"));
+    verify(stack, blueprint, topology);
+  }
+
+  private Configuration createConfigurations() {
+    Map<String, Map<String, String>> firstLevelConfig = Maps.newHashMap();
+    firstLevelConfig.put("hdfs-site", new HashMap<String, String>());
+    firstLevelConfig.put("yarn-site", new HashMap<String, String>());
+    firstLevelConfig.put("cluster-env", new HashMap<String, String>());
+    firstLevelConfig.put("global", new HashMap<String, String>());
+
+    Map<String, Map<String, Map<String, String>>> firstLevelAttributes = Maps.newHashMap();
+    firstLevelAttributes.put("hdfs-site", new HashMap<String, Map<String, String>>());
+
+    Map<String, Map<String, String>> secondLevelConfig = Maps.newHashMap();
+    secondLevelConfig.put("admin-properties", new HashMap<String, String>());
+    Map<String, Map<String, Map<String, String>>> secondLevelAttributes = Maps.newHashMap();
+    secondLevelAttributes.put("admin-properties", new HashMap<String, Map<String, String>>());
+    secondLevelAttributes.put("yarn-site", new HashMap<String, Map<String, String>>());
+
+    Configuration secondLevelConf = new Configuration(secondLevelConfig, secondLevelAttributes);
+    return new Configuration(firstLevelConfig, firstLevelAttributes, secondLevelConf);
+  }
 }

ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java

@@ -251,6 +251,8 @@ public class ClusterInstallWithoutStartTest {
     expect(stack.getConfiguration()).andReturn(stackConfig).anyTimes();
     expect(stack.getName()).andReturn(STACK_NAME).anyTimes();
     expect(stack.getVersion()).andReturn(STACK_VERSION).anyTimes();
+    expect(stack.getServiceForConfigType("service1-site")).andReturn("service1");
+    expect(stack.getServiceForConfigType("service2-site")).andReturn("service2");
     expect(stack.getExcludedConfigurationTypes("service1")).andReturn(Collections.<String>emptySet()).anyTimes();
     expect(stack.getExcludedConfigurationTypes("service2")).andReturn(Collections.<String>emptySet()).anyTimes();
 

ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurationTest.java

@@ -346,6 +346,20 @@ public class ConfigurationTest {
     assertNull(configuration.getPropertyValue("type1", "XXXXX"));
   }
 
+  @Test
+  public void testRemoveConfigTypes() {
+    Configuration configuration = createConfigurationWithParents_PropsOnly();
+    configuration.removeConfigType("type1");
+    assertNull(configuration.getProperties().get("type1"));
+  }
+
+  @Test
+  public void testRemoveConfigTypesForAttributes() {
+    Configuration configuration = createConfigurationWithParents_PropsOnly();
+    configuration.removeConfigType("type1");
+    assertNull(configuration.getAttributes().get("type1"));
+  }
+
   private Configuration createConfigurationWithParents_PropsOnly() {
     // parents parent config properties
     Map<String, Map<String, String>> parentParentProperties = new HashMap<String, Map<String, String>>();

ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java

@@ -228,6 +228,8 @@ public class TopologyManagerTest {
     expect(stack.getComponents()).andReturn(serviceComponents).anyTimes();
     expect(stack.getComponents("service1")).andReturn(serviceComponents.get("service1")).anyTimes();
     expect(stack.getComponents("service2")).andReturn(serviceComponents.get("service2")).anyTimes();
+    expect(stack.getServiceForConfigType("service1-site")).andReturn("service1");
+    expect(stack.getServiceForConfigType("service2-site")).andReturn("service2");
     expect(stack.getConfiguration()).andReturn(stackConfig).anyTimes();
     expect(stack.getName()).andReturn(STACK_NAME).anyTimes();
     expect(stack.getVersion()).andReturn(STACK_VERSION).anyTimes();