AMBARI-9097. Deploying Falcon via a Blueprint fails. (rnettleton)

ambari-server/src/main/java/org/apache/ambari/server/controller/StackServiceResponse.java

@@ -26,6 +26,7 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 public class StackServiceResponse {
 
@@ -40,6 +41,9 @@ public class StackServiceResponse {
   private List<String> customCommands;
 
   private Map<String, Map<String, Map<String, String>>> configTypes;
+
+  private Set<String> excludedConfigTypes;
+
   private List<String> requiredServices;
 
   /**
@@ -62,6 +66,7 @@ public class StackServiceResponse {
     comments = service.getComment();
     serviceVersion = service.getVersion();
     configTypes = service.getConfigTypeAttributes();
+    excludedConfigTypes = service.getExcludedConfigTypes();
     requiredServices = service.getRequiredServices();
     serviceCheckSupported = null != service.getCommandScript();
 
@@ -138,6 +143,10 @@ public class StackServiceResponse {
   public Map<String, Map<String, Map<String, String>>> getConfigTypes() {
     return configTypes;
   }
+
+  public Set<String> getExcludedConfigTypes() {
+    return excludedConfigTypes;
+  }
   
   public List<String> getRequiredServices() {
     return requiredServices;

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java

@@ -808,23 +808,27 @@ public class ClusterResourceProvider extends BaseBlueprintProcessor {
    *
    * @throws SystemException an unexpected exception occurred
    */
-  private void setConfigurationsOnCluster(String clusterName, Stack stack, Map<String, HostGroupImpl> blueprintHostGroups) throws SystemException {
+  void setConfigurationsOnCluster(String clusterName, Stack stack, Map<String, HostGroupImpl> blueprintHostGroups) throws SystemException {
     List<BlueprintServiceConfigRequest> listofConfigRequests =
       new LinkedList<BlueprintServiceConfigRequest>();
 
     // create a list of config requests on a per-service basis, in order
     // to properly support the new service configuration versioning mechanism
     // in Ambari
-    Collection<String> encounteredConfigTypes = new HashSet<String>();
     for (String service : getServicesToDeploy(stack, blueprintHostGroups)) {
       BlueprintServiceConfigRequest blueprintConfigRequest =
         new BlueprintServiceConfigRequest(service);
 
       for (String serviceConfigType : stack.getConfigurationTypes(service)) {
-        //todo: This is a temporary fix to ensure that we don't try to add the same
-        //todo: config type multiple times.
-        //todo: This is to unblock BUG-28939 and will be correctly fixed as part of BUG-29145.
-        if (encounteredConfigTypes.add(serviceConfigType)) {
+        Set<String> excludedConfigTypes = stack.getExcludedConfigurationTypes(service);
+        if (excludedConfigTypes == null) {
+          // if the service does not have excluded config types
+          // associated, then treat this as an empty set
+          excludedConfigTypes = Collections.emptySet();
+        }
+
+        // only include config types that are not excluded, re-introducing fix from AMBARI-8009
+        if (!excludedConfigTypes.contains(serviceConfigType)) {
           // skip handling of cluster-env here
           if (!serviceConfigType.equals("cluster-env")) {
             if (mapClusterConfigurations.containsKey(serviceConfigType)) {

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java

@@ -99,6 +99,12 @@ class Stack {
   private Map<String, Map<String, Map<String, ConfigProperty>>> serviceConfigurations =
       new HashMap<String, Map<String, Map<String, ConfigProperty>>>();
 
+  /**
+   * Map of service to set of excluded config types
+   */
+  private Map<String, Set<String>> excludedConfigurationTypes =
+    new HashMap<String, Set<String>>();
+
   /**
    * Ambari Management Controller, used to obtain Stack definitions
    */
@@ -154,6 +160,7 @@ class Stack {
     for (StackServiceResponse stackService : stackServices) {
       String serviceName = stackService.getServiceName();
       parseComponents(serviceName);
+      parseExcludedConfigurations(stackService);
       parseConfigurations(serviceName);
       registerConditionalDependencies();
     }
@@ -213,6 +220,18 @@ class Stack {
     return serviceConfigurations.get(service).keySet();
   }
 
+  /**
+   * Get the set of excluded configuration types
+   *   for this service
+   *
+   * @param service service name
+   *
+   * @return Set of names of excluded config types
+   */
+  public Set<String> getExcludedConfigurationTypes(String service) {
+    return excludedConfigurationTypes.get(service);
+  }
+
   /**
    * Get config properties for the specified service and configuration type.
    *
@@ -425,6 +444,15 @@ class Stack {
     }
   }
 
+  /**
+   * Obtain the excluded configuration types from the StackServiceResponse
+   *
+   * @param stackServiceResponse the response object associated with this stack service
+   */
+  private void parseExcludedConfigurations(StackServiceResponse stackServiceResponse) {
+    excludedConfigurationTypes.put(stackServiceResponse.getServiceName(), stackServiceResponse.getExcludedConfigTypes());
+  }
+
   /**
    * Register conditional dependencies.
    */

ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java

@@ -636,6 +636,7 @@ public class BlueprintResourceProviderTest {
     expect(stackServiceResponse.getServiceName()).andReturn("test-service").anyTimes();
     expect(stackServiceResponse.getStackName()).andReturn("test-stack-name").anyTimes();
     expect(stackServiceResponse.getStackVersion()).andReturn("test-stack-version").anyTimes();
+    expect(stackServiceResponse.getExcludedConfigTypes()).andReturn(Collections.<String>emptySet());
 
     expect(managementController.getStackComponents(capture(serviceComponentRequestCapture))).andReturn(setServiceComponents).anyTimes();
     expect(stackServiceComponentResponse.getCardinality()).andReturn("2").anyTimes();
@@ -746,6 +747,7 @@ public class BlueprintResourceProviderTest {
     expect(stackServiceResponse.getServiceName()).andReturn("test-service").anyTimes();
     expect(stackServiceResponse.getStackName()).andReturn("test-stack-name").anyTimes();
     expect(stackServiceResponse.getStackVersion()).andReturn("test-stack-version").anyTimes();
+    expect(stackServiceResponse.getExcludedConfigTypes()).andReturn(Collections.<String>emptySet());
 
     expect(managementController.getStackComponents(capture(serviceComponentRequestCapture))).andReturn(setServiceComponents).anyTimes();
     expect(stackServiceComponentResponse.getCardinality()).andReturn("2").anyTimes();
@@ -864,6 +866,7 @@ public class BlueprintResourceProviderTest {
     expect(stackServiceResponse.getServiceName()).andReturn("test-service").anyTimes();
     expect(stackServiceResponse.getStackName()).andReturn("test-stack-name").anyTimes();
     expect(stackServiceResponse.getStackVersion()).andReturn("test-stack-version").anyTimes();;
+    expect(stackServiceResponse.getExcludedConfigTypes()).andReturn(Collections.<String>emptySet());
 
     expect(managementController.getStackComponents(capture(serviceComponentRequestCapture))).andReturn(setServiceComponents).anyTimes();
     expect(stackServiceComponentResponse.getCardinality()).andReturn("2").anyTimes();
@@ -971,6 +974,7 @@ public class BlueprintResourceProviderTest {
     expect(stackServiceResponse.getServiceName()).andReturn("test-service").anyTimes();
     expect(stackServiceResponse.getStackName()).andReturn("test-stack-name").anyTimes();
     expect(stackServiceResponse.getStackVersion()).andReturn("test-stack-version").anyTimes();
+    expect(stackServiceResponse.getExcludedConfigTypes()).andReturn(Collections.<String>emptySet());
 
     expect(managementController.getStackComponents(capture(serviceComponentRequestCapture))).andReturn(setServiceComponents).anyTimes();
     expect(stackServiceComponentResponse.getCardinality()).andReturn("2").anyTimes();

ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterResourceProviderTest.java

@@ -35,6 +35,7 @@ import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
@@ -2963,6 +2964,7 @@ public class ClusterResourceProviderTest {
 
 
     expect(mockStackServiceResponseOne.getServiceName()).andReturn("OOZIE").atLeastOnce();
+    expect(mockStackServiceResponseOne.getExcludedConfigTypes()).andReturn(Collections.<String>emptySet()).atLeastOnce();
     expect(mockManagementController.getStackServices(isA(Set.class))).andReturn(Collections.singleton(mockStackServiceResponseOne));
     expect(mockManagementController.getStackComponents(isA(Set.class))).andReturn(Collections.singleton(mockStackComponentResponse));
     expect(mockManagementController.getStackConfigurations(isA(Set.class))).andReturn(Collections.<StackConfigurationResponse>emptySet());
@@ -3046,6 +3048,7 @@ public class ClusterResourceProviderTest {
 
 
     expect(mockStackServiceResponseOne.getServiceName()).andReturn("FALCON").atLeastOnce();
+    expect(mockStackServiceResponseOne.getExcludedConfigTypes()).andReturn(Collections.<String>emptySet()).atLeastOnce();
     expect(mockManagementController.getStackServices(isA(Set.class))).andReturn(Collections.singleton(mockStackServiceResponseOne));
     expect(mockManagementController.getStackComponents(isA(Set.class))).andReturn(Collections.singleton(mockStackComponentResponse));
     expect(mockManagementController.getStackConfigurations(isA(Set.class))).andReturn(Collections.<StackConfigurationResponse>emptySet());
@@ -3127,6 +3130,7 @@ public class ClusterResourceProviderTest {
 
 
     expect(mockStackServiceResponseOne.getServiceName()).andReturn("OOZIE").atLeastOnce();
+    expect(mockStackServiceResponseOne.getExcludedConfigTypes()).andReturn(Collections.<String>emptySet()).atLeastOnce();
     expect(mockManagementController.getStackServices(isA(Set.class))).andReturn(Collections.singleton(mockStackServiceResponseOne));
     expect(mockManagementController.getStackComponents(isA(Set.class))).andReturn(Collections.singleton(mockStackComponentResponse));
     expect(mockManagementController.getStackConfigurations(isA(Set.class))).andReturn(Collections.<StackConfigurationResponse>emptySet());
@@ -3203,6 +3207,7 @@ public class ClusterResourceProviderTest {
     expect(mockStackComponentResponse.getAutoDeploy()).andReturn(new AutoDeployInfo());
 
     expect(mockStackServiceResponseOne.getServiceName()).andReturn("FALCON").atLeastOnce();
+    expect(mockStackServiceResponseOne.getExcludedConfigTypes()).andReturn(Collections.<String>emptySet()).atLeastOnce();
 
     expect(mockManagementController.getStackServices(isA(Set.class))).andReturn(Collections.singleton(mockStackServiceResponseOne));
     expect(mockManagementController.getStackComponents(isA(Set.class))).andReturn(Collections.singleton(mockStackComponentResponse));
@@ -3280,6 +3285,7 @@ public class ClusterResourceProviderTest {
     expect(mockStackComponentResponse.getAutoDeploy()).andReturn(new AutoDeployInfo());
 
     expect(mockStackServiceResponseOne.getServiceName()).andReturn("HIVE").atLeastOnce();
+    expect(mockStackServiceResponseOne.getExcludedConfigTypes()).andReturn(Collections.<String>emptySet()).atLeastOnce();
     expect(mockManagementController.getStackServices(isA(Set.class))).andReturn(Collections.singleton(mockStackServiceResponseOne));
     expect(mockManagementController.getStackComponents(isA(Set.class))).andReturn(Collections.singleton(mockStackComponentResponse));
     expect(mockManagementController.getStackConfigurations(isA(Set.class))).andReturn(Collections.<StackConfigurationResponse>emptySet());
@@ -3356,6 +3362,7 @@ public class ClusterResourceProviderTest {
     expect(mockStackComponentResponse.getAutoDeploy()).andReturn(new AutoDeployInfo());
 
     expect(mockStackServiceResponseOne.getServiceName()).andReturn("HBASE").atLeastOnce();
+    expect(mockStackServiceResponseOne.getExcludedConfigTypes()).andReturn(Collections.<String>emptySet()).atLeastOnce();
 
     expect(mockManagementController.getStackServices(isA(Set.class))).andReturn(Collections.singleton(mockStackServiceResponseOne));
     expect(mockManagementController.getStackComponents(isA(Set.class))).andReturn(Collections.singleton(mockStackComponentResponse));
@@ -3402,6 +3409,215 @@ public class ClusterResourceProviderTest {
 
   }
 
+  @Test
+  public void testSetConfigurationsOnClusterWithExcludedTypes() throws Exception {
+    EasyMockSupport mockSupport = new EasyMockSupport();
+    AmbariManagementController mockMgmtController =
+      mockSupport.createMock(AmbariManagementController.class);
+    ResourceProvider mockServiceProvider =
+      mockSupport.createMock(ResourceProvider.class);
+    ResourceProvider mockComponentProvider =
+      mockSupport.createMock(ResourceProvider.class);
+    ResourceProvider mockHostProvider =
+      mockSupport.createMock(ResourceProvider.class);
+    ResourceProvider mockHostComponentProvider =
+      mockSupport.createMock(ResourceProvider.class);
+    ResourceProvider mockConfigGroupProvider =
+      mockSupport.createMock(ResourceProvider.class);
+    Stack mockStack =
+      mockSupport.createMock(Stack.class);
+    BaseBlueprintProcessor.HostGroupImpl mockHostGroupOne =
+      mockSupport.createMock(BaseBlueprintProcessor.HostGroupImpl.class);
+
+    ArrayList<Capture<Set<ClusterRequest>>> listOfRequestCaptures =
+      new ArrayList<Capture<Set<ClusterRequest>>>();
+    for (int i = 0; i < 2; i++) {
+      listOfRequestCaptures.add(new Capture<Set<ClusterRequest>>());
+    }
+
+    ArrayList<Capture<Map<String, String>>> listOfPropertiesCaptures =
+      new ArrayList<Capture<Map<String, String>>>();
+    for (int i = 0; i < 2; i++) {
+      listOfPropertiesCaptures.add(new Capture<Map<String, String>>());
+    }
+
+    expect(mockHostGroupOne.getHostInfo()).andReturn(Collections.singleton("c6401.ambari.apache.org")).atLeastOnce();
+    expect(mockHostGroupOne.getComponents()).andReturn(Arrays.asList("FALCON_SERVER", "FALCON_CLIENT"));
+    expect(mockStack.getServicesForComponents(Arrays.asList("FALCON_SERVER", "FALCON_CLIENT")))
+      .andReturn(Arrays.asList("FALCON")).atLeastOnce();
+    expect(mockStack.getConfigurationTypes("FALCON")).andReturn(Arrays.asList("falcon-site", "falcon-env", "oozie-site")).atLeastOnce();
+    // configure falcon to include a single excluded config type
+    expect(mockStack.getExcludedConfigurationTypes("FALCON")).andReturn(Collections.<String>singleton("oozie-site")).atLeastOnce();
+
+    // setup expectations for controller.updateClusters() calls
+    for (int i = 0; i < 2; i++) {
+      expect(mockMgmtController.updateClusters(capture(listOfRequestCaptures.get(i)), capture(listOfPropertiesCaptures.get(i)))).andReturn(null);
+    }
+
+    Map<String, BaseBlueprintProcessor.HostGroupImpl> testMapOfHostGroups =
+      new HashMap<String, BaseBlueprintProcessor.HostGroupImpl>();
+    testMapOfHostGroups.put("host-group-one", mockHostGroupOne);
+
+    mockSupport.replayAll();
+
+    ClusterResourceProvider clusterResourceProvider =
+      new TestClusterResourceProvider(mockMgmtController, mockServiceProvider,
+        mockComponentProvider, mockHostProvider, mockHostComponentProvider, mockConfigGroupProvider);
+
+    Map<String, Map<String, String>> clusterConfig =
+      clusterResourceProvider.getClusterConfigurations();
+    clusterConfig.put("falcon-site", Collections.singletonMap("key1", "value1"));
+    clusterConfig.put("falcon-env", Collections.singletonMap("envKey1", "envValue1"));
+    clusterConfig.put("oozie-site", Collections.singletonMap("oozie-key-one", "oozie-value-one"));
+    clusterConfig.put("cluster-env", Collections.<String, String>emptyMap());
+
+    // call the method being tested
+    clusterResourceProvider.setConfigurationsOnCluster("clusterone", mockStack, testMapOfHostGroups);
+
+    // verify that the ClusterRequest's passed to the controller include the expected information
+    for (Capture<Set<ClusterRequest>> requestCapture : listOfRequestCaptures) {
+      Set<ClusterRequest> request = requestCapture.getValue();
+      assertEquals("Incorrect number of cluster requests in this update",
+                   1, request.size());
+    }
+
+
+    for (Capture<Map<String, String>> propertiesCapture : listOfPropertiesCaptures) {
+      assertNull("Incorrect request properties sent with this update",
+                 propertiesCapture.getValue());
+    }
+
+    // verify that the config requests include the expected information
+    ClusterRequest requestOne = listOfRequestCaptures.get(0).getValue().iterator().next();
+    ClusterRequest requestTwo = listOfRequestCaptures.get(1).getValue().iterator().next();
+
+    if (requestOne.getDesiredConfig().size() == 1) {
+      verifyClusterRequest(requestOne, "cluster-env");
+      // verify that the falcon config does not include oozie-site, since it is excluded
+      verifyClusterRequest(requestTwo, "falcon-site", "falcon-env");
+    } else {
+      verifyClusterRequest(requestTwo, "cluster-env");
+      // verify that the falcon config does not include oozie-site, since it is excluded
+      verifyClusterRequest(requestOne, "falcon-site", "falcon-env");
+    }
+
+    mockSupport.verifyAll();
+  }
+
+  @Test
+  public void testSetConfigurationsOnClusterWithNoExcludedTypes() throws Exception {
+    EasyMockSupport mockSupport = new EasyMockSupport();
+    AmbariManagementController mockMgmtController =
+      mockSupport.createMock(AmbariManagementController.class);
+    ResourceProvider mockServiceProvider =
+      mockSupport.createMock(ResourceProvider.class);
+    ResourceProvider mockComponentProvider =
+      mockSupport.createMock(ResourceProvider.class);
+    ResourceProvider mockHostProvider =
+      mockSupport.createMock(ResourceProvider.class);
+    ResourceProvider mockHostComponentProvider =
+      mockSupport.createMock(ResourceProvider.class);
+    ResourceProvider mockConfigGroupProvider =
+      mockSupport.createMock(ResourceProvider.class);
+    Stack mockStack =
+      mockSupport.createMock(Stack.class);
+    BaseBlueprintProcessor.HostGroupImpl mockHostGroupOne =
+      mockSupport.createMock(BaseBlueprintProcessor.HostGroupImpl.class);
+
+    ArrayList<Capture<Set<ClusterRequest>>> listOfRequestCaptures =
+      new ArrayList<Capture<Set<ClusterRequest>>>();
+    for (int i = 0; i < 2; i++) {
+      listOfRequestCaptures.add(new Capture<Set<ClusterRequest>>());
+    }
+
+    ArrayList<Capture<Map<String, String>>> listOfPropertiesCaptures =
+      new ArrayList<Capture<Map<String, String>>>();
+    for (int i = 0; i < 2; i++) {
+      listOfPropertiesCaptures.add(new Capture<Map<String, String>>());
+    }
+
+    expect(mockHostGroupOne.getHostInfo()).andReturn(Collections.singleton("c6401.ambari.apache.org")).atLeastOnce();
+    expect(mockHostGroupOne.getComponents()).andReturn(Arrays.asList("FALCON_SERVER", "FALCON_CLIENT"));
+    expect(mockStack.getServicesForComponents(Arrays.asList("FALCON_SERVER", "FALCON_CLIENT")))
+      .andReturn(Arrays.asList("FALCON")).atLeastOnce();
+    expect(mockStack.getConfigurationTypes("FALCON")).andReturn(Arrays.asList("falcon-site", "falcon-env", "oozie-site")).atLeastOnce();
+    // configure falcon to NOT have any excluded types
+    expect(mockStack.getExcludedConfigurationTypes("FALCON")).andReturn(Collections.<String>emptySet()).atLeastOnce();
+
+    // setup expectations for controller.updateClusters() calls
+    for (int i = 0; i < 2; i++) {
+      expect(mockMgmtController.updateClusters(capture(listOfRequestCaptures.get(i)), capture(listOfPropertiesCaptures.get(i)))).andReturn(null);
+    }
+
+    Map<String, BaseBlueprintProcessor.HostGroupImpl> testMapOfHostGroups =
+      new HashMap<String, BaseBlueprintProcessor.HostGroupImpl>();
+    testMapOfHostGroups.put("host-group-one", mockHostGroupOne);
+
+    mockSupport.replayAll();
+
+    ClusterResourceProvider clusterResourceProvider =
+      new TestClusterResourceProvider(mockMgmtController, mockServiceProvider,
+        mockComponentProvider, mockHostProvider, mockHostComponentProvider, mockConfigGroupProvider);
+
+    Map<String, Map<String, String>> clusterConfig =
+      clusterResourceProvider.getClusterConfigurations();
+
+    clusterConfig.put("falcon-site", Collections.singletonMap("key1", "value1"));
+    clusterConfig.put("falcon-env", Collections.singletonMap("envKey1", "envValue1"));
+    clusterConfig.put("oozie-site", Collections.singletonMap("oozie-key-one", "oozie-value-one"));
+    clusterConfig.put("cluster-env", Collections.<String, String>emptyMap());
+
+    // call the method being tested
+    clusterResourceProvider.setConfigurationsOnCluster("clusterone", mockStack, testMapOfHostGroups);
+
+    // verify that the ClusterRequest's passed to the controller include the expected information
+    for (Capture<Set<ClusterRequest>> requestCapture : listOfRequestCaptures) {
+      Set<ClusterRequest> request = requestCapture.getValue();
+      assertEquals("Incorrect number of cluster requests in this update",
+        1, request.size());
+    }
+
+    for (Capture<Map<String, String>> propertiesCapture : listOfPropertiesCaptures) {
+      assertNull("Incorrect request properties sent with this update",
+        propertiesCapture.getValue());
+    }
+
+    // verify that the config requests include the expected information
+    ClusterRequest requestOne = listOfRequestCaptures.get(0).getValue().iterator().next();
+    ClusterRequest requestTwo = listOfRequestCaptures.get(1).getValue().iterator().next();
+
+    if (requestOne.getDesiredConfig().size() == 1) {
+      verifyClusterRequest(requestOne, "cluster-env");
+      // verify that the falcon config includes oozie-site, since nothing is excluded in this test
+      verifyClusterRequest(requestTwo, "falcon-site", "falcon-env", "oozie-site");
+    } else {
+      verifyClusterRequest(requestTwo, "cluster-env");
+      // verify that the falcon config includes oozie-site, since nothing is excluded in this test
+      verifyClusterRequest(requestOne, "falcon-site", "falcon-env", "oozie-site");
+    }
+
+    mockSupport.verifyAll();
+  }
+
+  private static void verifyClusterRequest(ClusterRequest request, String... expectedConfigTypes) throws Exception {
+    assertEquals("Incorrect number of cluster requests ",
+                 expectedConfigTypes.length, request.getDesiredConfig().size());
+
+    Set<String> foundConfigTypes = new HashSet<String>();
+    // build set of config types listed in this request
+    for (ConfigurationRequest configRequest : request.getDesiredConfig()) {
+      foundConfigTypes.add(configRequest.getType());
+    }
+
+    // verify that the expected types are found
+    for (String expectedType : expectedConfigTypes) {
+      assertTrue("Expected config type not found in this config request",
+                 foundConfigTypes.contains(expectedType));
+    }
+
+  }
+
+
 
   private class TestClusterResourceProvider extends ClusterResourceProvider {