AMBARI-6959. Stacks service API: configTypes filed should return contained configuration files for the service (aonishuk)

ambari-server/src/main/java/org/apache/ambari/server/api/util/StackExtensionHelper.java

@@ -24,6 +24,7 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.ListIterator;
@@ -143,12 +144,6 @@ public class StackExtensionHelper {
             parentService.getConfigDependencies() != null ?
                 parentService.getConfigDependencies() :
                 Collections.<String>emptyList());
-    mergedServiceInfo.setConfigTypes(
-        childService.getConfigTypes() != null ?
-            childService.getConfigTypes() :
-            parentService.getConfigTypes() != null ?
-                parentService.getConfigTypes() :
-                Collections.<String, Map<String, Map<String, String>>>emptyMap());
     mergedServiceInfo.setExcludedConfigTypes(
       childService.getExcludedConfigTypes() != null ?
         childService.getExcludedConfigTypes() :
@@ -179,7 +174,15 @@ public class StackExtensionHelper {
     } else {
       mergedServiceInfo.setOsSpecifics(parentService.getOsSpecifics());
     }
-
+    
+    mergedServiceInfo.setConfigTypes(new HashMap<String, Map<String, Map<String, String>>>());
+    if(childService.getConfigTypes() != null) {
+      mergedServiceInfo.getConfigTypes().putAll(childService.getConfigTypes());
+    }
+    if(parentService.getConfigTypes() != null) {
+      mergedServiceInfo.getConfigTypes().putAll(parentService.getConfigTypes());
+    }
+    
     CommandScriptDefinition commandScript = childService.getCommandScript();
     if (commandScript != null) {
        mergedServiceInfo.setCommandScript(childService.getCommandScript());
@@ -421,8 +424,21 @@ public class StackExtensionHelper {
           serviceInfoMap.put(service.getName(), newServiceInfo);
         }
         
-        // add action for service check
+        // remove 'excluded-config-types' from configTypes
         ServiceInfo serviceInfo = serviceInfoMap.get(service.getName());
+        if(serviceInfo.getExcludedConfigTypes() != null) { 
+          Iterator<Map.Entry<String,Map<String,Map<String,String>>>> configTypesItetator = serviceInfo.getConfigTypes().entrySet().iterator();
+          
+          while(configTypesItetator.hasNext()) {
+            Map.Entry<String,Map<String,Map<String,String>>> configTypeMap = configTypesItetator.next();
+            
+            if(serviceInfo.getExcludedConfigTypes().contains(configTypeMap.getKey())) {
+              configTypesItetator.remove();
+            }
+          }
+        }
+        
+        // add action for service check
         if(serviceInfo.getCommandScript() != null) {
           actionMetadata.addServiceCheckAction(serviceInfo.getName());
         }
@@ -503,7 +519,6 @@ public class StackExtensionHelper {
           List<ServiceInfo> serviceInfos = smiv2x.getServices();
           for (ServiceInfo serviceInfo : serviceInfos) {
             serviceInfo.setSchemaVersion(AmbariMetaInfo.SCHEMA_VERSION_2);
-            populateConfigTypesFromDependencies(serviceInfo);
 
             // Find service package folder
             String servicePackageDir = resolveServicePackageFolder(
@@ -706,6 +721,12 @@ public class StackExtensionHelper {
     serviceInfo.getProperties().addAll(getProperties(configuration, fileName));
     int extIndex = fileName.indexOf(AmbariMetaInfo.SERVICE_CONFIG_FILE_NAME_POSTFIX);
     String configType = fileName.substring(0, extIndex);
+   
+    addConfigType(serviceInfo, configType);
+    setConfigTypeAttributes(serviceInfo, configuration, configType);
+  }
+  
+  void setConfigTypeAttributes(ServiceInfo serviceInfo, ConfigurationXml configuration, String configType) {
     for (Map.Entry<QName, String> attribute : configuration.getAttributes().entrySet()) {
       for (Supports supportsProperty : Supports.values()) {
         String attributeName = attribute.getKey().getLocalPart();
@@ -717,23 +738,43 @@ public class StackExtensionHelper {
       }
     }
   }
+  
+  void addConfigType(ServiceInfo serviceInfo, String configType) {
+    if(serviceInfo.getConfigTypes() == null) {
+      serviceInfo.setConfigTypes(new HashMap<String, Map<String, Map<String, String>>>());
+    }
+    
+    Map<String, Map<String, Map<String, String>>> configTypes = serviceInfo.getConfigTypes();
+    configTypes.put(configType, new HashMap<String, Map<String, String>>());
+    
+    
+    Map<String, Map<String, String>> properties = configTypes.get(configType);
+    Map<String, String> supportsProperties = new HashMap<String, String>();
+    for (Supports supportsProperty : Supports.values()) {
+      supportsProperties.put(supportsProperty.getPropertyName(), supportsProperty.getDefaultValue());
+    }
+    properties.put(Supports.KEYWORD, supportsProperties); 
+  }
 
   /**
    * Populate ServiceInfo#configTypes with default entries based on ServiceInfo#configDependencies property
    */
   void populateConfigTypesFromDependencies(ServiceInfo serviceInfo) {
-    List<String> configDependencies = serviceInfo.getConfigDependenciesWithComponents();
-    if (configDependencies != null) {
+    List<PropertyInfo> configurations = serviceInfo.getProperties();
+    if (configurations != null) {
       Map<String, Map<String, Map<String, String>>> configTypes = new HashMap<String, Map<String, Map<String, String>>>();
-      for (String configDependency : configDependencies) {
-        if (!configTypes.containsKey(configDependency)) {
+      for (PropertyInfo configuration : configurations) {
+        int extIndex = configuration.getFilename().indexOf(AmbariMetaInfo.SERVICE_CONFIG_FILE_NAME_POSTFIX);
+        String configType = configuration.getFilename().substring(0, extIndex);
+        
+        if (!configTypes.containsKey(configType)) {
           Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
           Map<String, String> supportsProperties = new HashMap<String, String>();
           for (Supports supportsProperty : Supports.values()) {
             supportsProperties.put(supportsProperty.getPropertyName(), supportsProperty.getDefaultValue());
           }
           properties.put(Supports.KEYWORD, supportsProperties);
-          configTypes.put(configDependency, properties);
+          configTypes.put(configType, properties);
         }
       }
       serviceInfo.setConfigTypes(configTypes);

ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/configuration/core-site.xml

@@ -1,20 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-<configuration supports_final="true">
-</configuration>

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration-mapred/core-site.xml

@@ -1,20 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-<configuration supports_final="true">
-</configuration>

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration/core-site.xml

@@ -1,20 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-<configuration supports_final="true">
-</configuration>

ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/configuration/core-site.xml

@@ -1,20 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-<configuration supports_final="true">
-</configuration>

ambari-server/src/test/java/org/apache/ambari/server/api/util/StackExtensionHelperTest.java

@@ -23,6 +23,7 @@ import org.apache.ambari.server.metadata.ActionMetadata;
 import org.apache.ambari.server.state.*;
 
 import java.io.File;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -36,12 +37,16 @@ import static org.junit.Assert.*;
 
 import org.apache.ambari.server.state.stack.ConfigurationXml;
 import org.junit.Test;
+import org.xml.sax.SAXException;
 
 import com.google.inject.AbstractModule;
 import com.google.inject.Guice;
 import com.google.inject.Injector;
 
+import javax.xml.bind.JAXBException;
 import javax.xml.namespace.QName;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.xpath.XPathExpressionException;
 
 public class StackExtensionHelperTest {
 
@@ -244,7 +249,7 @@ public class StackExtensionHelperTest {
     for (ServiceInfo serviceInfo : allServices) {
       if (serviceInfo.getName().equals("HDFS")) {
         assertEquals(5, serviceInfo.getConfigDependencies().size());
-        assertEquals(5, serviceInfo.getConfigTypes().size());
+        assertEquals(4, serviceInfo.getConfigTypes().size());
         assertTrue(serviceInfo.getConfigDependencies().contains("core-site"));
         assertTrue(serviceInfo.getConfigDependencies().contains("global"));
         assertTrue(serviceInfo.getConfigDependencies().contains("hdfs-site"));
@@ -280,31 +285,6 @@ public class StackExtensionHelperTest {
       }
     }
   }
-  
-  @Test
-  public void testrequiredServicesPropertyInheritance() throws Exception{
-    File stackRoot = new File(stackRootStr);
-    StackInfo stackInfo = new StackInfo();
-    stackInfo.setName("HDP");
-    stackInfo.setVersion("2.0.7");
-    StackExtensionHelper helper = new StackExtensionHelper(injector, stackRoot);
-    helper.populateServicesForStack(stackInfo);
-    helper.fillInfo();
-    List<ServiceInfo> allServices = helper.getAllApplicableServices(stackInfo);
-    assertEquals(13, allServices.size());
-    
-    List<String> expectedRequiredServices = new ArrayList<String>();
-    expectedRequiredServices.add("HDFS");
-    expectedRequiredServices.add("TEZ");
-    
-    for (ServiceInfo serviceInfo : allServices) {
-      if (serviceInfo.getName().equals("HBASE")) {
-        assertTrue(serviceInfo.getRequiredServices().equals(expectedRequiredServices));
-      } else {
-        assertTrue((serviceInfo.getRequiredServices() == null || serviceInfo.getRequiredServices().isEmpty()));
-      }
-    }
-  }
 
   @Test
   public void getSchemaVersion() throws Exception {
@@ -320,60 +300,37 @@ public class StackExtensionHelperTest {
     version = helper.getSchemaVersion(v2MetaInfoFile);
     assertEquals("2.0", version);
   }
-
-  @Test
-  public void testPopulateConfigTypes() {
-    File stackRoot = new File(stackRootStr);
-    StackExtensionHelper helper = new StackExtensionHelper(injector, stackRoot);
-    List<String> configDependencies = Arrays.asList("dep1", "dep2");
-    ServiceInfo serviceInfo = new ServiceInfo();
-    serviceInfo.setConfigDependencies(configDependencies);
-    helper.populateConfigTypesFromDependencies(serviceInfo);
-
-    Map<String, Map<String, Map<String, String>>> configTypes = serviceInfo.getConfigTypes();
-    assertEquals(2, configTypes.size());
-    assertTrue(configTypes.containsKey("dep1"));
-    assertTrue(configTypes.containsKey("dep2"));
-    Map<String, Map<String, String>> properties;
-    properties= configTypes.get("dep1");
-    assertEquals(1, properties.size());
-    assertTrue(properties.containsKey("supports"));
-    assertEquals(1, properties.get("supports").size());
-    assertTrue(properties.get("supports").containsKey("final"));
-    assertEquals("false", properties.get("supports").get("final"));
-    properties= configTypes.get("dep2");
-    assertEquals(1, properties.size());
-    assertTrue(properties.containsKey("supports"));
-    assertEquals(1, properties.get("supports").size());
-    assertTrue(properties.get("supports").containsKey("final"));
-    assertEquals("false", properties.get("supports").get("final"));
+  
+  public StackExtensionHelper getStackExtensionHelper() {
+    File stackRoot = new File(stackRootStr);    
+    return new StackExtensionHelper(injector, stackRoot);
   }
-
-  @Test
-  public void testPopulateConfigTypes_emptyList() {
-    File stackRoot = new File(stackRootStr);
-    StackExtensionHelper helper = new StackExtensionHelper(injector, stackRoot);
-    List<String> configDependencies = Collections.emptyList();
-    ServiceInfo serviceInfo = new ServiceInfo();
-    serviceInfo.setConfigDependencies(configDependencies);
-    helper.populateConfigTypesFromDependencies(serviceInfo);
-
-    Map<String, Map<String, Map<String, String>>> configTypes = serviceInfo.getConfigTypes();
-    assertNotNull(configTypes);
-    assertEquals(0, configTypes.size());
+  
+  public ServiceInfo getServiceFromStack(StackExtensionHelper helper, String stackName, String stackVersion, String serviceName) throws XPathExpressionException, ParserConfigurationException, SAXException, IOException, JAXBException {
+    StackInfo stackInfo = new StackInfo();
+    stackInfo.setName(stackName);
+    stackInfo.setVersion(stackVersion);
+    
+    helper.populateServicesForStack(stackInfo);
+    
+    for(ServiceInfo service:stackInfo.getServices()) {
+      if(service.getName().equals(serviceName)) {
+        return service;
+      }
+    }
+    return null;
   }
 
   @Test
-  public void testPopulateConfigTypes_null() {
-    File stackRoot = new File(stackRootStr);
-    StackExtensionHelper helper = new StackExtensionHelper(injector, stackRoot);
-    List<String> configDependencies = null;
-    ServiceInfo serviceInfo = new ServiceInfo();
-    serviceInfo.setConfigDependencies(configDependencies);
-    helper.populateConfigTypesFromDependencies(serviceInfo);
+  public void testPopulateConfigTypes() throws XPathExpressionException, ParserConfigurationException, SAXException, IOException, JAXBException {
+    StackExtensionHelper helper = getStackExtensionHelper();
+    ServiceInfo serviceInfo = getServiceFromStack(helper, "HDP", "2.0.7", "HDFS");
+    String expectedConfigTypes = 
+        "{global={supports={final=true}}, hdfs-site={supports={final=false}}, hadoop-policy={supports={final=false}}, core-site={supports={final=false}}}";
 
     Map<String, Map<String, Map<String, String>>> configTypes = serviceInfo.getConfigTypes();
-    assertNull(configTypes);
+    assertEquals(4, configTypes.size());
+    assertEquals(expectedConfigTypes, configTypes.toString());
   }
 
   @Test
@@ -472,81 +429,38 @@ public class StackExtensionHelperTest {
 
   @Test
   public void testPopulateServiceProperties_noSupportsFinalFlag() throws Exception {
-    // init
-    File stackRoot = new File(stackRootStr);
-    StackExtensionHelper helper = createMockBuilder(StackExtensionHelper.class).addMockedMethod("addConfigTypeProperty")
-        .withConstructor(injector, stackRoot).createMock();
-    File config = new File(stackRootStr
+    StackExtensionHelper helper = getStackExtensionHelper();
+    ServiceInfo serviceInfo = getServiceFromStack(helper, "HDP", "2.0.7", "YARN");
+    
+    File configFile = new File(stackRootStr
         + "HDP/2.0.7/services/YARN/configuration/yarn-site.xml".replaceAll("/", File.separator));
-    ServiceInfo serviceInfo = createNiceMock(ServiceInfo.class);
-    List<PropertyInfo> properties = createNiceMock(List.class);
-
-    // expectations
-    expect(serviceInfo.getProperties()).andReturn(properties).times(1);
-    expect(properties.addAll((Collection) anyObject())).andReturn(true).times(1);
-    replay(properties);
-    replay(serviceInfo);
-    replay(helper);
-
-    // eval
-    helper.populateServiceProperties(config, serviceInfo);
-
-    // verification
-    verify(properties, serviceInfo, helper);
+    
+    helper.populateServiceProperties(configFile, serviceInfo);
+    
+    assertEquals("{yarn-site={supports={final=false}}}", serviceInfo.getConfigTypes().toString());
   }
 
   @Test
   public void testPopulateServiceProperties_supportsFinalTrue() throws Exception {
-    // init
-    File stackRoot = new File(stackRootStr);
-    StackExtensionHelper helper = createMockBuilder(StackExtensionHelper.class).addMockedMethod("addConfigTypeProperty")
-        .withConstructor(injector, stackRoot).createMock();
-    File config = new File(stackRootStr
+    StackExtensionHelper helper = getStackExtensionHelper();
+    ServiceInfo serviceInfo = getServiceFromStack(helper, "HDP", "2.0.7", "HDFS");
+    
+    File configFile = new File(stackRootStr
         + "HDP/2.0.7/services/HDFS/configuration/global.xml".replaceAll("/", File.separator));
-    ServiceInfo serviceInfo = createNiceMock(ServiceInfo.class);
-    List<PropertyInfo> properties = createNiceMock(List.class);
-
-    // expectations
-    expect(serviceInfo.getProperties()).andReturn(properties).times(1);
-    expect(properties.addAll((Collection) anyObject())).andReturn(true).times(1);
-    helper.addConfigTypeProperty(serviceInfo, "global", StackExtensionHelper.Supports.KEYWORD,
-        StackExtensionHelper.Supports.FINAL.getPropertyName(), "true");
-    replay(properties);
-    replay(serviceInfo);
-    replay(helper);
-
-    // eval
-    helper.populateServiceProperties(config, serviceInfo);
-
-    // verification
-    verify(properties, serviceInfo, helper);
+    helper.populateServiceProperties(configFile, serviceInfo);
+    
+    assertEquals("{global={supports={final=true}}, hdfs-site={supports={final=false}}, hadoop-policy={supports={final=false}}, core-site={supports={final=false}}}", serviceInfo.getConfigTypes().toString());
   }
 
   @Test
   public void testPopulateServiceProperties_supportsFinalFalse() throws Exception {
-    // init
-    File stackRoot = new File(stackRootStr);
-    StackExtensionHelper helper = createMockBuilder(StackExtensionHelper.class).addMockedMethod("addConfigTypeProperty")
-        .withConstructor(injector, stackRoot).createMock();
-    File config = new File(stackRootStr
-        + "HDP/2.0.7/services/HDFS/configuration/core-site.xml".replaceAll("/", File.separator));
-    ServiceInfo serviceInfo = createNiceMock(ServiceInfo.class);
-    List<PropertyInfo> properties = createNiceMock(List.class);
-
-    // expectations
-    expect(serviceInfo.getProperties()).andReturn(properties).times(1);
-    expect(properties.addAll((Collection) anyObject())).andReturn(true).times(1);
-    helper.addConfigTypeProperty(serviceInfo, "core-site", StackExtensionHelper.Supports.KEYWORD,
-        StackExtensionHelper.Supports.FINAL.getPropertyName(), "false");
-    replay(properties);
-    replay(serviceInfo);
-    replay(helper);
-
-    // eval
-    helper.populateServiceProperties(config, serviceInfo);
-
-    // verification
-    verify(properties, serviceInfo, helper);
+    StackExtensionHelper helper = getStackExtensionHelper();
+    ServiceInfo serviceInfo = getServiceFromStack(helper, "HDP", "2.0.7", "HDFS");
+    File configFile = new File(stackRootStr
+        + "HDP/2.0.7/services/YARN/configuration/yarn-site.xml".replaceAll("/", File.separator));
+    helper.populateServiceProperties(configFile, serviceInfo);
+    
+    assertEquals("{global={supports={final=true}}, hdfs-site={supports={final=false}}, yarn-site={supports={final=false}}, hadoop-policy={supports={final=false}}, core-site={supports={final=false}}}", serviceInfo.getConfigTypes().toString());
   }
 
   @Test
@@ -561,6 +475,7 @@ public class StackExtensionHelperTest {
     List<PropertyInfo> properties = createNiceMock(List.class);
 
     // expectations
+    expect(serviceInfo.getConfigTypes()).andReturn(new HashMap<String, Map<String, Map<String, String>>>()).times(2);
     expect(serviceInfo.getProperties()).andReturn(properties).times(1);
     expect(properties.addAll((Collection) anyObject())).andReturn(true).times(1);
     helper.addConfigTypeProperty(serviceInfo, "yarn-site", StackExtensionHelper.Supports.KEYWORD,
@@ -587,6 +502,7 @@ public class StackExtensionHelperTest {
     List<PropertyInfo> properties = createNiceMock(List.class);
 
     // expectations
+    expect(serviceInfo.getConfigTypes()).andReturn(new HashMap<String, Map<String, Map<String, String>>>()).times(2);
     expect(serviceInfo.getProperties()).andReturn(properties).times(1);
     expect(properties.addAll((Collection) anyObject())).andReturn(true).times(1);
     expect(serviceInfo.getConfigTypes()).andReturn(null).times(1);