Revert "AMBARI-6959. Stacks service API: configTypes filed should return contained configuration files for the service (aonishuk)"

This reverts commit f2d91b20fd61777b594311cbcf20343ee14b074e.

ambari-server/src/main/java/org/apache/ambari/server/api/util/StackExtensionHelper.java

@@ -24,7 +24,6 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.ListIterator;
@@ -144,6 +143,12 @@ public class StackExtensionHelper {
             parentService.getConfigDependencies() != null ?
                 parentService.getConfigDependencies() :
                 Collections.<String>emptyList());
+    mergedServiceInfo.setConfigTypes(
+        childService.getConfigTypes() != null ?
+            childService.getConfigTypes() :
+            parentService.getConfigTypes() != null ?
+                parentService.getConfigTypes() :
+                Collections.<String, Map<String, Map<String, String>>>emptyMap());
     mergedServiceInfo.setExcludedConfigTypes(
       childService.getExcludedConfigTypes() != null ?
         childService.getExcludedConfigTypes() :
@@ -174,15 +179,7 @@ public class StackExtensionHelper {
     } else {
       mergedServiceInfo.setOsSpecifics(parentService.getOsSpecifics());
     }
-    
-    mergedServiceInfo.setConfigTypes(new HashMap<String, Map<String, Map<String, String>>>());
-    if(childService.getConfigTypes() != null) {
-      mergedServiceInfo.getConfigTypes().putAll(childService.getConfigTypes());
-    }
-    if(parentService.getConfigTypes() != null) {
-      mergedServiceInfo.getConfigTypes().putAll(parentService.getConfigTypes());
-    }
-    
+
     CommandScriptDefinition commandScript = childService.getCommandScript();
     if (commandScript != null) {
        mergedServiceInfo.setCommandScript(childService.getCommandScript());
@@ -424,21 +421,8 @@ public class StackExtensionHelper {
           serviceInfoMap.put(service.getName(), newServiceInfo);
         }
         
-        // remove 'excluded-config-types' from configTypes
-        ServiceInfo serviceInfo = serviceInfoMap.get(service.getName());
-        if(serviceInfo.getExcludedConfigTypes() != null) { 
-          Iterator<Map.Entry<String,Map<String,Map<String,String>>>> configTypesItetator = serviceInfo.getConfigTypes().entrySet().iterator();
-          
-          while(configTypesItetator.hasNext()) {
-            Map.Entry<String,Map<String,Map<String,String>>> configTypeMap = configTypesItetator.next();
-            
-            if(serviceInfo.getExcludedConfigTypes().contains(configTypeMap.getKey())) {
-              configTypesItetator.remove();
-            }
-          }
-        }
-        
         // add action for service check
+        ServiceInfo serviceInfo = serviceInfoMap.get(service.getName());
         if(serviceInfo.getCommandScript() != null) {
           actionMetadata.addServiceCheckAction(serviceInfo.getName());
         }
@@ -519,6 +503,7 @@ public class StackExtensionHelper {
           List<ServiceInfo> serviceInfos = smiv2x.getServices();
           for (ServiceInfo serviceInfo : serviceInfos) {
             serviceInfo.setSchemaVersion(AmbariMetaInfo.SCHEMA_VERSION_2);
+            populateConfigTypesFromDependencies(serviceInfo);
 
             // Find service package folder
             String servicePackageDir = resolveServicePackageFolder(
@@ -721,12 +706,6 @@ public class StackExtensionHelper {
     serviceInfo.getProperties().addAll(getProperties(configuration, fileName));
     int extIndex = fileName.indexOf(AmbariMetaInfo.SERVICE_CONFIG_FILE_NAME_POSTFIX);
     String configType = fileName.substring(0, extIndex);
-   
-    addConfigType(serviceInfo, configType);
-    setConfigTypeAttributes(serviceInfo, configuration, configType);
-  }
-  
-  void setConfigTypeAttributes(ServiceInfo serviceInfo, ConfigurationXml configuration, String configType) {
     for (Map.Entry<QName, String> attribute : configuration.getAttributes().entrySet()) {
       for (Supports supportsProperty : Supports.values()) {
         String attributeName = attribute.getKey().getLocalPart();
@@ -738,43 +717,23 @@ public class StackExtensionHelper {
       }
     }
   }
-  
-  void addConfigType(ServiceInfo serviceInfo, String configType) {
-    if(serviceInfo.getConfigTypes() == null) {
-      serviceInfo.setConfigTypes(new HashMap<String, Map<String, Map<String, String>>>());
-    }
-    
-    Map<String, Map<String, Map<String, String>>> configTypes = serviceInfo.getConfigTypes();
-    configTypes.put(configType, new HashMap<String, Map<String, String>>());
-    
-    
-    Map<String, Map<String, String>> properties = configTypes.get(configType);
-    Map<String, String> supportsProperties = new HashMap<String, String>();
-    for (Supports supportsProperty : Supports.values()) {
-      supportsProperties.put(supportsProperty.getPropertyName(), supportsProperty.getDefaultValue());
-    }
-    properties.put(Supports.KEYWORD, supportsProperties); 
-  }
 
   /**
    * Populate ServiceInfo#configTypes with default entries based on ServiceInfo#configDependencies property
    */
   void populateConfigTypesFromDependencies(ServiceInfo serviceInfo) {
-    List<PropertyInfo> configurations = serviceInfo.getProperties();
-    if (configurations != null) {
+    List<String> configDependencies = serviceInfo.getConfigDependenciesWithComponents();
+    if (configDependencies != null) {
       Map<String, Map<String, Map<String, String>>> configTypes = new HashMap<String, Map<String, Map<String, String>>>();
-      for (PropertyInfo configuration : configurations) {
-        int extIndex = configuration.getFilename().indexOf(AmbariMetaInfo.SERVICE_CONFIG_FILE_NAME_POSTFIX);
-        String configType = configuration.getFilename().substring(0, extIndex);
-        
-        if (!configTypes.containsKey(configType)) {
+      for (String configDependency : configDependencies) {
+        if (!configTypes.containsKey(configDependency)) {
           Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
           Map<String, String> supportsProperties = new HashMap<String, String>();
           for (Supports supportsProperty : Supports.values()) {
             supportsProperties.put(supportsProperty.getPropertyName(), supportsProperty.getDefaultValue());
           }
           properties.put(Supports.KEYWORD, supportsProperties);
-          configTypes.put(configType, properties);
+          configTypes.put(configDependency, properties);
         }
       }
       serviceInfo.setConfigTypes(configTypes);

ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/configuration/core-site.xml

@@ -0,0 +1,20 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<configuration supports_final="true">
+</configuration>

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration-mapred/core-site.xml

@@ -0,0 +1,20 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<configuration supports_final="true">
+</configuration>

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration/core-site.xml

@@ -0,0 +1,20 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<configuration supports_final="true">
+</configuration>

ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/configuration/core-site.xml

@@ -0,0 +1,20 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<configuration supports_final="true">
+</configuration>

ambari-server/src/test/java/org/apache/ambari/server/api/util/StackExtensionHelperTest.java

@@ -23,7 +23,6 @@ import org.apache.ambari.server.metadata.ActionMetadata;
 import org.apache.ambari.server.state.*;
 
 import java.io.File;
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -37,16 +36,12 @@ import static org.junit.Assert.*;
 
 import org.apache.ambari.server.state.stack.ConfigurationXml;
 import org.junit.Test;
-import org.xml.sax.SAXException;
 
 import com.google.inject.AbstractModule;
 import com.google.inject.Guice;
 import com.google.inject.Injector;
 
-import javax.xml.bind.JAXBException;
 import javax.xml.namespace.QName;
-import javax.xml.parsers.ParserConfigurationException;
-import javax.xml.xpath.XPathExpressionException;
 
 public class StackExtensionHelperTest {
 
@@ -249,7 +244,7 @@ public class StackExtensionHelperTest {
     for (ServiceInfo serviceInfo : allServices) {
       if (serviceInfo.getName().equals("HDFS")) {
         assertEquals(5, serviceInfo.getConfigDependencies().size());
-        assertEquals(4, serviceInfo.getConfigTypes().size());
+        assertEquals(5, serviceInfo.getConfigTypes().size());
         assertTrue(serviceInfo.getConfigDependencies().contains("core-site"));
         assertTrue(serviceInfo.getConfigDependencies().contains("global"));
         assertTrue(serviceInfo.getConfigDependencies().contains("hdfs-site"));
@@ -285,6 +280,31 @@ public class StackExtensionHelperTest {
       }
     }
   }
+  
+  @Test
+  public void testrequiredServicesPropertyInheritance() throws Exception{
+    File stackRoot = new File(stackRootStr);
+    StackInfo stackInfo = new StackInfo();
+    stackInfo.setName("HDP");
+    stackInfo.setVersion("2.0.7");
+    StackExtensionHelper helper = new StackExtensionHelper(injector, stackRoot);
+    helper.populateServicesForStack(stackInfo);
+    helper.fillInfo();
+    List<ServiceInfo> allServices = helper.getAllApplicableServices(stackInfo);
+    assertEquals(13, allServices.size());
+    
+    List<String> expectedRequiredServices = new ArrayList<String>();
+    expectedRequiredServices.add("HDFS");
+    expectedRequiredServices.add("TEZ");
+    
+    for (ServiceInfo serviceInfo : allServices) {
+      if (serviceInfo.getName().equals("HBASE")) {
+        assertTrue(serviceInfo.getRequiredServices().equals(expectedRequiredServices));
+      } else {
+        assertTrue((serviceInfo.getRequiredServices() == null || serviceInfo.getRequiredServices().isEmpty()));
+      }
+    }
+  }
 
   @Test
   public void getSchemaVersion() throws Exception {
@@ -300,37 +320,60 @@ public class StackExtensionHelperTest {
     version = helper.getSchemaVersion(v2MetaInfoFile);
     assertEquals("2.0", version);
   }
-  
-  public StackExtensionHelper getStackExtensionHelper() {
-    File stackRoot = new File(stackRootStr);    
-    return new StackExtensionHelper(injector, stackRoot);
+
+  @Test
+  public void testPopulateConfigTypes() {
+    File stackRoot = new File(stackRootStr);
+    StackExtensionHelper helper = new StackExtensionHelper(injector, stackRoot);
+    List<String> configDependencies = Arrays.asList("dep1", "dep2");
+    ServiceInfo serviceInfo = new ServiceInfo();
+    serviceInfo.setConfigDependencies(configDependencies);
+    helper.populateConfigTypesFromDependencies(serviceInfo);
+
+    Map<String, Map<String, Map<String, String>>> configTypes = serviceInfo.getConfigTypes();
+    assertEquals(2, configTypes.size());
+    assertTrue(configTypes.containsKey("dep1"));
+    assertTrue(configTypes.containsKey("dep2"));
+    Map<String, Map<String, String>> properties;
+    properties= configTypes.get("dep1");
+    assertEquals(1, properties.size());
+    assertTrue(properties.containsKey("supports"));
+    assertEquals(1, properties.get("supports").size());
+    assertTrue(properties.get("supports").containsKey("final"));
+    assertEquals("false", properties.get("supports").get("final"));
+    properties= configTypes.get("dep2");
+    assertEquals(1, properties.size());
+    assertTrue(properties.containsKey("supports"));
+    assertEquals(1, properties.get("supports").size());
+    assertTrue(properties.get("supports").containsKey("final"));
+    assertEquals("false", properties.get("supports").get("final"));
   }
-  
-  public ServiceInfo getServiceFromStack(StackExtensionHelper helper, String stackName, String stackVersion, String serviceName) throws XPathExpressionException, ParserConfigurationException, SAXException, IOException, JAXBException {
-    StackInfo stackInfo = new StackInfo();
-    stackInfo.setName(stackName);
-    stackInfo.setVersion(stackVersion);
-    
-    helper.populateServicesForStack(stackInfo);
-    
-    for(ServiceInfo service:stackInfo.getServices()) {
-      if(service.getName().equals(serviceName)) {
-        return service;
-      }
-    }
-    return null;
+
+  @Test
+  public void testPopulateConfigTypes_emptyList() {
+    File stackRoot = new File(stackRootStr);
+    StackExtensionHelper helper = new StackExtensionHelper(injector, stackRoot);
+    List<String> configDependencies = Collections.emptyList();
+    ServiceInfo serviceInfo = new ServiceInfo();
+    serviceInfo.setConfigDependencies(configDependencies);
+    helper.populateConfigTypesFromDependencies(serviceInfo);
+
+    Map<String, Map<String, Map<String, String>>> configTypes = serviceInfo.getConfigTypes();
+    assertNotNull(configTypes);
+    assertEquals(0, configTypes.size());
   }
 
   @Test
-  public void testPopulateConfigTypes() throws XPathExpressionException, ParserConfigurationException, SAXException, IOException, JAXBException {
-    StackExtensionHelper helper = getStackExtensionHelper();
-    ServiceInfo serviceInfo = getServiceFromStack(helper, "HDP", "2.0.7", "HDFS");
-    String expectedConfigTypes = 
-        "{global={supports={final=true}}, hdfs-site={supports={final=false}}, hadoop-policy={supports={final=false}}, core-site={supports={final=false}}}";
+  public void testPopulateConfigTypes_null() {
+    File stackRoot = new File(stackRootStr);
+    StackExtensionHelper helper = new StackExtensionHelper(injector, stackRoot);
+    List<String> configDependencies = null;
+    ServiceInfo serviceInfo = new ServiceInfo();
+    serviceInfo.setConfigDependencies(configDependencies);
+    helper.populateConfigTypesFromDependencies(serviceInfo);
 
     Map<String, Map<String, Map<String, String>>> configTypes = serviceInfo.getConfigTypes();
-    assertEquals(4, configTypes.size());
-    assertEquals(expectedConfigTypes, configTypes.toString());
+    assertNull(configTypes);
   }
 
   @Test
@@ -429,38 +472,81 @@ public class StackExtensionHelperTest {
 
   @Test
   public void testPopulateServiceProperties_noSupportsFinalFlag() throws Exception {
-    StackExtensionHelper helper = getStackExtensionHelper();
-    ServiceInfo serviceInfo = getServiceFromStack(helper, "HDP", "2.0.7", "YARN");
-    
-    File configFile = new File(stackRootStr
+    // init
+    File stackRoot = new File(stackRootStr);
+    StackExtensionHelper helper = createMockBuilder(StackExtensionHelper.class).addMockedMethod("addConfigTypeProperty")
+        .withConstructor(injector, stackRoot).createMock();
+    File config = new File(stackRootStr
         + "HDP/2.0.7/services/YARN/configuration/yarn-site.xml".replaceAll("/", File.separator));
-    
-    helper.populateServiceProperties(configFile, serviceInfo);
-    
-    assertEquals("{yarn-site={supports={final=false}}}", serviceInfo.getConfigTypes().toString());
+    ServiceInfo serviceInfo = createNiceMock(ServiceInfo.class);
+    List<PropertyInfo> properties = createNiceMock(List.class);
+
+    // expectations
+    expect(serviceInfo.getProperties()).andReturn(properties).times(1);
+    expect(properties.addAll((Collection) anyObject())).andReturn(true).times(1);
+    replay(properties);
+    replay(serviceInfo);
+    replay(helper);
+
+    // eval
+    helper.populateServiceProperties(config, serviceInfo);
+
+    // verification
+    verify(properties, serviceInfo, helper);
   }
 
   @Test
   public void testPopulateServiceProperties_supportsFinalTrue() throws Exception {
-    StackExtensionHelper helper = getStackExtensionHelper();
-    ServiceInfo serviceInfo = getServiceFromStack(helper, "HDP", "2.0.7", "HDFS");
-    
-    File configFile = new File(stackRootStr
+    // init
+    File stackRoot = new File(stackRootStr);
+    StackExtensionHelper helper = createMockBuilder(StackExtensionHelper.class).addMockedMethod("addConfigTypeProperty")
+        .withConstructor(injector, stackRoot).createMock();
+    File config = new File(stackRootStr
         + "HDP/2.0.7/services/HDFS/configuration/global.xml".replaceAll("/", File.separator));
-    helper.populateServiceProperties(configFile, serviceInfo);
-    
-    assertEquals("{global={supports={final=true}}, hdfs-site={supports={final=false}}, hadoop-policy={supports={final=false}}, core-site={supports={final=false}}}", serviceInfo.getConfigTypes().toString());
+    ServiceInfo serviceInfo = createNiceMock(ServiceInfo.class);
+    List<PropertyInfo> properties = createNiceMock(List.class);
+
+    // expectations
+    expect(serviceInfo.getProperties()).andReturn(properties).times(1);
+    expect(properties.addAll((Collection) anyObject())).andReturn(true).times(1);
+    helper.addConfigTypeProperty(serviceInfo, "global", StackExtensionHelper.Supports.KEYWORD,
+        StackExtensionHelper.Supports.FINAL.getPropertyName(), "true");
+    replay(properties);
+    replay(serviceInfo);
+    replay(helper);
+
+    // eval
+    helper.populateServiceProperties(config, serviceInfo);
+
+    // verification
+    verify(properties, serviceInfo, helper);
   }
 
   @Test
   public void testPopulateServiceProperties_supportsFinalFalse() throws Exception {
-    StackExtensionHelper helper = getStackExtensionHelper();
-    ServiceInfo serviceInfo = getServiceFromStack(helper, "HDP", "2.0.7", "HDFS");
-    File configFile = new File(stackRootStr
-        + "HDP/2.0.7/services/YARN/configuration/yarn-site.xml".replaceAll("/", File.separator));
-    helper.populateServiceProperties(configFile, serviceInfo);
-    
-    assertEquals("{global={supports={final=true}}, hdfs-site={supports={final=false}}, yarn-site={supports={final=false}}, hadoop-policy={supports={final=false}}, core-site={supports={final=false}}}", serviceInfo.getConfigTypes().toString());
+    // init
+    File stackRoot = new File(stackRootStr);
+    StackExtensionHelper helper = createMockBuilder(StackExtensionHelper.class).addMockedMethod("addConfigTypeProperty")
+        .withConstructor(injector, stackRoot).createMock();
+    File config = new File(stackRootStr
+        + "HDP/2.0.7/services/HDFS/configuration/core-site.xml".replaceAll("/", File.separator));
+    ServiceInfo serviceInfo = createNiceMock(ServiceInfo.class);
+    List<PropertyInfo> properties = createNiceMock(List.class);
+
+    // expectations
+    expect(serviceInfo.getProperties()).andReturn(properties).times(1);
+    expect(properties.addAll((Collection) anyObject())).andReturn(true).times(1);
+    helper.addConfigTypeProperty(serviceInfo, "core-site", StackExtensionHelper.Supports.KEYWORD,
+        StackExtensionHelper.Supports.FINAL.getPropertyName(), "false");
+    replay(properties);
+    replay(serviceInfo);
+    replay(helper);
+
+    // eval
+    helper.populateServiceProperties(config, serviceInfo);
+
+    // verification
+    verify(properties, serviceInfo, helper);
   }
 
   @Test
@@ -475,7 +561,6 @@ public class StackExtensionHelperTest {
     List<PropertyInfo> properties = createNiceMock(List.class);
 
     // expectations
-    expect(serviceInfo.getConfigTypes()).andReturn(new HashMap<String, Map<String, Map<String, String>>>()).times(2);
     expect(serviceInfo.getProperties()).andReturn(properties).times(1);
     expect(properties.addAll((Collection) anyObject())).andReturn(true).times(1);
     helper.addConfigTypeProperty(serviceInfo, "yarn-site", StackExtensionHelper.Supports.KEYWORD,
@@ -502,7 +587,6 @@ public class StackExtensionHelperTest {
     List<PropertyInfo> properties = createNiceMock(List.class);
 
     // expectations
-    expect(serviceInfo.getConfigTypes()).andReturn(new HashMap<String, Map<String, Map<String, String>>>()).times(2);
     expect(serviceInfo.getProperties()).andReturn(properties).times(1);
     expect(properties.addAll((Collection) anyObject())).andReturn(true).times(1);
     expect(serviceInfo.getConfigTypes()).andReturn(null).times(1);