AMBARI-5494 - Ability to use custom jmx port from *-site.xml files to get jmx metrics

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java

@@ -171,7 +171,7 @@ public abstract class AbstractProviderModule implements ProviderModule, Resource
 
   @Override
   public ResourceProvider getResourceProvider(Resource.Type type) {
-    if (!propertyProviders.containsKey(type)) {
+    if (!resourceProviders.containsKey(type)) {
       registerResourceProvider(type);
     }
     return resourceProviders.get(type);

contrib/ambari-scom/ambari-scom-server/pom.xml

@@ -28,6 +28,7 @@
     <url>http://maven.apache.org</url>
     <properties>
         <ambari.version>1.3.0-SNAPSHOT</ambari.version>
+        <powermock.version>1.5.4</powermock.version>
     </properties>
     <dependencies>
         <dependency>
@@ -43,6 +44,25 @@
             <scope>test</scope>
         </dependency>
         <dependency>
+            <groupId>org.powermock</groupId>
+            <artifactId>powermock-module-junit4</artifactId>
+            <version>${powermock.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.powermock</groupId>
+            <artifactId>powermock-api-mockito</artifactId>
+            <version>${powermock.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+          <groupId>org.powermock</groupId>
+          <artifactId>powermock-api-easymock</artifactId>
+          <version>${powermock.version}</version>
+          <scope>test</scope>
+        </dependency>
+
+      <dependency>
             <groupId>org.apache.ambari</groupId>
             <artifactId>ambari-server</artifactId>
             <version>${ambari.version}</version>
@@ -52,6 +72,11 @@
             <artifactId>jersey-server</artifactId>
             <version>1.8</version>
         </dependency>
+        <dependency>
+          <groupId>com.thoughtworks.xstream</groupId>
+          <artifactId>xstream</artifactId>
+          <version>1.4.7</version>
+        </dependency>
     </dependencies>
 
     <build>

contrib/ambari-scom/ambari-scom-server/src/main/java/org/apache/ambari/msi/AbstractResourceProvider.java

@@ -266,6 +266,8 @@ public abstract class AbstractResourceProvider implements ResourceProvider {
       return new RequestProvider(clusterDefinition);
     } else if (type.equals(Resource.Type.Task)) {
       return new TaskProvider(clusterDefinition);
+    } else if (type.equals(Resource.Type.Configuration)) {
+      return new ConfigurationProvider(clusterDefinition);
     } else {
       return new NoOpProvider(type, clusterDefinition);
     }

contrib/ambari-scom/ambari-scom-server/src/main/java/org/apache/ambari/msi/ConfigurationProvider.java

@@ -0,0 +1,155 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.msi;
+
+import com.thoughtworks.xstream.XStream;
+import com.thoughtworks.xstream.converters.Converter;
+import com.thoughtworks.xstream.converters.MarshallingContext;
+import com.thoughtworks.xstream.converters.UnmarshallingContext;
+import com.thoughtworks.xstream.io.HierarchicalStreamReader;
+import com.thoughtworks.xstream.io.HierarchicalStreamWriter;
+import com.thoughtworks.xstream.io.xml.StaxDriver;
+import org.apache.ambari.server.controller.internal.ResourceImpl;
+import org.apache.ambari.server.controller.spi.Predicate;
+import org.apache.ambari.server.controller.spi.Request;
+import org.apache.ambari.server.controller.spi.Resource;
+import org.apache.ambari.server.controller.utilities.PropertyHelper;
+import org.apache.commons.lang.StringUtils;
+
+import java.io.InputStream;
+import java.util.AbstractMap;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Configuration provider for a MSI defined cluster.
+ */
+public class ConfigurationProvider extends BaseResourceProvider {
+
+  protected static final String CONFIGURATION_CLUSTER_NAME_PROPERTY_ID = PropertyHelper.getPropertyId("Config", "cluster_name");
+  public static final String CONFIGURATION_CONFIG_TYPE_PROPERTY_ID = PropertyHelper.getPropertyId(null, "type");
+  public static final String CONFIGURATION_CONFIG_TAG_PROPERTY_ID = PropertyHelper.getPropertyId(null, "tag");
+
+  private Map<String, Map<String, String>> allConfigs;
+
+  private static final String DESTINATION = "xml";
+  private static final Set<String> clusterConfigurationResources = new HashSet<String>();
+
+  static {
+    clusterConfigurationResources.add("hdfs-site");
+    clusterConfigurationResources.add("mapred-site");
+    clusterConfigurationResources.add("hbase-site");
+    clusterConfigurationResources.add("yarn-site");
+    clusterConfigurationResources.add("core-site");
+  }
+
+  // ----- AbstractResourceProvider ------------------------------------------
+
+  @Override
+  public void updateProperties(Resource resource, Request request, Predicate predicate) {
+    // Do nothing
+  }
+
+  @Override
+  public int updateProperties(Resource resource, Map<String, Object> properties) {
+    // Do nothing
+    return -1;
+  }
+
+  public ConfigurationProvider(ClusterDefinition clusterDefinition) {
+    super(Resource.Type.Configuration, clusterDefinition);
+    init();
+    initConfigurationResources();
+  }
+
+  class ScomConfigConverter implements Converter {
+    @Override
+    public void marshal(Object o, HierarchicalStreamWriter hierarchicalStreamWriter, MarshallingContext marshallingContext) {
+    }
+
+    @Override
+    public Object unmarshal(HierarchicalStreamReader hierarchicalStreamReader, UnmarshallingContext unmarshallingContext) {
+      Map<String, String> map = new HashMap<String, String>();
+
+      while (hierarchicalStreamReader.hasMoreChildren()) {
+        hierarchicalStreamReader.moveDown();
+        String name = "", value = "";
+        while (hierarchicalStreamReader.hasMoreChildren()) {
+          hierarchicalStreamReader.moveDown();
+          if ("name".equalsIgnoreCase(hierarchicalStreamReader.getNodeName())) {
+            name = hierarchicalStreamReader.getValue();
+          }
+          if ("value".equalsIgnoreCase(hierarchicalStreamReader.getNodeName())) {
+            value = hierarchicalStreamReader.getValue();
+          }
+          hierarchicalStreamReader.moveUp();
+        }
+
+        if (StringUtils.isNotEmpty(name) && StringUtils.isNotEmpty(value)) {
+          map.put(name, value);
+        }
+        hierarchicalStreamReader.moveUp();
+      }
+
+      return map;
+    }
+
+    @Override
+    public boolean canConvert(Class aClass) {
+      return AbstractMap.class.isAssignableFrom(aClass);
+    }
+  }
+
+  @SuppressWarnings("unchecked")
+  private void init() {
+    allConfigs = new HashMap<String, Map<String, String>>();
+
+    XStream xstream = new XStream(new StaxDriver());
+    xstream.alias("configuration", Map.class);
+    xstream.registerConverter(new ScomConfigConverter());
+
+    for (String configurationResource : clusterConfigurationResources) {
+      String configFileName = configurationResource + "." + DESTINATION;
+      InputStream is = ClassLoader.getSystemResourceAsStream(configFileName);
+      if (is == null) continue;
+      Map<String, String> properties = (HashMap<String, String>) xstream.fromXML(is);
+      allConfigs.put(configurationResource, properties);
+    }
+  }
+
+  private void initConfigurationResources() {
+    String clusterName = getClusterDefinition().getClusterName();
+
+    for (String type : allConfigs.keySet()) {
+      Resource resource = new ResourceImpl(Resource.Type.Configuration);
+      resource.setProperty(CONFIGURATION_CLUSTER_NAME_PROPERTY_ID, clusterName);
+      resource.setProperty(CONFIGURATION_CONFIG_TYPE_PROPERTY_ID, type);
+      resource.setProperty(CONFIGURATION_CONFIG_TAG_PROPERTY_ID, "version1");
+
+      Map<String, String> properties = allConfigs.get(type);
+      for (Map.Entry<String, String> entry : properties.entrySet()) {
+        String id = PropertyHelper.getPropertyId("properties", entry.getKey());
+        resource.setProperty(id, entry.getValue());
+      }
+
+      addResource(resource);
+    }
+  }
+}

contrib/ambari-scom/ambari-scom-server/src/test/java/org/apache/ambari/msi/AbstractResourceProviderTest.java

@@ -43,6 +43,7 @@ public class AbstractResourceProviderTest {
     types.add(Resource.Type.HostComponent);
     types.add(Resource.Type.Request);
     types.add(Resource.Type.Task);
+    types.add(Resource.Type.Configuration);
   }
 
   @Test

contrib/ambari-scom/ambari-scom-server/src/test/java/org/apache/ambari/msi/ConfigurationProviderTest.java

@@ -0,0 +1,171 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.msi;
+
+import com.thoughtworks.xstream.XStream;
+import com.thoughtworks.xstream.io.xml.StaxDriver;
+import org.apache.ambari.server.controller.spi.Predicate;
+import org.apache.ambari.server.controller.spi.Resource;
+import org.apache.ambari.server.controller.utilities.PredicateBuilder;
+import org.apache.ambari.server.controller.utilities.PropertyHelper;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.api.easymock.PowerMock;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import static org.easymock.EasyMock.*;
+
+/**
+ * Tests for ConfigurationProvider.
+ */
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({ConfigurationProvider.class, StaxDriver.class, XStream.class, ClassLoader.class, InputStream.class})
+public class ConfigurationProviderTest {
+
+  @Test
+  public void testConfigurationProvider_init_method_file_doesnt_exists() throws Exception {
+    ClusterDefinition clusterDefinitionMock = createStrictMock(ClusterDefinition.class);
+    PowerMock.suppress(PowerMock.methods(ConfigurationProvider.class, "initConfigurationResources"));
+
+    StaxDriver staxDriver = PowerMock.createStrictMock(StaxDriver.class);
+    XStream xstream = PowerMock.createStrictMock(XStream.class);
+
+    PowerMock.expectNew(StaxDriver.class).andReturn(staxDriver);
+    PowerMock.expectNew(XStream.class, staxDriver).andReturn(xstream);
+    xstream.alias("configuration", Map.class);
+    expectLastCall();
+    xstream.registerConverter(anyObject(ConfigurationProvider.ScomConfigConverter.class));
+    expectLastCall();
+
+    PowerMock.replay(staxDriver, StaxDriver.class, xstream, XStream.class);
+    replay(clusterDefinitionMock);
+    new ConfigurationProvider(clusterDefinitionMock);
+    PowerMock.verify(staxDriver, StaxDriver.class, xstream, XStream.class);
+    verify(clusterDefinitionMock);
+  }
+
+  @Test
+  public void testConfigurationProvider_init_method_file_exists() throws Exception {
+    ClusterDefinition clusterDefinitionMock = createStrictMock(ClusterDefinition.class);
+    PowerMock.suppress(PowerMock.methods(ConfigurationProvider.class, "initConfigurationResources"));
+
+    StaxDriver staxDriver = PowerMock.createStrictMock(StaxDriver.class);
+    XStream xstream = PowerMock.createStrictMock(XStream.class);
+    PowerMock.mockStatic(ClassLoader.class);
+    InputStream mockInputStream = createMock(InputStream.class);
+
+
+    PowerMock.expectNew(StaxDriver.class).andReturn(staxDriver);
+    PowerMock.expectNew(XStream.class, staxDriver).andReturn(xstream);
+    xstream.alias("configuration", Map.class);
+    expectLastCall();
+    xstream.registerConverter(anyObject(ConfigurationProvider.ScomConfigConverter.class));
+    expectLastCall();
+    expect(ClassLoader.getSystemResourceAsStream(anyObject(String.class))).andReturn(mockInputStream).times(5);
+    expect(xstream.fromXML(mockInputStream)).andReturn(new HashMap<String, String>()).times(5);
+
+    PowerMock.replay(staxDriver, StaxDriver.class, xstream, XStream.class, ClassLoader.class);
+    replay(clusterDefinitionMock, mockInputStream);
+
+    new ConfigurationProvider(clusterDefinitionMock);
+
+    PowerMock.verify(staxDriver, StaxDriver.class, xstream, XStream.class, ClassLoader.class);
+    verify(clusterDefinitionMock, mockInputStream);
+  }
+
+  @Test
+  public void testConfigurationProvider_initConfigurationResources_method() throws Exception {
+    ClusterDefinition clusterDefinitionMock = createStrictMock(ClusterDefinition.class);
+    StaxDriver staxDriver = PowerMock.createStrictMock(StaxDriver.class);
+    XStream xstream = PowerMock.createStrictMock(XStream.class);
+    PowerMock.mockStatic(ClassLoader.class);
+    InputStream mockInputStream = createMock(InputStream.class);
+
+
+    PowerMock.expectNew(StaxDriver.class).andReturn(staxDriver);
+    PowerMock.expectNew(XStream.class, staxDriver).andReturn(xstream);
+    xstream.alias("configuration", Map.class);
+    expectLastCall();
+    xstream.registerConverter(anyObject(ConfigurationProvider.ScomConfigConverter.class));
+    expectLastCall();
+    expect(ClassLoader.getSystemResourceAsStream(anyObject(String.class))).andReturn(mockInputStream).times(5);
+    expect(xstream.fromXML(mockInputStream)).andReturn(new HashMap<String, String>() {{
+      put("property_key", "propery_value");
+    }}).times(5);
+
+    expect(clusterDefinitionMock.getClusterName()).andReturn("ambari");
+
+    PowerMock.replay(staxDriver, StaxDriver.class, xstream, XStream.class, ClassLoader.class);
+    replay(clusterDefinitionMock, mockInputStream);
+
+    ConfigurationProvider configurationProvider = new ConfigurationProvider(clusterDefinitionMock);
+
+    PowerMock.verify(staxDriver, StaxDriver.class, xstream, XStream.class, ClassLoader.class);
+    verify(clusterDefinitionMock, mockInputStream);
+
+    Assert.assertEquals(5, configurationProvider.getResources().size());
+  }
+
+  @Test
+  public void testGetResourcesWithPredicate() throws Exception {
+    ClusterDefinition clusterDefinitionMock = createStrictMock(ClusterDefinition.class);
+    StaxDriver staxDriver = PowerMock.createStrictMock(StaxDriver.class);
+    XStream xstream = PowerMock.createStrictMock(XStream.class);
+    PowerMock.mockStatic(ClassLoader.class);
+    InputStream mockInputStream = createMock(InputStream.class);
+
+
+    PowerMock.expectNew(StaxDriver.class).andReturn(staxDriver);
+    PowerMock.expectNew(XStream.class, staxDriver).andReturn(xstream);
+    xstream.alias("configuration", Map.class);
+    expectLastCall();
+    xstream.registerConverter(anyObject(ConfigurationProvider.ScomConfigConverter.class));
+    expectLastCall();
+    expect(ClassLoader.getSystemResourceAsStream(anyObject(String.class))).andReturn(mockInputStream).times(5);
+    expect(xstream.fromXML(mockInputStream)).andReturn(new HashMap<String, String>() {{
+      put("property_key", "propery_value");
+    }}).times(5);
+
+    expect(clusterDefinitionMock.getClusterName()).andReturn("ambari");
+
+    PowerMock.replay(staxDriver, StaxDriver.class, xstream, XStream.class, ClassLoader.class);
+    replay(clusterDefinitionMock, mockInputStream);
+
+    ConfigurationProvider configurationProvider = new ConfigurationProvider(clusterDefinitionMock);
+
+    PowerMock.verify(staxDriver, StaxDriver.class, xstream, XStream.class, ClassLoader.class);
+    verify(clusterDefinitionMock, mockInputStream);
+
+    Predicate configPredicate = new PredicateBuilder().property
+            (ConfigurationProvider.CONFIGURATION_CLUSTER_NAME_PROPERTY_ID).equals("ambari").and()
+            .property(ConfigurationProvider.CONFIGURATION_CONFIG_TYPE_PROPERTY_ID).equals("yarn-site").and()
+            .property(ConfigurationProvider.CONFIGURATION_CONFIG_TAG_PROPERTY_ID).equals("version1").toPredicate();
+
+    Set<Resource> resources = configurationProvider.getResources(PropertyHelper.getReadRequest(), configPredicate);
+    Assert.assertNotNull(resources);
+    Assert.assertEquals(1, resources.size());
+  }
+}