AMBARI-4685. API doesn't return NN or RM metrics with customized port (ncole)

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java

@@ -50,6 +50,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.util.*;
+import java.util.Map.Entry;
 
 /**
  * An abstract provider module implementation.
@@ -68,12 +69,12 @@ public abstract class AbstractProviderModule implements ProviderModule, Resource
   private static final Map<Service.Type, String> serviceConfigVersions =
     Collections.synchronizedMap(new HashMap<Service.Type, String>());
   private static final Map<Service.Type, String> serviceConfigTypes = new HashMap<Service.Type, String>();
-  private static final Map<Service.Type, Map<String, String>> serviceDesiredProperties = new
-    HashMap<Service.Type, Map<String, String>>();
+  private static final Map<Service.Type, Map<String, String[]>> serviceDesiredProperties = new
+    HashMap<Service.Type, Map<String, String[]>>();
   private static final Map<String, Service.Type> componentServiceMap = new
     HashMap<String, Service.Type>();
   
-  private static final Map<String, Map<String, String>> jmxDesiredProperties = new HashMap<String, Map<String,String>>();
+  private static final Map<String, Map<String, String[]>> jmxDesiredProperties = new HashMap<String, Map<String,String[]>>();
   private volatile Map<String, String> clusterCoreSiteConfigVersionMap = new HashMap<String, String>();
   private volatile Map<String, String> clusterJmxProtocolMap = new HashMap<String, String>();
   
@@ -81,27 +82,35 @@ public abstract class AbstractProviderModule implements ProviderModule, Resource
     serviceConfigTypes.put(Service.Type.HDFS, "hdfs-site");
     serviceConfigTypes.put(Service.Type.MAPREDUCE, "mapred-site");
     serviceConfigTypes.put(Service.Type.HBASE, "hbase-site");
+    serviceConfigTypes.put(Service.Type.YARN, "yarn-site");
  
     componentServiceMap.put("NAMENODE", Service.Type.HDFS);
     componentServiceMap.put("DATANODE", Service.Type.HDFS);
     componentServiceMap.put("JOBTRACKER", Service.Type.MAPREDUCE);
     componentServiceMap.put("TASKTRACKER", Service.Type.MAPREDUCE);
     componentServiceMap.put("HBASE_MASTER", Service.Type.HBASE);
+    componentServiceMap.put("RESOURCEMANAGER", Service.Type.YARN);
 
-    Map<String, String> initPropMap = new HashMap<String, String>();
-    initPropMap.put("NAMENODE", "dfs.http.address");
-    initPropMap.put("DATANODE", "dfs.datanode.http.address");
+    Map<String, String[]> initPropMap = new HashMap<String, String[]>();
+    initPropMap.put("NAMENODE", new String[] {"dfs.http.address", "dfs.namenode.http-address"});
+    initPropMap.put("DATANODE", new String[] {"dfs.datanode.http.address"});
     serviceDesiredProperties.put(Service.Type.HDFS, initPropMap);
-    initPropMap = new HashMap<String, String>();
-    initPropMap.put("JOBTRACKER", "mapred.job.tracker.http.address");
-    initPropMap.put("TASKTRACKER", "mapred.task.tracker.http.address");
+    
+    initPropMap = new HashMap<String, String[]>();
+    initPropMap.put("JOBTRACKER", new String[] {"mapred.job.tracker.http.address"});
+    initPropMap.put("TASKTRACKER", new String[] {"mapred.task.tracker.http.address"});
     serviceDesiredProperties.put(Service.Type.MAPREDUCE, initPropMap);
-    initPropMap = new HashMap<String, String>();
-    initPropMap.put("HBASE_MASTER", "hbase.master.info.port");
+    
+    initPropMap = new HashMap<String, String[]>();
+    initPropMap.put("HBASE_MASTER", new String[] {"hbase.master.info.port"});
     serviceDesiredProperties.put(Service.Type.HBASE, initPropMap);
+
+    initPropMap = new HashMap<String, String[]>();
+    initPropMap.put("RESOURCEMANAGER", new String[] {"yarn.resourcemanager.webapp.address"});
+    serviceDesiredProperties.put(Service.Type.YARN, initPropMap);
     
-    initPropMap = new HashMap<String, String>();
-    initPropMap.put("NAMENODE", "hadoop.ssl.enabled");
+    initPropMap = new HashMap<String, String[]>();
+    initPropMap.put("NAMENODE", new String[] {"hadoop.ssl.enabled"});
     jmxDesiredProperties.put("NAMENODE", initPropMap);
   }
 
@@ -208,6 +217,7 @@ public abstract class AbstractProviderModule implements ProviderModule, Resource
       }
     }
     Service.Type service = componentServiceMap.get(componentName);
+    
     if (service != null) {
       try {
         String currVersion = getDesiredConfigVersion(clusterName,
@@ -216,9 +226,11 @@ public abstract class AbstractProviderModule implements ProviderModule, Resource
         String oldVersion = serviceConfigVersions.get(service);
         if (!currVersion.equals(oldVersion)) {
           serviceConfigVersions.put(service, currVersion);
+          
           Map<String, String> portMap = getDesiredConfigMap(clusterName,
             currVersion, serviceConfigTypes.get(service),
             serviceDesiredProperties.get(service));
+          
           for (String compName : portMap.keySet()) {
             clusterJmxPorts.put(compName, getPortString(portMap.get
               (compName)));
@@ -594,7 +606,7 @@ public abstract class AbstractProviderModule implements ProviderModule, Resource
   }
 
   private Map<String, String> getDesiredConfigMap(String clusterName,
-      String versionTag, String configType, Map<String, String> keys) throws
+      String versionTag, String configType, Map<String, String[]> keys) throws
         NoSuchParentResourceException, UnsupportedPropertyException,
         SystemException {
     // Get desired configs based on the tag
@@ -613,20 +625,30 @@ public abstract class AbstractProviderModule implements ProviderModule, Resource
       LOG.info("Resource for the desired config not found. " + e);
       return Collections.emptyMap();
     }
+    
     Map<String, String> mConfigs = new HashMap<String, String>();
     if (configResources != null) {
       for (Resource res : configResources) {
        Map<String, String> evalutedProperties = null;                      
-        for (String key : keys.keySet()) {
-          String value = (String) res.getPropertyValue
-            (PropertyHelper.getPropertyId(PROPERTIES_CATEGORY, keys.get(key)));
-          if (value != null && value.contains("${"))
+        for (Entry<String,String[]> entry : keys.entrySet()) {
+          String propName = null;
+          String value = null;
+          
+          for (String pname : entry.getValue()) {
+            propName = pname;
+            value = (String) res.getPropertyValue(PropertyHelper.getPropertyId(
+                PROPERTIES_CATEGORY, pname));
+            if (null != value)
+              break;
+          }
+          
+          if (value != null && value.contains("${")) {
             if (evalutedProperties == null){
               evalutedProperties = new HashMap<String, String>();
               Map<String, Object> properties = res.getPropertiesMap().get(PROPERTIES_CATEGORY);
-              for (Map.Entry<String, Object> entry : properties.entrySet()) {
-                String keyString = entry.getKey();
-                Object object = entry.getValue();
+              for (Map.Entry<String, Object> subentry : properties.entrySet()) {
+                String keyString = subentry.getKey();
+                Object object = subentry.getValue();
                 String valueString;
                 if (object != null && object instanceof String){
                   valueString = (String)object;
@@ -635,10 +657,11 @@ public abstract class AbstractProviderModule implements ProviderModule, Resource
                 }
               }              
             }
-          value = postProcessPropertyValue(keys.get(key), value, evalutedProperties, null);
-          LOG.debug("PROPERTY -> key: " + keys.get(key) + ", " +
-          "value: " + value);
-          mConfigs.put(key, value);
+          }
+          value = postProcessPropertyValue(propName, value, evalutedProperties, null);
+          LOG.debug("PROPERTY -> key: " + propName + ", " + "value: " + value);
+          
+          mConfigs.put(entry.getKey(), value);
         }
       }
     }

ambari-server/src/main/java/org/apache/ambari/server/state/Service.java

@@ -114,6 +114,7 @@ public interface Service {
     ZOOKEEPER,
     PIG,
     HCATALOG,
-    FLUME
+    FLUME,
+    YARN
   }
 }

ambari-server/src/test/java/org/apache/ambari/server/controller/internal/JMXHostProviderTest.java

@@ -17,13 +17,27 @@
  */
 package org.apache.ambari.server.controller.internal;
 
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import com.google.inject.persist.PersistService;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.api.services.AmbariMetaInfo;
-import org.apache.ambari.server.controller.*;
-import org.apache.ambari.server.controller.spi.*;
+import org.apache.ambari.server.controller.AmbariManagementController;
+import org.apache.ambari.server.controller.ClusterRequest;
+import org.apache.ambari.server.controller.ConfigurationRequest;
+import org.apache.ambari.server.controller.ServiceComponentHostRequest;
+import org.apache.ambari.server.controller.ServiceComponentRequest;
+import org.apache.ambari.server.controller.ServiceRequest;
+import org.apache.ambari.server.controller.spi.NoSuchParentResourceException;
+import org.apache.ambari.server.controller.spi.NoSuchResourceException;
+import org.apache.ambari.server.controller.spi.Resource;
+import org.apache.ambari.server.controller.spi.ResourceAlreadyExistsException;
+import org.apache.ambari.server.controller.spi.ResourceProvider;
+import org.apache.ambari.server.controller.spi.SystemException;
+import org.apache.ambari.server.controller.spi.UnsupportedPropertyException;
 import org.apache.ambari.server.controller.utilities.PropertyHelper;
 import org.apache.ambari.server.orm.GuiceJpaInitializer;
 import org.apache.ambari.server.orm.InMemoryDefaultTestModule;
@@ -36,17 +50,16 @@ import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.persist.PersistService;
 
 public class JMXHostProviderTest {
   private Injector injector;
   private Clusters clusters;
   static AmbariManagementController controller;
-  private static final String NAMENODE_PORT = "dfs.http.address";
+  private static final String NAMENODE_PORT_V1 = "dfs.http.address";
+  private static final String NAMENODE_PORT_V2 = "dfs.namenode.http-address";
   private static final String DATANODE_PORT = "dfs.datanode.http.address";
 
   @Before
@@ -106,7 +119,7 @@ public class JMXHostProviderTest {
     controller.createHostComponents(requests);
   }
 
-  private void createHDFSServiceConfigs() throws AmbariException {
+  private void createHDFSServiceConfigs(boolean version1) throws AmbariException {
     String clusterName = "c1";
     ClusterRequest r = new ClusterRequest(null, clusterName, "HDP-0.1", null);
     controller.createCluster(r);
@@ -148,21 +161,32 @@ public class JMXHostProviderTest {
       host1, null);
     createServiceComponentHost(clusterName, serviceName, componentName3,
       host2, null);
-
+    
     // Create configs
-    Map<String, String> configs = new HashMap<String, String>();
-    configs.put(NAMENODE_PORT, "localhost:${ambari.dfs.datanode.http.port}");
-    configs.put(DATANODE_PORT, "localhost:70075");
-    configs.put("ambari.dfs.datanode.http.port", "70070");
-    ConfigurationRequest cr = new ConfigurationRequest(clusterName,
-      "hdfs-site", "version1", configs);
-    controller.createConfiguration(cr);
-
-    Map<String, String> configVersions = new HashMap<String, String>();
-    Set<ServiceRequest> sReqs = new HashSet<ServiceRequest>();
-    configVersions.put("hdfs-site", "version1");
-    sReqs.add(new ServiceRequest(clusterName, serviceName, null));
-    ServiceResourceProviderTest.updateServices(controller, sReqs, mapRequestProps, true, false);
+    if (version1) {
+      Map<String, String> configs = new HashMap<String, String>();
+      configs.put(NAMENODE_PORT_V1, "localhost:${ambari.dfs.datanode.http.port}");
+      configs.put(DATANODE_PORT, "localhost:70075");
+      configs.put("ambari.dfs.datanode.http.port", "70070");
+      
+      ConfigurationRequest cr = new ConfigurationRequest(clusterName,
+        "hdfs-site", "version1", configs);
+      ClusterRequest crequest = new ClusterRequest(null, clusterName, null, null);
+      crequest.setDesiredConfig(cr);
+      controller.updateClusters(Collections.singleton(crequest), new HashMap<String,String>());
+      
+    } else {
+      Map<String, String> configs = new HashMap<String, String>();
+      configs.put(NAMENODE_PORT_V2, "localhost:70071");
+      configs.put(DATANODE_PORT, "localhost:70075");
+
+      ConfigurationRequest cr = new ConfigurationRequest(clusterName,
+        "hdfs-site", "version2", configs);
+      
+      ClusterRequest crequest = new ClusterRequest(null, clusterName, null, null);
+      crequest.setDesiredConfig(cr);
+      controller.updateClusters(Collections.singleton(crequest), new HashMap<String,String>());
+    }
   }
 
   private void createConfigs() throws AmbariException {
@@ -207,8 +231,9 @@ public class JMXHostProviderTest {
 
     // Create configs
     Map<String, String> configs = new HashMap<String, String>();
-    configs.put(NAMENODE_PORT, "localhost:70070");
+    configs.put(NAMENODE_PORT_V1, "localhost:${ambari.dfs.datanode.http.port}");
     configs.put(DATANODE_PORT, "localhost:70075");
+    configs.put("ambari.dfs.datanode.http.port", "70070");
 
     ConfigurationRequest cr1 = new ConfigurationRequest(clusterName,
       "hdfs-site", "versionN", configs);
@@ -219,18 +244,18 @@ public class JMXHostProviderTest {
     Cluster cluster = clusters.getCluster(clusterName);
     Assert.assertEquals("versionN", cluster.getDesiredConfigByType("hdfs-site")
       .getVersionTag());
-    Assert.assertEquals("localhost:70070", cluster.getDesiredConfigByType
-      ("hdfs-site").getProperties().get(NAMENODE_PORT));
+    Assert.assertEquals("localhost:${ambari.dfs.datanode.http.port}", cluster.getDesiredConfigByType
+      ("hdfs-site").getProperties().get(NAMENODE_PORT_V1));
   }
 
 
   @Test
-  public void testJMXPortMapInitAtServiceLevel() throws
+  public void testJMXPortMapInitAtServiceLevelVersion1() throws
     NoSuchParentResourceException,
     ResourceAlreadyExistsException, UnsupportedPropertyException,
     SystemException, AmbariException, NoSuchResourceException {
 
-    createHDFSServiceConfigs();
+    createHDFSServiceConfigs(true);
 
     JMXHostProviderModule providerModule = new JMXHostProviderModule();
     providerModule.registerResourceProvider(Resource.Type.Service);
@@ -243,6 +268,26 @@ public class JMXHostProviderTest {
     Assert.assertEquals(null, providerModule.getPort("c1", "TASKTRACKER"));
     Assert.assertEquals(null, providerModule.getPort("c1", "HBASE_MASTER"));
   }
+  
+  @Test
+  public void testJMXPortMapInitAtServiceLevelVersion2() throws
+    NoSuchParentResourceException,
+    ResourceAlreadyExistsException, UnsupportedPropertyException,
+    SystemException, AmbariException, NoSuchResourceException {
+
+    createHDFSServiceConfigs(false);
+
+    JMXHostProviderModule providerModule = new JMXHostProviderModule();
+    providerModule.registerResourceProvider(Resource.Type.Service);
+    providerModule.registerResourceProvider(Resource.Type.Configuration);
+    // Non default port addresses
+    Assert.assertEquals("70071", providerModule.getPort("c1", "NAMENODE"));
+    Assert.assertEquals("70075", providerModule.getPort("c1", "DATANODE"));
+    // Default port addresses
+    Assert.assertEquals(null, providerModule.getPort("c1", "JOBTRACKER"));
+    Assert.assertEquals(null, providerModule.getPort("c1", "TASKTRACKER"));
+    Assert.assertEquals(null, providerModule.getPort("c1", "HBASE_MASTER"));
+  }  
 
   @Test
   public void testJMXPortMapInitAtClusterLevel() throws