AMBARI-3347.Performance issue requesting metrics when dynamic metric name specified. (odiachenko)

ambari-agent/src/main/puppet/modules/hdp-ganglia/files/rrd.py

@@ -23,6 +23,7 @@ import os
 import rrdtool
 import sys
 import time
+import re
 
 # place this script in /var/www/cgi-bin of the Ganglia collector
 # requires 'yum install rrdtool-python' on the Ganglia collector
@@ -145,14 +146,21 @@ for cluster in clusterParts:
     if len(hostParts) == 0 or pathParts[-1] in hostParts:
       for file in files:
         for metric in metricParts:
+          doPrintMetric = False
           if file.endswith(metric + ".rrd"):
+            doPrintMetric = True
+          else:
+            metricRegex = metric + '.rrd$'
+            p = re.compile(metricRegex)
+            if p.match(file):
+              doPrintMetric = True
 
-            printMetric(pathParts[-2], pathParts[-1], file[:-4],
-                os.path.join(path, file), cf, start, end, resolution, pointInTime)
+          if doPrintMetric:
+            printMetric(pathParts[-2], pathParts[-1], file[:-4], os.path.join(path, file), cf, start, end, resolution, pointInTime)
 
 sys.stdout.write("[AMBARI_END]\n")
 # write end time
 sys.stdout.write(str(time.mktime(time.gmtime())))
 sys.stdout.write("\n")
 
-sys.stdout.flush
+sys.stdout.flush

ambari-server/src/main/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProvider.java

@@ -23,6 +23,7 @@ import org.apache.ambari.server.configuration.ComponentSSLConfiguration;
 import org.apache.ambari.server.controller.internal.PropertyInfo;
 import org.apache.ambari.server.controller.spi.*;
 import org.apache.ambari.server.controller.utilities.StreamProvider;
+import org.apache.http.client.utils.URIBuilder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -228,7 +229,7 @@ public abstract class GangliaPropertyProvider extends AbstractPropertyProvider {
             updateComponentMetricMap(componentMetricMap, id);
           }
 
-          boolean requestAll = getPropertyInfoMap(getComponentName(resource), id, propertyInfoMap);
+          getPropertyInfoMap(getComponentName(resource), id, propertyInfoMap);
 
           for (Map.Entry<String, PropertyInfo> entry : propertyInfoMap.entrySet()) {
             String propertyId = entry.getKey();
@@ -242,9 +243,8 @@ public abstract class GangliaPropertyProvider extends AbstractPropertyProvider {
                 rrdRequest = new RRDRequest(clusterName, temporalInfo);
                 requests.put(temporalInfo, rrdRequest);
               }
-              rrdRequest.putResource(key, resource);
+              rrdRequest.putResource(key, resource);              
               rrdRequest.putPropertyId(propertyInfo.getPropertyId(), propertyId);
-              rrdRequest.setRequestAllMetrics(requestAll);
             }
           }
         }
@@ -277,48 +277,52 @@ public abstract class GangliaPropertyProvider extends AbstractPropertyProvider {
     String hosts    = getSetString(hostSet, 100);
     String metrics  = getSetString(metricSet, 50);
 
-    StringBuilder sb = new StringBuilder();
+    URIBuilder uriBuilder = new URIBuilder();
 
     if (configuration.isGangliaSSL()) {
-      sb.append("https://");
+      uriBuilder.setScheme("https");
     } else {
-      sb.append("http://");
+      uriBuilder.setScheme("http");
     }
 
-    sb.append(hostProvider.getGangliaCollectorHostName(clusterName)).
-        append("/cgi-bin/rrd.py?c=").
-        append(clusters);
+    
+    uriBuilder.setHost(hostProvider.getGangliaCollectorHostName(clusterName));
+    
+    uriBuilder.setPath("/cgi-bin/rrd.py");
+    
+    uriBuilder.setParameter("c", clusters);
+    
 
     if (hosts.length() > 0) {
-      sb.append("&h=").append(hosts);
+      uriBuilder.setParameter("h", hosts);
     }
 
     if (metrics.length() > 0) {
-      sb.append("&m=").append(metrics);
+      uriBuilder.setParameter("m", metrics);
     }
 
     if (temporalInfo != null) {
       long startTime = temporalInfo.getStartTime();
       if (startTime != -1) {
-        sb.append("&s=").append(startTime);
+        uriBuilder.setParameter("s", String.valueOf(startTime));
       }
 
       long endTime = temporalInfo.getEndTime();
       if (endTime != -1) {
-        sb.append("&e=").append(endTime);
+        uriBuilder.setParameter("e", String.valueOf(endTime));
       }
 
       long step = temporalInfo.getStep();
       if (step != -1) {
-        sb.append("&r=").append(step);
+        uriBuilder.setParameter("r", String.valueOf(step));
       }
     }
     else {
-      sb.append("&e=now");
-      sb.append("&pt=true");
+      uriBuilder.setParameter("e", "now");
+      uriBuilder.setParameter("pt", "true");
     }
 
-    return sb.toString();
+    return uriBuilder.toString();
   }
 
   /**
@@ -382,7 +386,6 @@ public abstract class GangliaPropertyProvider extends AbstractPropertyProvider {
     private final Map<String, Set<String>> metrics = new HashMap<String, Set<String>>();
     private final Set<String> clusterSet = new HashSet<String>();
     private final Set<String> hostSet = new HashSet<String>();
-    private boolean requestAll = false;
 
 
     private RRDRequest(String clusterName, TemporalInfo temporalInfo) {
@@ -390,10 +393,6 @@ public abstract class GangliaPropertyProvider extends AbstractPropertyProvider {
       this.temporalInfo = temporalInfo;
     }
 
-    public void setRequestAllMetrics(boolean requestAll) {
-      this.requestAll = this.requestAll | requestAll;
-    }
-
     public void putResource(ResourceKey key, Resource resource) {
       clusterSet.add(key.getClusterName());
       hostSet.add(key.getHostName());
@@ -424,23 +423,22 @@ public abstract class GangliaPropertyProvider extends AbstractPropertyProvider {
      */
     public Collection<Resource> populateResources() throws SystemException {
 
-      String spec = getSpec(clusterName, clusterSet, hostSet,
-          requestAll ? Collections.<String>emptySet() : metrics.keySet(), temporalInfo);
+      String spec = getSpec(clusterName, clusterSet, hostSet, metrics.keySet(), temporalInfo);
+
       BufferedReader reader = null;
       try {
         
         //Check if host is live
         if (!hostProvider.isGangliaCollectorHostLive(clusterName)) {
           LOG.info("Ganglia host is not live");
-            return Collections.emptySet();
+          return Collections.emptySet();
         }
         
         //Check if Ganglia server component is live
         if (!hostProvider.isGangliaCollectorComponentLive(clusterName)) {
           LOG.info("Ganglia server component is not live");
-            return Collections.emptySet();
+          return Collections.emptySet();
         }
-        
 
         reader = new BufferedReader(new InputStreamReader(
             getStreamProvider().readFrom(spec)));

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractPropertyProvider.java

@@ -107,20 +107,19 @@ public abstract class AbstractPropertyProvider extends BaseProvider implements P
     return propertyInfoMap;
   }
 
-  // TODO : added for FLUME and reg exp property ids... revisit.
-  protected boolean getPropertyInfoMap(String componentName, String propertyId, Map<String, PropertyInfo> propertyInfoMap) {
+  protected void getPropertyInfoMap(String componentName, String propertyId, Map<String, PropertyInfo> propertyInfoMap) {
     Map<String, PropertyInfo> componentMetricMap = componentMetrics.get(componentName);
 
     propertyInfoMap.clear();
 
     if (componentMetricMap == null) {
-      return false;
+      return;
     }
 
     PropertyInfo propertyInfo = componentMetricMap.get(propertyId);
     if (propertyInfo != null) {
       propertyInfoMap.put(propertyId, propertyInfo);
-      return false;
+      return;
     }
 
     String regExpKey = getRegExpKey(propertyId);
@@ -129,13 +128,10 @@ public abstract class AbstractPropertyProvider extends BaseProvider implements P
       propertyInfo = componentMetricMap.get(regExpKey);
       if (propertyInfo != null) {
         propertyInfoMap.put(regExpKey, propertyInfo);
-        return true;
+        return;
       }
     }
 
-    // TODO : For now, if the property info map contains any metrics with regular expressions then get back all the metrics.
-    boolean requestAllMetrics = false;
-
     if (!propertyId.endsWith("/")){
       propertyId += "/";
     }
@@ -143,7 +139,6 @@ public abstract class AbstractPropertyProvider extends BaseProvider implements P
     for (Map.Entry<String, PropertyInfo> entry : componentMetricMap.entrySet()) {
       if (entry.getKey().startsWith(propertyId)) {
         String key = entry.getKey();
-        requestAllMetrics = isPatternKey(key);
         propertyInfoMap.put(key, entry.getValue());
       }
     }
@@ -153,14 +148,15 @@ public abstract class AbstractPropertyProvider extends BaseProvider implements P
         regExpKey += "/";
       }
 
+      
       for (Map.Entry<String, PropertyInfo> entry : componentMetricMap.entrySet()) {
         if (entry.getKey().startsWith(regExpKey)) {
-          requestAllMetrics = true;
           propertyInfoMap.put(entry.getKey(), entry.getValue());
         }
       }
     }
-    return requestAllMetrics;
+
+    return;
   }
 
   /**

ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java

@@ -19,6 +19,7 @@ package org.apache.ambari.server.controller.ganglia;
 
 import org.apache.ambari.server.configuration.ComponentSSLConfiguration;
 import org.apache.ambari.server.configuration.ComponentSSLConfigurationTest;
+import org.apache.ambari.server.controller.internal.PropertyInfo;
 import org.apache.ambari.server.controller.internal.ResourceImpl;
 import org.apache.ambari.server.controller.internal.TemporalInfoImpl;
 import org.apache.ambari.server.controller.spi.Request;
@@ -27,6 +28,10 @@ import org.apache.ambari.server.controller.spi.SystemException;
 import org.apache.ambari.server.controller.spi.TemporalInfo;
 import org.apache.ambari.server.controller.utilities.PropertyHelper;
 import org.apache.ambari.server.controller.utilities.PropertyHelper.MetricsVersion;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.Predicate;
+import org.apache.http.NameValuePair;
+import org.apache.http.client.utils.URIBuilder;
 import org.junit.Assert;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -37,12 +42,14 @@ import org.powermock.api.easymock.PowerMock;
 import org.powermock.core.classloader.annotations.PrepareForTest;
 
 
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
@@ -96,8 +103,9 @@ public class GangliaPropertyProviderTest {
     TestStreamProvider streamProvider  = new TestStreamProvider("temporal_ganglia_data.txt");
     TestGangliaHostProvider hostProvider = new TestGangliaHostProvider();
 
+    Map<String, Map<String, PropertyInfo>> gangliaPropertyIds = PropertyHelper.getGangliaPropertyIds(Resource.Type.HostComponent, PropertyHelper.MetricsVersion.HDP1);
     GangliaPropertyProvider propertyProvider = new GangliaHostComponentPropertyProvider(
-        PropertyHelper.getGangliaPropertyIds(Resource.Type.HostComponent, PropertyHelper.MetricsVersion.HDP1),
+        gangliaPropertyIds,
         streamProvider,
         configuration,
         hostProvider,
@@ -135,8 +143,6 @@ public class GangliaPropertyProviderTest {
     // only ask for one property
     temporalInfoMap = new HashMap<String, TemporalInfo>();
 
-    //http://ec2-174-129-152-147.compute-1.amazonaws.com/cgi-bin/rrd.py?c=HDPSlaves&m=jvm.metrics.gcCount,mapred.shuffleOutput.shuffle_exceptions_caught,mapred.shuffleOutput.shuffle_failed_outputs,mapred.shuffleOutput.shuffle_output_bytes,mapred.shuffleOutput.shuffle_success_outputs&s=10&e=20&r=1&h=ip-10-85-111-149.ec2.internal
-
     Set<String> properties = new HashSet<String>();
     String shuffle_exceptions_caught = PropertyHelper.getPropertyId("metrics/mapred/shuffleOutput", "shuffle_exceptions_caught");
     String shuffle_failed_outputs    = PropertyHelper.getPropertyId("metrics/mapred/shuffleOutput", "shuffle_failed_outputs");
@@ -156,9 +162,37 @@ public class GangliaPropertyProviderTest {
 
     Assert.assertEquals(1, propertyProvider.populateResources(Collections.singleton(resource), request, null).size());
 
-    expected = (configuration.isGangliaSSL() ? "https" : "http") +
-        "://domU-12-31-39-0E-34-E1.compute-1.internal/cgi-bin/rrd.py?c=HDPSlaves&h=domU-12-31-39-0E-34-E1.compute-1.internal&m=mapred.shuffleOutput.shuffle_output_bytes,mapred.shuffleOutput.shuffle_success_outputs,mapred.shuffleOutput.shuffle_failed_outputs,mapred.shuffleOutput.shuffle_exceptions_caught&s=10&e=20&r=1";
-    Assert.assertEquals(expected, streamProvider.getLastSpec());
+    
+    List<String> metricsRegexes = new ArrayList<String>();
+    
+    metricsRegexes.add("metrics/mapred/shuffleOutput/shuffle_exceptions_caught");
+    metricsRegexes.add("metrics/mapred/shuffleOutput/shuffle_failed_outputs");
+    metricsRegexes.add("metrics/mapred/shuffleOutput/shuffle_output_bytes");
+    metricsRegexes.add("metrics/mapred/shuffleOutput/shuffle_success_outputs");
+    
+    
+    String metricsList = getMetricsRegexes(metricsRegexes, gangliaPropertyIds, "TASKTRACKER");
+    
+    URIBuilder expectedUri = new URIBuilder();
+
+    expectedUri.setScheme((configuration.isGangliaSSL() ? "https" : "http"));
+    expectedUri.setHost("domU-12-31-39-0E-34-E1.compute-1.internal");
+    expectedUri.setPath("/cgi-bin/rrd.py");
+    expectedUri.setParameter("c", "HDPSlaves");
+    expectedUri.setParameter("h", "domU-12-31-39-0E-34-E1.compute-1.internal");
+    expectedUri.setParameter("m", metricsList);
+    expectedUri.setParameter("s", "10");
+    expectedUri.setParameter("e", "20");
+    expectedUri.setParameter("r", "1");
+    
+
+    URIBuilder actualUri = new URIBuilder(streamProvider.getLastSpec());
+
+    Assert.assertEquals(expectedUri.getScheme(), actualUri.getScheme());
+    Assert.assertEquals(expectedUri.getHost(), actualUri.getHost());
+    Assert.assertEquals(expectedUri.getPath(), actualUri.getPath());
+    
+    Assert.assertTrue(isUrlParamsEquals(actualUri, expectedUri));
 
 
     Assert.assertEquals(6, PropertyHelper.getProperties(resource).size());
@@ -245,9 +279,22 @@ public class GangliaPropertyProviderTest {
     Request  request = PropertyHelper.getReadRequest(Collections.singleton(PROPERTY_ID), temporalInfoMap);
 
     Assert.assertEquals(3, propertyProvider.populateResources(resources, request, null).size());
-
-    String expected = (configuration.isGangliaSSL() ? "https" : "http") +
-        "://domU-12-31-39-0E-34-E1.compute-1.internal/cgi-bin/rrd.py?c=HDPJobTracker,HDPHBaseMaster,HDPResourceManager,HDPSlaves,HDPHistoryServer,HDPNameNode&h=domU-12-31-39-0E-34-E3.compute-1.internal,domU-12-31-39-0E-34-E1.compute-1.internal,domU-12-31-39-0E-34-E2.compute-1.internal&m=jvm.metrics.gcCount&s=10&e=20&r=1";
+    
+    URIBuilder uriBuilder = new URIBuilder();
+
+    uriBuilder.setScheme((configuration.isGangliaSSL() ? "https" : "http"));
+    uriBuilder.setHost("domU-12-31-39-0E-34-E1.compute-1.internal");
+    uriBuilder.setPath("/cgi-bin/rrd.py");
+    uriBuilder.setParameter("c", "HDPJobTracker,HDPHBaseMaster,HDPResourceManager,HDPSlaves,HDPHistoryServer,HDPNameNode");
+    uriBuilder.setParameter("h", "domU-12-31-39-0E-34-E3.compute-1.internal,domU-12-31-39-0E-34-E1.compute-1.internal,domU-12-31-39-0E-34-E2.compute-1.internal");
+    uriBuilder.setParameter("m", "jvm.metrics.gcCount");
+    uriBuilder.setParameter("s", "10");
+    uriBuilder.setParameter("e", "20");
+    uriBuilder.setParameter("r", "1");
+
+    String expected = uriBuilder.toString();
+    
+    
     Assert.assertEquals(expected, streamProvider.getLastSpec());
 
     for (Resource res : resources) {
@@ -285,8 +332,20 @@ public class GangliaPropertyProviderTest {
 
     Assert.assertEquals(150, propertyProvider.populateResources(resources, request, null).size());
 
-    String expected = (configuration.isGangliaSSL() ? "https" : "http") +
-        "://domU-12-31-39-0E-34-E1.compute-1.internal/cgi-bin/rrd.py?c=HDPJobTracker,HDPHBaseMaster,HDPResourceManager,HDPSlaves,HDPHistoryServer,HDPNameNode&m=jvm.metrics.gcCount&s=10&e=20&r=1";
+    
+    URIBuilder uriBuilder = new URIBuilder();
+    
+    uriBuilder.setScheme((configuration.isGangliaSSL() ? "https" : "http"));
+    uriBuilder.setHost("domU-12-31-39-0E-34-E1.compute-1.internal");
+    uriBuilder.setPath("/cgi-bin/rrd.py");
+    uriBuilder.setParameter("c", "HDPJobTracker,HDPHBaseMaster,HDPResourceManager,HDPSlaves,HDPHistoryServer,HDPNameNode");
+    uriBuilder.setParameter("m", "jvm.metrics.gcCount");
+    uriBuilder.setParameter("s", "10");
+    uriBuilder.setParameter("e", "20");
+    uriBuilder.setParameter("r", "1");
+    
+    String expected = uriBuilder.toString();
+    
     Assert.assertEquals(expected, streamProvider.getLastSpec());
 
   }
@@ -343,8 +402,9 @@ public class GangliaPropertyProviderTest {
     TestStreamProvider streamProvider  = new TestStreamProvider("flume_ganglia_data.txt");
     TestGangliaHostProvider hostProvider = new TestGangliaHostProvider();
 
+    Map<String, Map<String, PropertyInfo>> gangliaPropertyIds = PropertyHelper.getGangliaPropertyIds(Resource.Type.HostComponent, MetricsVersion.HDP1);
     GangliaPropertyProvider propertyProvider = new GangliaHostComponentPropertyProvider(
-        PropertyHelper.getGangliaPropertyIds(Resource.Type.HostComponent, MetricsVersion.HDP1),
+        gangliaPropertyIds,
         streamProvider,
         configuration,
         hostProvider,
@@ -365,10 +425,32 @@ public class GangliaPropertyProviderTest {
 
     Assert.assertEquals(1, propertyProvider.populateResources(Collections.singleton(resource), request, null).size());
 
-    String expected = (configuration.isGangliaSSL() ? "https" : "http") +
-        "://domU-12-31-39-0E-34-E1.compute-1.internal/cgi-bin/rrd.py?c=HDPSlaves&h=ip-10-39-113-33.ec2.internal&m=flume.CHANNEL.c1.ChannelCapacity&s=10&e=20&r=1";
-    Assert.assertEquals(expected, streamProvider.getLastSpec());
+    List<String> metricsRegexes = new ArrayList<String>();
+    
+    metricsRegexes.add(FLUME_CHANNEL_CAPACITY_PROPERTY);
+
+    String metricsList = getMetricsRegexes(metricsRegexes, gangliaPropertyIds, "FLUME_SERVER");
+    
+    URIBuilder expectedUri = new URIBuilder();
+    
+    expectedUri.setScheme((configuration.isGangliaSSL() ? "https" : "http"));
+    expectedUri.setHost("domU-12-31-39-0E-34-E1.compute-1.internal");
+    expectedUri.setPath("/cgi-bin/rrd.py");
+    expectedUri.setParameter("c", "HDPSlaves");
+    expectedUri.setParameter("h", "ip-10-39-113-33.ec2.internal");
+    expectedUri.setParameter("m", metricsList);
+    expectedUri.setParameter("s", "10");
+    expectedUri.setParameter("e", "20");
+    expectedUri.setParameter("r", "1");
+    
+    URIBuilder actualUri = new URIBuilder(streamProvider.getLastSpec());
 
+    Assert.assertEquals(expectedUri.getScheme(), actualUri.getScheme());
+    Assert.assertEquals(expectedUri.getHost(), actualUri.getHost());
+    Assert.assertEquals(expectedUri.getPath(), actualUri.getPath());
+    
+    Assert.assertTrue(isUrlParamsEquals(actualUri, expectedUri));    
+    
     Assert.assertEquals(3, PropertyHelper.getProperties(resource).size());
     Assert.assertNotNull(resource.getPropertyValue(FLUME_CHANNEL_CAPACITY_PROPERTY));
   }
@@ -378,15 +460,16 @@ public class GangliaPropertyProviderTest {
     TestStreamProvider streamProvider  = new TestStreamProvider("flume_ganglia_data.txt");
     TestGangliaHostProvider hostProvider = new TestGangliaHostProvider();
 
+    Map<String, Map<String, PropertyInfo>> gangliaPropertyIds = PropertyHelper.getGangliaPropertyIds(Resource.Type.HostComponent, MetricsVersion.HDP1);
     GangliaPropertyProvider propertyProvider = new GangliaHostComponentPropertyProvider(
-        PropertyHelper.getGangliaPropertyIds(Resource.Type.HostComponent, MetricsVersion.HDP1),
+        gangliaPropertyIds,
         streamProvider,
         configuration,
         hostProvider,
         CLUSTER_NAME_PROPERTY_ID,
         HOST_NAME_PROPERTY_ID,
         COMPONENT_NAME_PROPERTY_ID);
-
+    
     // flume
     Resource resource = new ResourceImpl(Resource.Type.HostComponent);
 
@@ -400,14 +483,36 @@ public class GangliaPropertyProviderTest {
     ids.add(FLUME_CATEGORY2);
     ids.add(PROPERTY_ID2);
 
-    Request  request = PropertyHelper.getReadRequest(ids, temporalInfoMap);
+    Request request = PropertyHelper.getReadRequest(ids, temporalInfoMap);
 
     Assert.assertEquals(1, propertyProvider.populateResources(Collections.singleton(resource), request, null).size());
 
-    String expected = (configuration.isGangliaSSL() ? "https" : "http") +
-        "://domU-12-31-39-0E-34-E1.compute-1.internal/cgi-bin/rrd.py?c=HDPSlaves&h=ip-10-39-113-33.ec2.internal&e=now&pt=true";
-    Assert.assertEquals(expected, streamProvider.getLastSpec());
+    List<String> metricsRegexes = new ArrayList<String>();
+    
+    metricsRegexes.add("metrics/flume");
+    metricsRegexes.add("metrics/cpu/cpu_wio");
+    
+    String metricsList = getMetricsRegexes(metricsRegexes, gangliaPropertyIds, "FLUME_SERVER");
+    
+    URIBuilder expectedUri = new URIBuilder();
+
+    expectedUri.setScheme((configuration.isGangliaSSL() ? "https" : "http"));
+    expectedUri.setHost("domU-12-31-39-0E-34-E1.compute-1.internal");
+    expectedUri.setPath("/cgi-bin/rrd.py");
+    expectedUri.setParameter("c", "HDPSlaves");
+    expectedUri.setParameter("h", "ip-10-39-113-33.ec2.internal");
+    expectedUri.setParameter("m", metricsList);
+    expectedUri.setParameter("e", "now");
+    expectedUri.setParameter("pt", "true");
+    
+    URIBuilder actualUri = new URIBuilder(streamProvider.getLastSpec());
 
+    Assert.assertEquals(expectedUri.getScheme(), actualUri.getScheme());
+    Assert.assertEquals(expectedUri.getHost(), actualUri.getHost());
+    Assert.assertEquals(expectedUri.getPath(), actualUri.getPath());
+    
+    Assert.assertTrue(isUrlParamsEquals(actualUri, expectedUri));
+       
     Assert.assertEquals(22, PropertyHelper.getProperties(resource).size());
     Assert.assertNotNull(resource.getPropertyValue(PROPERTY_ID2));
     Assert.assertNotNull(resource.getPropertyValue(FLUME_CHANNEL_CAPACITY_PROPERTY));
@@ -451,8 +556,9 @@ public class GangliaPropertyProviderTest {
     TestStreamProvider streamProvider  = new TestStreamProvider("flume_ganglia_data.txt");
     TestGangliaHostProvider hostProvider = new TestGangliaHostProvider();
 
+    Map<String, Map<String, PropertyInfo>> gangliaPropertyIds = PropertyHelper.getGangliaPropertyIds(Resource.Type.HostComponent, MetricsVersion.HDP1);
     GangliaPropertyProvider propertyProvider = new GangliaHostComponentPropertyProvider(
-        PropertyHelper.getGangliaPropertyIds(Resource.Type.HostComponent, MetricsVersion.HDP1),
+        gangliaPropertyIds,
         streamProvider,
         configuration,
         hostProvider,
@@ -473,9 +579,31 @@ public class GangliaPropertyProviderTest {
 
     Assert.assertEquals(1, propertyProvider.populateResources(Collections.singleton(resource), request, null).size());
 
-    String expected = (configuration.isGangliaSSL() ? "https" : "http") +
-        "://domU-12-31-39-0E-34-E1.compute-1.internal/cgi-bin/rrd.py?c=HDPSlaves&h=ip-10-39-113-33.ec2.internal&s=10&e=20&r=1";
-    Assert.assertEquals(expected, streamProvider.getLastSpec());
+    List<String> metricsRegexes = new ArrayList<String>();
+    
+    metricsRegexes.add("metrics/flume");
+    
+    String metricsList = getMetricsRegexes(metricsRegexes, gangliaPropertyIds, "FLUME_SERVER");
+    
+    URIBuilder expectedUri = new URIBuilder();
+
+    expectedUri.setScheme((configuration.isGangliaSSL() ? "https" : "http"));
+    expectedUri.setHost("domU-12-31-39-0E-34-E1.compute-1.internal");
+    expectedUri.setPath("/cgi-bin/rrd.py");
+    expectedUri.setParameter("c", "HDPSlaves");
+    expectedUri.setParameter("h", "ip-10-39-113-33.ec2.internal");
+    expectedUri.setParameter("m", metricsList);
+    expectedUri.setParameter("s", "10");
+    expectedUri.setParameter("e", "20");
+    expectedUri.setParameter("r", "1");
+    
+    URIBuilder actualUri = new URIBuilder(streamProvider.getLastSpec());
+
+    Assert.assertEquals(expectedUri.getScheme(), actualUri.getScheme());
+    Assert.assertEquals(expectedUri.getHost(), actualUri.getHost());
+    Assert.assertEquals(expectedUri.getPath(), actualUri.getPath());
+    
+    Assert.assertTrue(isUrlParamsEquals(actualUri, expectedUri));    
 
     Assert.assertEquals(21, PropertyHelper.getProperties(resource).size());
     Assert.assertNotNull(resource.getPropertyValue(FLUME_CHANNEL_CAPACITY_PROPERTY));
@@ -486,8 +614,9 @@ public class GangliaPropertyProviderTest {
     TestStreamProvider streamProvider  = new TestStreamProvider("flume_ganglia_data.txt");
     TestGangliaHostProvider hostProvider = new TestGangliaHostProvider();
 
+    Map<String, Map<String, PropertyInfo>> gangliaPropertyIds = PropertyHelper.getGangliaPropertyIds(Resource.Type.HostComponent, MetricsVersion.HDP1);
     GangliaPropertyProvider propertyProvider = new GangliaHostComponentPropertyProvider(
-        PropertyHelper.getGangliaPropertyIds(Resource.Type.HostComponent, MetricsVersion.HDP1),
+        gangliaPropertyIds,
         streamProvider,
         configuration,
         hostProvider,
@@ -508,9 +637,31 @@ public class GangliaPropertyProviderTest {
 
     Assert.assertEquals(1, propertyProvider.populateResources(Collections.singleton(resource), request, null).size());
 
-    String expected = (configuration.isGangliaSSL() ? "https" : "http") +
-        "://domU-12-31-39-0E-34-E1.compute-1.internal/cgi-bin/rrd.py?c=HDPSlaves&h=ip-10-39-113-33.ec2.internal&s=10&e=20&r=1";
-    Assert.assertEquals(expected, streamProvider.getLastSpec());
+    List<String> metricsRegexes = new ArrayList<String>();
+    
+    metricsRegexes.add("metrics/flume/");
+    
+    String metricsList = getMetricsRegexes(metricsRegexes, gangliaPropertyIds, "FLUME_SERVER");
+    
+    URIBuilder expectedUri = new URIBuilder();
+
+    expectedUri.setScheme((configuration.isGangliaSSL() ? "https" : "http"));
+    expectedUri.setHost("domU-12-31-39-0E-34-E1.compute-1.internal");
+    expectedUri.setPath("/cgi-bin/rrd.py");
+    expectedUri.setParameter("c", "HDPSlaves");
+    expectedUri.setParameter("h", "ip-10-39-113-33.ec2.internal");
+    expectedUri.setParameter("m", metricsList);
+    expectedUri.setParameter("s", "10");
+    expectedUri.setParameter("e", "20");
+    expectedUri.setParameter("r", "1");
+    
+    URIBuilder actualUri = new URIBuilder(streamProvider.getLastSpec());
+
+    Assert.assertEquals(expectedUri.getScheme(), actualUri.getScheme());
+    Assert.assertEquals(expectedUri.getHost(), actualUri.getHost());
+    Assert.assertEquals(expectedUri.getPath(), actualUri.getPath());
+    
+    Assert.assertTrue(isUrlParamsEquals(actualUri, expectedUri));
 
     Assert.assertEquals(21, PropertyHelper.getProperties(resource).size());
     Assert.assertNotNull(resource.getPropertyValue(FLUME_CHANNEL_CAPACITY_PROPERTY));
@@ -521,8 +672,9 @@ public class GangliaPropertyProviderTest {
     TestStreamProvider streamProvider  = new TestStreamProvider("flume_ganglia_data.txt");
     TestGangliaHostProvider hostProvider = new TestGangliaHostProvider();
 
+    Map<String, Map<String, PropertyInfo>> gangliaPropertyIds = PropertyHelper.getGangliaPropertyIds(Resource.Type.HostComponent, MetricsVersion.HDP1);
     GangliaPropertyProvider propertyProvider = new GangliaHostComponentPropertyProvider(
-        PropertyHelper.getGangliaPropertyIds(Resource.Type.HostComponent, MetricsVersion.HDP1),
+        gangliaPropertyIds,
         streamProvider,
         configuration,
         hostProvider,
@@ -542,10 +694,33 @@ public class GangliaPropertyProviderTest {
     Request  request = PropertyHelper.getReadRequest(Collections.singleton(FLUME_CATEGORY3), temporalInfoMap);
 
     Assert.assertEquals(1, propertyProvider.populateResources(Collections.singleton(resource), request, null).size());
+    
+    List<String> metricsRegexes = new ArrayList<String>();
+    
+    metricsRegexes.add("metrics/flume/$1/CHANNEL/$2/");
+    metricsRegexes.add(FLUME_CHANNEL_CAPACITY_PROPERTY);
 
-    String expected = (configuration.isGangliaSSL() ? "https" : "http") +
-        "://domU-12-31-39-0E-34-E1.compute-1.internal/cgi-bin/rrd.py?c=HDPSlaves&h=ip-10-39-113-33.ec2.internal&s=10&e=20&r=1";
-    Assert.assertEquals(expected, streamProvider.getLastSpec());
+    String metricsList = getMetricsRegexes(metricsRegexes, gangliaPropertyIds, "FLUME_SERVER");
+    
+    URIBuilder expectedUri = new URIBuilder();
+
+    expectedUri.setScheme((configuration.isGangliaSSL() ? "https" : "http"));
+    expectedUri.setHost("domU-12-31-39-0E-34-E1.compute-1.internal");
+    expectedUri.setPath("/cgi-bin/rrd.py");
+    expectedUri.setParameter("c", "HDPSlaves");
+    expectedUri.setParameter("h", "ip-10-39-113-33.ec2.internal");
+    expectedUri.setParameter("m", metricsList);
+    expectedUri.setParameter("s", "10");
+    expectedUri.setParameter("e", "20");
+    expectedUri.setParameter("r", "1");
+    
+    URIBuilder actualUri = new URIBuilder(streamProvider.getLastSpec());
+
+    Assert.assertEquals(expectedUri.getScheme(), actualUri.getScheme());
+    Assert.assertEquals(expectedUri.getHost(), actualUri.getHost());
+    Assert.assertEquals(expectedUri.getPath(), actualUri.getPath());
+    
+    Assert.assertTrue(isUrlParamsEquals(actualUri, expectedUri));    
 
     Assert.assertEquals(11, PropertyHelper.getProperties(resource).size());
     Assert.assertNotNull(resource.getPropertyValue(FLUME_CHANNEL_CAPACITY_PROPERTY));
@@ -556,8 +731,9 @@ public class GangliaPropertyProviderTest {
     TestStreamProvider streamProvider  = new TestStreamProvider("flume_ganglia_data.txt");
     TestGangliaHostProvider hostProvider = new TestGangliaHostProvider();
 
+    Map<String, Map<String, PropertyInfo>> gangliaPropertyIds = PropertyHelper.getGangliaPropertyIds(Resource.Type.HostComponent, MetricsVersion.HDP1);
     GangliaPropertyProvider propertyProvider = new GangliaHostComponentPropertyProvider(
-        PropertyHelper.getGangliaPropertyIds(Resource.Type.HostComponent, MetricsVersion.HDP1),
+        gangliaPropertyIds,
         streamProvider,
         configuration,
         hostProvider,
@@ -577,11 +753,34 @@ public class GangliaPropertyProviderTest {
     Request  request = PropertyHelper.getReadRequest(Collections.singleton(FLUME_CATEGORY4), temporalInfoMap);
 
     Assert.assertEquals(1, propertyProvider.populateResources(Collections.singleton(resource), request, null).size());
+    
+    List<String> metricsRegexes = new ArrayList<String>();
+    
+    metricsRegexes.add("metrics/flume/$1/CHANNEL/$2");
+    metricsRegexes.add(FLUME_CHANNEL_CAPACITY_PROPERTY);
 
-    String expected = (configuration.isGangliaSSL() ? "https" : "http") +
-        "://domU-12-31-39-0E-34-E1.compute-1.internal/cgi-bin/rrd.py?c=HDPSlaves&h=ip-10-39-113-33.ec2.internal&s=10&e=20&r=1";
-    Assert.assertEquals(expected, streamProvider.getLastSpec());
+    String metricsList = getMetricsRegexes(metricsRegexes, gangliaPropertyIds, "FLUME_SERVER");
+    
+    URIBuilder expectedUri = new URIBuilder();
+
+    expectedUri.setScheme((configuration.isGangliaSSL() ? "https" : "http"));
+    expectedUri.setHost("domU-12-31-39-0E-34-E1.compute-1.internal");
+    expectedUri.setPath("/cgi-bin/rrd.py");
+    expectedUri.setParameter("c", "HDPSlaves");
+    expectedUri.setParameter("h", "ip-10-39-113-33.ec2.internal");
+    expectedUri.setParameter("m", metricsList);
+    expectedUri.setParameter("s", "10");
+    expectedUri.setParameter("e", "20");
+    expectedUri.setParameter("r", "1");
+    
+    URIBuilder actualUri = new URIBuilder(streamProvider.getLastSpec());
 
+    Assert.assertEquals(expectedUri.getScheme(), actualUri.getScheme());
+    Assert.assertEquals(expectedUri.getHost(), actualUri.getHost());
+    Assert.assertEquals(expectedUri.getPath(), actualUri.getPath());
+    
+    Assert.assertTrue(isUrlParamsEquals(actualUri, expectedUri));    
+    
     Assert.assertEquals(11, PropertyHelper.getProperties(resource).size());
     Assert.assertNotNull(resource.getPropertyValue(FLUME_CHANNEL_CAPACITY_PROPERTY));
   }
@@ -591,8 +790,9 @@ public class GangliaPropertyProviderTest {
     TestStreamProvider streamProvider  = new TestStreamProvider("temporal_ganglia_data_yarn_queues.txt");
     TestGangliaHostProvider hostProvider = new TestGangliaHostProvider();
 
+    Map<String, Map<String, PropertyInfo>> gangliaPropertyIds = PropertyHelper.getGangliaPropertyIds(Resource.Type.HostComponent, MetricsVersion.HDP2);
     GangliaPropertyProvider propertyProvider = new GangliaHostComponentPropertyProvider(
-        PropertyHelper.getGangliaPropertyIds(Resource.Type.HostComponent, MetricsVersion.HDP2),
+        gangliaPropertyIds,
         streamProvider,
         configuration,
         hostProvider,
@@ -612,14 +812,37 @@ public class GangliaPropertyProviderTest {
     Request  request = PropertyHelper.getReadRequest(Collections.singleton(RM_CATEGORY_1), temporalInfoMap);
 
     Assert.assertEquals(1, propertyProvider.populateResources(Collections.singleton(resource), request, null).size());
+    
+    List<String> metricsRegexes = new ArrayList<String>();
+    
+    metricsRegexes.add("metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/");
 
-    String expected = (configuration.isGangliaSSL() ? "https" : "http") +
-        "://domU-12-31-39-0E-34-E1.compute-1.internal/cgi-bin/rrd.py?c=HDPResourceManager&h=dev01.ambari.apache.org&s=10&e=20&r=1";
-    Assert.assertEquals(expected, streamProvider.getLastSpec());    
+    String metricsList = getMetricsRegexes(metricsRegexes, gangliaPropertyIds, "RESOURCEMANAGER");
+    
+    URIBuilder expectedUri = new URIBuilder();
+
+    expectedUri.setScheme((configuration.isGangliaSSL() ? "https" : "http"));
+    expectedUri.setHost("domU-12-31-39-0E-34-E1.compute-1.internal");
+    expectedUri.setPath("/cgi-bin/rrd.py");
+    expectedUri.setParameter("c", "HDPResourceManager");
+    expectedUri.setParameter("h", "dev01.ambari.apache.org");
+    expectedUri.setParameter("m", metricsList);
+    expectedUri.setParameter("s", "10");
+    expectedUri.setParameter("e", "20");
+    expectedUri.setParameter("r", "1");
+    
+    URIBuilder actualUri = new URIBuilder(streamProvider.getLastSpec());
+
+    Assert.assertEquals(expectedUri.getScheme(), actualUri.getScheme());
+    Assert.assertEquals(expectedUri.getHost(), actualUri.getHost());
+    Assert.assertEquals(expectedUri.getPath(), actualUri.getPath());
+    
+    Assert.assertTrue(isUrlParamsEquals(actualUri, expectedUri));    
+    
     
     Assert.assertTrue(PropertyHelper.getProperties(resource).size() > 2);
     Assert.assertNotNull(resource.getPropertyValue(RM_AVAILABLE_MEMORY_PROPERTY));
-  }  
+  }
 
   @Test
   public void testPopulateResources_journalNode() throws Exception {
@@ -763,6 +986,52 @@ public class GangliaPropertyProviderTest {
       Assert.assertEquals(testData[i++][2], resource.getPropertyValue(property));
     }
   }
+  
+  private boolean isUrlParamsEquals(URIBuilder actualUri, URIBuilder expectedUri) {
+    for (final NameValuePair expectedParam : expectedUri.getQueryParams()) {
+      NameValuePair actualParam = (NameValuePair) CollectionUtils.find(actualUri.getQueryParams(), new Predicate() {
+        
+        @Override
+        public boolean evaluate(Object arg0) {
+          if (!(arg0 instanceof NameValuePair))
+            return false;
+          
+          NameValuePair otherObj = (NameValuePair) arg0;
+          return otherObj.getName().equals(expectedParam.getName());
+        }
+      });
+      
+
+      List<String> actualParamList = new ArrayList<String>(Arrays.asList(actualParam.getValue().split(",")));
+      List<String> expectedParamList = new ArrayList<String>(Arrays.asList(expectedParam.getValue().split(",")));
+      
+      Collections.sort(actualParamList);
+      Collections.sort(expectedParamList);
+      
+      if (!actualParamList.equals(expectedParamList))
+        return false;
+    }
+    
+    return true;
+  }
+  
+  private String getMetricsRegexes(List<String> metricsRegexes,
+      Map<String, Map<String, PropertyInfo>> gangliaPropertyIds,
+      String componentName) {
+    
+    StringBuilder metricsBuilder = new StringBuilder();
+    
+    for (Map.Entry<String, PropertyInfo> entry : gangliaPropertyIds.get(componentName).entrySet())
+    {
+      for (String metricRegex: metricsRegexes)
+      {
+        if (entry.getKey().startsWith(metricRegex)) {
+          metricsBuilder.append(entry.getValue().getPropertyId() + ",");
+        }
+      }
+    }
+    return metricsBuilder.toString();
+  }
 
   private static class TestGangliaHostProvider implements GangliaHostProvider {