AMBARI-2786. YARN time series data needed for NodeManager statuses. (Andrew Onischuk via smohanty)

ambari-server/src/main/resources/ganglia_properties_2.json

@@ -6578,6 +6578,31 @@
         "pointInTime":false,
         "temporal":true
       },
+      "metrics/yarn/ClusterMetrics/NumActiveNMs":{
+        "metric":"yarn.ClusterMetrics.NumActiveNMs",
+        "pointInTime":false,
+        "temporal":true
+      },
+      "metrics/yarn/ClusterMetrics/NumDecommissionedNMs":{
+        "metric":"yarn.ClusterMetrics.NumDecommissionedNMs",
+        "pointInTime":false,
+        "temporal":true
+      },
+      "metrics/yarn/ClusterMetrics/NumLostNMs":{
+        "metric":"yarn.ClusterMetrics.NumLostNMs",
+        "pointInTime":false,
+        "temporal":true
+      },
+      "metrics/yarn/ClusterMetrics/NumUnhealthyNMs":{
+        "metric":"yarn.ClusterMetrics.NumUnhealthyNMs",
+        "pointInTime":false,
+        "temporal":true
+      },
+      "metrics/yarn/ClusterMetrics/NumRebootedNMs":{
+        "metric":"yarn.ClusterMetrics.NumRebootedNMs",
+        "pointInTime":false,
+        "temporal":true
+      },
       "metrics/yarn/Queue/AllocatedVCores":{
         "metric":"yarn.QueueMetrics.AllocatedVCores",
         "pointInTime":false,
@@ -16698,6 +16723,31 @@
         "pointInTime":false,
         "temporal":true
       },
+      "metrics/yarn/ClusterMetrics/NumActiveNMs":{
+        "metric":"yarn.ClusterMetrics.NumActiveNMs",
+        "pointInTime":false,
+        "temporal":true
+      },
+      "metrics/yarn/ClusterMetrics/NumDecommissionedNMs":{
+        "metric":"yarn.ClusterMetrics.NumDecommissionedNMs",
+        "pointInTime":false,
+        "temporal":true
+      },
+      "metrics/yarn/ClusterMetrics/NumLostNMs":{
+        "metric":"yarn.ClusterMetrics.NumLostNMs",
+        "pointInTime":false,
+        "temporal":true
+      },
+      "metrics/yarn/ClusterMetrics/NumUnhealthyNMs":{
+        "metric":"yarn.ClusterMetrics.NumUnhealthyNMs",
+        "pointInTime":false,
+        "temporal":true
+      },
+      "metrics/yarn/ClusterMetrics/NumRebootedNMs":{
+        "metric":"yarn.ClusterMetrics.NumRebootedNMs",
+        "pointInTime":false,
+        "temporal":true
+      },
       "metrics/yarn/Queue/AllocatedVCores":{
         "metric":"yarn.QueueMetrics.AllocatedVCores",
         "pointInTime":false,

ambari-server/src/main/resources/jmx_properties_2.json

@@ -3560,6 +3560,31 @@
         "metric" : "Hadoop:service=ResourceManager,name=RMNMInfo.LiveNodeManagers",
         "pointInTime" : true,
         "temporal" : false
+       },
+       "metrics/yarn/ClusterMetrics/NumActiveNMs":{
+        "metric" : "Hadoop:service=ResourceManager,name=ClusterMetrics.NumActiveNMs",
+        "pointInTime":true,
+        "temporal":false
+      },
+      "metrics/yarn/ClusterMetrics/NumDecommissionedNMs":{
+        "metric" : "Hadoop:service=ResourceManager,name=ClusterMetrics.NumDecommissionedNMs",
+        "pointInTime":true,
+        "temporal":false
+      },
+      "metrics/yarn/ClusterMetrics/NumLostNMs":{
+        "metric" : "Hadoop:service=ResourceManager,name=ClusterMetrics.NumLostNMs",
+        "pointInTime":true,
+        "temporal":false
+      },
+      "metrics/yarn/ClusterMetrics/NumUnhealthyNMs":{
+        "metric" : "Hadoop:service=ResourceManager,name=ClusterMetrics.NumUnhealthyNMs",
+        "pointInTime":true,
+        "temporal":false
+      },
+      "metrics/yarn/ClusterMetrics/NumRebootedNMs":{
+        "metric" : "Hadoop:service=ResourceManager,name=ClusterMetrics.NumRebootedNMs",
+        "pointInTime":true,
+        "temporal":false
       },
       "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/running_60":{
         "metric" : "Hadoop:service=ResourceManager,name=QueueMetrics(.+).running_60",
@@ -7914,6 +7939,31 @@
         "pointInTime" : true,
         "temporal" : false
       },
+      "metrics/yarn/ClusterMetrics/NumActiveNMs":{
+        "metric" : "Hadoop:service=ResourceManager,name=ClusterMetrics.NumActiveNMs",
+        "pointInTime":true,
+        "temporal":false
+      },
+      "metrics/yarn/ClusterMetrics/NumDecommissionedNMs":{
+        "metric" : "Hadoop:service=ResourceManager,name=ClusterMetrics.NumDecommissionedNMs",
+        "pointInTime":true,
+        "temporal":false
+      },
+      "metrics/yarn/ClusterMetrics/NumLostNMs":{
+        "metric" : "Hadoop:service=ResourceManager,name=ClusterMetrics.NumLostNMs",
+        "pointInTime":true,
+        "temporal":false
+      },
+      "metrics/yarn/ClusterMetrics/NumUnhealthyNMs":{
+        "metric" : "Hadoop:service=ResourceManager,name=ClusterMetrics.NumUnhealthyNMs",
+        "pointInTime":true,
+        "temporal":false
+      },
+      "metrics/yarn/ClusterMetrics/NumRebootedNMs":{
+        "metric" : "Hadoop:service=ResourceManager,name=ClusterMetrics.NumRebootedNMs",
+        "pointInTime":true,
+        "temporal":false
+      },
       "metrics/yarn/Queue/$1.replaceAll(\",q(\\d+)=\",\"/\").substring(1)/running_60":{
         "metric" : "Hadoop:service=ResourceManager,name=QueueMetrics(.+).running_60",
         "pointInTime" : true,

ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java

@@ -23,6 +23,7 @@ import org.apache.ambari.server.controller.internal.ResourceImpl;
 import org.apache.ambari.server.controller.internal.TemporalInfoImpl;
 import org.apache.ambari.server.controller.spi.Request;
 import org.apache.ambari.server.controller.spi.Resource;
+import org.apache.ambari.server.controller.spi.SystemException;
 import org.apache.ambari.server.controller.spi.TemporalInfo;
 import org.apache.ambari.server.controller.utilities.PropertyHelper;
 import org.apache.ambari.server.controller.utilities.PropertyHelper.MetricsVersion;
@@ -238,6 +239,52 @@ public class GangliaPropertyProviderTest {
     Assert.assertEquals(expected, streamProvider.getLastSpec());
 
   }
+  
+  class PopulateResourceManagerResourcesTest{
+    public void testProperty(String property, String jmxMetricName) throws SystemException{
+      TestStreamProvider streamProvider  = new TestStreamProvider("yarn_ganglia_data.txt");
+      TestGangliaHostProvider hostProvider = new TestGangliaHostProvider();
+
+      GangliaPropertyProvider propertyProvider = new GangliaHostComponentPropertyProvider(
+          PropertyHelper.getGangliaPropertyIds(Resource.Type.HostComponent, MetricsVersion.HDP2),
+          streamProvider,
+          configuration,
+          hostProvider,
+          CLUSTER_NAME_PROPERTY_ID,
+          HOST_NAME_PROPERTY_ID,
+          COMPONENT_NAME_PROPERTY_ID);
+
+      // flume
+      Resource resource = new ResourceImpl(Resource.Type.HostComponent);
+
+      resource.setProperty(HOST_NAME_PROPERTY_ID, "ip-10-39-113-33.ec2.internal");
+      resource.setProperty(COMPONENT_NAME_PROPERTY_ID, "RESOURCEMANAGER");
+
+      // only ask for one property
+      Map<String, TemporalInfo> temporalInfoMap = new HashMap<String, TemporalInfo>();
+      temporalInfoMap.put(property, new TemporalInfoImpl(10L, 20L, 1L));
+      Request  request = PropertyHelper.getReadRequest(Collections.singleton(property), temporalInfoMap);
+
+      Assert.assertEquals(1, propertyProvider.populateResources(Collections.singleton(resource), request, null).size());
+
+      String expected = (configuration.isGangliaSSL() ? "https" : "http") +
+          String.format("://domU-12-31-39-0E-34-E1.compute-1.internal/cgi-bin/rrd.py?c=HDPResourceManager&h=ip-10-39-113-33.ec2.internal&m=%s&s=10&e=20&r=1",jmxMetricName);
+      Assert.assertEquals(expected, streamProvider.getLastSpec());
+
+      Assert.assertEquals(3, PropertyHelper.getProperties(resource).size());
+      Assert.assertNotNull(resource.getPropertyValue(property));
+    }
+  }
+  
+  @Test
+  public void testPopulateResources_resourcemanager_clustermetrics() throws Exception {
+	  PopulateResourceManagerResourcesTest tester = new PopulateResourceManagerResourcesTest();
+	  tester.testProperty("metrics/yarn/ClusterMetrics/NumActiveNMs","yarn.ClusterMetrics.NumActiveNMs");
+	  tester.testProperty("metrics/yarn/ClusterMetrics/NumDecommissionedNMs","yarn.ClusterMetrics.NumDecommissionedNMs");
+	  tester.testProperty("metrics/yarn/ClusterMetrics/NumLostNMs", "yarn.ClusterMetrics.NumLostNMs");
+	  tester.testProperty("metrics/yarn/ClusterMetrics/NumUnhealthyNMs", "yarn.ClusterMetrics.NumUnhealthyNMs");
+	  tester.testProperty("metrics/yarn/ClusterMetrics/NumRebootedNMs", "yarn.ClusterMetrics.NumRebootedNMs");
+  }
 
 
   @Test

ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/JMXPropertyProviderTest.java

@@ -375,6 +375,12 @@ public class JMXPropertyProviderTest {
     Assert.assertEquals(8192,  resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/yarn/Queue/root", "AvailableMB")));
     Assert.assertEquals(1,  resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/yarn/Queue/root", "AvailableVCores")));
     Assert.assertEquals(2,  resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/yarn/Queue/root", "AppsSubmitted")));
+    
+    Assert.assertEquals(1,  resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/yarn/ClusterMetrics", "NumActiveNMs")));
+    Assert.assertEquals(0,  resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/yarn/ClusterMetrics", "NumDecommissionedNMs")));
+    Assert.assertEquals(0,  resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/yarn/ClusterMetrics", "NumLostNMs")));
+    Assert.assertEquals(0,  resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/yarn/ClusterMetrics", "NumUnhealthyNMs")));
+    Assert.assertEquals(0,  resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/yarn/ClusterMetrics", "NumRebootedNMs")));
   }
   
   @Test