AMBARI-1206. Expose missing metrics on host components. (tbeerbower via mahadev)

git-svn-id: https://svn.apache.org/repos/asf/incubator/ambari/trunk@1435464 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -82,6 +82,9 @@ Trunk (unreleased changes):
  AMBARI-1213. Cleanup python test cases and introduce third party library for
  mock testing python code. (mahadev)
 
+ AMBARI-1206. Expose missing metrics on host components. (tbeerbower via
+ mahadev)
+
  BUG FIXES
 
  AMBARI-1230. There is a big gap in the lower part of the Jobs table header.

ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/datanode.pp

@@ -91,7 +91,7 @@ class hdp-hadoop::datanode(
 define hdp-hadoop::datanode::create_data_dirs($service_state)
 {
   $dirs = hdp_array_from_comma_list($name)
-  hdp::directory_recursive_create { $dirs :
+  hdp::directory_recursive_create_ignore_failure { $dirs :
     owner => $hdp-hadoop::params::hdfs_user,
     mode => '0750',
     service_state => $service_state,

ambari-server/src/main/resources/ganglia_properties.json

@@ -13918,11 +13918,31 @@
         "pointInTime":true,
         "temporal":true
       },
+      "metrics/mapred/shuffleOutput/shuffle_exceptions_caught":{
+        "metric":"mapred.shuffleOutput.shuffle_exceptions_caught",
+        "pointInTime":true,
+        "temporal":true
+      },
+      "metrics/mapred/shuffleOutput/shuffle_failed_outputs":{
+        "metric":"mapred.shuffleOutput.shuffle_failed_outputs",
+        "pointInTime":true,
+        "temporal":true
+      },
       "metrics/mapred/shuffleOutput/shuffle_handler_busy_percent":{
         "metric":"mapred.shuffleOutput.shuffle_handler_busy_percent",
         "pointInTime":true,
         "temporal":true
       },
+      "metrics/mapred/shuffleOutput/shuffle_output_bytes":{
+        "metric":"mapred.shuffleOutput.shuffle_output_bytes",
+        "pointInTime":true,
+        "temporal":true
+      },
+      "metrics/mapred/shuffleOutput/shuffle_success_outputs":{
+        "metric":"mapred.shuffleOutput.shuffle_success_outputs",
+        "pointInTime":true,
+        "temporal":true
+      },
       "metrics/mapred/tasktracker/mapTaskSlots":{
         "metric":"mapred.tasktracker.mapTaskSlots",
         "pointInTime":true,

ambari-server/src/main/resources/jmx_properties.json

@@ -49,6 +49,16 @@
         "pointInTime" : true,
         "temporal" : false
       },
+      "ServiceComponentInfo/NonHeapMemoryMax":{
+        "metric" : "java.lang:type=Memory.NonHeapMemoryUsage[max]",
+        "pointInTime" : true,
+        "temporal" : false
+      },
+      "ServiceComponentInfo/NonHeapMemoryUsed":{
+        "metric" : "java.lang:type=Memory.NonHeapMemoryUsage[used]",
+        "pointInTime" : true,
+        "temporal" : false
+      },
       "ServiceComponentInfo/LiveNodes":{
         "metric" : "Hadoop:service=NameNode,name=NameNodeInfo.LiveNodes",
         "pointInTime" : true,
@@ -558,6 +568,16 @@
         "pointInTime" : true,
         "temporal" : false
       },
+      "ServiceComponentInfo/NonHeapMemoryMax":{
+        "metric" : "java.lang:type=Memory.NonHeapMemoryUsage[max]",
+        "pointInTime" : true,
+        "temporal" : false
+      },
+      "ServiceComponentInfo/NonHeapMemoryUsed":{
+        "metric" : "java.lang:type=Memory.NonHeapMemoryUsage[used]",
+        "pointInTime" : true,
+        "temporal" : false
+      },
       "ServiceComponentInfo/Queue/jobs_completed":{
         "metric" : "Hadoop:service=JobTracker,name=QueueMetrics,q=default.jobs_completed",
         "pointInTime" : true,
@@ -971,7 +991,6 @@
     },
 
     "HBASE_MASTER":{
-
       "ServiceComponentInfo/AverageLoad":{
         "metric" : "hadoop:service=Master,name=Master.AverageLoad",
         "pointInTime" : true,
@@ -987,6 +1006,16 @@
         "pointInTime" : true,
         "temporal" : false
       },
+      "ServiceComponentInfo/NonHeapMemoryMax":{
+        "metric" : "java.lang:type=Memory.NonHeapMemoryUsage[max]",
+        "pointInTime" : true,
+        "temporal" : false
+      },
+      "ServiceComponentInfo/NonHeapMemoryUsed":{
+        "metric" : "java.lang:type=Memory.NonHeapMemoryUsage[used]",
+        "pointInTime" : true,
+        "temporal" : false
+      },
       "ServiceComponentInfo/MasterActiveTime":{
         "metric" : "hadoop:service=Master,name=Master.MasterActiveTime",
         "pointInTime" : true,
@@ -1151,6 +1180,26 @@
         "pointInTime" : true,
         "temporal" : false
       },
+      "metrics/jvm/HeapMemoryMax":{
+        "metric" : "java.lang:type=Memory.HeapMemoryUsage[max]",
+        "pointInTime" : true,
+        "temporal" : false
+      },
+      "metrics/jvm/HeapMemoryUsed":{
+        "metric" : "java.lang:type=Memory.HeapMemoryUsage[used]",
+        "pointInTime" : true,
+        "temporal" : false
+      },
+      "metrics/jvm/NonHeapMemoryMax":{
+        "metric" : "java.lang:type=Memory.NonHeapMemoryUsage[max]",
+        "pointInTime" : true,
+        "temporal" : false
+      },
+      "metrics/jvm/NonHeapMemoryUsed":{
+        "metric" : "java.lang:type=Memory.NonHeapMemoryUsage[used]",
+        "pointInTime" : true,
+        "temporal" : false
+      },
       "metrics/jvm/gcCount":{
         "metric" : "Hadoop:service=NameNode,name=jvm.gcCount",
         "pointInTime" : true,
@@ -1513,6 +1562,26 @@
         "pointInTime" : true,
         "temporal" : false
       },
+      "metrics/jvm/HeapMemoryMax":{
+        "metric" : "java.lang:type=Memory.HeapMemoryUsage[max]",
+        "pointInTime" : true,
+        "temporal" : false
+      },
+      "metrics/jvm/HeapMemoryUsed":{
+        "metric" : "java.lang:type=Memory.HeapMemoryUsage[used]",
+        "pointInTime" : true,
+        "temporal" : false
+      },
+      "metrics/jvm/NonHeapMemoryMax":{
+        "metric" : "java.lang:type=Memory.NonHeapMemoryUsage[max]",
+        "pointInTime" : true,
+        "temporal" : false
+      },
+      "metrics/jvm/NonHeapMemoryUsed":{
+        "metric" : "java.lang:type=Memory.NonHeapMemoryUsage[used]",
+        "pointInTime" : true,
+        "temporal" : false
+      },
       "metrics/jvm/gcCount":{
         "metric" : "Hadoop:service=DataNode,name=jvm.gcCount",
         "pointInTime" : true,
@@ -1805,6 +1874,26 @@
       }
     },
     "JOBTRACKER":{
+      "metrics/jvm/HeapMemoryMax":{
+        "metric" : "java.lang:type=Memory.HeapMemoryUsage[max]",
+        "pointInTime" : true,
+        "temporal" : false
+      },
+      "metrics/jvm/HeapMemoryUsed":{
+        "metric" : "java.lang:type=Memory.HeapMemoryUsage[used]",
+        "pointInTime" : true,
+        "temporal" : false
+      },
+      "metrics/jvm/NonHeapMemoryMax":{
+        "metric" : "java.lang:type=Memory.NonHeapMemoryUsage[max]",
+        "pointInTime" : true,
+        "temporal" : false
+      },
+      "metrics/jvm/NonHeapMemoryUsed":{
+        "metric" : "java.lang:type=Memory.NonHeapMemoryUsage[used]",
+        "pointInTime" : true,
+        "temporal" : false
+      },
       "metrics/jvm/gcCount":{
         "metric" : "Hadoop:service=JobTracker,name=jvm.gcCount",
         "pointInTime" : true,
@@ -2097,6 +2186,26 @@
       }
     },
     "TASKTRACKER":{
+      "metrics/jvm/HeapMemoryMax":{
+        "metric" : "java.lang:type=Memory.HeapMemoryUsage[max]",
+        "pointInTime" : true,
+        "temporal" : false
+      },
+      "metrics/jvm/HeapMemoryUsed":{
+        "metric" : "java.lang:type=Memory.HeapMemoryUsage[used]",
+        "pointInTime" : true,
+        "temporal" : false
+      },
+      "metrics/jvm/NonHeapMemoryMax":{
+        "metric" : "java.lang:type=Memory.NonHeapMemoryUsage[max]",
+        "pointInTime" : true,
+        "temporal" : false
+      },
+      "metrics/jvm/NonHeapMemoryUsed":{
+        "metric" : "java.lang:type=Memory.NonHeapMemoryUsage[used]",
+        "pointInTime" : true,
+        "temporal" : false
+      },
       "metrics/jvm/gcCount":{
         "metric" : "Hadoop:service=TaskTracker,name=jvm.gcCount",
         "pointInTime" : true,
@@ -2239,6 +2348,31 @@
       }
     },
     "HBASE_MASTER":{
+      "metrics/load/AverageLoad":{
+        "metric" : "hadoop:service=Master,name=Master.AverageLoad",
+        "pointInTime" : true,
+        "temporal" : false
+      },
+      "metrics/jvm/HeapMemoryMax":{
+        "metric" : "java.lang:type=Memory.HeapMemoryUsage[max]",
+        "pointInTime" : true,
+        "temporal" : false
+      },
+      "metrics/jvm/HeapMemoryUsed":{
+        "metric" : "java.lang:type=Memory.HeapMemoryUsage[used]",
+        "pointInTime" : true,
+        "temporal" : false
+      },
+      "metrics/jvm/NonHeapMemoryMax":{
+        "metric" : "java.lang:type=Memory.NonHeapMemoryUsage[max]",
+        "pointInTime" : true,
+        "temporal" : false
+      },
+      "metrics/jvm/NonHeapMemoryUsed":{
+        "metric" : "java.lang:type=Memory.NonHeapMemoryUsage[used]",
+        "pointInTime" : true,
+        "temporal" : false
+      }
     }
   }
 }

ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java

@@ -73,6 +73,35 @@ public class GangliaPropertyProviderTest {
 
     Assert.assertEquals(3, PropertyHelper.getProperties(resource).size());
     Assert.assertNotNull(resource.getPropertyValue(PROPERTY_ID));
+
+
+    // tasktracker
+    resource = new ResourceImpl(Resource.Type.HostComponent);
+    resource.setProperty(HOST_NAME_PROPERTY_ID, "domU-12-31-39-0E-34-E1.compute-1.internal");
+    resource.setProperty(COMPONENT_NAME_PROPERTY_ID, "TASKTRACKER");
+
+    // only ask for one property
+    temporalInfoMap = new HashMap<String, TemporalInfo>();
+
+    //http://ec2-174-129-152-147.compute-1.amazonaws.com/cgi-bin/rrd.py?c=HDPSlaves&m=jvm.metrics.gcCount,mapred.shuffleOutput.shuffle_exceptions_caught,mapred.shuffleOutput.shuffle_failed_outputs,mapred.shuffleOutput.shuffle_output_bytes,mapred.shuffleOutput.shuffle_success_outputs&s=10&e=20&r=1&h=ip-10-85-111-149.ec2.internal
+
+    Set<String> properties = new HashSet<String>();
+    properties.add(PropertyHelper.getPropertyId("metrics/mapred/shuffleOutput", "shuffle_exceptions_caught"));
+    properties.add(PropertyHelper.getPropertyId("metrics/mapred/shuffleOutput", "shuffle_failed_outputs"));
+    properties.add(PropertyHelper.getPropertyId("metrics/mapred/shuffleOutput", "shuffle_output_bytes"));
+    properties.add(PropertyHelper.getPropertyId("metrics/mapred/shuffleOutput", "shuffle_success_outputs"));
+    request = PropertyHelper.getReadRequest(properties, temporalInfoMap);
+
+    Assert.assertEquals(1, propertyProvider.populateResources(Collections.singleton(resource), request, null).size());
+
+    Assert.assertEquals("http://domU-12-31-39-0E-34-E1.compute-1.internal/cgi-bin/rrd.py?c=HDPSlaves&h=domU-12-31-39-0E-34-E1.compute-1.internal&m=mapred.shuffleOutput.shuffle_output_bytes,mapred.shuffleOutput.shuffle_success_outputs,mapred.shuffleOutput.shuffle_failed_outputs,mapred.shuffleOutput.shuffle_exceptions_caught&e=now&pt=true",
+        streamProvider.getLastSpec());
+
+    Assert.assertEquals(6, PropertyHelper.getProperties(resource).size());
+    Assert.assertNotNull(resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/mapred/shuffleOutput", "shuffle_exceptions_caught")));
+    Assert.assertNotNull(resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/mapred/shuffleOutput", "shuffle_failed_outputs")));
+    Assert.assertNotNull(resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/mapred/shuffleOutput", "shuffle_output_bytes")));
+    Assert.assertNotNull(resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/mapred/shuffleOutput", "shuffle_success_outputs")));
   }
 
 
@@ -91,6 +120,7 @@ public class GangliaPropertyProviderTest {
 
     Set<Resource> resources = new HashSet<Resource>();
 
+    // host
     Resource resource = new ResourceImpl(Resource.Type.Host);
     resource.setProperty(HOST_NAME_PROPERTY_ID, "domU-12-31-39-0E-34-E1.compute-1.internal");
     resources.add(resource);

ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/JMXPropertyProviderTest.java

@@ -26,6 +26,8 @@ import org.junit.Assert;
 import org.junit.Test;
 
 import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
 
 /**
  * JMX property provider tests.
@@ -60,9 +62,12 @@ public class JMXPropertyProviderTest {
     Assert.assertEquals(propertyProvider.getSpec("domu-12-31-39-0e-34-e1.compute-1.internal:50070"), streamProvider.getLastSpec());
 
     // see test/resources/hdfs_namenode_jmx.json for values
-    Assert.assertEquals(1084287,  resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/rpc", "ReceivedBytes")));
-    Assert.assertEquals(173,      resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/dfs/namenode", "CreateFileOps")));
-    Assert.assertEquals(405.8686, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "memHeapUsedM")));
+    Assert.assertEquals(13670605,  resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/rpc", "ReceivedBytes")));
+    Assert.assertEquals(28,      resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/dfs/namenode", "CreateFileOps")));
+    Assert.assertEquals(1006632960, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "HeapMemoryMax")));
+    Assert.assertEquals(473433016, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "HeapMemoryUsed")));
+    Assert.assertEquals(136314880, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "NonHeapMemoryMax")));
+    Assert.assertEquals(23634400, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "NonHeapMemoryUsed")));
 
 
     // datanode
@@ -79,8 +84,11 @@ public class JMXPropertyProviderTest {
     Assert.assertEquals(propertyProvider.getSpec("domu-12-31-39-14-ee-b3.compute-1.internal:50075"), streamProvider.getLastSpec());
 
     // see test/resources/hdfs_datanode_jmx.json for values
-    Assert.assertEquals(0,  resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/rpc", "ReceivedBytes")));
-    Assert.assertEquals(16.870667, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "memHeapUsedM")));
+    Assert.assertEquals(856,  resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/rpc", "ReceivedBytes")));
+    Assert.assertEquals(954466304, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "HeapMemoryMax")));
+    Assert.assertEquals(9772616, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "HeapMemoryUsed")));
+    Assert.assertEquals(136314880, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "NonHeapMemoryMax")));
+    Assert.assertEquals(21933376, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "NonHeapMemoryUsed")));
 
 
     // jobtracker
@@ -89,17 +97,81 @@ public class JMXPropertyProviderTest {
     resource.setProperty(HOST_COMPONENT_HOST_NAME_PROPERTY_ID, "domu-12-31-39-14-ee-b3.compute-1.internal");
     resource.setProperty(HOST_COMPONENT_COMPONENT_NAME_PROPERTY_ID, "JOBTRACKER");
 
-    // only ask for one property
-    request = PropertyHelper.getReadRequest(Collections.singleton(PropertyHelper.getPropertyId("metrics/jvm", "threadsWaiting")));
+    // only ask for specific properties
+    Set<String> properties = new HashSet<String>();
+    properties.add(PropertyHelper.getPropertyId("metrics/jvm", "threadsWaiting"));
+    properties.add(PropertyHelper.getPropertyId("metrics/jvm", "HeapMemoryMax"));
+    properties.add(PropertyHelper.getPropertyId("metrics/jvm", "HeapMemoryUsed"));
+    properties.add(PropertyHelper.getPropertyId("metrics/jvm", "NonHeapMemoryMax"));
+    properties.add(PropertyHelper.getPropertyId("metrics/jvm", "NonHeapMemoryUsed"));
+    request = PropertyHelper.getReadRequest(properties);
 
     propertyProvider.populateResources(Collections.singleton(resource), request, null);
 
     Assert.assertEquals(propertyProvider.getSpec("domu-12-31-39-14-ee-b3.compute-1.internal:50030"), streamProvider.getLastSpec());
 
     // see test/resources/mapreduce_jobtracker_jmx.json for values
-    // resource should now contain 3 properties... host name, component name, and jvm.threadsWaiting (from request)
-    Assert.assertEquals(3, PropertyHelper.getProperties(resource).size());
+    Assert.assertEquals(7, PropertyHelper.getProperties(resource).size());
     Assert.assertEquals(59, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "threadsWaiting")));
+    Assert.assertEquals(1052770304, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "HeapMemoryMax")));
+    Assert.assertEquals(43580400, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "HeapMemoryUsed")));
+    Assert.assertEquals(136314880, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "NonHeapMemoryMax")));
+    Assert.assertEquals(29602888, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "NonHeapMemoryUsed")));
+
+    Assert.assertNull(resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "gcCount")));
+
+    // tasktracker
+    resource = new ResourceImpl(Resource.Type.HostComponent);
+
+    resource.setProperty(HOST_COMPONENT_HOST_NAME_PROPERTY_ID, "domu-12-31-39-14-ee-b3.compute-1.internal");
+    resource.setProperty(HOST_COMPONENT_COMPONENT_NAME_PROPERTY_ID, "TASKTRACKER");
+
+    // only ask for specific properties
+    properties = new HashSet<String>();
+    properties.add(PropertyHelper.getPropertyId("metrics/jvm", "HeapMemoryMax"));
+    properties.add(PropertyHelper.getPropertyId("metrics/jvm", "HeapMemoryUsed"));
+    properties.add(PropertyHelper.getPropertyId("metrics/jvm", "NonHeapMemoryMax"));
+    properties.add(PropertyHelper.getPropertyId("metrics/jvm", "NonHeapMemoryUsed"));
+    request = PropertyHelper.getReadRequest(properties);
+
+    propertyProvider.populateResources(Collections.singleton(resource), request, null);
+
+    Assert.assertEquals(propertyProvider.getSpec("domu-12-31-39-14-ee-b3.compute-1.internal:50060"), streamProvider.getLastSpec());
+
+    Assert.assertEquals(6, PropertyHelper.getProperties(resource).size());
+    Assert.assertEquals(954466304, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "HeapMemoryMax")));
+    Assert.assertEquals(18330984, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "HeapMemoryUsed")));
+    Assert.assertEquals(136314880, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "NonHeapMemoryMax")));
+    Assert.assertEquals(24235104, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "NonHeapMemoryUsed")));
+
+    Assert.assertNull(resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "gcCount")));
+
+    // hbase master
+    resource = new ResourceImpl(Resource.Type.HostComponent);
+
+    resource.setProperty(HOST_COMPONENT_HOST_NAME_PROPERTY_ID, "domu-12-31-39-14-ee-b3.compute-1.internal");
+    resource.setProperty(HOST_COMPONENT_COMPONENT_NAME_PROPERTY_ID, "HBASE_MASTER");
+
+    // only ask for specific properties
+    properties = new HashSet<String>();
+    properties.add(PropertyHelper.getPropertyId("metrics/jvm", "HeapMemoryMax"));
+    properties.add(PropertyHelper.getPropertyId("metrics/jvm", "HeapMemoryUsed"));
+    properties.add(PropertyHelper.getPropertyId("metrics/jvm", "NonHeapMemoryMax"));
+    properties.add(PropertyHelper.getPropertyId("metrics/jvm", "NonHeapMemoryUsed"));
+    properties.add(PropertyHelper.getPropertyId("metrics/load", "AverageLoad"));
+    request = PropertyHelper.getReadRequest(properties);
+
+    propertyProvider.populateResources(Collections.singleton(resource), request, null);
+
+    Assert.assertEquals(propertyProvider.getSpec("domu-12-31-39-14-ee-b3.compute-1.internal:60010"), streamProvider.getLastSpec());
+
+    Assert.assertEquals(7, PropertyHelper.getProperties(resource).size());
+    Assert.assertEquals(1069416448, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "HeapMemoryMax")));
+    Assert.assertEquals(4806976, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "HeapMemoryUsed")));
+    Assert.assertEquals(136314880, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "NonHeapMemoryMax")));
+    Assert.assertEquals(28971240, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "NonHeapMemoryUsed")));
+    Assert.assertEquals(3.0, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/load", "AverageLoad")));
+
     Assert.assertNull(resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "gcCount")));
   }
 

ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/TestStreamProvider.java

@@ -34,7 +34,7 @@ public class TestStreamProvider implements StreamProvider {
     FILE_MAPPING.put("50075", "hdfs_datanode_jmx.json");
     FILE_MAPPING.put("50030", "mapreduce_jobtracker_jmx.json");
     FILE_MAPPING.put("50060", "mapreduce_tasktracker_jmx.json");
-    FILE_MAPPING.put("60010", "hdfs_namenode_jmx.json");
+    FILE_MAPPING.put("60010", "hbase_hbasemaster_jmx.json");
   }
 
   private String lastSpec;

ambari-server/src/test/resources/temporal_ganglia_data.txt

@@ -1,747 +1,719 @@
-1111111111
+1358487599.0
+sum
+HDPSlaves
+domU-12-31-39-0E-34-E1.compute-1.internal
+mapred.shuffleOutput.shuffle_exceptions_caught
+1358434800
+360
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+[AMBARI_DP_END]
 sum
 HDPSlaves
 domU-12-31-39-0E-34-E1.compute-1.internal
 jvm.metrics.gcCount
-1349899845
-15
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-1.85333333
-12.0466667
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
+1358434800
+360
+83513252.9778
+119191023.641
+94591541.6419
+122656559.37
+142370213.761
+107374182.528
+131235111.95
+130489436.186
+143115866.264
+132030498.116
+131235096.191
+145722098.374
+128678583.679
+131235111.822
+69196695.2917
+133621204.781
+95443717.7111
+115895949.467
+134643809.447
+135211933.4
+142370212.222
+128053678.05
+141574824.397
+97034446.3556
+92035020.3
+39200091.525
+129644383.156
+143960940.847
+132030476.178
+127826414.616
+59879570.0701
+142370213.906
+147142389.835
+143165576.536
+134963382.052
+143165576.534
+141141658.42
+141461233.555
+119304637.975
+119304655.727
+146347025.188
+143392823.479
+131007864.879
+139984129.803
+131235108.855
+107374174.855
+95443717.671
+131235111.822
+143165576.538
+83513263.8167
+111350993.135
+118509282.802
+142370212.222
+120100011.423
+93852989.0687
+23065591.0807
+118509256.794
+142370212.222
+120100011.461
+143165576.518
+143165576.532
+71582797.8174
+146148183.173
+116322039.185
+131235103.56
+131235123.489
+95443706.0444
+71582788.25
+139984127.162
+134416571.797
+143165577.856
+107374182.454
+143165564.577
+119304647.097
+128053662.656
+84990350.0306
+117827550.129
+132030467.969
+143960940.848
+93852997.3374
+142938330.213
+109192149.075
+120895375.756
+138691664.073
+135709034.825
+146148182.177
+94847194.4554
+128849018.873
+107374182.416
+71582801.1622
+134416556.176
+128053654.561
+83513252.9926
+63380593.77
+115576389.233
 [AMBARI_DP_END]
 sum
 HDPSlaves
 domU-12-31-39-0E-34-E2.compute-1.internal
 jvm.metrics.gcCount
-1349899845
-15
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-1.85333333
-12.0466667
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
+1358434800
+360
+83513252.9778
+119191023.641
+94591541.6419
+122656559.37
+142370213.761
+107374182.528
+131235111.95
+130489436.186
+143115866.264
+132030498.116
+131235096.191
+145722098.374
+128678583.679
+131235111.822
+69196695.2917
+133621204.781
+95443717.7111
+115895949.467
+134643809.447
+135211933.4
+142370212.222
+128053678.05
+141574824.397
+97034446.3556
+92035020.3
+39200091.525
+129644383.156
+143960940.847
+132030476.178
+127826414.616
+59879570.0701
+142370213.906
+147142389.835
+143165576.536
+134963382.052
+143165576.534
+141141658.42
+141461233.555
+119304637.975
+119304655.727
+146347025.188
+143392823.479
+131007864.879
+139984129.803
+131235108.855
+107374174.855
+95443717.671
+131235111.822
+143165576.538
+83513263.8167
+111350993.135
+118509282.802
+142370212.222
+120100011.423
+93852989.0687
+23065591.0807
+118509256.794
+142370212.222
+120100011.461
+143165576.518
+143165576.532
+71582797.8174
+146148183.173
+116322039.185
+131235103.56
+131235123.489
+95443706.0444
+71582788.25
+139984127.162
+134416571.797
+143165577.856
+107374182.454
+143165564.577
+119304647.097
+128053662.656
+84990350.0306
+117827550.129
+132030467.969
+143960940.848
+93852997.3374
+142938330.213
+109192149.075
+120895375.756
+138691664.073
+135709034.825
+146148182.177
+94847194.4554
+128849018.873
+107374182.416
+71582801.1622
+134416556.176
+128053654.561
+83513252.9926
+63380593.77
+115576389.233
 [AMBARI_DP_END]
 sum
 HDPSlaves
 domU-12-31-39-0E-34-E3.compute-1.internal
 jvm.metrics.gcCount
-1349899845
-15
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-1.85333333
-12.0466667
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
+1358434800
+360
+83513252.9778
+119191023.641
+94591541.6419
+122656559.37
+142370213.761
+107374182.528
+131235111.95
+130489436.186
+143115866.264
+132030498.116
+131235096.191
+145722098.374
+128678583.679
+131235111.822
+69196695.2917
+133621204.781
+95443717.7111
+115895949.467
+134643809.447
+135211933.4
+142370212.222
+128053678.05
+141574824.397
+97034446.3556
+92035020.3
+39200091.525
+129644383.156
+143960940.847
+132030476.178
+127826414.616
+59879570.0701
+142370213.906
+147142389.835
+143165576.536
+134963382.052
+143165576.534
+141141658.42
+141461233.555
+119304637.975
+119304655.727
+146347025.188
+143392823.479
+131007864.879
+139984129.803
+131235108.855
+107374174.855
+95443717.671
+131235111.822
+143165576.538
+83513263.8167
+111350993.135
+118509282.802
+142370212.222
+120100011.423
+93852989.0687
+23065591.0807
+118509256.794
+142370212.222
+120100011.461
+143165576.518
+143165576.532
+71582797.8174
+146148183.173
+116322039.185
+131235103.56
+131235123.489
+95443706.0444
+71582788.25
+139984127.162
+134416571.797
+143165577.856
+107374182.454
+143165564.577
+119304647.097
+128053662.656
+84990350.0306
+117827550.129
+132030467.969
+143960940.848
+93852997.3374
+142938330.213
+109192149.075
+120895375.756
+138691664.073
+135709034.825
+146148182.177
+94847194.4554
+128849018.873
+107374182.416
+71582801.1622
+134416556.176
+128053654.561
+83513252.9926
+63380593.77
+115576389.233
+[AMBARI_DP_END]
+sum
+HDPSlaves
+domU-12-31-39-0E-34-E1.compute-1.internal
+mapred.shuffleOutput.shuffle_output_bytes
+1358434800
+360
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+[AMBARI_DP_END]
+sum
+HDPSlaves
+domU-12-31-39-0E-34-E1.compute-1.internal
+mapred.shuffleOutput.shuffle_failed_outputs
+1358434800
+360
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+[AMBARI_DP_END]
+sum
+HDPSlaves
+domU-12-31-39-0E-34-E1.compute-1.internal
+mapred.shuffleOutput.shuffle_success_outputs
+1358434800
+360
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
 [AMBARI_DP_END]
 [AMBARI_END]
-999999999
+1358487599.0