AMBARI-13517. Ambari Server JVM crashed after several clicks in Web UI to navigate graph timerange. (swagle)

ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java

@@ -27,6 +27,7 @@ import javax.xml.bind.annotation.XmlRootElement;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.codehaus.jackson.map.annotate.JsonDeserialize;
 
 @XmlRootElement(name = "metric")
 @XmlAccessorType(XmlAccessType.NONE)
@@ -41,7 +42,7 @@ public class TimelineMetric implements Comparable<TimelineMetric> {
   private long timestamp;
   private long startTime;
   private String type;
-  private Map<Long, Double> metricValues = new TreeMap<Long, Double>();
+  private TreeMap<Long, Double> metricValues = new TreeMap<Long, Double>();
 
   // default
   public TimelineMetric() {
@@ -124,11 +125,11 @@ public class TimelineMetric implements Comparable<TimelineMetric> {
   }
 
   @XmlElement(name = "metrics")
-  public Map<Long, Double> getMetricValues() {
+  public TreeMap<Long, Double> getMetricValues() {
     return metricValues;
   }
 
-  public void setMetricValues(Map<Long, Double> metricValues) {
+  public void setMetricValues(TreeMap<Long, Double> metricValues) {
     this.metricValues = metricValues;
   }
 

ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsCache.java

@@ -161,7 +161,7 @@ public class TimelineMetricsCache {
     Double value = counterMetricLastValue.get(metricName);
     double previousValue = value != null ? value : firstValue;
     Map<Long, Double> metricValues = timelineMetric.getMetricValues();
-    Map<Long, Double>   newMetricValues = new TreeMap<Long, Double>();
+    TreeMap<Long, Double>   newMetricValues = new TreeMap<Long, Double>();
     for (Map.Entry<Long, Double> entry : metricValues.entrySet()) {
       newMetricValues.put(entry.getKey(), entry.getValue() - previousValue);
       previousValue = entry.getValue();

ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsCacheTest.java

@@ -80,7 +80,7 @@ public class TimelineMetricsCacheTest {
     timelineMetric.setAppId("test serviceName");
     timelineMetric.setStartTime(startTime);
     timelineMetric.setType("Number");
-    timelineMetric.setMetricValues(metricValues);
+    timelineMetric.setMetricValues(new TreeMap<Long, Double>(metricValues));
     return timelineMetric;
   }
 

ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java

@@ -275,7 +275,7 @@ public class HBaseTimelineMetricStore extends AbstractService implements Timelin
       metric.setHostName(metricList.get(0).getHostName());
       // Assumption that metrics are ordered by start time
       metric.setStartTime(metricList.get(0).getStartTime());
-      Map<Long, Double> metricRecords = new TreeMap<Long, Double>();
+      TreeMap<Long, Double> metricRecords = new TreeMap<Long, Double>();
       for (TimelineMetric timelineMetric : metricList) {
         metricRecords.putAll(timelineMetric.getMetricValues());
       }

ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java

@@ -52,6 +52,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.TreeMap;
 import java.util.concurrent.TimeUnit;
 
 import static java.util.concurrent.TimeUnit.SECONDS;
@@ -187,19 +188,19 @@ public class PhoenixHBaseAccessor {
     return metric;
   }
 
-  private static Map<Long, Double> readLastMetricValueFromJSON(String json)
-    throws IOException {
-    Map<Long, Double> values = readMetricFromJSON(json);
-    Long lastTimeStamp = Collections.max(values.keySet());
+  private static TreeMap<Long, Double> readLastMetricValueFromJSON(String json)
+      throws IOException {
+    TreeMap<Long, Double> values = readMetricFromJSON(json);
+    Long lastTimeStamp = values.lastKey();
 
-    HashMap<Long, Double> valueMap = new HashMap<Long, Double>(1);
+    TreeMap<Long, Double> valueMap = new TreeMap<Long, Double>();
     valueMap.put(lastTimeStamp, values.get(lastTimeStamp));
     return valueMap;
   }
 
   @SuppressWarnings("unchecked")
-  public static Map<Long, Double>  readMetricFromJSON(String json) throws IOException {
-    return (Map<Long, Double>) mapper.readValue(json, metricValuesTypeRef);
+  public static TreeMap<Long, Double>  readMetricFromJSON(String json) throws IOException {
+    return (TreeMap<Long, Double>) mapper.readValue(json, metricValuesTypeRef);
   }
 
   private Connection getConnectionRetryingOnException()
@@ -467,8 +468,10 @@ public class PhoenixHBaseAccessor {
       // which is thrown in hbase TimeRange.java
       Throwable io = ex.getCause();
       String className = null;
-      for (StackTraceElement ste : io.getStackTrace()) {
-        className = ste.getClassName();
+      if (io != null) {
+        for (StackTraceElement ste : io.getStackTrace()) {
+          className = ste.getClassName();
+        }
       }
       if (className != null && className.equals("TimeRange")) {
         // This is "maxStamp is smaller than minStamp" exception

ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricReadHelper.java

@@ -40,7 +40,7 @@ public class TimelineMetricReadHelper {
   public TimelineMetric getTimelineMetricFromResultSet(ResultSet rs)
       throws SQLException, IOException {
     TimelineMetric metric = getTimelineMetricCommonsFromResultSet(rs);
-    Map<Long, Double> sortedByTimeMetrics = new TreeMap<Long, Double>(
+    TreeMap<Long, Double> sortedByTimeMetrics = new TreeMap<Long, Double>(
         PhoenixHBaseAccessor.readMetricFromJSON(rs.getString("METRICS")));
     metric.setMetricValues(sortedByTimeMetrics);
     return metric;

ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITClusterAggregator.java

@@ -47,6 +47,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.TreeMap;
 
 import static junit.framework.Assert.assertEquals;
 import static junit.framework.Assert.assertNotNull;
@@ -503,7 +504,7 @@ public class ITClusterAggregator extends AbstractMiniHBaseClusterTest {
     metric1.setAppId("resourcemanager");
     metric1.setHostName("h1");
     metric1.setStartTime(1431372311811l);
-    metric1.setMetricValues(new HashMap<Long, Double>() {{
+    metric1.setMetricValues(new TreeMap<Long, Double>() {{
       put(1431372311811l, 1.0);
       put(1431372321811l, 1.0);
       put(1431372331811l, 1.0);
@@ -518,7 +519,7 @@ public class ITClusterAggregator extends AbstractMiniHBaseClusterTest {
     metric2.setAppId("resourcemanager");
     metric2.setHostName("h1");
     metric2.setStartTime(1431372381810l);
-    metric2.setMetricValues(new HashMap<Long, Double>() {{
+    metric2.setMetricValues(new TreeMap<Long, Double>() {{
       put(1431372381810l, 1.0);
       put(1431372391811l, 1.0);
       put(1431372401811l, 1.0);

ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITMetricAggregator.java

@@ -42,6 +42,7 @@ import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.TreeMap;
 
 import static junit.framework.Assert.assertEquals;
 import static junit.framework.Assert.assertTrue;
@@ -383,7 +384,7 @@ public class ITMetricAggregator extends AbstractMiniHBaseClusterTest {
     m.setHostName(host);
     m.setMetricName(metricName);
     m.setStartTime(startTime);
-    Map<Long, Double> vals = new HashMap<Long, Double>();
+    TreeMap<Long, Double> vals = new TreeMap<Long, Double>();
     vals.put(startTime + 15000l, 0.0);
     vals.put(startTime + 30000l, 0.0);
     vals.put(startTime + 45000l, 1.0);

ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricTestHelper.java

@@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.TreeMap;
 
 public class MetricTestHelper {
 
@@ -83,7 +84,7 @@ public class MetricTestHelper {
     m.setInstanceId(instanceId);
     m.setMetricName(metricName);
     m.setStartTime(startTime);
-    Map<Long, Double> vals = new HashMap<Long, Double>();
+    TreeMap<Long, Double> vals = new TreeMap<Long, Double>();
     vals.put(startTime + 15000l, val);
     vals.put(startTime + 30000l, val);
     vals.put(startTime + 45000l, val);

ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java

@@ -25,6 +25,7 @@ import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
+import java.util.TreeMap;
 
 public class TestTimelineMetricStore implements TimelineMetricStore {
   @Override
@@ -44,7 +45,7 @@ public class TestTimelineMetricStore implements TimelineMetricStore {
     metric1.setInstanceId(null);
     metric1.setHostName("c6401");
     metric1.setStartTime(1407949812L);
-    metric1.setMetricValues(new HashMap<Long, Double>() {{
+    metric1.setMetricValues(new TreeMap<Long, Double>() {{
       put(1407949812L, 1.0d);
       put(1407949912L, 1.8d);
       put(1407950002L, 0.7d);
@@ -55,7 +56,7 @@ public class TestTimelineMetricStore implements TimelineMetricStore {
     metric2.setInstanceId("3");
     metric2.setHostName("c6401");
     metric2.setStartTime(1407949812L);
-    metric2.setMetricValues(new HashMap<Long, Double>() {{
+    metric2.setMetricValues(new TreeMap<Long, Double>() {{
       put(1407949812L, 2.5d);
       put(1407949912L, 3.0d);
       put(1407950002L, 0.9d);

ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPaddingMethod.java

@@ -52,15 +52,7 @@ public class MetricsPaddingMethod {
       return;
     }
 
-    TreeMap<Long, Double> values;
-    Map<Long, Double> metricValuesMap = metric.getMetricValues();
-    if (metricValuesMap instanceof TreeMap) {
-      values = (TreeMap<Long, Double>) metricValuesMap;
-    }
-    else {
-      // JSON dser returns LinkedHashMap that is not Navigable
-      values = new TreeMap<Long, Double>(metricValuesMap);
-    }
+    TreeMap<Long, Double> values = metric.getMetricValues();
 
     long dataInterval = getTimelineMetricInterval(values);
 

ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheProvider.java

@@ -23,6 +23,8 @@ import net.sf.ehcache.Cache;
 import net.sf.ehcache.CacheManager;
 import net.sf.ehcache.config.CacheConfiguration;
 import net.sf.ehcache.config.PersistenceConfiguration;
+import net.sf.ehcache.config.SizeOfPolicyConfiguration;
+import net.sf.ehcache.config.SizeOfPolicyConfiguration.MaxDepthExceededBehavior;
 import net.sf.ehcache.store.MemoryStoreEvictionPolicy;
 import org.apache.ambari.server.configuration.Configuration;
 import org.apache.commons.lang.StringUtils;
@@ -40,6 +42,7 @@ import static net.sf.ehcache.config.PersistenceConfiguration.*;
 public class TimelineMetricCacheProvider {
   private TimelineMetricCache timelineMetricsCache;
   private volatile boolean isCacheInitialized = false;
+  public static final String TIMELINE_METRIC_CACHE_MANAGER_NAME = "timelineMetricCacheManager";
   public static final String TIMELINE_METRIC_CACHE_INSTANCE_NAME = "timelineMetricCache";
 
   Configuration configuration;
@@ -61,8 +64,13 @@ public class TimelineMetricCacheProvider {
     }
 
     System.setProperty("net.sf.ehcache.skipUpdateCheck", "true");
+    // Use custom sizing engine to speed cache sizing calculations
+    System.setProperty("net.sf.ehcache.sizeofengine." + TIMELINE_METRIC_CACHE_MANAGER_NAME,
+      "org.apache.ambari.server.controller.metrics.timeline.cache.TimelineMetricsCacheSizeOfEngine");
+
     net.sf.ehcache.config.Configuration managerConfig =
       new net.sf.ehcache.config.Configuration();
+    managerConfig.setName(TIMELINE_METRIC_CACHE_MANAGER_NAME);
 
     // Set max heap available to the cache manager
     managerConfig.setMaxBytesLocalHeap(configuration.getMetricsCacheManagerHeapPercent());
@@ -74,17 +82,18 @@ public class TimelineMetricCacheProvider {
       configuration.getMetricCacheTTLSeconds() + ", idle = " +
       configuration.getMetricCacheIdleSeconds());
 
-    PersistenceConfiguration persistenceConfiguration = new PersistenceConfiguration();
-    persistenceConfiguration.setStrategy(Strategy.NONE.name());
-
-    //Create a Cache specifying its configuration.
+    // Create a Cache specifying its configuration.
     CacheConfiguration cacheConfiguration = new CacheConfiguration()
-        .name(TIMELINE_METRIC_CACHE_INSTANCE_NAME)
-        .timeToLiveSeconds(configuration.getMetricCacheTTLSeconds()) // 1 hour
-        .timeToIdleSeconds(configuration.getMetricCacheIdleSeconds()) // 5 minutes
-        .memoryStoreEvictionPolicy(MemoryStoreEvictionPolicy.LRU)
-        .eternal(false)
-        .persistence(persistenceConfiguration);
+      .name(TIMELINE_METRIC_CACHE_INSTANCE_NAME)
+      .timeToLiveSeconds(configuration.getMetricCacheTTLSeconds()) // 1 hour
+      .timeToIdleSeconds(configuration.getMetricCacheIdleSeconds()) // 5 minutes
+      .memoryStoreEvictionPolicy(MemoryStoreEvictionPolicy.LRU)
+      .sizeOfPolicy(new SizeOfPolicyConfiguration() // Set sizeOf policy to continue on max depth reached - avoid OOM
+        .maxDepth(10000)
+        .maxDepthExceededBehavior(MaxDepthExceededBehavior.CONTINUE))
+      .eternal(false)
+      .persistence(new PersistenceConfiguration()
+        .strategy(Strategy.NONE.name()));
 
     Cache cache = new Cache(cacheConfiguration);
 

ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricsCacheSizeOfEngine.java

@@ -0,0 +1,137 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.controller.metrics.timeline.cache;
+
+import net.sf.ehcache.pool.Size;
+import net.sf.ehcache.pool.SizeOfEngine;
+import net.sf.ehcache.pool.impl.DefaultSizeOfEngine;
+import net.sf.ehcache.pool.sizeof.ReflectionSizeOf;
+import net.sf.ehcache.pool.sizeof.SizeOf;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.util.Map;
+
+/**
+ * Cache sizing engine that reduces reflective calls over the Object graph to
+ * find total Heap usage.
+ */
+public class TimelineMetricsCacheSizeOfEngine implements SizeOfEngine {
+
+  private final static Logger LOG = LoggerFactory.getLogger(TimelineMetricsCacheSizeOfEngine.class);
+  public static int DEFAULT_MAX_DEPTH = 1000;
+  public static boolean DEFAULT_ABORT_WHEN_MAX_DEPTH_EXCEEDED = false;
+
+  private SizeOfEngine underlying = null;
+  SizeOf reflectionSizeOf = new ReflectionSizeOf();
+
+  // Optimizations
+  private volatile long timelineMetricPrimitivesApproximation = 0;
+
+  private TimelineMetricsCacheSizeOfEngine(SizeOfEngine underlying) {
+    this.underlying = underlying;
+  }
+
+  public TimelineMetricsCacheSizeOfEngine() {
+    this(new DefaultSizeOfEngine(DEFAULT_MAX_DEPTH, DEFAULT_ABORT_WHEN_MAX_DEPTH_EXCEEDED));
+
+    LOG.info("Creating custom sizeof engine for TimelineMetrics.");
+  }
+
+  @Override
+  public Size sizeOf(Object key, Object value, Object container) {
+    try {
+      LOG.debug("BEGIN - Sizeof, key: {}, value: {}", key, value);
+
+      long size = 0;
+
+      if (key instanceof TimelineAppMetricCacheKey) {
+        size += getTimelineMetricCacheKeySize((TimelineAppMetricCacheKey) key);
+      }
+
+      if (value instanceof TimelineMetricsCacheValue) {
+        size += getTimelineMetricCacheValueSize((TimelineMetricsCacheValue) value);
+      }
+      // Mark size as not being exact
+      return new Size(size, false);
+    } finally {
+      LOG.debug("END - Sizeof, key: {}", key);
+    }
+  }
+
+  private long getTimelineMetricCacheKeySize(TimelineAppMetricCacheKey key) {
+    long size = reflectionSizeOf.sizeOf(key.getAppId());
+    size += key.getMetricNames() != null && !key.getMetricNames().isEmpty() ?
+      reflectionSizeOf.deepSizeOf(1000, false, key.getMetricNames()).getCalculated() : 0;
+    size += key.getSpec() != null ?
+      reflectionSizeOf.deepSizeOf(1000, false, key.getSpec()).getCalculated() : 0;
+    // 4 fixed longs of @TemporalInfo + reference
+    size += 40;
+    size += 8; // Object overhead
+
+    return size;
+  }
+
+  private long getTimelineMetricCacheValueSize(TimelineMetricsCacheValue value) {
+    long size = 16; // startTime + endTime
+    Map<String, TimelineMetric> metrics = value.getTimelineMetrics();
+    size += 8; // Object reference
+
+    if (metrics != null) {
+      for (Map.Entry<String, TimelineMetric> metricEntry : metrics.entrySet()) {
+        size += reflectionSizeOf.sizeOf(metricEntry.getKey());
+
+        TimelineMetric metric = metricEntry.getValue();
+
+        if (timelineMetricPrimitivesApproximation == 0) {
+          timelineMetricPrimitivesApproximation += reflectionSizeOf.sizeOf(metric.getMetricName());
+          timelineMetricPrimitivesApproximation += reflectionSizeOf.sizeOf(metric.getAppId());
+          timelineMetricPrimitivesApproximation += reflectionSizeOf.sizeOf(metric.getHostName());
+          timelineMetricPrimitivesApproximation += reflectionSizeOf.sizeOf(metric.getInstanceId());
+          timelineMetricPrimitivesApproximation += reflectionSizeOf.sizeOf(metric.getTimestamp());
+          timelineMetricPrimitivesApproximation += reflectionSizeOf.sizeOf(metric.getStartTime());
+          timelineMetricPrimitivesApproximation += reflectionSizeOf.sizeOf(metric.getType());
+          timelineMetricPrimitivesApproximation += 8; // Object overhead
+
+          LOG.debug("timelineMetricPrimitivesApproximation bytes = " + timelineMetricPrimitivesApproximation);
+        }
+        size += timelineMetricPrimitivesApproximation;
+
+        Map<Long, Double> metricValues = metric.getMetricValues();
+        if (metricValues != null && !metricValues.isEmpty()) {
+          // Numeric wrapper: 12 bytes + 8 bytes Data type + 4 bytes alignment = 48 (Long, Double)
+          // Tree Map: 12 bytes for header + 20 bytes for 5 object fields : pointers + 1 byte for flag = 40
+          LOG.debug("Size of metric value: " + (48 + 40) * metricValues.size());
+          size += (48 + 40) * metricValues.size(); // Treemap size is O(1)
+        }
+      }
+      LOG.debug("Total Size of metric values in cache: " + size);
+    }
+
+    return size;
+  }
+
+  @Override
+  public SizeOfEngine copyWith(int maxDepth, boolean abortWhenMaxDepthExceeded) {
+    LOG.debug("Copying tracing sizeof engine, maxdepth: {}, abort: {}",
+      maxDepth, abortWhenMaxDepthExceeded);
+
+    return new TimelineMetricsCacheSizeOfEngine(
+      underlying.copyWith(maxDepth, abortWhenMaxDepthExceeded));
+  }
+}

ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/MetricsPaddingMethodTest.java

@@ -174,7 +174,7 @@ public class MetricsPaddingMethodTest {
     timelineMetric.setHostName("h1");
     timelineMetric.setAppId("a1");
     timelineMetric.setTimestamp(now);
-    Map<Long, Double> inputValues = new TreeMap<Long, Double>();
+    TreeMap<Long, Double> inputValues = new TreeMap<Long, Double>();
     inputValues.put(now - 100, 1.0d);
     inputValues.put(now - 200, 2.0d);
     inputValues.put(now - 300, 3.0d);

ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheSizingTest.java

@@ -0,0 +1,110 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.controller.metrics.timeline.cache;
+
+import net.sf.ehcache.pool.sizeof.ReflectionSizeOf;
+import net.sf.ehcache.pool.sizeof.SizeOf;
+import org.apache.ambari.server.controller.internal.TemporalInfoImpl;
+import org.apache.ambari.server.controller.spi.TemporalInfo;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.junit.Assert;
+import org.junit.Test;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+public class TimelineMetricCacheSizingTest {
+
+  SizeOf reflectionSizeOf = new ReflectionSizeOf();
+
+  private TimelineMetric getSampleTimelineMetric(String metricName) {
+    TimelineMetric metric = new TimelineMetric();
+    metric.setMetricName(metricName);
+    metric.setAppId("KAFKA_BROKER");
+    metric.setInstanceId("NULL");
+    metric.setHostName("my.privatehostname.of.average.length");
+    metric.setTimestamp(System.currentTimeMillis());
+    metric.setStartTime(System.currentTimeMillis());
+    metric.setType("LONG");
+
+    // JSON dser gives a LinkedHashMap
+    TreeMap<Long, Double> valueMap = new TreeMap<>();
+    long now = System.currentTimeMillis();
+    for (int i = 0; i < 25000; i++) {
+      valueMap.put(new Long(now + i), new Double(1.0 + i));
+    }
+
+    metric.setMetricValues(valueMap);
+
+    return metric;
+  }
+
+  @Test
+  public void testTimelineMetricCacheSizing() throws Exception {
+    Set<String> metricNames = new HashSet<>();
+    String metric1 = "prefix1.suffix1.suffix2.actualNamePrefix.longMetricName1";
+    String metric2 = "prefix1.suffix1.suffix2.actualNamePrefix.longMetricName2";
+    String metric3 = "prefix1.suffix1.suffix2.actualNamePrefix.longMetricName3";
+    String metric4 = "prefix1.suffix1.suffix2.actualNamePrefix.longMetricName4";
+    String metric5 = "prefix1.suffix1.suffix2.actualNamePrefix.longMetricName5";
+    String metric6 = "prefix1.suffix1.suffix2.actualNamePrefix.longMetricName6";
+
+    metricNames.add(metric1);
+    metricNames.add(metric2);
+    metricNames.add(metric3);
+    metricNames.add(metric4);
+    metricNames.add(metric5);
+    metricNames.add(metric6);
+
+    long now = System.currentTimeMillis();
+    TemporalInfo temporalInfo = new TemporalInfoImpl(now - 1000, now, 15);
+
+    TimelineAppMetricCacheKey key = new TimelineAppMetricCacheKey(
+      metricNames, "KAFKA_BROKER", temporalInfo);
+    // Some random spec
+    key.setSpec("http://104.196.94.129:6188/ws/v1/timeline/metrics?metricNames=" +
+      "jvm.JvmMetrics.MemHeapCommittedM&appId=RESOURCEMANAGER&" +
+      "startTime=1439522640000&endTime=1440127440000&precision=hours");
+
+    Map<String, TimelineMetric> metricMap = new HashMap<>();
+    metricMap.put(metric1, getSampleTimelineMetric(metric1));
+    metricMap.put(metric2, getSampleTimelineMetric(metric2));
+    metricMap.put(metric3, getSampleTimelineMetric(metric3));
+    metricMap.put(metric4, getSampleTimelineMetric(metric4));
+    metricMap.put(metric5, getSampleTimelineMetric(metric5));
+    metricMap.put(metric6, getSampleTimelineMetric(metric6));
+
+    TimelineMetricsCacheValue value = new TimelineMetricsCacheValue(now - 1000, now, metricMap);
+
+    TimelineMetricsCacheSizeOfEngine customSizeOfEngine = new TimelineMetricsCacheSizeOfEngine();
+
+    long bytesFromReflectionEngine =
+      reflectionSizeOf.deepSizeOf(1000, false, key).getCalculated() +
+      reflectionSizeOf.deepSizeOf(1000, false, value).getCalculated();
+
+    long bytesFromCustomSizeOfEngine = customSizeOfEngine.sizeOf(key, value, null).getCalculated();
+
+    long sampleSizeInMB = bytesFromReflectionEngine / (1024 * 1024);
+    long discrepancyInKB = Math.abs(bytesFromCustomSizeOfEngine - bytesFromReflectionEngine) / 1024;
+
+    Assert.assertTrue("Sample size is greater that 10 MB", sampleSizeInMB > 10);
+    Assert.assertTrue("Discrepancy in values is less than 10K", discrepancyInKB  < 10);
+  }
+}

ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheTest.java

@@ -132,7 +132,7 @@ public class TimelineMetricCacheTest {
     TimelineMetric timelineMetric = new TimelineMetric();
     timelineMetric.setMetricName("cpu_user");
     timelineMetric.setAppId("app1");
-    Map<Long, Double> metricValues = new HashMap<Long, Double>();
+    TreeMap<Long, Double> metricValues = new TreeMap<Long, Double>();
     metricValues.put(now + 100, 1.0);
     metricValues.put(now + 200, 2.0);
     metricValues.put(now + 300, 3.0);
@@ -297,7 +297,7 @@ public class TimelineMetricCacheTest {
     final TimelineMetric timelineMetric1 = new TimelineMetric();
     timelineMetric1.setMetricName("cpu_user");
     timelineMetric1.setAppId("app1");
-    Map<Long, Double> metricValues = new TreeMap<Long, Double>();
+    TreeMap<Long, Double> metricValues = new TreeMap<Long, Double>();
     metricValues.put(now - 100, 1.0);
     metricValues.put(now - 200, 2.0);
     metricValues.put(now - 300, 3.0);