YARN-3551. Consolidate data model change according to the backend implementation (Zhijie Shen via sale)

(cherry picked from commit 557a3950bddc837469244835f5577899080115d8)

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java

@@ -261,7 +261,7 @@ public class TimelineServicePerformanceV2 extends Configured implements Tool {
           // add a metric
           TimelineMetric metric = new TimelineMetric();
           metric.setId("foo_metric");
-          metric.setSingleData(123456789L);
+          metric.addValue(System.currentTimeMillis(), 123456789L);
           entity.addMetric(metric);
           // add a config
           entity.addConfig("foo", "bar");

hadoop-yarn-project/CHANGES.txt

@@ -58,6 +58,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
     YARN-3431. Sub resources of timeline entity needs to be passed to a separate 
     endpoint. (Zhijie Shen via junping_du)
 
+    YARN-3551. Consolidate data model change according to the backend
+    implementation (Zhijie Shen via sjlee)
+
   IMPROVEMENTS
 
   OPTIMIZATIONS

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java

@@ -80,7 +80,7 @@ public class TimelineEntity {
   private TimelineEntity real;
   private Identifier identifier;
   private HashMap<String, Object> info = new HashMap<>();
-  private HashMap<String, Object> configs = new HashMap<>();
+  private HashMap<String, String> configs = new HashMap<>();
   private Set<TimelineMetric> metrics = new HashSet<>();
   private Set<TimelineEvent> events = new HashSet<>();
   private HashMap<String, Set<String>> isRelatedToEntities = new HashMap<>();
@@ -213,7 +213,7 @@ public class TimelineEntity {
   // required by JAXB
   @InterfaceAudience.Private
   @XmlElement(name = "configs")
-  public HashMap<String, Object> getConfigsJAXB() {
+  public HashMap<String, String> getConfigsJAXB() {
     if (real == null) {
       return configs;
     } else {
@@ -221,7 +221,7 @@ public class TimelineEntity {
     }
   }
 
-  public Map<String, Object> getConfigs() {
+  public Map<String, String> getConfigs() {
     if (real == null) {
       return configs;
     } else {
@@ -229,19 +229,19 @@ public class TimelineEntity {
     }
   }
 
-  public void setConfigs(Map<String, Object> configs) {
+  public void setConfigs(Map<String, String> configs) {
     if (real == null) {
       if (configs != null && !(configs instanceof HashMap)) {
-        this.configs = new HashMap<String, Object>(configs);
+        this.configs = new HashMap<String, String>(configs);
       } else {
-        this.configs = (HashMap<String, Object>) configs;
+        this.configs = (HashMap<String, String>) configs;
       }
     } else {
       real.setConfigs(configs);
     }
   }
 
-  public void addConfigs(Map<String, Object> configs) {
+  public void addConfigs(Map<String, String> configs) {
     if (real == null) {
       this.configs.putAll(configs);
     } else {
@@ -249,7 +249,7 @@ public class TimelineEntity {
     }
   }
 
-  public void addConfig(String key, Object value) {
+  public void addConfig(String key, String value) {
     if (real == null) {
       configs.put(key, value);
     } else {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java

@@ -24,113 +24,104 @@ import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
 import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlRootElement;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
+import java.util.Comparator;
 import java.util.Map;
+import java.util.TreeMap;
 
 @XmlRootElement(name = "metric")
 @XmlAccessorType(XmlAccessType.NONE)
 @InterfaceAudience.Public
 @InterfaceStability.Unstable
 public class TimelineMetric {
-  private String id;
-  private HashMap<String, Object> info = new HashMap<>();
-  private Object singleData;
-  private HashMap<Long, Object> timeSeries = new LinkedHashMap<>();
-  private long startTime;
-  private long endTime;
-
-  public TimelineMetric() {
 
+  public static enum Type {
+    SINGLE_VALUE,
+    TIME_SERIES
   }
 
-  @XmlElement(name = "id")
-  public String getId() {
-    return id;
-  }
-
-  public void setId(String id) {
-    this.id = id;
-  }
+  private Type type;
+  private String id;
+  private Comparator<Long> reverseComparator = new Comparator<Long>() {
+    @Override
+    public int compare(Long l1, Long l2) {
+      return -l1.compareTo(l2);
+    }
+  };
+  private TreeMap<Long, Number> values = new TreeMap<>(reverseComparator);
 
-  // required by JAXB
-  @InterfaceAudience.Private
-  @XmlElement(name = "info")
-  public HashMap<String, Object> getInfoJAXB() {
-    return info;
+  public TimelineMetric() {
+    this(Type.SINGLE_VALUE);
   }
 
-  public Map<String, Object> getInfo() {
-    return info;
+  public TimelineMetric(Type type) {
+    this.type = type;
   }
 
-  public void setInfo(Map<String, Object> info) {
-    if (info != null && !(info instanceof HashMap)) {
-      this.info = new HashMap<String, Object>(info);
-    } else {
-      this.info = (HashMap<String, Object>) info;
-    }
-  }
 
-  public void addInfo(Map<String, Object> info) {
-    this.info.putAll(info);
+  @XmlElement(name = "type")
+  public Type getType() {
+    return type;
   }
 
-  public void addInfo(String key, Object value) {
-    info.put(key, value);
+  public void setType(Type type) {
+    this.type = type;
   }
 
-  @XmlElement(name = "data")
-  public Object getSingleData() {
-    return singleData;
+  @XmlElement(name = "id")
+  public String getId() {
+    return id;
   }
 
-  public void setSingleData(Object singleData) {
-    this.singleData = singleData;
+  public void setId(String id) {
+    this.id = id;
   }
 
   // required by JAXB
   @InterfaceAudience.Private
-  @XmlElement(name = "timeseries")
-  public HashMap<Long, Object> getTimeSeriesJAXB() {
-    return timeSeries;
+  @XmlElement(name = "values")
+  public TreeMap<Long, Number> getValuesJAXB() {
+    return values;
   }
 
-  public Map<Long, Object> getTimeSeries() {
-    return timeSeries;
+  public Map<Long, Number> getValues() {
+    return values;
   }
 
-  public void setTimeSeries(Map<Long, Object> timeSeries) {
-    if (timeSeries != null && !(timeSeries instanceof LinkedHashMap)) {
-      this.timeSeries = new LinkedHashMap<Long, Object>(timeSeries);
+  public void setValues(Map<Long, Number> values) {
+    if (type == Type.SINGLE_VALUE) {
+      overwrite(values);
     } else {
-      this.timeSeries = (LinkedHashMap<Long, Object>) timeSeries;
+      if (values != null) {
+        this.values = new TreeMap<Long, Number>(reverseComparator);
+        this.values.putAll(values);
+      } else {
+        this.values = null;
+      }
     }
   }
 
-  public void addTimeSeries(Map<Long, Object> timeSeries) {
-    this.timeSeries.putAll(timeSeries);
-  }
-
-  public void addTimeSeriesData(long timestamp, Object value) {
-    timeSeries.put(timestamp, value);
-  }
-
-  @XmlElement(name = "starttime")
-  public long getStartTime() {
-    return startTime;
-  }
-
-  public void setStartTime(long startTime) {
-    this.startTime = startTime;
+  public void addValues(Map<Long, Number> values) {
+    if (type == Type.SINGLE_VALUE) {
+      overwrite(values);
+    } else {
+      this.values.putAll(values);
+    }
   }
 
-  @XmlElement(name = "endtime")
-  public long getEndTime() {
-    return endTime;
+  public void addValue(long timestamp, Number value) {
+    if (type == Type.SINGLE_VALUE) {
+      values.clear();
+    }
+    values.put(timestamp, value);
   }
 
-  public void setEndTime(long endTime) {
-    this.endTime = endTime;
+  private void overwrite(Map<Long, Number> values) {
+    if (values.size() > 1) {
+      throw new IllegalArgumentException(
+          "Values cannot contain more than one point in " +
+              Type.SINGLE_VALUE + " mode");
+    }
+    this.values.clear();
+    this.values.putAll(values);
   }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java

@@ -28,7 +28,11 @@ import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
 import org.junit.Test;
 import org.junit.Assert;
 
+import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
 
 
 public class TestTimelineServiceRecords {
@@ -41,38 +45,83 @@ public class TestTimelineServiceRecords {
     entity.setType("test type 1");
     entity.setId("test id 1");
     entity.addInfo("test info key 1", "test info value 1");
-    entity.addInfo("test info key 2", "test info value 2");
+    entity.addInfo("test info key 2",
+        Arrays.asList("test info value 2", "test info value 3"));
+    entity.addInfo("test info key 3", true);
+    Assert.assertTrue(
+        entity.getInfo().get("test info key 3") instanceof Boolean);
     entity.addConfig("test config key 1", "test config value 1");
     entity.addConfig("test config key 2", "test config value 2");
-    TimelineMetric metric1 = new TimelineMetric();
+
+    TimelineMetric metric1 =
+        new TimelineMetric(TimelineMetric.Type.TIME_SERIES);
     metric1.setId("test metric id 1");
-    metric1.addInfo("test info key 1", "test info value 1");
-    metric1.addInfo("test info key 2", "test info value 2");
-    metric1.addTimeSeriesData(1L, "test time series 1");
-    metric1.addTimeSeriesData(2L, "test time series 2");
-    metric1.setStartTime(0L);
-    metric1.setEndTime(1L);
+    metric1.addValue(1L, 1.0F);
+    metric1.addValue(3L, 3.0D);
+    metric1.addValue(2L, 2);
+    Assert.assertEquals(TimelineMetric.Type.TIME_SERIES, metric1.getType());
+    Iterator<Map.Entry<Long, Number>> itr =
+        metric1.getValues().entrySet().iterator();
+    Map.Entry<Long, Number> entry = itr.next();
+    Assert.assertEquals(new Long(3L), entry.getKey());
+    Assert.assertEquals(new Double(3.0D), entry.getValue());
+    entry = itr.next();
+    Assert.assertEquals(new Long(2L), entry.getKey());
+    Assert.assertEquals(new Integer(2), entry.getValue());
+    entry = itr.next();
+    Assert.assertEquals(new Long(1L), entry.getKey());
+    Assert.assertEquals(new Float(1.0F), entry.getValue());
+    Assert.assertFalse(itr.hasNext());
     entity.addMetric(metric1);
-    TimelineMetric metric2 = new TimelineMetric();
+
+    TimelineMetric metric2 =
+        new TimelineMetric(TimelineMetric.Type.SINGLE_VALUE);
     metric2.setId("test metric id 1");
-    metric2.addInfo("test info key 1", "test info value 1");
-    metric2.addInfo("test info key 2", "test info value 2");
-    metric2.setSingleData("test info value 3");
-    metric1.setStartTime(0L);
-    metric1.setEndTime(1L);
+    metric2.addValue(3L, (short) 3);
+    Assert.assertEquals(TimelineMetric.Type.SINGLE_VALUE, metric2.getType());
+    Assert.assertTrue(
+        metric2.getValues().values().iterator().next() instanceof Short);
+    Map<Long, Number> points = new HashMap<>();
+    points.put(4L, 4.0D);
+    points.put(5L, 5.0D);
+    try {
+      metric2.setValues(points);
+      Assert.fail();
+    } catch (IllegalArgumentException e) {
+      Assert.assertTrue(e.getMessage().contains(
+          "Values cannot contain more than one point in"));
+    }
+    try {
+      metric2.addValues(points);
+      Assert.fail();
+    } catch (IllegalArgumentException e) {
+      Assert.assertTrue(e.getMessage().contains(
+          "Values cannot contain more than one point in"));
+    }
     entity.addMetric(metric2);
+
     TimelineEvent event1 = new TimelineEvent();
     event1.setId("test event id 1");
     event1.addInfo("test info key 1", "test info value 1");
-    event1.addInfo("test info key 2", "test info value 2");
+    event1.addInfo("test info key 2",
+        Arrays.asList("test info value 2", "test info value 3"));
+    event1.addInfo("test info key 3", true);
+    Assert.assertTrue(
+        event1.getInfo().get("test info key 3") instanceof Boolean);
     event1.setTimestamp(0L);
     entity.addEvent(event1);
+
     TimelineEvent event2 = new TimelineEvent();
     event2.setId("test event id 2");
     event2.addInfo("test info key 1", "test info value 1");
-    event2.addInfo("test info key 2", "test info value 2");
+    event2.addInfo("test info key 2",
+        Arrays.asList("test info value 2", "test info value 3"));
+    event2.addInfo("test info key 3", true);
+    Assert.assertTrue(
+        event2.getInfo().get("test info key 3") instanceof Boolean);
     event2.setTimestamp(1L);
     entity.addEvent(event2);
+
     entity.setCreatedTime(0L);
     entity.setModifiedTime(1L);
     entity.addRelatesToEntity("test type 2", "test id 2");

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java

@@ -559,7 +559,7 @@ public class ContainersMonitorImpl extends AbstractService implements
                   ResourceCalculatorProcessTree.UNAVAILABLE) {
                 TimelineMetric memoryMetric = new TimelineMetric();
                 memoryMetric.setId(ContainerMetric.MEMORY.toString() + pId);
-                memoryMetric.addTimeSeriesData(currentTime, currentPmemUsage);
+                memoryMetric.addValue(currentTime, currentPmemUsage);
                 entity.addMetric(memoryMetric);
               }
               // if cpuUsageTotalCoresPercentage data is available
@@ -567,7 +567,7 @@ public class ContainersMonitorImpl extends AbstractService implements
                 ResourceCalculatorProcessTree.UNAVAILABLE) {
                 TimelineMetric cpuMetric = new TimelineMetric();
                 cpuMetric.setId(ContainerMetric.CPU.toString() + pId);
-                cpuMetric.addTimeSeriesData(currentTime, 
+                cpuMetric.addValue(currentTime,
                     cpuUsageTotalCoresPercentage);
                 entity.addMetric(cpuMetric);
               }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java

@@ -81,6 +81,12 @@ public class TestTimelineServiceClientIntegration {
       TimelineEntity entity = new TimelineEntity();
       entity.setType("test entity type");
       entity.setId("test entity id");
+      TimelineMetric metric =
+          new TimelineMetric(TimelineMetric.Type.TIME_SERIES);
+      metric.setId("test metric id");
+      metric.addValue(1L, 1.0D);
+      metric.addValue(2L, 2.0D);
+      entity.addMetric(metric);
       client.putEntities(entity);
       client.putEntitiesAsync(entity);
     } finally {