YARN-2543. Made resource usage be published to the timeline server too. Contributed by Naganarasimha G R.

hadoop-yarn-project/CHANGES.txt

@@ -470,6 +470,9 @@ Release 2.7.0 - UNRELEASED
     YARN-3056. Add verification for containerLaunchDuration
     in TestNodeManagerMetrics. (zhihai xu via xgong)
 
+    YARN-2543. Made resource usage be published to the timeline server too.
+    (Naganarasimha G R via zjshen)
+
 Release 2.6.0 - 2014-11-18
 
   INCOMPATIBLE CHANGES

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java

@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
 import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerReport;
@@ -222,6 +223,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
     String diagnosticsInfo = null;
     FinalApplicationStatus finalStatus = FinalApplicationStatus.UNDEFINED;
     YarnApplicationState state = null;
+    ApplicationResourceUsageReport appResources = null;
     Map<ApplicationAccessType, String> appViewACLs =
         new HashMap<ApplicationAccessType, String>();
     Map<String, Object> entityInfo = entity.getOtherInfo();
@@ -260,6 +262,14 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
             entityInfo.get(ApplicationMetricsConstants.TYPE_ENTITY_INFO)
                 .toString();
       }
+      if (entityInfo.containsKey(ApplicationMetricsConstants.APP_CPU_METRICS)) {
+        long vcoreSeconds=Long.parseLong(entityInfo.get(
+                ApplicationMetricsConstants.APP_CPU_METRICS).toString());
+        long memorySeconds=Long.parseLong(entityInfo.get(
+                ApplicationMetricsConstants.APP_MEM_METRICS).toString());
+        appResources=ApplicationResourceUsageReport
+            .newInstance(0, 0, null, null, null, memorySeconds, vcoreSeconds);
+      }
     }
     List<TimelineEvent> events = entity.getEvents();
     if (events != null) {
@@ -310,7 +320,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
     return new ApplicationReportExt(ApplicationReport.newInstance(
         ConverterUtils.toApplicationId(entity.getEntityId()),
         latestApplicationAttemptId, user, queue, name, null, -1, null, state,
-        diagnosticsInfo, null, createdTime, finishedTime, finalStatus, null,
+        diagnosticsInfo, null, createdTime, finishedTime, finalStatus, appResources,
         null, 1.0F, type, null), appViewACLs);
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java

@@ -80,6 +80,10 @@ public class TestApplicationHistoryClientService {
         clientService.getClientHandler().getApplicationReport(request);
     ApplicationReport appReport = response.getApplicationReport();
     Assert.assertNotNull(appReport);
+    Assert.assertEquals(123, appReport.getApplicationResourceUsageReport()
+        .getMemorySeconds());
+    Assert.assertEquals(345, appReport.getApplicationResourceUsageReport()
+        .getVcoreSeconds());
     Assert.assertEquals("application_0_0001", appReport.getApplicationId()
       .toString());
     Assert.assertEquals("test app type",

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java

@@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerReport;
 import org.apache.hadoop.yarn.api.records.ContainerState;
@@ -201,6 +202,12 @@ public class TestApplicationHistoryManagerOnTimelineStore {
             app.getOriginalTrackingUrl());
         Assert.assertEquals("test diagnostics info", app.getDiagnostics());
       }
+      ApplicationResourceUsageReport applicationResourceUsageReport =
+          app.getApplicationResourceUsageReport();
+      Assert.assertEquals(123,
+          applicationResourceUsageReport.getMemorySeconds());
+      Assert
+          .assertEquals(345, applicationResourceUsageReport.getVcoreSeconds());
       Assert.assertEquals(FinalApplicationStatus.UNDEFINED,
           app.getFinalApplicationStatus());
       Assert.assertEquals(YarnApplicationState.FINISHED,
@@ -416,6 +423,8 @@ public class TestApplicationHistoryManagerOnTimelineStore {
     entityInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO, "test queue");
     entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO,
         Integer.MAX_VALUE + 1L);
+    entityInfo.put(ApplicationMetricsConstants.APP_MEM_METRICS,123);
+    entityInfo.put(ApplicationMetricsConstants.APP_CPU_METRICS,345);
     if (emptyACLs) {
       entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO, "");
     } else {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java

@@ -63,6 +63,12 @@ public class ApplicationMetricsConstants {
 
   public static final String STATE_EVENT_INFO =
       "YARN_APPLICATION_STATE";
+  
+  public static final String APP_CPU_METRICS =
+      "YARN_APPLICATION_CPU_METRIC";
+  
+  public static final String APP_MEM_METRICS =
+      "YARN_APPLICATION_MEM_METRIC";
 
   public static final String LATEST_APP_ATTEMPT_EVENT_INFO =
       "YARN_APPLICATION_LATEST_APP_ATTEMPT";

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationFinishedEvent.java

@@ -22,6 +22,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
 
 public class ApplicationFinishedEvent extends
     SystemMetricsEvent {
@@ -31,6 +32,7 @@ public class ApplicationFinishedEvent extends
   private FinalApplicationStatus appStatus;
   private YarnApplicationState state;
   private ApplicationAttemptId latestAppAttemptId;
+  private RMAppMetrics appMetrics;
 
   public ApplicationFinishedEvent(
       ApplicationId appId,
@@ -38,13 +40,15 @@ public class ApplicationFinishedEvent extends
       FinalApplicationStatus appStatus,
       YarnApplicationState state,
       ApplicationAttemptId latestAppAttemptId,
-      long finishedTime) {
+      long finishedTime,
+      RMAppMetrics appMetrics) {
     super(SystemMetricsEventType.APP_FINISHED, finishedTime);
     this.appId = appId;
     this.diagnosticsInfo = diagnosticsInfo;
     this.appStatus = appStatus;
     this.latestAppAttemptId = latestAppAttemptId;
     this.state = state;
+    this.appMetrics=appMetrics;
   }
 
   @Override
@@ -72,4 +76,7 @@ public class ApplicationFinishedEvent extends
     return latestAppAttemptId;
   }
 
+  public RMAppMetrics getAppMetrics() {
+    return appMetrics;
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java

@@ -45,6 +45,7 @@ import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
 import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants;
 import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
@@ -120,7 +121,8 @@ public class SystemMetricsPublisher extends CompositeService {
               RMServerUtils.createApplicationState(state),
               app.getCurrentAppAttempt() == null ?
                   null : app.getCurrentAppAttempt().getAppAttemptId(),
-              finishedTime));
+              finishedTime,
+              app.getRMAppMetrics()));
     }
   }
 
@@ -276,6 +278,12 @@ public class SystemMetricsPublisher extends CompositeService {
       eventInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO,
           event.getLatestApplicationAttemptId().toString());
     }
+    RMAppMetrics appMetrics = event.getAppMetrics();
+    entity.addOtherInfo(ApplicationMetricsConstants.APP_CPU_METRICS,
+        appMetrics.getVcoreSeconds());
+    entity.addOtherInfo(ApplicationMetricsConstants.APP_MEM_METRICS,
+        appMetrics.getMemorySeconds());
+    
     tEvent.setEventInfo(eventInfo);
     entity.addEvent(tEvent);
     putEntity(entity);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java

@@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants;
 import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
 import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
@@ -148,8 +149,18 @@ public class TestSystemMetricsPublisher {
             entity.getOtherInfo().get(
                 ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO));
       } else {
-        Assert.assertEquals("", entity.getOtherInfo().get(
-            ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO));
+        Assert.assertEquals(
+            "",
+            entity.getOtherInfo().get(
+                ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO));
+        Assert.assertEquals(
+            app.getRMAppMetrics().getMemorySeconds(),
+            Long.parseLong(entity.getOtherInfo()
+                .get(ApplicationMetricsConstants.APP_MEM_METRICS).toString()));
+        Assert.assertEquals(
+            app.getRMAppMetrics().getVcoreSeconds(),
+            Long.parseLong(entity.getOtherInfo()
+                .get(ApplicationMetricsConstants.APP_CPU_METRICS).toString()));
       }
       boolean hasCreatedEvent = false;
       boolean hasFinishedEvent = false;
@@ -340,6 +351,8 @@ public class TestSystemMetricsPublisher {
     when(app.getCurrentAppAttempt()).thenReturn(appAttempt);
     when(app.getFinalApplicationStatus()).thenReturn(
         FinalApplicationStatus.UNDEFINED);
+    when(app.getRMAppMetrics()).thenReturn(
+        new RMAppMetrics(null, 0, 0, Integer.MAX_VALUE, Long.MAX_VALUE));
     return app;
   }