YARN-6616: YARN AHS shows submitTime for jobs same as startTime. Contributed by Prabhu Joseph

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java

@@ -57,7 +57,8 @@ public abstract class ApplicationReport {
       ApplicationAttemptId applicationAttemptId, String user, String queue,
       String name, String host, int rpcPort, Token clientToAMToken,
       YarnApplicationState state, String diagnostics, String url,
-      long startTime, long finishTime, FinalApplicationStatus finalStatus,
+      long startTime, long submitTime, long finishTime,
+      FinalApplicationStatus finalStatus,
       ApplicationResourceUsageReport appResources, String origTrackingUrl,
       float progress, String applicationType, Token amRmToken) {
     ApplicationReport report = Records.newRecord(ApplicationReport.class);
@@ -73,6 +74,7 @@ public abstract class ApplicationReport {
     report.setDiagnostics(diagnostics);
     report.setTrackingUrl(url);
     report.setStartTime(startTime);
+    report.setSubmitTime(submitTime);
     report.setFinishTime(finishTime);
     report.setFinalApplicationStatus(finalStatus);
     report.setApplicationResourceUsageReport(appResources);
@@ -91,14 +93,30 @@ public abstract class ApplicationReport {
       YarnApplicationState state, String diagnostics, String url,
       long startTime, long finishTime, FinalApplicationStatus finalStatus,
       ApplicationResourceUsageReport appResources, String origTrackingUrl,
+      float progress, String applicationType, Token amRmToken) {
+    return newInstance(applicationId, applicationAttemptId, user, queue,
+        name, host, rpcPort, clientToAMToken, state, diagnostics, url,
+        startTime, startTime, finishTime, finalStatus, appResources,
+        origTrackingUrl, progress, applicationType, amRmToken);
+  }
+
+  @Private
+  @Unstable
+  public static ApplicationReport newInstance(ApplicationId applicationId,
+      ApplicationAttemptId applicationAttemptId, String user, String queue,
+      String name, String host, int rpcPort, Token clientToAMToken,
+      YarnApplicationState state, String diagnostics, String url,
+      long startTime, long submitTime, long finishTime,
+      FinalApplicationStatus finalStatus,
+      ApplicationResourceUsageReport appResources, String origTrackingUrl,
       float progress, String applicationType, Token amRmToken, Set<String> tags,
       boolean unmanagedApplication, Priority priority,
       String appNodeLabelExpression, String amNodeLabelExpression) {
     ApplicationReport report =
         newInstance(applicationId, applicationAttemptId, user, queue, name,
           host, rpcPort, clientToAMToken, state, diagnostics, url, startTime,
-          finishTime, finalStatus, appResources, origTrackingUrl, progress,
-          applicationType, amRmToken);
+          submitTime, finishTime, finalStatus, appResources, origTrackingUrl,
+          progress, applicationType, amRmToken);
     report.setApplicationTags(tags);
     report.setUnmanagedApp(unmanagedApplication);
     report.setPriority(priority);
@@ -107,6 +125,25 @@ public abstract class ApplicationReport {
     return report;
   }
 
+  @Private
+  @Unstable
+  public static ApplicationReport newInstance(ApplicationId applicationId,
+      ApplicationAttemptId applicationAttemptId, String user, String queue,
+      String name, String host, int rpcPort, Token clientToAMToken,
+      YarnApplicationState state, String diagnostics, String url,
+      long startTime, long finishTime, FinalApplicationStatus finalStatus,
+      ApplicationResourceUsageReport appResources, String origTrackingUrl,
+      float progress, String applicationType, Token amRmToken, Set<String> tags,
+      boolean unmanagedApplication, Priority priority,
+      String appNodeLabelExpression, String amNodeLabelExpression) {
+    return newInstance(applicationId, applicationAttemptId, user, queue,
+        name, host, rpcPort, clientToAMToken, state, diagnostics, url,
+        startTime, startTime, finishTime, finalStatus, appResources,
+        origTrackingUrl, progress, applicationType, amRmToken, tags,
+        unmanagedApplication, priority, appNodeLabelExpression,
+        amNodeLabelExpression);
+  }
+
   /**
    * Get the <code>ApplicationId</code> of the application.
    * @return <code>ApplicationId</code> of the application
@@ -281,6 +318,18 @@ public abstract class ApplicationReport {
   @Unstable
   public abstract void setStartTime(long startTime);
 
+  /**
+   * Get the <em>submit time</em> of the application.
+   * @return <em>submit time</em> of the application
+   */
+  @Public
+  @Stable
+  public abstract long getSubmitTime();
+
+  @Private
+  @Unstable
+  public abstract void setSubmitTime(long submitTime);
+
   /**
    * Get the <em>finish time</em> of the application.
    * @return <em>finish time</em> of the application

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto

@@ -211,6 +211,7 @@ message ApplicationReportProto {
   optional PriorityProto priority = 23;
   optional string appNodeLabelExpression = 24;
   optional string amNodeLabelExpression = 25;
+  optional int64 submitTime = 26;
 }
 
 enum LogAggregationStatusProto {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java

@@ -102,6 +102,8 @@ public class TestAHSClient {
     Assert.assertEquals(report, expectedReports.get(0));
     Assert.assertEquals(report.getApplicationId().toString(), expectedReports
       .get(0).getApplicationId().toString());
+    Assert.assertEquals(report.getSubmitTime(), expectedReports.get(0)
+      .getSubmitTime());
     client.stop();
   }
 
@@ -332,7 +334,7 @@ public class TestAHSClient {
           ApplicationReport.newInstance(applicationId,
             ApplicationAttemptId.newInstance(applicationId, 1), "user",
             "queue", "appname", "host", 124, null,
-            YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
+            YarnApplicationState.RUNNING, "diagnostics", "url", 1, 2, 3,
             FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN",
             null);
       List<ApplicationReport> applicationReports =

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java

@@ -211,6 +211,12 @@ public class ApplicationReportPBImpl extends ApplicationReport {
     return p.getStartTime();
   }
 
+  @Override
+  public long getSubmitTime() {
+    ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
+    return p.getSubmitTime();
+  }
+
   @Override
   public long getFinishTime() {
     ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
@@ -404,6 +410,12 @@ public class ApplicationReportPBImpl extends ApplicationReport {
     builder.setStartTime(startTime);
   }
 
+  @Override
+  public void setSubmitTime(long submitTime) {
+    maybeInitBuilder();
+    builder.setSubmitTime(submitTime);
+  }
+
   @Override
   public void setFinishTime(long finishTime) {
     maybeInitBuilder();

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java

@@ -137,9 +137,9 @@ public class ApplicationHistoryManagerImpl extends AbstractService implements
       currentApplicationAttemptId, appHistory.getUser(), appHistory.getQueue(),
       appHistory.getApplicationName(), host, rpcPort, null,
       appHistory.getYarnApplicationState(), appHistory.getDiagnosticsInfo(),
-      trackingUrl, appHistory.getStartTime(), appHistory.getFinishTime(),
-      appHistory.getFinalApplicationStatus(), null, "", 100,
-      appHistory.getApplicationType(), null);
+      trackingUrl, appHistory.getStartTime(), appHistory.getSubmitTime(),
+      appHistory.getFinishTime(), appHistory.getFinalApplicationStatus(),
+      null, "", 100, appHistory.getApplicationType(), null);
   }
 
   private ApplicationAttemptHistoryData getLastAttempt(ApplicationId appId)

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java

@@ -249,6 +249,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
     String type = null;
     boolean unmanagedApplication = false;
     long createdTime = 0;
+    long submittedTime = 0;
     long finishedTime = 0;
     float progress = 0.0f;
     int applicationPriority = 0;
@@ -280,10 +281,11 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
         return new ApplicationReportExt(ApplicationReport.newInstance(
             ApplicationId.fromString(entity.getEntityId()),
             latestApplicationAttemptId, user, queue, name, null, -1, null,
-            state, diagnosticsInfo, null, createdTime, finishedTime,
-            finalStatus, null, null, progress, type, null, appTags,
-            unmanagedApplication, Priority.newInstance(applicationPriority),
-            appNodeLabelExpression, amNodeLabelExpression), appViewACLs);
+            state, diagnosticsInfo, null, createdTime, submittedTime,
+            finishedTime, finalStatus, null, null, progress, type, null,
+            appTags, unmanagedApplication, Priority.newInstance(
+            applicationPriority), appNodeLabelExpression,
+            amNodeLabelExpression), appViewACLs);
       }
       if (entityInfo.containsKey(ApplicationMetricsConstants.QUEUE_ENTITY_INFO)) {
         queue =
@@ -328,6 +330,8 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
             entityInfo.get(ApplicationMetricsConstants.AM_NODE_LABEL_EXPRESSION)
                 .toString();
       }
+      submittedTime = parseLong(entityInfo,
+          ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO);
 
       if (entityInfo.containsKey(ApplicationMetricsConstants.APP_CPU_METRICS)) {
         long vcoreSeconds = parseLong(entityInfo,
@@ -438,10 +442,10 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
     return new ApplicationReportExt(ApplicationReport.newInstance(
         ApplicationId.fromString(entity.getEntityId()),
         latestApplicationAttemptId, user, queue, name, null, -1, null, state,
-        diagnosticsInfo, null, createdTime, finishedTime, finalStatus,
-        appResources, null, progress, type, null, appTags, unmanagedApplication,
-        Priority.newInstance(applicationPriority), appNodeLabelExpression,
-        amNodeLabelExpression), appViewACLs);
+        diagnosticsInfo, null, createdTime, submittedTime, finishedTime,
+        finalStatus, appResources, null, progress, type, null, appTags,
+        unmanagedApplication, Priority.newInstance(applicationPriority),
+        appNodeLabelExpression, amNodeLabelExpression), appViewACLs);
   }
 
   private static long parseLong(Map<String, Object> entityInfo,

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java

@@ -197,6 +197,7 @@ public class TestApplicationHistoryManagerOnTimelineStore {
       }
       Assert.assertEquals(Integer.MAX_VALUE + 2L
           + app.getApplicationId().getId(), app.getStartTime());
+      Assert.assertEquals(Integer.MAX_VALUE + 1L, app.getSubmitTime());
       Assert.assertEquals(Integer.MAX_VALUE + 3L
           + +app.getApplicationId().getId(), app.getFinishTime());
       Assert.assertTrue(Math.abs(app.getProgress() - 1.0F) < 0.0001);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java

@@ -84,7 +84,7 @@ public class AppInfo {
     diagnosticsInfo = app.getDiagnostics();
     trackingUrl = app.getTrackingUrl();
     originalTrackingUrl = app.getOriginalTrackingUrl();
-    submittedTime = app.getStartTime();
+    submittedTime = app.getSubmitTime();
     startedTime = app.getStartTime();
     finishedTime = app.getFinishTime();
     elapsedTime = Times.elapsed(startedTime, finishedTime);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java

@@ -230,6 +230,7 @@ public abstract class MockAsm extends MockApps {
     final String name = newAppName();
     final String queue = newQueue();
     final long start = 123456 + i * 1000;
+    final long submit = start + i * 50;
     final long finish = 234567 + i * 1000;
     final String type = YarnConfiguration.DEFAULT_APPLICATION_TYPE;
     YarnApplicationState[] allStates = YarnApplicationState.values();
@@ -265,6 +266,11 @@ public abstract class MockAsm extends MockApps {
         return start;
       }
 
+      @Override
+      public long getSubmitTime() {
+        return submit;
+      }
+
       @Override
       public long getFinishTime() {
         return finish;
@@ -312,10 +318,10 @@ public abstract class MockAsm extends MockApps {
             ApplicationResourceUsageReport.newInstance(0, 0, null, null, null, 
             0, 0, 0, 0, 0, 0);
         ApplicationReport report = ApplicationReport.newInstance(
-            getApplicationId(), appAttemptId, getUser(), getQueue(), 
-            getName(), null, 0, null, null, getDiagnostics().toString(), 
-            getTrackingUrl(), getStartTime(), getFinishTime(), 
-            getFinalApplicationStatus(), usageReport , null, getProgress(),
+            getApplicationId(), appAttemptId, getUser(), getQueue(),
+            getName(), null, 0, null, null, getDiagnostics().toString(),
+            getTrackingUrl(), getStartTime(), getSubmitTime(), getFinishTime(),
+            getFinalApplicationStatus(), usageReport, null, getProgress(),
             type, null);
         return report;
       }