Merge -c 1240886 from trunk to branch-0.23 to fix MAPREDUCE-3746. Initialize queue metrics upfront and added start/finish time to RM Web-UI.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1240887 13f79535-47bb-0310-9956-ffa450edef68

hadoop-mapreduce-project/CHANGES.txt

@@ -660,6 +660,9 @@ Release 0.23.1 - Unreleased
     MAPREDUCE-3765. FifoScheduler does not respect yarn.scheduler.fifo.minimum-
     allocation-mb setting (Hitesh Shah via mahadev)
 
+    MAPREDUCE-3747. Initialize queue metrics upfront and added start/finish
+    time to RM Web-UI. (acmurthy) 
+
 Release 0.23.0 - 2011-11-01 
 
   INCOMPATIBLE CHANGES

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java

@@ -98,6 +98,12 @@ extends org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue {
    */
   public float getUsedCapacity();
   
+  /**
+   * Set used capacity of the queue.
+   * @param usedCapacity used capacity of the queue
+   */
+  public void setUsedCapacity(float usedCapacity);
+  
   /**
    * Get the currently utilized resources in the cluster 
    * by the queue and children (if any).
@@ -114,6 +120,12 @@ extends org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue {
    */
   public float getUtilization();
   
+  /**
+   * Get the current <em>utilization</em> of the queue.
+   * @param utilization queue utilization
+   */
+  public void setUtilization(float utilization);
+  
   /**
    * Get the current run-state of the queue
    * @return current run-state

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java

@@ -17,7 +17,9 @@
 */
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
 
+import org.apache.hadoop.yarn.Lock;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
 
 class CSQueueUtils {
   
@@ -65,4 +67,40 @@ class CSQueueUtils {
         1);
   }
   
+  @Lock(CSQueue.class)
+  public static void updateQueueStatistics(
+      final CSQueue childQueue, final CSQueue parentQueue, 
+      final Resource clusterResource, final Resource minimumAllocation) {
+    final int clusterMemory = clusterResource.getMemory();
+    final int usedMemory = childQueue.getUsedResources().getMemory();
+    
+    float queueLimit = 0.0f;
+    float utilization = 0.0f;
+    float usedCapacity = 0.0f;
+    if (clusterMemory > 0) {
+      queueLimit = clusterMemory * childQueue.getAbsoluteCapacity();
+      final float parentAbsoluteCapacity = 
+          (parentQueue == null) ? 1.0f : parentQueue.getAbsoluteCapacity();
+      utilization = (usedMemory / queueLimit);
+      usedCapacity = (usedMemory / (clusterMemory * parentAbsoluteCapacity));
+    }
+    
+    childQueue.setUtilization(utilization);
+    childQueue.setUsedCapacity(usedCapacity);
+    
+    int available = 
+        Math.max((roundUp(minimumAllocation, (int)queueLimit) - usedMemory), 0); 
+    childQueue.getMetrics().setAvailableResourcesToQueue(
+        Resources.createResource(available));
+  }
+
+  public static int roundUp(Resource minimumAllocation, int memory) {
+    int minMemory = minimumAllocation.getMemory();
+    return LeafQueue.divideAndCeil(memory, minMemory) * minMemory; 
+  }
+
+  public static int roundDown(Resource minimumAllocation, int memory) {
+    int minMemory = minimumAllocation.getMemory();
+    return (memory / minMemory) * minMemory;
+  }
 }

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java

@@ -180,7 +180,9 @@ public class LeafQueue implements CSQueue {
     Map<QueueACL, AccessControlList> acls = 
       cs.getConfiguration().getAcls(getQueuePath());
 
-    setupQueueConfigs(capacity, absoluteCapacity, 
+    setupQueueConfigs(
+        cs.getClusterResources(),
+        capacity, absoluteCapacity, 
         maximumCapacity, absoluteMaxCapacity, 
         userLimit, userLimitFactor, 
         maxApplications, maxApplicationsPerUser,
@@ -198,6 +200,7 @@ public class LeafQueue implements CSQueue {
   }
 
   private synchronized void setupQueueConfigs(
+      Resource clusterResource,
       float capacity, float absoluteCapacity, 
       float maximumCapacity, float absoluteMaxCapacity,
       int userLimit, float userLimitFactor,
@@ -235,6 +238,10 @@ public class LeafQueue implements CSQueue {
     for (Map.Entry<QueueACL, AccessControlList> e : acls.entrySet()) {
       aclsString.append(e.getKey() + ":" + e.getValue().getAclString());
     }
+    
+    // Update metrics
+    CSQueueUtils.updateQueueStatistics(
+        this, parent, clusterResource, minimumAllocation);
 
     LOG.info("Initializing " + queueName + "\n" +
         "capacity = " + capacity +
@@ -386,11 +393,11 @@ public class LeafQueue implements CSQueue {
     return null;
   }
 
-  synchronized void setUtilization(float utilization) {
+  public synchronized void setUtilization(float utilization) {
     this.utilization = utilization;
   }
 
-  synchronized void setUsedCapacity(float usedCapacity) {
+  public synchronized void setUsedCapacity(float usedCapacity) {
     this.usedCapacity = usedCapacity;
   }
 
@@ -534,7 +541,9 @@ public class LeafQueue implements CSQueue {
     }
 
     LeafQueue leafQueue = (LeafQueue)queue;
-    setupQueueConfigs(leafQueue.capacity, leafQueue.absoluteCapacity, 
+    setupQueueConfigs(
+        clusterResource,
+        leafQueue.capacity, leafQueue.absoluteCapacity, 
         leafQueue.maximumCapacity, leafQueue.absoluteMaxCapacity, 
         leafQueue.userLimit, leafQueue.userLimitFactor, 
         leafQueue.maxApplications,
@@ -542,8 +551,6 @@ public class LeafQueue implements CSQueue {
         leafQueue.getMaximumActiveApplications(), 
         leafQueue.getMaximumActiveApplicationsPerUser(),
         leafQueue.state, leafQueue.acls);
-    
-    updateResource(clusterResource);
   }
 
   @Override
@@ -883,7 +890,8 @@ public class LeafQueue implements CSQueue {
 
     Resource queueMaxCap =                        // Queue Max-Capacity
         Resources.createResource(
-            roundDown((int)(absoluteMaxCapacity * clusterResource.getMemory()))
+            CSQueueUtils.roundDown(minimumAllocation, 
+                (int)(absoluteMaxCapacity * clusterResource.getMemory()))
             );
     
     Resource userConsumed = getUser(user).getConsumedResources(); 
@@ -904,16 +912,6 @@ public class LeafQueue implements CSQueue {
     return userLimit;
   }
   
-  private int roundUp(int memory) {
-    int minMemory = minimumAllocation.getMemory();
-    return divideAndCeil(memory, minMemory) * minMemory; 
-  }
-  
-  private int roundDown(int memory) {
-    int minMemory = minimumAllocation.getMemory();
-    return (memory / minMemory) * minMemory;
-  }
-  
   @Lock(NoLock.class)
   private Resource computeUserLimit(SchedulerApp application, 
       Resource clusterResource, Resource required) {
@@ -927,8 +925,11 @@ public class LeafQueue implements CSQueue {
     // Allow progress for queues with miniscule capacity
     final int queueCapacity = 
       Math.max(
-          roundUp((int)(absoluteCapacity * clusterResource.getMemory())), 
-          required.getMemory());
+          CSQueueUtils.roundUp(
+              minimumAllocation, 
+              (int)(absoluteCapacity * clusterResource.getMemory())), 
+          required.getMemory()
+          );
 
     final int consumed = usedResources.getMemory();
     final int currentCapacity = 
@@ -943,7 +944,8 @@ public class LeafQueue implements CSQueue {
     final int activeUsers = activeUsersManager.getNumActiveUsers();  
 
     int limit = 
-      roundUp(
+      CSQueueUtils.roundUp(
+          minimumAllocation,
           Math.min(
               Math.max(divideAndCeil(currentCapacity, activeUsers), 
                        divideAndCeil((int)userLimit*currentCapacity, 100)),
@@ -991,7 +993,7 @@ public class LeafQueue implements CSQueue {
     return true;
   }
 
-  private static int divideAndCeil(int a, int b) {
+  static int divideAndCeil(int a, int b) {
     if (b == 0) {
       LOG.info("divideAndCeil called with a=" + a + " b=" + b);
       return 0;
@@ -1325,7 +1327,8 @@ public class LeafQueue implements CSQueue {
       SchedulerApp application, Resource resource) {
     // Update queue metrics
     Resources.addTo(usedResources, resource);
-    updateResource(clusterResource);
+    CSQueueUtils.updateQueueStatistics(
+        this, parent, clusterResource, minimumAllocation);
     ++numContainers;
 
     // Update user metrics
@@ -1349,7 +1352,8 @@ public class LeafQueue implements CSQueue {
       SchedulerApp application, Resource resource) {
     // Update queue metrics
     Resources.subtractFrom(usedResources, resource);
-    updateResource(clusterResource);
+    CSQueueUtils.updateQueueStatistics(
+        this, parent, clusterResource, minimumAllocation);
     --numContainers;
 
     // Update user metrics
@@ -1374,6 +1378,10 @@ public class LeafQueue implements CSQueue {
         CSQueueUtils.computeMaxActiveApplicationsPerUser(
             maxActiveApplications, userLimit, userLimitFactor);
     
+    // Update metrics
+    CSQueueUtils.updateQueueStatistics(
+        this, parent, clusterResource, minimumAllocation);
+    
     // Update application properties
     for (SchedulerApp application : activeApplications) {
       synchronized (application) {
@@ -1383,18 +1391,6 @@ public class LeafQueue implements CSQueue {
     }
   }
   
-  private synchronized void updateResource(Resource clusterResource) {
-    float queueLimit = clusterResource.getMemory() * absoluteCapacity;
-    setUtilization(usedResources.getMemory() / queueLimit);
-    setUsedCapacity(usedResources.getMemory()
-        / (clusterResource.getMemory() * parent.getAbsoluteCapacity()));
-
-    Resource resourceLimit = 
-      Resources.createResource(roundUp((int)queueLimit));
-    metrics.setAvailableResourcesToQueue(
-      Resources.subtractFrom(resourceLimit, usedResources));
-  }
-
   @Override
   public QueueMetrics getMetrics() {
     return metrics;

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java

@@ -97,7 +97,8 @@ public class ParentQueue implements CSQueue {
     RecordFactoryProvider.getRecordFactory(null);
 
   public ParentQueue(CapacitySchedulerContext cs, 
-      String queueName, Comparator<CSQueue> comparator, CSQueue parent, CSQueue old) {
+      String queueName, Comparator<CSQueue> comparator, 
+      CSQueue parent, CSQueue old) {
     minimumAllocation = cs.getMinimumResourceCapability();
     
     this.parent = parent;
@@ -137,7 +138,8 @@ public class ParentQueue implements CSQueue {
     this.queueInfo.setQueueName(queueName);
     this.queueInfo.setChildQueues(new ArrayList<QueueInfo>());
 
-    setupQueueConfigs(capacity, absoluteCapacity, 
+    setupQueueConfigs(cs.getClusterResources(),
+        capacity, absoluteCapacity, 
         maximumCapacity, absoluteMaxCapacity, state, acls);
     
     this.queueComparator = comparator;
@@ -149,9 +151,10 @@ public class ParentQueue implements CSQueue {
   }
 
   private synchronized void setupQueueConfigs(
-          float capacity, float absoluteCapacity, 
-          float maximumCapacity, float absoluteMaxCapacity,
-          QueueState state, Map<QueueACL, AccessControlList> acls
+      Resource clusterResource,
+      float capacity, float absoluteCapacity, 
+      float maximumCapacity, float absoluteMaxCapacity,
+      QueueState state, Map<QueueACL, AccessControlList> acls
   ) {
     // Sanity check
     CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity);
@@ -174,6 +177,10 @@ public class ParentQueue implements CSQueue {
       aclsString.append(e.getKey() + ":" + e.getValue().getAclString());
     }
 
+    // Update metrics
+    CSQueueUtils.updateQueueStatistics(
+        this, parent, clusterResource, minimumAllocation);
+
     LOG.info(queueName +
         ", capacity=" + capacity +
         ", asboluteCapacity=" + absoluteCapacity +
@@ -384,12 +391,10 @@ public class ParentQueue implements CSQueue {
     childQueues.addAll(currentChildQueues.values());
 
     // Set new configs
-    setupQueueConfigs(parentQueue.capacity, parentQueue.absoluteCapacity,
+    setupQueueConfigs(clusterResource,
+        parentQueue.capacity, parentQueue.absoluteCapacity,
         parentQueue.maximumCapacity, parentQueue.absoluteMaxCapacity,
         parentQueue.state, parentQueue.acls);
-
-    // Update
-    updateResource(clusterResource);
   }
 
   Map<String, CSQueue> getQueues(Set<CSQueue> queues) {
@@ -485,11 +490,11 @@ public class ParentQueue implements CSQueue {
         " #applications: " + getNumApplications());
   }
   
-  synchronized void setUsedCapacity(float usedCapacity) {
+  public synchronized void setUsedCapacity(float usedCapacity) {
     this.usedCapacity = usedCapacity;
   }
   
-  synchronized void setUtilization(float utilization) {
+  public synchronized void setUtilization(float utilization) {
     this.utilization = utilization;
   }
 
@@ -674,14 +679,16 @@ public class ParentQueue implements CSQueue {
   synchronized void allocateResource(Resource clusterResource, 
       Resource resource) {
     Resources.addTo(usedResources, resource);
-    updateResource(clusterResource);
+    CSQueueUtils.updateQueueStatistics(
+        this, parent, clusterResource, minimumAllocation);
     ++numContainers;
   }
   
   synchronized void releaseResource(Resource clusterResource, 
       Resource resource) {
     Resources.subtractFrom(usedResources, resource);
-    updateResource(clusterResource);
+    CSQueueUtils.updateQueueStatistics(
+        this, parent, clusterResource, minimumAllocation);
     --numContainers;
   }
 
@@ -691,22 +698,12 @@ public class ParentQueue implements CSQueue {
     for (CSQueue childQueue : childQueues) {
       childQueue.updateClusterResource(clusterResource);
     }
+    
+    // Update metrics
+    CSQueueUtils.updateQueueStatistics(
+        this, parent, clusterResource, minimumAllocation);
   }
   
-  private synchronized void updateResource(Resource clusterResource) {
-    float queueLimit = clusterResource.getMemory() * absoluteCapacity;
-    float parentAbsoluteCapacity = 
-        (rootQueue) ? 1.0f : parent.getAbsoluteCapacity();
-    setUtilization(usedResources.getMemory() / queueLimit);
-    setUsedCapacity(usedResources.getMemory() 
-        / (clusterResource.getMemory() * parentAbsoluteCapacity));
-  
-    Resource resourceLimit = 
-      Resources.createResource((int)queueLimit);
-    metrics.setAvailableResourcesToQueue(
-        Resources.subtractFrom(resourceLimit, usedResources));
-  }
-
   @Override
   public QueueMetrics getMetrics() {
     return metrics;

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java

@@ -19,7 +19,6 @@
 package org.apache.hadoop.yarn.server.resourcemanager.webapp;
 
 import static org.apache.hadoop.yarn.util.StringHelper.join;
-import static org.apache.hadoop.yarn.util.StringHelper.sjoin;
 import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE;
 import static org.apache.hadoop.yarn.webapp.view.JQueryUI._PROGRESSBAR;
 import static org.apache.hadoop.yarn.webapp.view.JQueryUI._PROGRESSBAR_VALUE;
@@ -27,6 +26,7 @@ import static org.apache.hadoop.yarn.webapp.view.JQueryUI._PROGRESSBAR_VALUE;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
+import org.apache.hadoop.yarn.util.Times;
 import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
 import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
 import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
@@ -52,11 +52,12 @@ class AppsBlock extends HtmlBlock {
             th(".user", "User").
             th(".name", "Name").
             th(".queue", "Queue").
+            th(".starttime", "StartTime").
+            th(".finishtime", "FinishTime").
             th(".state", "State").
             th(".finalstatus", "FinalStatus").
             th(".progress", "Progress").
-            th(".ui", "Tracking UI").
-            th(".note", "Note")._()._().
+            th(".ui", "Tracking UI")._()._().
         tbody();
     int i = 0;
     String reqState = $(APP_STATE);
@@ -67,6 +68,8 @@ class AppsBlock extends HtmlBlock {
       }
       AppInfo appInfo = new AppInfo(app, true);
       String percent = String.format("%.1f", appInfo.getProgress());
+      String startTime = Times.format(appInfo.getStartTime());
+      String finishTime = Times.format(appInfo.getFinishTime());
       tbody.
         tr().
           td().
@@ -75,6 +78,10 @@ class AppsBlock extends HtmlBlock {
           td(appInfo.getUser()).
           td(appInfo.getName()).
           td(appInfo.getQueue()).
+          td().
+            br().$title(startTime)._()._(startTime)._().
+          td().
+          br().$title(startTime)._()._(finishTime)._().
           td(appInfo.getState()).
           td(appInfo.getFinalStatus()).
           td().
@@ -85,8 +92,7 @@ class AppsBlock extends HtmlBlock {
                 $style(join("width:", percent, '%'))._()._()._().
           td().
             a(!appInfo.isTrackingUrlReady()?
-              "#" : appInfo.getTrackingUrlPretty(), appInfo.getTrackingUI())._().
-          td(appInfo.getNote())._();
+              "#" : appInfo.getTrackingUrlPretty(), appInfo.getTrackingUI())._()._();
       if (list.rendering != Render.HTML && ++i >= 20) break;
     }
     tbody._()._();

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java

@@ -55,15 +55,19 @@ public class MetricsOverviewTable extends HtmlBlock {
     //CSS in the correct spot
     html.style(".metrics {margin-bottom:5px}"); 
     
-    ClusterMetricsInfo clusterMetrics = new ClusterMetricsInfo(this.rm, this.rmContext);
-
+    ClusterMetricsInfo clusterMetrics = 
+        new ClusterMetricsInfo(this.rm, this.rmContext);
     
     DIV<Hamlet> div = html.div().$class("metrics");
     
-    div.table("#metricsoverview").
+    div.h3("Cluster Metrics").
+    table("#metricsoverview").
     thead().$class("ui-widget-header").
       tr().
         th().$class("ui-state-default")._("Apps Submitted")._().
+        th().$class("ui-state-default")._("Apps Pending")._().
+        th().$class("ui-state-default")._("Apps Running")._().
+        th().$class("ui-state-default")._("Apps Completed")._().
         th().$class("ui-state-default")._("Containers Running")._().
         th().$class("ui-state-default")._("Memory Used")._().
         th().$class("ui-state-default")._("Memory Total")._().
@@ -78,6 +82,14 @@ public class MetricsOverviewTable extends HtmlBlock {
     tbody().$class("ui-widget-content").
       tr().
         td(String.valueOf(clusterMetrics.getAppsSubmitted())).
+        td(String.valueOf(clusterMetrics.getAppsPending())).
+        td(String.valueOf(clusterMetrics.getAppsRunning())).
+        td(
+            String.valueOf(
+                clusterMetrics.getAppsCompleted() + 
+                clusterMetrics.getAppsFailed() + clusterMetrics.getAppsKilled()
+                )
+            ).
         td(String.valueOf(clusterMetrics.getContainersAllocated())).
         td(StringUtils.byteDesc(clusterMetrics.getAllocatedMB() * BYTES_IN_MB)).
         td(StringUtils.byteDesc(clusterMetrics.getTotalMB() * BYTES_IN_MB)).
@@ -89,26 +101,38 @@ public class MetricsOverviewTable extends HtmlBlock {
         td().a(url("nodes/rebooted"),String.valueOf(clusterMetrics.getRebootedNodes()))._().
       _().
     _()._();
-
+    
     String user = request().getRemoteUser();
     if (user != null) {
       UserMetricsInfo userMetrics = new UserMetricsInfo(this.rm, this.rmContext, user);
       if (userMetrics.metricsAvailable()) {
-        div.table("#usermetricsoverview").
+        div.h3("User Metrics for " + user).
+        table("#usermetricsoverview").
         thead().$class("ui-widget-header").
           tr().
-            th().$class("ui-state-default")._("Apps Submitted ("+user+")")._().
-            th().$class("ui-state-default")._("Containers Running ("+user+")")._().
-            th().$class("ui-state-default")._("Containers Pending ("+user+")")._().
-            th().$class("ui-state-default")._("Containers Reserved ("+user+")")._().
-            th().$class("ui-state-default")._("Memory Used ("+user+")")._().
-            th().$class("ui-state-default")._("Memory Pending ("+user+")")._().
-            th().$class("ui-state-default")._("Memory Reserved ("+user+")")._().
+            th().$class("ui-state-default")._("Apps Submitted")._().
+            th().$class("ui-state-default")._("Apps Pending")._().
+            th().$class("ui-state-default")._("Apps Running")._().
+            th().$class("ui-state-default")._("Apps Completed")._().
+            th().$class("ui-state-default")._("Containers Running")._().
+            th().$class("ui-state-default")._("Containers Pending")._().
+            th().$class("ui-state-default")._("Containers Reserved")._().
+            th().$class("ui-state-default")._("Memory Used")._().
+            th().$class("ui-state-default")._("Memory Pending")._().
+            th().$class("ui-state-default")._("Memory Reserved")._().
           _().
         _().
         tbody().$class("ui-widget-content").
           tr().
             td(String.valueOf(userMetrics.getAppsSubmitted())).
+            td(String.valueOf(userMetrics.getAppsPending())).
+            td(String.valueOf(userMetrics.getAppsRunning())).
+            td(
+                String.valueOf(
+                    (userMetrics.getAppsCompleted() + 
+                     userMetrics.getAppsFailed() + userMetrics.getAppsKilled())
+                    )
+              ).
             td(String.valueOf(userMetrics.getRunningContainers())).
             td(String.valueOf(userMetrics.getPendingContainers())).
             td(String.valueOf(userMetrics.getReservedContainers())).
@@ -117,6 +141,7 @@ public class MetricsOverviewTable extends HtmlBlock {
             td(StringUtils.byteDesc(userMetrics.getReservedMB() * BYTES_IN_MB)).
           _().
         _()._();
+        
       }
     }
 

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmView.java

@@ -63,10 +63,11 @@ public class RmView extends TwoColumnLayout {
 
   private String appsTableInit() {
     AppsList list = getInstance(AppsList.class);
-    // id, user, name, queue, state, progress, ui, note
+    // id, user, name, queue, starttime, finishtime, state, progress, ui
     StringBuilder init = tableInit().
-        append(", aoColumns:[{sType:'title-numeric'}, null, null, null, null,").
-        append("null,{sType:'title-numeric', bSearchable:false}, null, null]");
+        append(", aoColumns:[{sType:'title-numeric'}, null, null, null, ").
+        append("null, null , null, ").
+        append("null,{sType:'title-numeric', bSearchable:false}, null]");
 
     // Sort by id upon page load
     init.append(", aaSorting: [[0, 'asc']]");

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java

@@ -32,10 +32,20 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler
 public class ClusterMetricsInfo {
 
   protected int appsSubmitted;
+  protected int appsCompleted;
+  protected int appsPending;
+  protected int appsRunning;
+  protected int appsFailed;
+  protected int appsKilled;
+  
   protected long reservedMB;
   protected long availableMB;
   protected long allocatedMB;
+  
   protected int containersAllocated;
+  protected int containersReserved;
+  protected int containersPending;
+  
   protected long totalMB;
   protected int totalNodes;
   protected int lostNodes;
@@ -53,10 +63,20 @@ public class ClusterMetricsInfo {
     ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics();
 
     this.appsSubmitted = metrics.getAppsSubmitted();
+    this.appsCompleted = metrics.getAppsCompleted();
+    this.appsPending = metrics.getAppsPending();
+    this.appsRunning = metrics.getAppsRunning();
+    this.appsFailed = metrics.getAppsFailed();
+    this.appsKilled = metrics.getAppsKilled();
+    
     this.reservedMB = metrics.getReservedMB();
     this.availableMB = metrics.getAvailableMB();
     this.allocatedMB = metrics.getAllocatedMB();
+    
     this.containersAllocated = metrics.getAllocatedContainers();
+    this.containersPending = metrics.getPendingContainers();
+    this.containersReserved = metrics.getReservedContainers();
+    
     this.totalMB = availableMB + reservedMB + allocatedMB;
     this.activeNodes = clusterMetrics.getNumActiveNMs();
     this.lostNodes = clusterMetrics.getNumLostNMs();
@@ -71,6 +91,26 @@ public class ClusterMetricsInfo {
     return this.appsSubmitted;
   }
 
+  public int getAppsCompleted() {
+    return appsCompleted;
+  }
+
+  public int getAppsPending() {
+    return appsPending;
+  }
+
+  public int getAppsRunning() {
+    return appsRunning;
+  }
+
+  public int getAppsFailed() {
+    return appsFailed;
+  }
+
+  public int getAppsKilled() {
+    return appsKilled;
+  }
+
   public long getReservedMB() {
     return this.reservedMB;
   }
@@ -87,6 +127,14 @@ public class ClusterMetricsInfo {
     return this.containersAllocated;
   }
 
+  public int getReservedContainers() {
+    return this.containersReserved;
+  }
+
+  public int getPendingContainers() {
+    return this.containersPending;
+  }
+  
   public long getTotalMB() {
     return this.totalMB;
   }

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UserMetricsInfo.java

@@ -32,6 +32,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler
 public class UserMetricsInfo {
 
   protected int appsSubmitted;
+  protected int appsCompleted;
+  protected int appsPending;
+  protected int appsRunning;
+  protected int appsFailed;
+  protected int appsKilled;
   protected int runningContainers;
   protected int pendingContainers;
   protected int reservedContainers;
@@ -54,10 +59,18 @@ public class UserMetricsInfo {
 
     if (userMetrics != null) {
       this.userMetricsAvailable = true;
+      
       this.appsSubmitted = userMetrics.getAppsSubmitted();
+      this.appsCompleted = metrics.getAppsCompleted();
+      this.appsPending = metrics.getAppsPending();
+      this.appsRunning = metrics.getAppsRunning();
+      this.appsFailed = metrics.getAppsFailed();
+      this.appsKilled = metrics.getAppsKilled();
+
       this.runningContainers = userMetrics.getAllocatedContainers();
       this.pendingContainers = userMetrics.getPendingContainers();
       this.reservedContainers = userMetrics.getReservedContainers();
+      
       this.reservedMB = userMetrics.getReservedMB();
       this.pendingMB = userMetrics.getPendingMB();
       this.allocatedMB = userMetrics.getAllocatedMB();
@@ -72,6 +85,26 @@ public class UserMetricsInfo {
     return this.appsSubmitted;
   }
 
+  public int getAppsCompleted() {
+    return appsCompleted;
+  }
+
+  public int getAppsPending() {
+    return appsPending;
+  }
+
+  public int getAppsRunning() {
+    return appsRunning;
+  }
+
+  public int getAppsFailed() {
+    return appsFailed;
+  }
+
+  public int getAppsKilled() {
+    return appsKilled;
+  }
+
   public long getReservedMB() {
     return this.reservedMB;
   }

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java

@@ -142,7 +142,7 @@ public class TestApplicationLimits {
         CapacityScheduler.parseQueue(csContext, csConf, null, "root", 
             queues, queues, 
             CapacityScheduler.queueComparator, 
-            CapacityScheduler.applicationComparator, 
+            CapacityScheduler.applicationComparator,
             TestUtils.spyHook);
 
     LeafQueue queue = (LeafQueue)queues.get(A);
@@ -163,6 +163,10 @@ public class TestApplicationLimits {
             expectedMaxActiveApps * (queue.getUserLimit() / 100.0f) * 
             queue.getUserLimitFactor()), 
         queue.getMaximumActiveApplicationsPerUser());
+    assertEquals(
+        (int)(clusterResource.getMemory() * queue.getAbsoluteCapacity()),
+        queue.getMetrics().getAvailableMB()
+        );
     
     // Add some nodes to the cluster & test new limits
     clusterResource = Resources.createResource(120 * 16 * GB);
@@ -178,6 +182,10 @@ public class TestApplicationLimits {
         (int)Math.ceil(expectedMaxActiveApps * 
             (queue.getUserLimit() / 100.0f) * queue.getUserLimitFactor()), 
         queue.getMaximumActiveApplicationsPerUser());
+    assertEquals(
+        (int)(clusterResource.getMemory() * queue.getAbsoluteCapacity()),
+        queue.getMetrics().getAvailableMB()
+        );
   }
   
   @Test

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java

@@ -48,7 +48,7 @@ public class TestNodesPage {
 
   // Number of Actual Table Headers for NodesPage.NodesBlock might change in
   // future. In that case this value should be adjusted to the new value.
-  final int numberOfThInMetricsTable = 10;
+  final int numberOfThInMetricsTable = 13;
   final int numberOfActualTableHeaders = 10;
 
   private Injector injector;

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java

@@ -361,6 +361,7 @@ public class TestRMWebServices extends JerseyTest {
 
       verifyClusterMetrics(
           WebServicesTestUtils.getXmlInt(element, "appsSubmitted"),
+          WebServicesTestUtils.getXmlInt(element, "appsCompleted"),
           WebServicesTestUtils.getXmlInt(element, "reservedMB"),
           WebServicesTestUtils.getXmlInt(element, "availableMB"),
           WebServicesTestUtils.getXmlInt(element, "allocatedMB"),
@@ -379,8 +380,9 @@ public class TestRMWebServices extends JerseyTest {
       Exception {
     assertEquals("incorrect number of elements", 1, json.length());
     JSONObject clusterinfo = json.getJSONObject("clusterMetrics");
-    assertEquals("incorrect number of elements", 12, clusterinfo.length());
-    verifyClusterMetrics(clusterinfo.getInt("appsSubmitted"),
+    assertEquals("incorrect number of elements", 19, clusterinfo.length());
+    verifyClusterMetrics(
+        clusterinfo.getInt("appsSubmitted"), clusterinfo.getInt("appsCompleted"),
         clusterinfo.getInt("reservedMB"), clusterinfo.getInt("availableMB"),
         clusterinfo.getInt("allocatedMB"),
         clusterinfo.getInt("containersAllocated"),
@@ -390,7 +392,8 @@ public class TestRMWebServices extends JerseyTest {
         clusterinfo.getInt("rebootedNodes"),clusterinfo.getInt("activeNodes"));
   }
 
-  public void verifyClusterMetrics(int sub, int reservedMB, int availableMB,
+  public void verifyClusterMetrics(int submittedApps, int completedApps,
+      int reservedMB, int availableMB,
       int allocMB, int containersAlloc, int totalMB, int totalNodes,
       int lostNodes, int unhealthyNodes, int decommissionedNodes,
       int rebootedNodes, int activeNodes) throws JSONException, Exception {
@@ -404,7 +407,9 @@ public class TestRMWebServices extends JerseyTest {
         + metrics.getAllocatedMB();
 
     assertEquals("appsSubmitted doesn't match", 
-        metrics.getAppsSubmitted(), sub);
+        metrics.getAppsSubmitted(), submittedApps);
+    assertEquals("appsCompleted doesn't match", 
+        metrics.getAppsCompleted(), completedApps);
     assertEquals("reservedMB doesn't match",
         metrics.getReservedMB(), reservedMB);
     assertEquals("availableMB doesn't match",