YARN-3961. Expose pending, running and reserved containers of a queue in REST api and yarn top (adhoot via asuresh)

(cherry picked from commit 154c9d2e421390c808dea3b2826cdaae492ea337)

hadoop-yarn-project/CHANGES.txt

@@ -319,6 +319,9 @@ Release 2.8.0 - UNRELEASED
     YARN-4004. container-executor should print output of docker logs if the docker
     container exits with non-0 exit status. (Varun Vasudev via xgong)
 
+    YARN-3961. Expose pending, running and reserved containers of a queue in REST
+    api and yarn top (adhoot via asuresh)
+
   OPTIMIZATIONS
 
     YARN-3339. TestDockerContainerExecutor should pull a single image and not

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueStatistics.java

@@ -262,6 +262,42 @@ public abstract class QueueStatistics {
    */
   public abstract void setPendingVCores(long pendingVCores);
 
+  /**
+   * Get the number of pending containers.
+   * @return the number of pending containers.
+   */
+  public abstract long getPendingContainers();
+
+  /**
+   * Set the number of pending containers.
+   * @param pendingContainers the pending containers.
+   */
+  public abstract void setPendingContainers(long pendingContainers);
+
+  /**
+   * Get the number of allocated containers.
+   * @return the number of allocated containers.
+   */
+  public abstract long getAllocatedContainers();
+
+  /**
+   * Set the number of allocated containers.
+   * @param allocatedContainers the allocated containers.
+   */
+  public abstract void setAllocatedContainers(long allocatedContainers);
+
+  /**
+   * Get the number of reserved containers.
+   * @return the number of reserved containers.
+   */
+  public abstract long getReservedContainers();
+
+  /**
+   * Set the number of reserved containers.
+   * @param reservedContainers the reserved containers.
+   */
+  public abstract void setReservedContainers(long reservedContainers);
+
   /**
    * Get the reserved vcores
    * 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto

@@ -384,6 +384,9 @@ message QueueStatisticsProto {
   optional int64 allocatedVCores = 13;
   optional int64 pendingVCores = 14;
   optional int64 reservedVCores = 15;
+  optional int64 allocatedContainers = 16;
+  optional int64 pendingContainers = 17;
+  optional int64 reservedContainers = 18;
 }
 
 message QueueInfoProto {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java

@@ -71,7 +71,7 @@ public class TopCLI extends YarnCLI {
   private String SET_CURSOR_HOME = "\u001b[H";
   private String CHANGE_BACKGROUND = "\u001b[7m";
   private String RESET_BACKGROUND = "\u001b[0m";
-  private String SET_CURSOR_LINE_6_COLUMN_0 = "\u001b[6;0f";
+  private String SET_CURSOR_LINE_7_COLUMN_0 = "\u001b[7;0f";
 
   // guava cache for getapplications call
   protected Cache<GetApplicationsRequest, List<ApplicationReport>>
@@ -331,6 +331,9 @@ public class TopCLI extends YarnCLI {
     long allocatedVCores;
     long pendingVCores;
     long reservedVCores;
+    long allocatedContainers;
+    long reservedContainers;
+    long pendingContainers;
   }
 
   private class KeyboardMonitor extends Thread {
@@ -596,14 +599,14 @@ public class TopCLI extends YarnCLI {
     String[] tput_cursor_home = { "tput", "cup", "0", "0" };
     String[] tput_clear = { "tput", "clear" };
     String[] tput_clear_line = { "tput", "el" };
-    String[] tput_set_cursor_line_6_column_0 = { "tput", "cup", "5", "0" };
+    String[] tput_set_cursor_line_7_column_0 = { "tput", "cup", "6", "0" };
     String[] tput_change_background = { "tput", "smso" };
     String[] tput_reset_background = { "tput", "rmso" };
     SET_CURSOR_HOME = getCommandOutput(tput_cursor_home);
     CLEAR = getCommandOutput(tput_clear);
     CLEAR_LINE = getCommandOutput(tput_clear_line);
-    SET_CURSOR_LINE_6_COLUMN_0 =
-        getCommandOutput(tput_set_cursor_line_6_column_0);
+    SET_CURSOR_LINE_7_COLUMN_0 =
+        getCommandOutput(tput_set_cursor_line_7_column_0);
     CHANGE_BACKGROUND = getCommandOutput(tput_change_background);
     RESET_BACKGROUND = getCommandOutput(tput_reset_background);
   }
@@ -712,6 +715,9 @@ public class TopCLI extends YarnCLI {
         queueMetrics.allocatedVCores += stats.getAllocatedVCores();
         queueMetrics.pendingVCores += stats.getPendingVCores();
         queueMetrics.reservedVCores += stats.getReservedVCores();
+        queueMetrics.allocatedContainers += stats.getAllocatedContainers();
+        queueMetrics.pendingContainers += stats.getPendingContainers();
+        queueMetrics.reservedContainers += stats.getReservedContainers();
       }
     }
     queueMetrics.availableMemoryGB = queueMetrics.availableMemoryGB / 1024;
@@ -793,12 +799,18 @@ public class TopCLI extends YarnCLI {
       queueMetrics.availableVCores, queueMetrics.allocatedVCores,
       queueMetrics.pendingVCores, queueMetrics.reservedVCores), terminalWidth,
       true));
+
+    ret.append(CLEAR_LINE);
+    ret.append(limitLineLength(String.format(
+        "Queue(s) Containers: %d allocated, %d pending, %d reserved%n",
+            queueMetrics.allocatedContainers, queueMetrics.pendingContainers,
+            queueMetrics.reservedContainers), terminalWidth, true));
     return ret.toString();
   }
 
   String getPrintableAppInformation(List<ApplicationInformation> appsInfo) {
     StringBuilder ret = new StringBuilder();
-    int limit = terminalHeight - 8;
+    int limit = terminalHeight - 9;
     List<String> columns = new ArrayList<>();
     for (int i = 0; i < limit; ++i) {
       ret.append(CLEAR_LINE);
@@ -944,7 +956,7 @@ public class TopCLI extends YarnCLI {
     synchronized (lock) {
       printHeader(header);
       printApps(appsStr);
-      System.out.print(SET_CURSOR_LINE_6_COLUMN_0);
+      System.out.print(SET_CURSOR_LINE_7_COLUMN_0);
       System.out.print(CLEAR_LINE);
     }
   }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueStatisticsPBImpl.java

@@ -254,4 +254,40 @@ public class QueueStatisticsPBImpl extends QueueStatistics {
     maybeInitBuilder();
     builder.setReservedVCores(reservedVCores);
   }
+
+  @Override
+  public long getPendingContainers() {
+    QueueStatisticsProtoOrBuilder p = viaProto ? proto : builder;
+    return (p.hasPendingContainers()) ? p.getPendingContainers() : -1;
+  }
+
+  @Override
+  public void setPendingContainers(long pendingContainers) {
+    maybeInitBuilder();
+    builder.setPendingContainers(pendingContainers);
+  }
+
+  @Override
+  public long getAllocatedContainers() {
+    QueueStatisticsProtoOrBuilder p = viaProto ? proto : builder;
+    return (p.hasAllocatedContainers()) ? p.getAllocatedContainers() : -1;
+  }
+
+  @Override
+  public void setAllocatedContainers(long allocatedContainers) {
+    maybeInitBuilder();
+    builder.setAllocatedContainers(allocatedContainers);
+  }
+
+  @Override
+  public long getReservedContainers() {
+    QueueStatisticsProtoOrBuilder p = viaProto ? proto : builder;
+    return (p.hasReservedContainers()) ? p.getReservedContainers() : -1;
+  }
+
+  @Override
+  public void setReservedContainers(long reservedContainers) {
+    maybeInitBuilder();
+    builder.setReservedContainers(reservedContainers);
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java

@@ -323,6 +323,9 @@ public abstract class AbstractCSQueue implements CSQueue {
     stats.setAllocatedVCores(getMetrics().getAllocatedVirtualCores());
     stats.setPendingVCores(getMetrics().getPendingVirtualCores());
     stats.setReservedVCores(getMetrics().getReservedVirtualCores());
+    stats.setPendingContainers(getMetrics().getPendingContainers());
+    stats.setAllocatedContainers(getMetrics().getAllocatedContainers());
+    stats.setReservedContainers(getMetrics().getReservedContainers());
     return stats;
   }
   

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java

@@ -57,6 +57,9 @@ public class CapacitySchedulerQueueInfo {
   protected ResourceInfo resourcesUsed;
   private boolean hideReservationQueues = false;
   protected ArrayList<String> nodeLabels = new ArrayList<String>();
+  protected long allocatedContainers;
+  protected long reservedContainers;
+  protected long pendingContainers;
 
   CapacitySchedulerQueueInfo() {
   };
@@ -81,6 +84,9 @@ public class CapacitySchedulerQueueInfo {
     absoluteUsedCapacity =
         cap(qCapacities.getAbsoluteUsedCapacity(nodeLabel), 0f, 1f) * 100;
     numApplications = q.getNumApplications();
+    allocatedContainers = q.getMetrics().getAllocatedContainers();
+    pendingContainers = q.getMetrics().getPendingContainers();
+    reservedContainers = q.getMetrics().getReservedContainers();
     queueName = q.getQueueName();
     state = q.getState();
     resourcesUsed = new ResourceInfo(queueResourceUsage.getUsed(nodeLabel));
@@ -124,6 +130,18 @@ public class CapacitySchedulerQueueInfo {
     return numApplications;
   }
 
+  public long getAllocatedContainers() {
+    return allocatedContainers;
+  }
+
+  public long getReservedContainers() {
+    return reservedContainers;
+  }
+
+  public long getPendingContainers() {
+    return pendingContainers;
+  }
+
   public String getQueueName() {
     return this.queueName;
   }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java

@@ -57,7 +57,11 @@ public class FairSchedulerQueueInfo {
   private ResourceInfo steadyFairResources;
   private ResourceInfo fairResources;
   private ResourceInfo clusterResources;
-  
+
+  private long pendingContainers;
+  private long allocatedContainers;
+  private long reservedContainers;
+
   private String queueName;
   private String schedulingPolicy;
 
@@ -95,6 +99,10 @@ public class FairSchedulerQueueInfo {
     
     maxApps = allocConf.getQueueMaxApps(queueName);
 
+    pendingContainers = queue.getMetrics().getPendingContainers();
+    allocatedContainers = queue.getMetrics().getAllocatedContainers();
+    reservedContainers = queue.getMetrics().getReservedContainers();
+
     if (allocConf.isReservable(queueName) &&
         !allocConf.getShowReservationAsQueues(queueName)) {
       return;
@@ -103,6 +111,18 @@ public class FairSchedulerQueueInfo {
     childQueues = getChildQueues(queue, scheduler);
   }
 
+  public long getPendingContainers() {
+    return pendingContainers;
+  }
+
+  public long getAllocatedContainers() {
+    return allocatedContainers;
+  }
+
+  public long getReservedContainers() {
+    return reservedContainers;
+  }
+
   protected FairSchedulerQueueInfoList getChildQueues(FSQueue queue,
                                                       FairScheduler scheduler) {
     // Return null to omit 'childQueues' field from the return value of
@@ -172,7 +192,7 @@ public class FairSchedulerQueueInfo {
   public ResourceInfo getUsedResources() {
     return usedResources;
   }
-  
+
   /**
    * Returns the queue's min share in as a fraction of the entire
    * cluster capacity.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java

@@ -349,10 +349,10 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
   private void verifySubQueue(JSONObject info, String q, 
       float parentAbsCapacity, float parentAbsMaxCapacity)
       throws JSONException, Exception {
-    int numExpectedElements = 13;
+    int numExpectedElements = 16;
     boolean isParentQueue = true;
     if (!info.has("queues")) {
-      numExpectedElements = 25;
+      numExpectedElements = 28;
       isParentQueue = false;
     }
     assertEquals("incorrect number of elements", numExpectedElements, info.length());