YARN-4751. In 2.7, Labeled queue usage not shown properly in capacity scheduler UI. Contributed by Eric Payne

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java

@@ -87,6 +87,8 @@ public abstract class AbstractCSQueue implements CSQueue {
   protected CapacitySchedulerContext csContext;
   protected YarnAuthorizationProvider authorizer = null;
 
+  private Resource clusterResource;
+
   public AbstractCSQueue(CapacitySchedulerContext cs, 
       String queueName, CSQueue parent, CSQueue old) throws IOException {
     this.labelManager = cs.getRMContext().getNodeLabelManager();
@@ -291,6 +293,7 @@ public abstract class AbstractCSQueue implements CSQueue {
         .getReservationContinueLook();
 
     this.preemptionDisabled = isQueueHierarchyPreemptionDisabled(this);
+    this.clusterResource = clusterResource;
   }
   
   protected QueueInfo getQueueInfo() {
@@ -322,7 +325,9 @@ public abstract class AbstractCSQueue implements CSQueue {
     if (nodeLabels == null || nodeLabels.isEmpty()) {
       queueUsage.incUsed(resource);
     } else {
-      for (String label : Sets.intersection(accessibleLabels, nodeLabels)) {
+      Set<String> anls = (accessibleLabels.contains(RMNodeLabelsManager.ANY))
+          ? labelManager.getClusterNodeLabels() : accessibleLabels;
+      for (String label : Sets.intersection(anls, nodeLabels)) {
         queueUsage.incUsed(label, resource);
       }
     }
@@ -338,7 +343,9 @@ public abstract class AbstractCSQueue implements CSQueue {
     if (null == nodeLabels || nodeLabels.isEmpty()) {
       queueUsage.decUsed(resource);
     } else {
-      for (String label : Sets.intersection(accessibleLabels, nodeLabels)) {
+      Set<String> anls = (accessibleLabels.contains(RMNodeLabelsManager.ANY))
+          ? labelManager.getClusterNodeLabels() : accessibleLabels;
+      for (String label : Sets.intersection(anls, nodeLabels)) {
         queueUsage.decUsed(label, resource);
       }
     }
@@ -527,4 +534,43 @@ public abstract class AbstractCSQueue implements CSQueue {
     // sorry, you cannot access
     return false;
   }
+
+  /**
+   * Retrieve used resources by this queue for the specified node label.
+   * Used capacity of label =
+   *   (queue's used resources labeled by nodeLabel)
+   *      / (   (all resources labeled by nodLabel)
+   *          X (percent of labeled resources allocated to this queue)  )
+   * @param nodeLabel label for which to get used resources
+   * @return used resources by this queue for specified label
+   */
+  public final synchronized float getUsedCapacity(final String nodeLabel) {
+    Resource availableToQueue =
+        Resources.multiply(
+            labelManager.getResourceByLabel(nodeLabel, this.clusterResource),
+            queueCapacities.getAbsoluteCapacity(nodeLabel));
+    if (!Resources.greaterThan(resourceCalculator, this.clusterResource,
+         availableToQueue, Resources.none())) {
+      return 0.0f;
+    }
+    return
+        Resources.divide(resourceCalculator, this.clusterResource,
+            queueUsage.getUsed(nodeLabel), availableToQueue);
+  }
+
+  /**
+   * Retrieve absolute used resources by this queue for the specified node label.
+   * @param nodeLabel label for which to get absolute used resources
+   * @return absolute used resources by this queue for specified label
+   */
+  public final synchronized float getAbsoluteUsedCapacity(final String nodeLabel) {
+    Resource labeledResources =
+               labelManager.getResourceByLabel(nodeLabel, this.clusterResource);
+    if (!Resources.greaterThan(resourceCalculator, this.clusterResource,
+        labeledResources, Resources.none())) {
+      return 0.0f;
+    }
+    return Resources.divide(resourceCalculator, this.clusterResource,
+        queueUsage.getUsed(nodeLabel), labeledResources);
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java

@@ -284,4 +284,18 @@ extends org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue {
    * @return resourceUsage
    */
   public ResourceUsage getQueueResourceUsage();
+
+  /**
+   * Get used capacity for the specified label.
+   * @param nodeLabel
+   * @return used capacity
+   */
+  public float getUsedCapacity(String nodeLabel);
+
+  /**
+   * Get absolute used capacity for the specified label.
+   * @param nodeLabel
+   * @return absolute used capacity
+   */
+  public float getAbsoluteUsedCapacity(String nodeLabel);
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java

@@ -424,8 +424,12 @@ public class LeafQueue extends AbstractCSQueue {
     ArrayList<UserInfo> usersToReturn = new ArrayList<UserInfo>();
     for (Map.Entry<String, User> entry : users.entrySet()) {
       User user = entry.getValue();
-      usersToReturn.add(new UserInfo(entry.getKey(), Resources.clone(user
-          .getUsed()), user.getActiveApplications(), user
+      Resource usedRes = Resource.newInstance(0, 0);
+      for (String nl : getAccessibleLabelSet()) {
+        Resources.addTo(usedRes, user.getUsed(nl));
+      }
+      usersToReturn.add(new UserInfo(entry.getKey(), usedRes,
+          user.getActiveApplications(), user
           .getPendingApplications(), Resources.clone(user
           .getConsumedAMResources()), Resources.clone(user
           .getUserResourceLimit())));
@@ -433,6 +437,22 @@ public class LeafQueue extends AbstractCSQueue {
     return usersToReturn;
   }
 
+  /**
+   * Gets the labels which are accessible by this queue. If ANY label can be
+   * accessed, put all labels in the set.
+   * @return accessiglbe node labels
+   */
+  protected final Set<String> getAccessibleLabelSet() {
+    Set<String> nodeLabels = new HashSet<String>();
+    if (this.getAccessibleNodeLabels().contains(RMNodeLabelsManager.ANY)) {
+      nodeLabels.addAll(labelManager.getClusterNodeLabels());
+    } else {
+      nodeLabels.addAll(this.getAccessibleNodeLabels());
+    }
+    nodeLabels.add(RMNodeLabelsManager.NO_LABEL);
+    return nodeLabels;
+  }
+
   @Override
   public synchronized void reinitialize(
       CSQueue newlyParsedQueue, Resource clusterResource) 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java

@@ -29,7 +29,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UserInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerLeafQueueInfo;
@@ -319,8 +318,7 @@ class CapacitySchedulerPage extends RmView {
             String nodeLabel =
                 csqinfo.label.length() == 0 ? "<DEFAULT_PARTITION>"
                     : csqinfo.label;
-            QueueCapacities queueCapacities = root.getQueueCapacities();
-            used = queueCapacities.getUsedCapacity(label.getLabelName());
+            used = root.getUsedCapacity(label.getLabelName());
             String partitionUiTag =
                 "Partition: " + nodeLabel + " " + label.getResource();
             ul.li().

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java

@@ -58,7 +58,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
     String label = nodeLabel.getLabelName();
     QueueCapacities parentQueueCapacities = parent.getQueueCapacities();
     this.queueName = parent.getQueueName();
-    this.usedCapacity = parentQueueCapacities.getUsedCapacity(label) * 100;
+    this.usedCapacity = parent.getUsedCapacity(label) * 100;
     this.capacity = parentQueueCapacities.getCapacity(label) * 100;
     float max = parentQueueCapacities.getMaximumCapacity(label);
     if (max < EPSILON || max > 1f)
@@ -106,8 +106,8 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
     List<CSQueue> childNonLeafQueues = new ArrayList<>();
     for (CSQueue queue : parent.getChildQueues()) {
       if (!((AbstractCSQueue) queue).accessibleToPartition(nodeLabel
-              .getLabelName())) {
-        // Skip displaying the hierarchy for the queues for which the exclusive
+          .getLabelName())) {
+        // Skip displaying the hierarchy for the queues for which the
         // labels are not accessible
         continue;
       }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java

@@ -71,7 +71,7 @@ public class CapacitySchedulerQueueInfo {
 
     queuePath = q.getQueuePath();
     capacity = qCapacities.getCapacity(nodeLabel) * 100;
-    usedCapacity = qCapacities.getUsedCapacity(nodeLabel) * 100;
+    usedCapacity = q.getUsedCapacity(nodeLabel) * 100;
 
     maxCapacity = qCapacities.getMaximumCapacity(nodeLabel);
     if (maxCapacity < EPSILON || maxCapacity > 1f)
@@ -82,8 +82,7 @@ public class CapacitySchedulerQueueInfo {
         cap(qCapacities.getAbsoluteCapacity(nodeLabel), 0f, 1f) * 100;
     absoluteMaxCapacity =
         cap(qCapacities.getAbsoluteMaximumCapacity(nodeLabel), 0f, 1f) * 100;
-    absoluteUsedCapacity =
-        cap(qCapacities.getAbsoluteUsedCapacity(nodeLabel), 0f, 1f) * 100;
+    absoluteUsedCapacity = q.getAbsoluteUsedCapacity(nodeLabel) * 100;
     numApplications = q.getNumApplications();
     queueName = q.getQueueName();
     state = q.getState();