YARN-7254. UI and metrics changes related to absolute resource configuration. (Sunil G via wangda)

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueConfigurations.java

@@ -147,4 +147,80 @@ public abstract class QueueConfigurations {
   @Private
   @Unstable
   public abstract void setMaxAMPercentage(float maxAMPercentage);
+
+  /**
+   * Get the effective minimum capacity of queue (from absolute resource).
+   *
+   * @return minimum resource capability
+   */
+  @Public
+  @Unstable
+  public abstract Resource getEffectiveMinCapacity();
+
+  /**
+   * Set the effective minimum capacity of queue (from absolute resource).
+   *
+   * @param capacity
+   *          minimum resource capability
+   */
+  @Private
+  @Unstable
+  public abstract void setEffectiveMinCapacity(Resource capacity);
+
+  /**
+   * Get the effective maximum capacity of queue (from absolute resource).
+   *
+   * @return maximum resource capability
+   */
+  @Public
+  @Unstable
+  public abstract Resource getEffectiveMaxCapacity();
+
+  /**
+   * Set the effective maximum capacity of queue (from absolute resource).
+   *
+   * @param capacity
+   *          maximum resource capability
+   */
+  @Private
+  @Unstable
+  public abstract void setEffectiveMaxCapacity(Resource capacity);
+
+  /**
+   * Get the configured minimum capacity of queue (from absolute resource).
+   *
+   * @return minimum resource capability
+   */
+  @Public
+  @Unstable
+  public abstract Resource getConfiguredMinCapacity();
+
+  /**
+   * Set the configured minimum capacity of queue (from absolute resource).
+   *
+   * @param configuredMinResource
+   *          minimum resource capability
+   */
+  @Public
+  @Unstable
+  public abstract void setConfiguredMinCapacity(Resource configuredMinResource);
+
+  /**
+   * Get the configured maximum capacity of queue (from absolute resource).
+   *
+   * @return maximum resource capability
+   */
+  @Public
+  @Unstable
+  public abstract Resource getConfiguredMaxCapacity();
+
+  /**
+   * Set the configured maximum capacity of queue (from absolute resource).
+   *
+   * @param configuredMaxResource
+   *          maximum resource capability
+   */
+  @Public
+  @Unstable
+  public abstract void setConfiguredMaxCapacity(Resource configuredMaxResource);
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java

@@ -225,6 +225,12 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
         Long.MAX_VALUE);
   }
 
+  public static ResourceInformation newInstance(String name, String units,
+      long minRes, long maxRes) {
+    return ResourceInformation.newInstance(name, units, 0L,
+        ResourceTypes.COUNTABLE, minRes, maxRes);
+  }
+
   public static ResourceInformation newInstance(String name, long value) {
     return ResourceInformation
         .newInstance(name, "", value, ResourceTypes.COUNTABLE, 0L,

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto

@@ -551,6 +551,10 @@ message QueueConfigurationsProto {
     optional float maxCapacity = 3;
     optional float absoluteMaxCapacity = 4;
     optional float maxAMPercentage = 5;
+    optional ResourceProto effectiveMinCapacity = 6;
+    optional ResourceProto effectiveMaxCapacity = 7;
+    optional ResourceProto configuredMinCapacity = 8;
+    optional ResourceProto configuredMaxCapacity = 9;
 }
 
 message QueueConfigurationsMapProto {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java

@@ -19,16 +19,22 @@
 package org.apache.hadoop.yarn.api.records.impl.pb;
 
 import org.apache.hadoop.yarn.api.records.QueueConfigurations;
+import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.proto.YarnProtos.QueueConfigurationsProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.QueueConfigurationsProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
 
 import com.google.protobuf.TextFormat;
 
 public class QueueConfigurationsPBImpl extends QueueConfigurations {
 
-  QueueConfigurationsProto proto =
-      QueueConfigurationsProto.getDefaultInstance();
+  QueueConfigurationsProto proto = QueueConfigurationsProto
+      .getDefaultInstance();
   QueueConfigurationsProto.Builder builder = null;
+  Resource configuredMinResource = null;
+  Resource configuredMaxResource = null;
+  Resource effMinResource = null;
+  Resource effMaxResource = null;
   boolean viaProto = false;
 
   public QueueConfigurationsPBImpl() {
@@ -41,11 +47,40 @@ public class QueueConfigurationsPBImpl extends QueueConfigurations {
   }
 
   public QueueConfigurationsProto getProto() {
+    mergeLocalToProto();
     proto = viaProto ? proto : builder.build();
     viaProto = true;
     return proto;
   }
 
+  private void mergeLocalToProto() {
+    if (viaProto) {
+      maybeInitBuilder();
+    }
+    mergeLocalToBuilder();
+    proto = builder.build();
+    viaProto = true;
+  }
+
+  private void mergeLocalToBuilder() {
+    if (this.effMinResource != null) {
+      builder
+          .setEffectiveMinCapacity(convertToProtoFormat(this.effMinResource));
+    }
+    if (this.effMaxResource != null) {
+      builder
+          .setEffectiveMaxCapacity(convertToProtoFormat(this.effMaxResource));
+    }
+    if (this.configuredMinResource != null) {
+      builder.setEffectiveMinCapacity(
+          convertToProtoFormat(this.configuredMinResource));
+    }
+    if (this.configuredMaxResource != null) {
+      builder.setEffectiveMaxCapacity(
+          convertToProtoFormat(this.configuredMaxResource));
+    }
+  }
+
   @Override
   public float getCapacity() {
     QueueConfigurationsProtoOrBuilder p = viaProto ? proto : builder;
@@ -106,6 +141,58 @@ public class QueueConfigurationsPBImpl extends QueueConfigurations {
     builder.setMaxAMPercentage(maxAMPercentage);
   }
 
+  @Override
+  public Resource getEffectiveMinCapacity() {
+    QueueConfigurationsProtoOrBuilder p = viaProto ? proto : builder;
+    if (this.effMinResource != null) {
+      return this.effMinResource;
+    }
+    if (!p.hasEffectiveMinCapacity()) {
+      return null;
+    }
+    this.effMinResource = convertFromProtoFormat(p.getEffectiveMinCapacity());
+    return this.effMinResource;
+  }
+
+  @Override
+  public void setEffectiveMinCapacity(Resource capacity) {
+    maybeInitBuilder();
+    if (capacity == null) {
+      builder.clearEffectiveMinCapacity();
+    }
+    this.effMinResource = capacity;
+  }
+
+  @Override
+  public Resource getEffectiveMaxCapacity() {
+    QueueConfigurationsProtoOrBuilder p = viaProto ? proto : builder;
+    if (this.effMaxResource != null) {
+      return this.effMaxResource;
+    }
+    if (!p.hasEffectiveMaxCapacity()) {
+      return null;
+    }
+    this.effMaxResource = convertFromProtoFormat(p.getEffectiveMaxCapacity());
+    return this.effMaxResource;
+  }
+
+  @Override
+  public void setEffectiveMaxCapacity(Resource capacity) {
+    maybeInitBuilder();
+    if (capacity == null) {
+      builder.clearEffectiveMaxCapacity();
+    }
+    this.effMaxResource = capacity;
+  }
+
+  private ResourcePBImpl convertFromProtoFormat(ResourceProto p) {
+    return new ResourcePBImpl(p);
+  }
+
+  private ResourceProto convertToProtoFormat(Resource t) {
+    return ProtoUtils.convertToProtoFormat(t);
+  }
+
   private void maybeInitBuilder() {
     if (viaProto || builder == null) {
       builder = QueueConfigurationsProto.newBuilder(proto);
@@ -134,4 +221,49 @@ public class QueueConfigurationsPBImpl extends QueueConfigurations {
     return false;
   }
 
+  @Override
+  public Resource getConfiguredMinCapacity() {
+    QueueConfigurationsProtoOrBuilder p = viaProto ? proto : builder;
+    if (this.configuredMinResource != null) {
+      return this.configuredMinResource;
+    }
+    if (!p.hasConfiguredMinCapacity()) {
+      return null;
+    }
+    this.configuredMinResource = convertFromProtoFormat(
+        p.getConfiguredMinCapacity());
+    return this.configuredMinResource;
+  }
+
+  @Override
+  public void setConfiguredMinCapacity(Resource configuredMinResource) {
+    maybeInitBuilder();
+    if (configuredMinResource == null) {
+      builder.clearConfiguredMinCapacity();
+    }
+    this.configuredMinResource = configuredMinResource;
+  }
+
+  @Override
+  public Resource getConfiguredMaxCapacity() {
+    QueueConfigurationsProtoOrBuilder p = viaProto ? proto : builder;
+    if (this.configuredMaxResource != null) {
+      return this.configuredMaxResource;
+    }
+    if (!p.hasConfiguredMaxCapacity()) {
+      return null;
+    }
+    this.configuredMaxResource = convertFromProtoFormat(
+        p.getConfiguredMaxCapacity());
+    return this.configuredMaxResource;
+  }
+
+  @Override
+  public void setConfiguredMaxCapacity(Resource configuredMaxResource) {
+    maybeInitBuilder();
+    if (configuredMaxResource == null) {
+      builder.clearConfiguredMaxCapacity();
+    }
+    this.configuredMaxResource = configuredMaxResource;
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java

@@ -570,8 +570,26 @@ public class DominantResourceCalculator extends ResourceCalculator {
 
   @Override
   public Resource normalizeDown(Resource r, Resource stepFactor) {
-    return Resources.createResource(
-        roundDown(r.getMemorySize(), stepFactor.getMemorySize()),
-        roundDown(r.getVirtualCores(), stepFactor.getVirtualCores()));
+    Resource ret = Resource.newInstance(r);
+    int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
+    for (int i = 0; i < maxLength; i++) {
+      ResourceInformation rResourceInformation = r.getResourceInformation(i);
+      ResourceInformation stepFactorResourceInformation = stepFactor
+          .getResourceInformation(i);
+      ResourceInformation tmp = ret.getResourceInformation(i);
+
+      long rValue = rResourceInformation.getValue();
+      long stepFactorValue = UnitsConversionUtil.convert(
+          stepFactorResourceInformation.getUnits(),
+          rResourceInformation.getUnits(),
+          stepFactorResourceInformation.getValue());
+
+      long value = rValue;
+      if (stepFactorValue != 0) {
+        value = roundDown(rValue, stepFactorValue);
+      }
+      tmp.setValue(value);
+    }
+    return ret;
   }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java

@@ -112,42 +112,4 @@ public class QueueResourceQuotas extends AbstractResourceUsage {
   public void setEffectiveMaxResource(String label, Resource res) {
     _set(label, ResourceType.EFF_MAX_RESOURCE, res);
   }
-
-  /*
-   * Effective Minimum Resource
-   */
-  public Resource getEffectiveMinResourceUp() {
-    return _get(NL, ResourceType.EFF_MIN_RESOURCE_UP);
-  }
-
-  public Resource getEffectiveMinResourceUp(String label) {
-    return _get(label, ResourceType.EFF_MIN_RESOURCE_UP);
-  }
-
-  public void setEffectiveMinResourceUp(String label, Resource res) {
-    _set(label, ResourceType.EFF_MIN_RESOURCE_UP, res);
-  }
-
-  public void setEffectiveMinResourceUp(Resource res) {
-    _set(NL, ResourceType.EFF_MIN_RESOURCE_UP, res);
-  }
-
-  /*
-   * Effective Maximum Resource
-   */
-  public Resource getEffectiveMaxResourceUp() {
-    return getEffectiveMaxResourceUp(NL);
-  }
-
-  public Resource getEffectiveMaxResourceUp(String label) {
-    return _get(label, ResourceType.EFF_MAX_RESOURCE_UP);
-  }
-
-  public void setEffectiveMaxResourceUp(Resource res) {
-    setEffectiveMaxResourceUp(NL, res);
-  }
-
-  public void setEffectiveMaxResourceUp(String label, Resource res) {
-    _set(label, ResourceType.EFF_MAX_RESOURCE_UP, res);
-  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java

@@ -398,6 +398,10 @@ public abstract class AbstractCSQueue implements CSQueue {
       Resource maxResource = conf.getMaximumResourceRequirement(label,
           queuePath, resourceTypes);
 
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("capacityConfigType is '" + capacityConfigType
+            + "' for queue '" + getQueueName());
+      }
       if (this.capacityConfigType.equals(CapacityConfigType.NONE)) {
         this.capacityConfigType = (!minResource.equals(Resources.none())
             && queueCapacities.getAbsoluteCapacity(label) == 0f)
@@ -480,12 +484,6 @@ public abstract class AbstractCSQueue implements CSQueue {
         .clone(getQueueResourceQuotas().getEffectiveMinResource(label));
   }
 
-  @Override
-  public Resource getEffectiveCapacityUp(String label) {
-    return Resources
-        .clone(getQueueResourceQuotas().getEffectiveMinResourceUp(label));
-  }
-
   @Override
   public Resource getEffectiveCapacityDown(String label, Resource factor) {
     return Resources.normalizeDown(resourceCalculator,
@@ -621,6 +619,14 @@ public abstract class AbstractCSQueue implements CSQueue {
       queueConfiguration.setMaxCapacity(maxCapacity);
       queueConfiguration.setAbsoluteMaxCapacity(absMaxCapacity);
       queueConfiguration.setMaxAMPercentage(maxAMPercentage);
+      queueConfiguration.setConfiguredMinCapacity(
+          queueResourceQuotas.getConfiguredMinResource(nodeLabel));
+      queueConfiguration.setConfiguredMaxCapacity(
+          queueResourceQuotas.getConfiguredMaxResource(nodeLabel));
+      queueConfiguration.setEffectiveMinCapacity(
+          queueResourceQuotas.getEffectiveMinResource(nodeLabel));
+      queueConfiguration.setEffectiveMaxCapacity(
+          queueResourceQuotas.getEffectiveMaxResource(nodeLabel));
       queueConfigurations.put(nodeLabel, queueConfiguration);
     }
     return queueConfigurations;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java

@@ -25,7 +25,6 @@ import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 
-import com.google.common.annotations.VisibleForTesting;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Stable;
 import org.apache.hadoop.security.AccessControlException;
@@ -52,6 +51,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaS
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSet;
 
+import com.google.common.annotations.VisibleForTesting;
+
 /**
  * <code>CSQueue</code> represents a node in the tree of 
  * hierarchical queues in the {@link CapacityScheduler}.
@@ -380,7 +381,6 @@ public interface CSQueue extends SchedulerQueue<CSQueue> {
    * @return effective queue capacity
    */
   Resource getEffectiveCapacity(String label);
-  Resource getEffectiveCapacityUp(String label);
   Resource getEffectiveCapacityDown(String label, Resource factor);
 
   /**

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java

@@ -671,7 +671,7 @@ public class LeafQueue extends AbstractCSQueue {
           1.0f / Math.max(getAbstractUsersManager().getNumActiveUsers(), 1));
       effectiveUserLimit = Math.min(effectiveUserLimit * userWeight, 1.0f);
 
-      Resource queuePartitionResource = getEffectiveCapacityUp(nodePartition);
+      Resource queuePartitionResource = getEffectiveCapacity(nodePartition);
 
       Resource userAMLimit = Resources.multiplyAndNormalizeUp(
           resourceCalculator, queuePartitionResource,
@@ -700,7 +700,7 @@ public class LeafQueue extends AbstractCSQueue {
        * non-labeled), * with per-partition am-resource-percent to get the max am
        * resource limit for this queue and partition.
        */
-      Resource queuePartitionResource = getEffectiveCapacityUp(nodePartition);
+      Resource queuePartitionResource = getEffectiveCapacity(nodePartition);
 
       Resource queueCurrentLimit = Resources.none();
       // For non-labeled partition, we need to consider the current queue

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java

@@ -914,9 +914,11 @@ public class ParentQueue extends AbstractCSQueue {
       queueResourceQuotas.setConfiguredMaxResource(label, resourceByLabel);
       queueResourceQuotas.setEffectiveMinResource(label, resourceByLabel);
       queueResourceQuotas.setEffectiveMaxResource(label, resourceByLabel);
+      queueCapacities.setAbsoluteCapacity(label, 1.0f);
     }
 
-    // Total configured min resources of direct children of queue
+    // Total configured min resources of direct children of this given parent
+    // queue.
     Resource configuredMinResources = Resource.newInstance(0L, 0);
     for (CSQueue childQueue : getChildQueues()) {
       Resources.addTo(configuredMinResources,
@@ -960,7 +962,7 @@ public class ParentQueue extends AbstractCSQueue {
         // present could also be taken from effective max resource of parent.
         Resource parentMaxRes = queueResourceQuotas
             .getConfiguredMaxResource(label);
-        if (parentMaxRes.equals(Resources.none())) {
+        if (parent != null && parentMaxRes.equals(Resources.none())) {
           parentMaxRes = parent.getQueueResourceQuotas()
               .getEffectiveMaxResource(label);
         }
@@ -976,6 +978,11 @@ public class ParentQueue extends AbstractCSQueue {
             parentMaxRes);
         childQueue.getQueueResourceQuotas().setEffectiveMaxResource(label,
             Resources.clone(effMaxResource));
+
+        // In cases where we still need to update some units based on
+        // percentage, we have to calculate percentage and update.
+        deriveCapacityFromAbsoluteConfigurations(label, clusterResource, rc,
+            childQueue);
       } else {
         childQueue.getQueueResourceQuotas().setEffectiveMinResource(label,
             Resources.multiply(resourceByLabel,
@@ -983,16 +990,6 @@ public class ParentQueue extends AbstractCSQueue {
         childQueue.getQueueResourceQuotas().setEffectiveMaxResource(label,
             Resources.multiply(resourceByLabel, childQueue.getQueueCapacities()
                 .getAbsoluteMaximumCapacity(label)));
-
-        childQueue.getQueueResourceQuotas().setEffectiveMinResourceUp(label,
-            Resources.multiplyAndNormalizeUp(rc, resourceByLabel,
-                childQueue.getQueueCapacities().getAbsoluteCapacity(label),
-                minimumAllocation));
-        childQueue.getQueueResourceQuotas().setEffectiveMaxResourceUp(label,
-            Resources.multiplyAndNormalizeUp(rc,
-                resourceByLabel, childQueue.getQueueCapacities()
-                    .getAbsoluteMaximumCapacity(label),
-                    minimumAllocation));
       }
 
       if (LOG.isDebugEnabled()) {
@@ -1006,6 +1003,41 @@ public class ParentQueue extends AbstractCSQueue {
     }
   }
 
+  private void deriveCapacityFromAbsoluteConfigurations(String label,
+      Resource clusterResource, ResourceCalculator rc, CSQueue childQueue) {
+
+    /*
+     * In case when queues are configured with absolute resources, it is better
+     * to update capacity/max-capacity etc w.r.t absolute resource as well. In
+     * case of computation, these values wont be used any more. However for
+     * metrics and UI, its better these values are pre-computed here itself.
+     */
+
+    // 1. Update capacity as a float based on parent's minResource
+    childQueue.getQueueCapacities().setCapacity(label,
+        rc.divide(clusterResource,
+            childQueue.getQueueResourceQuotas().getEffectiveMinResource(label),
+            getQueueResourceQuotas().getEffectiveMinResource(label)));
+
+    // 2. Update max-capacity as a float based on parent's maxResource
+    childQueue.getQueueCapacities().setMaximumCapacity(label,
+        rc.divide(clusterResource,
+            childQueue.getQueueResourceQuotas().getEffectiveMaxResource(label),
+            getQueueResourceQuotas().getEffectiveMaxResource(label)));
+
+    // 3. Update absolute capacity as a float based on parent's minResource and
+    // cluster resource.
+    childQueue.getQueueCapacities().setAbsoluteCapacity(label,
+        (float) childQueue.getQueueCapacities().getCapacity()
+            / getQueueCapacities().getAbsoluteCapacity(label));
+
+    // 4. Update absolute max-capacity as a float based on parent's maxResource
+    // and cluster resource.
+    childQueue.getQueueCapacities().setAbsoluteMaximumCapacity(label,
+        (float) childQueue.getQueueCapacities().getMaximumCapacity(label)
+            / getQueueCapacities().getAbsoluteMaximumCapacity(label));
+  }
+
   @Override
   public List<CSQueue> getChildQueues() {
     try {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java

@@ -686,7 +686,7 @@ public class UsersManager implements AbstractUsersManager {
      * * If we're running over capacity, then its (usedResources + required)
      * (which extra resources we are allocating)
      */
-    Resource queueCapacity = lQueue.getEffectiveCapacityUp(nodePartition);
+    Resource queueCapacity = lQueue.getEffectiveCapacity(nodePartition);
 
     /*
      * Assume we have required resource equals to minimumAllocation, this can

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java

@@ -57,6 +57,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerStat
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractUsersManager;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedContainerChangeRequest;
@@ -908,6 +909,7 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
       StringBuilder diagnosticMessage) {
     LeafQueue queue = getCSLeafQueue();
     QueueCapacities queueCapacities = queue.getQueueCapacities();
+    QueueResourceQuotas queueResourceQuotas = queue.getQueueResourceQuotas();
     diagnosticMessage.append(" Details : AM Partition = ");
     diagnosticMessage.append(appAMNodePartitionName.isEmpty()
         ? NodeLabel.DEFAULT_NODE_LABEL_PARTITION : appAMNodePartitionName);
@@ -929,6 +931,18 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
         queueCapacities.getAbsoluteMaximumCapacity(appAMNodePartitionName)
             * 100);
     diagnosticMessage.append(" % ; ");
+    diagnosticMessage.append("Queue's capacity (absolute resource) = ");
+    diagnosticMessage.append(
+        queueResourceQuotas.getEffectiveMinResource(appAMNodePartitionName));
+    diagnosticMessage.append(" ; ");
+    diagnosticMessage.append("Queue's used capacity (absolute resource) = ");
+    diagnosticMessage
+        .append(queue.getQueueResourceUsage().getUsed(appAMNodePartitionName));
+    diagnosticMessage.append(" ; ");
+    diagnosticMessage.append("Queue's max capacity (absolute resource) = ");
+    diagnosticMessage.append(
+        queueResourceQuotas.getEffectiveMaxResource(appAMNodePartitionName));
+    diagnosticMessage.append(" ; ");
   }
 
   /**
@@ -992,13 +1006,10 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
       ResourceCalculator calc =
           rmContext.getScheduler().getResourceCalculator();
       if (!calc.isInvalidDivisor(totalPartitionRes)) {
-        float queueAbsMaxCapPerPartition =
-            ((AbstractCSQueue) getQueue()).getQueueCapacities()
-                .getAbsoluteCapacity(getAppAMNodePartitionName());
+        Resource effCap = ((AbstractCSQueue) getQueue())
+            .getEffectiveCapacity(getAppAMNodePartitionName());
         float queueUsagePerc = calc.divide(totalPartitionRes,
-            report.getUsedResources(),
-            Resources.multiply(totalPartitionRes, queueAbsMaxCapPerPartition))
-            * 100;
+            report.getUsedResources(), effCap) * 100;
         report.setQueueUsagePercentage(queueUsagePerc);
       }
       return report;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java

@@ -155,21 +155,36 @@ class CapacitySchedulerPage extends RmView {
           ? new ResourceInfo(Resources.none())
           : resourceUsages.getAmUsed();
       ri.
-          __("Used Capacity:", percent(capacities.getUsedCapacity() / 100)).
-          __("Configured Capacity:", percent(capacities.getCapacity() / 100)).
-          __("Configured Max Capacity:", percent(capacities.getMaxCapacity() / 100)).
-          __("Absolute Used Capacity:", percent(capacities.getAbsoluteUsedCapacity() / 100)).
-          __("Absolute Configured Capacity:", percent(capacities.getAbsoluteCapacity() / 100)).
-          __("Absolute Configured Max Capacity:", percent(capacities.getAbsoluteMaxCapacity() / 100)).
-          __("Used Resources:", resourceUsages.getUsed().toString()).
-          __("Configured Max Application Master Limit:", StringUtils.format("%.1f",
-          capacities.getMaxAMLimitPercentage())).
-          __("Max Application Master Resources:",
-          resourceUsages.getAMLimit().toString()).
-          __("Used Application Master Resources:",
-          amUsed.toString()).
-          __("Max Application Master Resources Per User:",
-          userAMResourceLimit.toString());
+          __("Used Capacity:",
+              appendPercent(resourceUsages.getUsed().toString(),
+                  capacities.getUsedCapacity() / 100))
+          .__("Configured Capacity:",
+              capacities.getConfiguredMinResource().toString())
+          .__("Configured Max Capacity:",
+              capacities.getConfiguredMaxResource().getResource()
+                  .equals(Resources.none())
+                      ? "unlimited"
+                      : capacities.getConfiguredMaxResource().toString())
+          .__("Effective Capacity:",
+              appendPercent(capacities.getEffectiveMinResource().toString(),
+                  capacities.getCapacity() / 100))
+          .__("Effective Max Capacity:",
+              appendPercent(capacities.getEffectiveMaxResource().toString(),
+                  capacities.getMaxCapacity() / 100))
+          .__("Absolute Used Capacity:",
+              percent(capacities.getAbsoluteUsedCapacity() / 100))
+          .__("Absolute Configured Capacity:",
+              percent(capacities.getAbsoluteCapacity() / 100))
+          .__("Absolute Configured Max Capacity:",
+              percent(capacities.getAbsoluteMaxCapacity() / 100))
+          .__("Used Resources:", resourceUsages.getUsed().toString())
+          .__("Configured Max Application Master Limit:",
+              StringUtils.format("%.1f", capacities.getMaxAMLimitPercentage()))
+          .__("Max Application Master Resources:",
+              resourceUsages.getAMLimit().toString())
+          .__("Used Application Master Resources:", amUsed.toString())
+          .__("Max Application Master Resources Per User:",
+              userAMResourceLimit.toString());
     }
 
     private void renderCommonLeafQueueInfo(ResponseInfo ri) {
@@ -615,6 +630,10 @@ class CapacitySchedulerPage extends RmView {
     return QueuesBlock.class;
   }
 
+  static String appendPercent(String message, float f) {
+    return message + " (" + StringUtils.formatPercent(f, 1) + ")";
+  }
+
   static String percent(float f) {
     return StringUtils.formatPercent(f, 1);
   }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java

@@ -59,7 +59,8 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
       max = 1f;
     this.maxCapacity = max * 100;
 
-    capacities = new QueueCapacitiesInfo(parent.getQueueCapacities(), false);
+    capacities = new QueueCapacitiesInfo(parent.getQueueCapacities(),
+        parent.getQueueResourceQuotas(), false);
     queues = getQueues(parent);
     health = new CapacitySchedulerHealthInfo(cs);
   }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java

@@ -25,6 +25,7 @@ import javax.xml.bind.annotation.XmlRootElement;
 import javax.xml.bind.annotation.XmlTransient;
 
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
@@ -89,8 +90,9 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
   }
 
   @Override
-  protected void populateQueueCapacities(QueueCapacities qCapacities) {
-    capacities = new QueueCapacitiesInfo(qCapacities);
+  protected void populateQueueCapacities(QueueCapacities qCapacities,
+      QueueResourceQuotas qResQuotas) {
+    capacities = new QueueCapacitiesInfo(qCapacities, qResQuotas);
   }
 
   public int getNumActiveApplications() {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java

@@ -28,6 +28,7 @@ import javax.xml.bind.annotation.XmlSeeAlso;
 import javax.xml.bind.annotation.XmlTransient;
 
 import org.apache.hadoop.yarn.api.records.QueueState;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueue;
@@ -103,7 +104,8 @@ public class CapacitySchedulerQueueInfo {
       Collections.sort(nodeLabels);
     }
     QueueCapacities qCapacities = q.getQueueCapacities();
-    populateQueueCapacities(qCapacities);
+    QueueResourceQuotas qResQuotas = q.getQueueResourceQuotas();
+    populateQueueCapacities(qCapacities, qResQuotas);
 
     ResourceUsage queueResourceUsage = q.getQueueResourceUsage();
     populateQueueResourceUsage(queueResourceUsage);
@@ -118,8 +120,10 @@ public class CapacitySchedulerQueueInfo {
     resources = new ResourcesInfo(queueResourceUsage, false);
   }
 
-  protected void populateQueueCapacities(QueueCapacities qCapacities) {
-    capacities = new QueueCapacitiesInfo(qCapacities, false);
+  protected void populateQueueCapacities(QueueCapacities qCapacities,
+      QueueResourceQuotas qResQuotas) {
+    capacities = new QueueCapacitiesInfo(qCapacities, qResQuotas,
+        false);
   }
 
   public float getCapacity() {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java

@@ -21,6 +21,9 @@ import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
 import javax.xml.bind.annotation.XmlRootElement;
 
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.resource.Resources;
+
 /**
  * This class represents queue capacities for a given partition
  */
@@ -36,13 +39,19 @@ public class PartitionQueueCapacitiesInfo {
   private float absoluteUsedCapacity;
   private float absoluteMaxCapacity  = 100;
   private float maxAMLimitPercentage;
+  private ResourceInfo configuredMinResource;
+  private ResourceInfo configuredMaxResource;
+  private ResourceInfo effectiveMinResource;
+  private ResourceInfo effectiveMaxResource;
 
   public PartitionQueueCapacitiesInfo() {
   }
 
   public PartitionQueueCapacitiesInfo(String partitionName, float capacity,
       float usedCapacity, float maxCapacity, float absCapacity,
-      float absUsedCapacity, float absMaxCapacity, float maxAMLimitPercentage) {
+      float absUsedCapacity, float absMaxCapacity, float maxAMLimitPercentage,
+      Resource confMinRes, Resource confMaxRes, Resource effMinRes,
+      Resource effMaxRes) {
     super();
     this.partitionName = partitionName;
     this.capacity = capacity;
@@ -52,6 +61,10 @@ public class PartitionQueueCapacitiesInfo {
     this.absoluteUsedCapacity = absUsedCapacity;
     this.absoluteMaxCapacity = absMaxCapacity;
     this.maxAMLimitPercentage = maxAMLimitPercentage;
+    this.configuredMinResource = new ResourceInfo(confMinRes);
+    this.configuredMaxResource = new ResourceInfo(confMaxRes);
+    this.effectiveMinResource = new ResourceInfo(effMinRes);
+    this.effectiveMaxResource = new ResourceInfo(effMaxRes);
   }
 
   public float getCapacity() {
@@ -117,4 +130,23 @@ public class PartitionQueueCapacitiesInfo {
   public void setMaxAMLimitPercentage(float maxAMLimitPercentage) {
     this.maxAMLimitPercentage = maxAMLimitPercentage;
   }
+
+  public ResourceInfo getConfiguredMinResource() {
+    return configuredMinResource;
+  }
+
+  public ResourceInfo getConfiguredMaxResource() {
+    if (configuredMaxResource.getResource().equals(Resources.none())) {
+      return null;
+    }
+    return configuredMaxResource;
+  }
+
+  public ResourceInfo getEffectiveMinResource() {
+    return effectiveMinResource;
+  }
+
+  public ResourceInfo getEffectiveMaxResource() {
+    return effectiveMaxResource;
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java

@@ -24,6 +24,7 @@ import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
 import javax.xml.bind.annotation.XmlRootElement;
 
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
 
 /**
@@ -39,6 +40,7 @@ public class QueueCapacitiesInfo {
   }
 
   public QueueCapacitiesInfo(QueueCapacities capacities,
+      QueueResourceQuotas resourceQuotas,
       boolean considerAMUsage) {
     if (capacities == null) {
       return;
@@ -68,12 +70,17 @@ public class QueueCapacitiesInfo {
       queueCapacitiesByPartition.add(new PartitionQueueCapacitiesInfo(
           partitionName, capacity, usedCapacity, maxCapacity, absCapacity,
           absUsedCapacity, absMaxCapacity,
-          considerAMUsage ? maxAMLimitPercentage : 0f));
+          considerAMUsage ? maxAMLimitPercentage : 0f,
+          resourceQuotas.getConfiguredMinResource(partitionName),
+          resourceQuotas.getConfiguredMaxResource(partitionName),
+          resourceQuotas.getEffectiveMinResource(partitionName),
+          resourceQuotas.getEffectiveMaxResource(partitionName)));
     }
   }
 
-  public QueueCapacitiesInfo(QueueCapacities capacities) {
-    this(capacities, true);
+  public QueueCapacitiesInfo(QueueCapacities capacities,
+      QueueResourceQuotas resourceQuotas) {
+    this(capacities, resourceQuotas, true);
   }
 
   public void add(PartitionQueueCapacitiesInfo partitionQueueCapacitiesInfo) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java

@@ -61,11 +61,20 @@ public class TestCapacitySchedulerWithMultiResourceTypes {
     Map<String, ResourceInformation> riMap = new HashMap<>();
 
     // Initialize mandatory resources
-    riMap.put(ResourceInformation.MEMORY_URI, ResourceInformation.MEMORY_MB);
-    riMap.put(ResourceInformation.VCORES_URI, ResourceInformation.VCORES);
-    riMap.put(RESOURCE_1, ResourceInformation
-        .newInstance(RESOURCE_1, "", 0, ResourceTypes.COUNTABLE, 0,
-            Integer.MAX_VALUE));
+    ResourceInformation memory = ResourceInformation.newInstance(
+        ResourceInformation.MEMORY_MB.getName(),
+        ResourceInformation.MEMORY_MB.getUnits(),
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB);
+    ResourceInformation vcores = ResourceInformation.newInstance(
+        ResourceInformation.VCORES.getName(),
+        ResourceInformation.VCORES.getUnits(),
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
+    riMap.put(ResourceInformation.MEMORY_URI, memory);
+    riMap.put(ResourceInformation.VCORES_URI, vcores);
+    riMap.put(RESOURCE_1, ResourceInformation.newInstance(RESOURCE_1, "", 0,
+        ResourceTypes.COUNTABLE, 0, Integer.MAX_VALUE));
 
     ResourceUtils.initializeResourcesFromResourceInformationMap(riMap);
 
@@ -107,7 +116,7 @@ public class TestCapacitySchedulerWithMultiResourceTypes {
     RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default");
     MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
 
-    Assert.assertEquals(Resource.newInstance(1 * GB, 0),
+    Assert.assertEquals(Resource.newInstance(1 * GB, 1),
         leafQueue.getUsedResources());
 
     RMNode rmNode1 = rm.getRMContext().getRMNodes().get(nm1.getNodeId());
@@ -123,9 +132,9 @@ public class TestCapacitySchedulerWithMultiResourceTypes {
     // Do node heartbeats 1 time and check container allocated.
     cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
 
-    // Now used resource = <mem=1GB, vcore=0> + <mem=2GB,vcore=2,res_1=2>
+    // Now used resource = <mem=1GB, vcore=1> + <mem=2GB,vcore=2,res_1=2>
     Assert.assertEquals(
-        TestUtils.createResource(3 * GB, 2, ImmutableMap.of(RESOURCE_1, 2)),
+        TestUtils.createResource(3 * GB, 3, ImmutableMap.of(RESOURCE_1, 2)),
         leafQueue.getUsedResources());
 
     // Acquire container

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java

@@ -74,6 +74,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManage
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueStateManager;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
@@ -1355,7 +1356,7 @@ public class TestLeafQueue {
     // TODO, fix headroom in the future patch
     assertEquals(1*GB, app_0.getHeadroom().getMemorySize());
       // User limit = 2G, 2 in use
-    assertEquals(0*GB, app_1.getHeadroom().getMemorySize());
+    assertEquals(1*GB, app_1.getHeadroom().getMemorySize());
       // the application is not yet active
 
     // Again one to user_0 since he hasn't exceeded user limit yet
@@ -1366,8 +1367,8 @@ public class TestLeafQueue {
     assertEquals(3*GB, a.getUsedResources().getMemorySize());
     assertEquals(2*GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(1*GB, app_1.getCurrentConsumption().getMemorySize());
-    assertEquals(1*GB, app_0.getHeadroom().getMemorySize()); // 4G - 3G
-    assertEquals(1*GB, app_1.getHeadroom().getMemorySize()); // 4G - 3G
+    assertEquals(0*GB, app_0.getHeadroom().getMemorySize());
+    assertEquals(0*GB, app_1.getHeadroom().getMemorySize()); // 4G - 3G
     
     // Submit requests for app_1 and set max-cap
     a.setMaxCapacity(.1f);
@@ -3874,7 +3875,7 @@ public class TestLeafQueue {
 
     // Queue "test" consumes 100% of the cluster, so its capacity and absolute
     // capacity are both 1.0f.
-    Queue queue = createQueue("test", null, 1.0f, 1.0f);
+    Queue queue = createQueue("test", null, 1.0f, 1.0f, res);
     final String user = "user1";
     FiCaSchedulerApp app =
         new FiCaSchedulerApp(appAttId, user, queue,
@@ -3891,7 +3892,8 @@ public class TestLeafQueue {
 
     // Queue "test2" is a child of root and its capacity is 50% of root. As a
     // child of root, its absolute capaicty is also 50%.
-    queue = createQueue("test2", null, 0.5f, 0.5f);
+    queue = createQueue("test2", null, 0.5f, 0.5f,
+        Resources.divideAndCeil(dominantResourceCalculator, res, 2));
     app = new FiCaSchedulerApp(appAttId, user, queue,
         queue.getAbstractUsersManager(), rmContext);
     app.getAppAttemptResourceUsage().incUsed(requestedResource);
@@ -3903,7 +3905,8 @@ public class TestLeafQueue {
 
     // Queue "test2.1" is 50% of queue "test2", which is 50% of the cluster.
     // Therefore, "test2.1" capacity is 50% and absolute capacity is 25%.
-    AbstractCSQueue qChild = createQueue("test2.1", queue, 0.5f, 0.25f);
+    AbstractCSQueue qChild = createQueue("test2.1", queue, 0.5f, 0.25f,
+        Resources.divideAndCeil(dominantResourceCalculator, res, 4));
     app = new FiCaSchedulerApp(appAttId, user, qChild,
         qChild.getAbstractUsersManager(), rmContext);
     app.getAppAttemptResourceUsage().incUsed(requestedResource);
@@ -3922,7 +3925,7 @@ public class TestLeafQueue {
   }
 
   private AbstractCSQueue createQueue(String name, Queue parent, float capacity,
-      float absCap) {
+      float absCap, Resource res) {
     CSQueueMetrics metrics = CSQueueMetrics.forQueue(name, parent, false, cs.getConf());
     QueueInfo queueInfo = QueueInfo.newInstance(name, capacity, 1.0f, 0, null,
         null, QueueState.RUNNING, null, "", null, false);
@@ -3934,6 +3937,13 @@ public class TestLeafQueue {
     QueueCapacities qCaps = mock(QueueCapacities.class);
     when(qCaps.getAbsoluteCapacity(any())).thenReturn(absCap);
     when(queue.getQueueCapacities()).thenReturn(qCaps);
+    QueueResourceQuotas qQuotas = mock(QueueResourceQuotas.class);
+    when(qQuotas.getConfiguredMinResource(any())).thenReturn(res);
+    when(qQuotas.getConfiguredMaxResource(any())).thenReturn(res);
+    when(qQuotas.getEffectiveMinResource(any())).thenReturn(res);
+    when(qQuotas.getEffectiveMaxResource(any())).thenReturn(res);
+    when(queue.getQueueResourceQuotas()).thenReturn(qQuotas);
+    when(queue.getEffectiveCapacity(any())).thenReturn(res);
     return queue;
   }