YARN-7337. Expose per-node over-allocation info in Node Report Contributed by Haibo Chen.

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java

@@ -124,11 +124,17 @@ public class TestRMNMInfo {
       Assert.assertNotNull(n.get("NodeManagerVersion"));
       Assert.assertNotNull(n.get("NumContainers"));
       Assert.assertEquals(
-              n.get("NodeId") + ": Unexpected number of used containers",
-              0, n.get("NumContainers").asInt());
+          n.get("NodeId") + ": Unexpected number of guaranteed containers used",
+          0, n.get("NumContainers").asInt());
+      Assert.assertEquals(n.get("NodeId") +
+              ": Unexpected number of opportunistic containers used",
+          0, n.get("NumOpportunisticContainers").asInt());
       Assert.assertEquals(
-              n.get("NodeId") + ": Unexpected amount of used memory",
+              n.get("NodeId") + ": Unexpected amount of guaranteed memory used",
               0, n.get("UsedMemoryMB").asInt());
+      Assert.assertEquals(
+          n.get("NodeId") + ": Unexpected amount of used opportunistic memory",
+          0, n.get("UsedOpportunisticMemoryMB").asInt());
       Assert.assertNotNull(n.get("AvailableMemoryMB"));
     }
   }
@@ -161,6 +167,8 @@ public class TestRMNMInfo {
       Assert.assertNotNull(n.get("NodeManagerVersion"));
       Assert.assertNull(n.get("NumContainers"));
       Assert.assertNull(n.get("UsedMemoryMB"));
+      Assert.assertNull(n.get("NumOpportunisticContainers"));
+      Assert.assertNull(n.get("UsedOpportunisticMemoryMB"));
       Assert.assertNull(n.get("AvailableMemoryMB"));
     }
   }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java

@@ -36,9 +36,12 @@ import org.apache.hadoop.yarn.util.Records;
  *   <li>{@link NodeId} of the node.</li>
  *   <li>HTTP Tracking URL of the node.</li>
  *   <li>Rack name for the node.</li>
- *   <li>Used {@link Resource} on the node.</li>
+ *   <li>Used guaranteed {@link Resource} on the node.</li>
+ *   <li>Used opportunistic {@link Resource} on the node.</li>
  *   <li>Total available {@link Resource} of the node.</li>
- *   <li>Number of running containers on the node.</li>
+ *   <li>Number of total running containers on the node.</li>
+ *   <li>Number of running guaranteed containers on the node.</li>
+ *   <li>Number of running opportunistic containers on the node.</li>
  * </ul>
  *
  * @see ApplicationClientProtocol#getClusterNodes(org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest)
@@ -50,33 +53,37 @@ public abstract class NodeReport {
   @Private
   @Unstable
   public static NodeReport newInstance(NodeId nodeId, NodeState nodeState,
-      String httpAddress, String rackName, Resource used, Resource capability,
-      int numContainers, String healthReport, long lastHealthReportTime) {
-    return newInstance(nodeId, nodeState, httpAddress, rackName, used,
-        capability, numContainers, healthReport, lastHealthReportTime,
-        null, null, null);
+      String httpAddress, String rackName, Resource guaranteedUsed,
+      Resource capability, int numGuaranteedContainers, String healthReport,
+      long lastHealthReportTime) {
+    return newInstance(nodeId, nodeState, httpAddress, rackName, guaranteedUsed,
+        capability, numGuaranteedContainers, healthReport, lastHealthReportTime,
+        null, null, null, null, 0);
   }
 
   @Private
   @Unstable
   public static NodeReport newInstance(NodeId nodeId, NodeState nodeState,
-      String httpAddress, String rackName, Resource used, Resource capability,
-      int numContainers, String healthReport, long lastHealthReportTime,
-      Set<String> nodeLabels, Integer decommissioningTimeout,
-      NodeUpdateType nodeUpdateType) {
+      String httpAddress, String rackName, Resource guaranteedUsed,
+      Resource capability, int numGuaranteedContainers, String healthReport,
+      long lastHealthReportTime, Set<String> nodeLabels,
+      Integer decommissioningTimeout, NodeUpdateType nodeUpdateType,
+      Resource opportunisticUsed, int numOpportunisticContainers) {
     NodeReport nodeReport = Records.newRecord(NodeReport.class);
     nodeReport.setNodeId(nodeId);
     nodeReport.setNodeState(nodeState);
     nodeReport.setHttpAddress(httpAddress);
     nodeReport.setRackName(rackName);
-    nodeReport.setUsed(used);
     nodeReport.setCapability(capability);
-    nodeReport.setNumContainers(numContainers);
+    nodeReport.setGuaranteedResourceUsed(guaranteedUsed);
+    nodeReport.setNumGuaranteedContainers(numGuaranteedContainers);
     nodeReport.setHealthReport(healthReport);
     nodeReport.setLastHealthReportTime(lastHealthReportTime);
     nodeReport.setNodeLabels(nodeLabels);
     nodeReport.setDecommissioningTimeout(decommissioningTimeout);
     nodeReport.setNodeUpdateType(nodeUpdateType);
+    nodeReport.setOpportunisticResourceUsed(opportunisticUsed);
+    nodeReport.setNumOpportunisticContainers(numOpportunisticContainers);
     return nodeReport;
   }
 
@@ -129,17 +136,43 @@ public abstract class NodeReport {
   public abstract void setRackName(String rackName);
   
   /**
-   * Get <em>used</em> <code>Resource</code> on the node.
-   * @return <em>used</em> <code>Resource</code> on the node
+   * Get <em>guaranteed</em> <code>Resource</code> used on the node.
+   * @return <em>guaranteed</em> <code>Resource</code> used on the node
    */
   @Public
   @Stable
+  @Deprecated
   public abstract Resource getUsed();
   
   @Private
   @Unstable
+  @Deprecated
   public abstract void setUsed(Resource used);
-  
+
+  /**
+   * Get <em>guaranteed</em> <code>Resource</code> used on the node.
+   * @return <em>guaranteed</em> <code>Resource</code> used on the node
+   */
+  @Public
+  @Unstable
+  public abstract Resource getGuaranteedResourceUsed();
+
+  @Private
+  @Unstable
+  public abstract void setGuaranteedResourceUsed(Resource guaranteed);
+
+  /**
+   * Get <em>opportunistic</em> <code>Resource</code> used on the node.
+   * @return <em>opportunistic</em> <code>Resource</code> used on the node
+   */
+  @Public
+  @Unstable
+  public abstract Resource getOpportunisticResourceUsed();
+
+  @Private
+  @Unstable
+  public abstract void setOpportunisticResourceUsed(Resource opportunistic);
+
   /**
    * Get the <em>total</em> <code>Resource</code> on the node.
    * @return <em>total</em> <code>Resource</code> on the node
@@ -153,19 +186,40 @@ public abstract class NodeReport {
   public abstract void setCapability(Resource capability);
   
   /**
-   * Get the <em>number of allocated containers</em> on the node.
-   * @return <em>number of allocated containers</em> on the node
+   * Get the <em>number of guaranteed containers</em> allocated on the node.
+   * @return <em>number of guaranteed containers</em> allocated on the node
    */
   @Private
   @Unstable
-  public abstract int getNumContainers();
+  public abstract int getNumGuaranteedContainers();
   
   @Private
   @Unstable
-  public abstract void setNumContainers(int numContainers);
-  
+  public abstract void setNumGuaranteedContainers(int numContainers);
 
-  /** 
+  /**
+   * Get the <em>number of opportunistic containers</em> allocated on the node.
+   * @return <em>number of opportunistic containers</em> allocated on the node
+   */
+  @Private
+  @Unstable
+  public abstract int getNumOpportunisticContainers();
+
+  @Private
+  @Unstable
+  public abstract void setNumOpportunisticContainers(int numContainers);
+
+  /**
+   * Get the <em>number of containers</em> allocated on the node.
+   * @return <em>number of containers</em> allocated on the node
+   */
+  @Private
+  @Unstable
+  public int getNumTotalContainers() {
+    return getNumGuaranteedContainers() + getNumOpportunisticContainers();
+  }
+
+  /**
    * Get the <em>diagnostic health report</em> of the node.
    * @return <em>diagnostic health report</em> of the node
    */

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto

@@ -344,9 +344,9 @@ message NodeReportProto {
   optional NodeIdProto nodeId = 1;
   optional string httpAddress = 2;
   optional string rackName = 3;
-  optional ResourceProto used = 4;
+  optional ResourceProto guaranteedResourceUsed = 4;
   optional ResourceProto capability = 5;
-  optional int32 numContainers = 6;
+  optional int32 numGuaranteedContainers = 6;
   optional NodeStateProto node_state = 7;
   optional string health_report = 8;
   optional int64 last_health_report_time = 9;
@@ -356,6 +356,9 @@ message NodeReportProto {
   optional uint32 decommissioning_timeout = 13;
   optional NodeUpdateTypeProto node_update_type = 14;
   repeated NodeAttributeProto node_attributes = 15;
+  optional int32 numOpportunisticContainers = 16;
+  optional ResourceProto opportunisticResourceUsed = 17;
+  optional int32 numTotalContainers = 18;
 }
 
 message NodeIdToLabelsProto {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java

@@ -647,7 +647,7 @@ public class Client {
           + ", nodeId=" + node.getNodeId() 
           + ", nodeAddress=" + node.getHttpAddress()
           + ", nodeRackName=" + node.getRackName()
-          + ", nodeNumContainers=" + node.getNumContainers());
+          + ", nodeNumContainers=" + node.getNumGuaranteedContainers());
     }
 
     QueueInfo queueInfo = yarnClient.getQueueInfo(this.amQueue);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java

@@ -43,6 +43,7 @@ import org.apache.hadoop.util.ToolRunner;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.NodeReport;
 import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 
 @Private
@@ -184,7 +185,7 @@ public class NodeCLI extends YarnCLI {
     for (NodeReport nodeReport : nodesReport) {
       writer.printf(NODES_PATTERN, nodeReport.getNodeId(), nodeReport
           .getNodeState(), nodeReport.getHttpAddress(), nodeReport
-          .getNumContainers());
+          .getNumGuaranteedContainers());
     }
     writer.flush();
   }
@@ -209,13 +210,18 @@ public class NodeCLI extends YarnCLI {
     for (NodeReport nodeReport : nodesReport) {
       writer.printf(NODES_PATTERN, nodeReport.getNodeId(),
           nodeReport.getNodeState(), nodeReport.getHttpAddress(),
-          nodeReport.getNumContainers());
+          nodeReport.getNumGuaranteedContainers());
       writer.println("Detailed Node Information :");
       writer.print("\tConfigured Resources : ");
       writer.println(nodeReport.getCapability());
-      writer.print("\tAllocated Resources : ");
-      if (nodeReport.getUsed() != null) {
-        writer.print(nodeReport.getUsed());
+      writer.print("\tAllocated Guaranteed Resources : ");
+      if (nodeReport.getGuaranteedResourceUsed() != null) {
+        writer.print(nodeReport.getGuaranteedResourceUsed());
+      }
+      writer.println();
+      writer.print("\tAllocated Opportunistic Resources : ");
+      if (nodeReport.getOpportunisticResourceUsed() != null) {
+        writer.print(nodeReport.getOpportunisticResourceUsed());
       }
       writer.println();
 
@@ -286,16 +292,32 @@ public class NodeCLI extends YarnCLI {
       nodeReportStr.print("\tHealth-Report : ");
       nodeReportStr
           .println(nodeReport.getHealthReport());
-      nodeReportStr.print("\tContainers : ");
-      nodeReportStr.println(nodeReport.getNumContainers());
-      nodeReportStr.print("\tMemory-Used : ");
-      nodeReportStr.println((nodeReport.getUsed() == null) ? "0MB"
-          : (nodeReport.getUsed().getMemorySize() + "MB"));
+      nodeReportStr.print("\tGuaranteed Containers : ");
+      nodeReportStr.println(nodeReport.getNumGuaranteedContainers());
+      nodeReportStr.print("\tOpportunistic Containers : ");
+      nodeReportStr.println(nodeReport.getNumOpportunisticContainers());
+      Resource guaranteedResourceUsed =
+          nodeReport.getGuaranteedResourceUsed();
+      Resource opportunisticResourceUsed =
+          nodeReport.getOpportunisticResourceUsed();
+      nodeReportStr.print("\tGuaranteed-Memory-Used : ");
+      nodeReportStr.println(
+          (guaranteedResourceUsed == null) ? "0MB" :
+              (guaranteedResourceUsed.getMemorySize() + "MB"));
+      nodeReportStr.print("\tOpportunistic-Memory-Used : ");
+      nodeReportStr.println(
+          (opportunisticResourceUsed == null) ? "0MB" :
+              (opportunisticResourceUsed.getMemorySize() + "MB"));
       nodeReportStr.print("\tMemory-Capacity : ");
       nodeReportStr.println(nodeReport.getCapability().getMemorySize() + "MB");
-      nodeReportStr.print("\tCPU-Used : ");
-      nodeReportStr.println((nodeReport.getUsed() == null) ? "0 vcores"
-          : (nodeReport.getUsed().getVirtualCores() + " vcores"));
+      nodeReportStr.print("\tGuaranteed-CPU-Used : ");
+      nodeReportStr.println(
+          (guaranteedResourceUsed == null) ? "0 vcores" :
+              (guaranteedResourceUsed.getVirtualCores() + " vcores"));
+      nodeReportStr.print("\tOpportunistic-CPU-Used : ");
+      nodeReportStr.println(
+          (opportunisticResourceUsed == null) ? "0 vcores" :
+              (opportunisticResourceUsed.getVirtualCores() + " vcores"));
       nodeReportStr.print("\tCPU-Capacity : ");
       nodeReportStr.println(nodeReport.getCapability().getVirtualCores() + " vcores");
       nodeReportStr.print("\tNode-Labels : ");

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java

@@ -1355,7 +1355,8 @@ public class TestYarnCLI {
     pw.println("                           0");
     pw.println("Detailed Node Information :");
     pw.println("\tConfigured Resources : <memory:0, vCores:0>");
-    pw.println("\tAllocated Resources : <memory:0, vCores:0>");
+    pw.println("\tAllocated Guaranteed Resources : <memory:0, vCores:0>");
+    pw.println("\tAllocated Opportunistic Resources : <memory:0, vCores:0>");
     pw.println("\tResource Utilization by Node : PMem:2048 MB, VMem:4096 MB, VCores:8.0");
     pw.println("\tResource Utilization by Containers : PMem:1024 MB, VMem:2048 MB, VCores:4.0");
     pw.println("\tNode-Labels : ");
@@ -1363,7 +1364,8 @@ public class TestYarnCLI {
     pw.println("                           0");
     pw.println("Detailed Node Information :");
     pw.println("\tConfigured Resources : <memory:0, vCores:0>");
-    pw.println("\tAllocated Resources : <memory:0, vCores:0>");
+    pw.println("\tAllocated Guaranteed Resources : <memory:0, vCores:0>");
+    pw.println("\tAllocated Opportunistic Resources : <memory:0, vCores:0>");
     pw.println("\tResource Utilization by Node : PMem:2048 MB, VMem:4096 MB, VCores:8.0");
     pw.println("\tResource Utilization by Containers : PMem:1024 MB, VMem:2048 MB, VCores:4.0");
     pw.println("\tNode-Labels : ");
@@ -1564,10 +1566,13 @@ public class TestYarnCLI {
     pw.println("\tLast-Health-Update : "
       + DateFormatUtils.format(new Date(0), "E dd/MMM/yy hh:mm:ss:SSzz"));
     pw.println("\tHealth-Report : ");
-    pw.println("\tContainers : 0");
-    pw.println("\tMemory-Used : 0MB");
+    pw.println("\tGuaranteed Containers : 0");
+    pw.println("\tOpportunistic Containers : 0");
+    pw.println("\tGuaranteed-Memory-Used : 0MB");
+    pw.println("\tOpportunistic-Memory-Used : 0MB");
     pw.println("\tMemory-Capacity : 0MB");
-    pw.println("\tCPU-Used : 0 vcores");
+    pw.println("\tGuaranteed-CPU-Used : 0 vcores");
+    pw.println("\tOpportunistic-CPU-Used : 0 vcores");
     pw.println("\tCPU-Capacity : 0 vcores");
     pw.println("\tNode-Labels : a,b,c,x,y,z");
     pw.println("\tNode Attributes : rm.yarn.io/GPU(STRING)=ARM");
@@ -1602,10 +1607,13 @@ public class TestYarnCLI {
     pw.println("\tLast-Health-Update : "
       + DateFormatUtils.format(new Date(0), "E dd/MMM/yy hh:mm:ss:SSzz"));
     pw.println("\tHealth-Report : ");
-    pw.println("\tContainers : 0");
-    pw.println("\tMemory-Used : 0MB");
+    pw.println("\tGuaranteed Containers : 0");
+    pw.println("\tOpportunistic Containers : 0");
+    pw.println("\tGuaranteed-Memory-Used : 0MB");
+    pw.println("\tOpportunistic-Memory-Used : 0MB");
     pw.println("\tMemory-Capacity : 0MB");
-    pw.println("\tCPU-Used : 0 vcores");
+    pw.println("\tGuaranteed-CPU-Used : 0 vcores");
+    pw.println("\tOpportunistic-CPU-Used : 0 vcores");
     pw.println("\tCPU-Capacity : 0 vcores");
     pw.println("\tNode-Labels : ");
     pw.println("\tNode Attributes : ");
@@ -1639,10 +1647,13 @@ public class TestYarnCLI {
     pw.println("\tLast-Health-Update : "
       + DateFormatUtils.format(new Date(0), "E dd/MMM/yy hh:mm:ss:SSzz"));
     pw.println("\tHealth-Report : ");
-    pw.println("\tContainers : 0");
-    pw.println("\tMemory-Used : 0MB");
+    pw.println("\tGuaranteed Containers : 0");
+    pw.println("\tOpportunistic Containers : 0");
+    pw.println("\tGuaranteed-Memory-Used : 0MB");
+    pw.println("\tOpportunistic-Memory-Used : 0MB");
     pw.println("\tMemory-Capacity : 0MB");
-    pw.println("\tCPU-Used : 0 vcores");
+    pw.println("\tGuaranteed-CPU-Used : 0 vcores");
+    pw.println("\tOpportunistic-CPU-Used : 0 vcores");
     pw.println("\tCPU-Capacity : 0 vcores");
     pw.println("\tNode-Labels : a,b,c,x,y,z");
     pw.println("\tNode Attributes : ");
@@ -2078,10 +2089,12 @@ public class TestYarnCLI {
         // ordered
         nodeLabels = ImmutableSet.of("c", "b", "a", "x", "z", "y");
       }
-      NodeReport nodeReport = NodeReport.newInstance(NodeId
-        .newInstance("host" + i, 0), state, "host" + 1 + ":8888",
-          "rack1", Records.newRecord(Resource.class), Records
-              .newRecord(Resource.class), 0, "", 0, nodeLabels, null, null);
+      NodeReport nodeReport = NodeReport.newInstance(
+          NodeId.newInstance("host" + i, 0), state, "host" + 1 + ":8888",
+          "rack1", Records.newRecord(Resource.class),
+          Records.newRecord(Resource.class), 0, "",
+          0, nodeLabels, null, null,
+          Records.newRecord(Resource.class), 0);
       if (!emptyResourceUtilization) {
         ResourceUtilization containersUtilization = ResourceUtilization
             .newInstance(1024, 2048, 4);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java

@@ -49,7 +49,8 @@ public class NodeReportPBImpl extends NodeReport {
   private NodeReportProto.Builder builder = null;
   private boolean viaProto = false;
   private NodeId nodeId;
-  private Resource used;
+  private Resource guaranteedResourceUsed;
+  private Resource opportunisticResourceUsed;
   private Resource capability;
   private ResourceUtilization containersUtilization = null;
   private ResourceUtilization nodeUtilization = null;
@@ -114,9 +115,17 @@ public class NodeReportPBImpl extends NodeReport {
   }
 
   @Override
-  public int getNumContainers() {
+  public int getNumGuaranteedContainers() {
     NodeReportProtoOrBuilder p = viaProto ? proto : builder;
-    return (p.hasNumContainers()) ? p.getNumContainers() : 0;
+    return (p.hasNumGuaranteedContainers()) ?
+        p.getNumGuaranteedContainers() : 0;
+  }
+
+  @Override
+  public int getNumOpportunisticContainers() {
+    NodeReportProtoOrBuilder p = viaProto ? proto : builder;
+    return (p.hasNumOpportunisticContainers()) ?
+        p.getNumOpportunisticContainers() : 0;
   }
 
   @Override
@@ -125,18 +134,25 @@ public class NodeReportPBImpl extends NodeReport {
     return (p.hasRackName()) ? p.getRackName() : null;
   }
 
+  @Deprecated
   @Override
   public Resource getUsed() {
-    if (this.used != null) {
-      return this.used;
+    return getGuaranteedResourceUsed();
+  }
+
+  @Override
+  public Resource getOpportunisticResourceUsed() {
+    if (this.opportunisticResourceUsed != null) {
+      return this.opportunisticResourceUsed;
     }
 
     NodeReportProtoOrBuilder p = viaProto ? proto : builder;
-    if (!p.hasUsed()) {
+    if (!p.hasOpportunisticResourceUsed()) {
       return null;
     }
-    this.used = convertFromProtoFormat(p.getUsed());
-    return this.used;
+    this.opportunisticResourceUsed =
+        convertFromProtoFormat(p.getOpportunisticResourceUsed());
+    return this.opportunisticResourceUsed;
   }
 
   @Override
@@ -201,13 +217,24 @@ public class NodeReportPBImpl extends NodeReport {
   }
 
   @Override
-  public void setNumContainers(int numContainers) {
+  public void setNumGuaranteedContainers(int numContainers) {
+    maybeInitBuilder();
+    if (numContainers == 0) {
+      builder.clearNumGuaranteedContainers();
+      return;
+    }
+    builder.setNumGuaranteedContainers(numContainers);
+  }
+
+  @Override
+  public void setNumOpportunisticContainers(int numContainers) {
     maybeInitBuilder();
     if (numContainers == 0) {
-      builder.clearNumContainers();
+      builder.clearNumOpportunisticContainers();
       return;
     }
-    builder.setNumContainers(numContainers);
+    builder.setNumOpportunisticContainers(numContainers);
+
   }
 
   @Override
@@ -220,13 +247,43 @@ public class NodeReportPBImpl extends NodeReport {
     builder.setRackName(rackName);
   }
 
+  @Deprecated
   @Override
   public void setUsed(Resource used) {
+    setGuaranteedResourceUsed(used);
+  }
+
+  @Override
+  public Resource getGuaranteedResourceUsed() {
+    if (this.guaranteedResourceUsed != null) {
+      return this.guaranteedResourceUsed;
+    }
+
+    NodeReportProtoOrBuilder p = viaProto ? proto : builder;
+    if (!p.hasGuaranteedResourceUsed()) {
+      return null;
+    }
+    this.guaranteedResourceUsed =
+        convertFromProtoFormat(p.getGuaranteedResourceUsed());
+    return this.guaranteedResourceUsed;
+  }
+
+  @Override
+  public void setGuaranteedResourceUsed(Resource guaranteed) {
+    maybeInitBuilder();
+    if (guaranteedResourceUsed == null) {
+      builder.clearGuaranteedResourceUsed();
+    }
+    this.guaranteedResourceUsed = guaranteed;
+  }
+
+  @Override
+  public void setOpportunisticResourceUsed(Resource opportunisticUsed) {
     maybeInitBuilder();
-    if (used == null) {
-      builder.clearUsed();
+    if (opportunisticUsed == null) {
+      builder.clearOpportunisticResourceUsed();
     }
-    this.used = used;
+    this.opportunisticResourceUsed = opportunisticUsed;
   }
 
   public NodeReportProto getProto() {
@@ -263,8 +320,13 @@ public class NodeReportPBImpl extends NodeReport {
             builder.getNodeId())) {
       builder.setNodeId(convertToProtoFormat(this.nodeId));
     }
-    if (this.used != null) {
-      builder.setUsed(convertToProtoFormat(this.used));
+    if (this.guaranteedResourceUsed != null) {
+      builder.setGuaranteedResourceUsed(
+          convertToProtoFormat(this.guaranteedResourceUsed));
+    }
+    if (this.opportunisticResourceUsed != null) {
+      builder.setOpportunisticResourceUsed(
+          convertToProtoFormat(this.opportunisticResourceUsed));
     }
     if (this.capability != null) {
       builder.setCapability(convertToProtoFormat(this.capability));

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java

@@ -186,26 +186,23 @@ public class BuilderUtils {
   }
 
   public static NodeReport newNodeReport(NodeId nodeId, NodeState nodeState,
-      String httpAddress, String rackName, Resource used, Resource capability,
-      int numContainers, String healthReport, long lastHealthReportTime) {
-    return newNodeReport(nodeId, nodeState, httpAddress, rackName, used,
-        capability, numContainers, healthReport, lastHealthReportTime,
-        null, null, null);
+      String httpAddress, String rackName, Resource guaranteedResourceUsed,
+      Resource capability, int numGuaranteedContainers,
+      Resource opportunisticResourceUsed, int numOpportunisticContainer,
+      String healthReport, long lastHealthReportTime, Set<String> nodeLabels,
+      Integer decommissioningTimeout, NodeUpdateType nodeUpdateType) {
+    return newNodeReport(nodeId, nodeState, httpAddress, rackName,
+        guaranteedResourceUsed, capability, numGuaranteedContainers,
+        opportunisticResourceUsed, numOpportunisticContainer, healthReport,
+        lastHealthReportTime, nodeLabels, null, null,
+        decommissioningTimeout, nodeUpdateType, null);
   }
 
   public static NodeReport newNodeReport(NodeId nodeId, NodeState nodeState,
-      String httpAddress, String rackName, Resource used, Resource capability,
-      int numContainers, String healthReport, long lastHealthReportTime,
-      Set<String> nodeLabels, Integer decommissioningTimeout,
-      NodeUpdateType nodeUpdateType) {
-    return newNodeReport(nodeId, nodeState, httpAddress, rackName, used,
-        capability, numContainers, healthReport, lastHealthReportTime,
-        nodeLabels, null, null, decommissioningTimeout, nodeUpdateType, null);
-  }
-
-  public static NodeReport newNodeReport(NodeId nodeId, NodeState nodeState,
-      String httpAddress, String rackName, Resource used, Resource capability,
-      int numContainers, String healthReport, long lastHealthReportTime,
+      String httpAddress, String rackName, Resource guaranteedResourceUsed,
+      Resource capability, int numGuaranteedContainers,
+      Resource opportunisticResourceUsed, int numOpportunisticContainers,
+      String healthReport, long lastHealthReportTime,
       Set<String> nodeLabels, ResourceUtilization containersUtilization,
       ResourceUtilization nodeUtilization, Integer decommissioningTimeout,
       NodeUpdateType nodeUpdateType, Set<NodeAttribute> attrs) {
@@ -214,9 +211,11 @@ public class BuilderUtils {
     nodeReport.setNodeState(nodeState);
     nodeReport.setHttpAddress(httpAddress);
     nodeReport.setRackName(rackName);
-    nodeReport.setUsed(used);
+    nodeReport.setGuaranteedResourceUsed(guaranteedResourceUsed);
     nodeReport.setCapability(capability);
-    nodeReport.setNumContainers(numContainers);
+    nodeReport.setNumGuaranteedContainers(numGuaranteedContainers);
+    nodeReport.setOpportunisticResourceUsed(opportunisticResourceUsed);
+    nodeReport.setNumOpportunisticContainers(numOpportunisticContainers);
     nodeReport.setHealthReport(healthReport);
     nodeReport.setLastHealthReportTime(lastHealthReportTime);
     nodeReport.setNodeLabels(nodeLabels);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java

@@ -1045,18 +1045,27 @@ public class ClientRMService extends AbstractService implements
   private NodeReport createNodeReports(RMNode rmNode) {
     SchedulerNodeReport schedulerNodeReport = 
         scheduler.getNodeReport(rmNode.getNodeID());
-    Resource used = BuilderUtils.newResource(0, 0);
-    int numContainers = 0;
+    Resource guaranteedResourceUsed = BuilderUtils.newResource(0, 0);
+    int numGuaranteedContainers = 0;
+    Resource opportunisticResourceUsed = BuilderUtils.newResource(0, 0);
+    int numOpportunisticContainers = 0;
     if (schedulerNodeReport != null) {
-      used = schedulerNodeReport.getUsedResource();
-      numContainers = schedulerNodeReport.getNumContainers();
-    }
+      guaranteedResourceUsed = schedulerNodeReport.getGuaranteedResourceUsed();
+      opportunisticResourceUsed =
+          schedulerNodeReport.getOpportunisticResourceUsed();
+      numGuaranteedContainers =
+          schedulerNodeReport.getNumGuaranteedContainers();
+      numOpportunisticContainers =
+          schedulerNodeReport.getNumOpportunisticContainers();
+    } 
 
     Set<NodeAttribute> attrs = rmNode.getAllNodeAttributes();
     NodeReport report =
         BuilderUtils.newNodeReport(rmNode.getNodeID(), rmNode.getState(),
-            rmNode.getHttpAddress(), rmNode.getRackName(), used,
-            rmNode.getTotalCapability(), numContainers,
+            rmNode.getHttpAddress(), rmNode.getRackName(),
+            guaranteedResourceUsed,
+            rmNode.getTotalCapability(), numGuaranteedContainers,
+            opportunisticResourceUsed, numOpportunisticContainers,
             rmNode.getHealthReport(), rmNode.getLastHealthReportTime(),
             rmNode.getNodeLabels(), rmNode.getAggregatedContainersUtilization(),
             rmNode.getNodeUtilization(), rmNode.getDecommissioningTimeout(),

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java

@@ -365,17 +365,27 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor {
         RMNode rmNode = rmNodeEntry.getKey();
         SchedulerNodeReport schedulerNodeReport =
             getScheduler().getNodeReport(rmNode.getNodeID());
-        Resource used = BuilderUtils.newResource(0, 0);
-        int numContainers = 0;
+        Resource guaranteedResourceUsed = BuilderUtils.newResource(0, 0);
+        int numGuaranteedContainers = 0;
+        Resource opportunisticResourceUsed = BuilderUtils.newResource(0, 0);
+        int numOpportunisticContainers = 0;
         if (schedulerNodeReport != null) {
-          used = schedulerNodeReport.getUsedResource();
-          numContainers = schedulerNodeReport.getNumContainers();
+          opportunisticResourceUsed =
+              schedulerNodeReport.getOpportunisticResourceUsed();
+          guaranteedResourceUsed =
+              schedulerNodeReport.getGuaranteedResourceUsed();
+          numGuaranteedContainers =
+              schedulerNodeReport.getNumGuaranteedContainers();
+          numOpportunisticContainers =
+              schedulerNodeReport.getNumOpportunisticContainers();
         }
         NodeId nodeId = rmNode.getNodeID();
         NodeReport report =
             BuilderUtils.newNodeReport(nodeId, rmNode.getState(),
-                rmNode.getHttpAddress(), rmNode.getRackName(), used,
-                rmNode.getTotalCapability(), numContainers,
+                rmNode.getHttpAddress(), rmNode.getRackName(),
+                guaranteedResourceUsed, rmNode.getTotalCapability(),
+                numGuaranteedContainers, opportunisticResourceUsed,
+                numOpportunisticContainers,
                 rmNode.getHealthReport(), rmNode.getLastHealthReportTime(),
                 rmNode.getNodeLabels(), rmNode.getDecommissioningTimeout(),
                 rmNodeEntry.getValue());

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java

@@ -79,27 +79,32 @@ public class RMNMInfo implements RMNMInfoBeans {
     List<InfoMap> nodesInfo = new ArrayList<InfoMap>();
 
     for (final RMNode ni : nodes) {
-        SchedulerNodeReport report = scheduler.getNodeReport(ni.getNodeID());
-        InfoMap info = new InfoMap();
-        info.put("HostName", ni.getHostName());
-        info.put("Rack", ni.getRackName());
-        info.put("State", ni.getState().toString());
-        info.put("NodeId", ni.getNodeID());
-        info.put("NodeHTTPAddress", ni.getHttpAddress());
-        info.put("LastHealthUpdate",
-                        ni.getLastHealthReportTime());
-        info.put("HealthReport",
-                        ni.getHealthReport());
-        info.put("NodeManagerVersion",
-                ni.getNodeManagerVersion());
-        if(report != null) {
-          info.put("NumContainers", report.getNumContainers());
-          info.put("UsedMemoryMB", report.getUsedResource().getMemorySize());
-          info.put("AvailableMemoryMB",
-              report.getAvailableResource().getMemorySize());
-        }
+      SchedulerNodeReport report = scheduler.getNodeReport(ni.getNodeID());
+      InfoMap info = new InfoMap();
+      info.put("HostName", ni.getHostName());
+      info.put("Rack", ni.getRackName());
+      info.put("State", ni.getState().toString());
+      info.put("NodeId", ni.getNodeID());
+      info.put("NodeHTTPAddress", ni.getHttpAddress());
+      info.put("LastHealthUpdate",
+                      ni.getLastHealthReportTime());
+      info.put("HealthReport",
+                      ni.getHealthReport());
+      info.put("NodeManagerVersion",
+              ni.getNodeManagerVersion());
+      if(report != null) {
+        info.put("NumContainers", report.getNumGuaranteedContainers());
+        info.put("NumOpportunisticContainers",
+            report.getNumOpportunisticContainers());
+        info.put("UsedMemoryMB",
+            report.getGuaranteedResourceUsed().getMemorySize());
+        info.put("UsedOpportunisticMemoryMB",
+            report.getOpportunisticResourceUsed().getMemorySize());
+        info.put("AvailableMemoryMB",
+            report.getAvailableGuaranteedResource().getMemorySize());
+      }
 
-        nodesInfo.add(info);
+      nodesInfo.add(info);
     }
 
     return JSON.toString(nodesInfo);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java

@@ -142,7 +142,7 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
   /** Physical resources in the node. */
   private volatile Resource physicalResource;
 
-  /* Container Queue Information for the node.. Used by Distributed Scheduler */
+  /* Container Queue Information for the node..*/
   private OpportunisticContainersStatus opportunisticContainersStatus;
 
   private final ContainerAllocationExpirer containerAllocationExpirer;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java

@@ -28,34 +28,54 @@ import org.apache.hadoop.yarn.api.records.Resource;
 @Private
 @Stable
 public class SchedulerNodeReport {
-  private final Resource used;
-  private final Resource avail;
-  private final int num;
+  private final Resource guaranteedResourceUsage;
+  private final Resource opportunisticResourceUsage;
+  private final Resource guaranteedResourceAvail;
+  private final int numOfGuaranteedContainers;
+  private final int numOfOpportunisticContainers;
 
   public SchedulerNodeReport(SchedulerNode node) {
-    this.used = node.getAllocatedResource();
-    this.avail = node.getUnallocatedResource();
-    this.num = node.getNumGuaranteedContainers();
+    this.guaranteedResourceUsage = node.getAllocatedResource();
+    this.opportunisticResourceUsage = node.getOpportunisticResourceAllocated();
+    this.guaranteedResourceAvail = node.getUnallocatedResource();
+    this.numOfGuaranteedContainers = node.getNumGuaranteedContainers();
+    this.numOfOpportunisticContainers = node.getNumOpportunisticContainers();
   }
   
   /**
-   * @return the amount of resources currently used by the node.
+   * @return the amount of guaranteed resources currently used by the node.
    */
-  public Resource getUsedResource() {
-    return used;
+  public Resource getGuaranteedResourceUsed() {
+    return guaranteedResourceUsage;
   }
 
   /**
-   * @return the amount of resources currently available on the node
+   * @return the amount of opportunistic resources currently used by the node.
    */
-  public Resource getAvailableResource() {
-    return avail;
+  public Resource getOpportunisticResourceUsed() {
+    return opportunisticResourceUsage;
   }
 
   /**
-   * @return the number of containers currently running on this node.
+   * @return the amount of guaranteed resources currently available on the node
    */
-  public int getNumContainers() {
-    return num;
+  public Resource getAvailableGuaranteedResource() {
+    return guaranteedResourceAvail;
+  }
+
+  /**
+   * @return the number of guaranteed containers currently running on
+   *         this node.
+   */
+  public int getNumGuaranteedContainers() {
+    return numOfGuaranteedContainers;
+  }
+
+  /**
+   * @return the number of opportunistic containers currently running on
+   *         this node.
+   */
+  public int getNumOpportunisticContainers() {
+    return numOfOpportunisticContainers;
   }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FifoSchedulerInfo.java

@@ -79,10 +79,13 @@ public class FifoSchedulerInfo extends SchedulerInfo {
 
     for (RMNode ni : rmContext.getRMNodes().values()) {
       SchedulerNodeReport report = fs.getNodeReport(ni.getNodeID());
-      this.usedNodeCapacity += report.getUsedResource().getMemorySize();
-      this.availNodeCapacity += report.getAvailableResource().getMemorySize();
+      this.usedNodeCapacity +=
+          report.getGuaranteedResourceUsed().getMemorySize();
+      this.availNodeCapacity +=
+          report.getAvailableGuaranteedResource().getMemorySize();
       this.totalNodeCapacity += ni.getTotalCapability().getMemorySize();
-      this.numContainers += fs.getNodeReport(ni.getNodeID()).getNumContainers();
+      this.numContainers +=
+          fs.getNodeReport(ni.getNodeID()).getNumGuaranteedContainers();
     }
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java

@@ -75,14 +75,17 @@ public class NodeInfo {
     this.usedMemoryMB = 0;
     this.availMemoryMB = 0;
     if (report != null) {
-      this.numContainers = report.getNumContainers();
-      this.usedMemoryMB = report.getUsedResource().getMemorySize();
-      this.availMemoryMB = report.getAvailableResource().getMemorySize();
-      this.usedVirtualCores = report.getUsedResource().getVirtualCores();
+      this.numContainers = report.getNumGuaranteedContainers();
+      this.usedMemoryMB = report.getGuaranteedResourceUsed().getMemorySize();
+      this.availMemoryMB =
+          report.getAvailableGuaranteedResource().getMemorySize();
+      this.usedVirtualCores =
+          report.getGuaranteedResourceUsed().getVirtualCores();
       this.availableVirtualCores =
-          report.getAvailableResource().getVirtualCores();
-      this.usedResource = new ResourceInfo(report.getUsedResource());
-      this.availableResource = new ResourceInfo(report.getAvailableResource());
+          report.getAvailableGuaranteedResource().getVirtualCores();
+      this.usedResource = new ResourceInfo(report.getGuaranteedResourceUsed());
+      this.availableResource =
+          new ResourceInfo(report.getAvailableGuaranteedResource());
     }
     this.id = id.toString();
     this.rack = ni.getRackName();

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java

@@ -225,10 +225,10 @@ public class NodeManager implements ContainerManagementProtocol {
     LOG.info("Checking resource usage for " + containerManagerAddress);
     Assert.assertEquals(available.getMemorySize(),
         resourceManager.getResourceScheduler().getNodeReport(
-            this.nodeId).getAvailableResource().getMemorySize());
+            this.nodeId).getAvailableGuaranteedResource().getMemorySize());
     Assert.assertEquals(used.getMemorySize(),
         resourceManager.getResourceScheduler().getNodeReport(
-            this.nodeId).getUsedResource().getMemorySize());
+            this.nodeId).getGuaranteedResourceUsed().getMemorySize());
   }
   
   @Override

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java

@@ -176,8 +176,10 @@ public class TestApplicationPriority {
     // check node report, 15 GB used (1 AM and 7 containers) and 1 GB available
     SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport(
         nm1.getNodeId());
-    Assert.assertEquals(15 * GB, report_nm1.getUsedResource().getMemorySize());
-    Assert.assertEquals(1 * GB, report_nm1.getAvailableResource().getMemorySize());
+    Assert.assertEquals(15 * GB,
+        report_nm1.getGuaranteedResourceUsed().getMemorySize());
+    Assert.assertEquals(1 * GB,
+        report_nm1.getAvailableGuaranteedResource().getMemorySize());
 
     // Submit the second app App2 with priority 8 (Higher than App1)
     Priority appPriority2 = Priority.newInstance(8);
@@ -189,8 +191,10 @@ public class TestApplicationPriority {
 
     // check node report, 16 GB used and 0 GB available
     report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
-    Assert.assertEquals(16 * GB, report_nm1.getUsedResource().getMemorySize());
-    Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+    Assert.assertEquals(16 * GB,
+        report_nm1.getGuaranteedResourceUsed().getMemorySize());
+    Assert.assertEquals(0 * GB,
+        report_nm1.getAvailableGuaranteedResource().getMemorySize());
 
     // get scheduler
     CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
@@ -210,8 +214,10 @@ public class TestApplicationPriority {
 
     // check node report, 12 GB used and 4 GB available
     report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
-    Assert.assertEquals(12 * GB, report_nm1.getUsedResource().getMemorySize());
-    Assert.assertEquals(4 * GB, report_nm1.getAvailableResource().getMemorySize());
+    Assert.assertEquals(12 * GB,
+        report_nm1.getGuaranteedResourceUsed().getMemorySize());
+    Assert.assertEquals(4 * GB,
+        report_nm1.getAvailableGuaranteedResource().getMemorySize());
 
     // send updated request for App1
     am1.allocate("127.0.0.1", 2 * GB, 10, new ArrayList<ContainerId>());
@@ -226,8 +232,10 @@ public class TestApplicationPriority {
 
     // check node report, 16 GB used and 0 GB available
     report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
-    Assert.assertEquals(16 * GB, report_nm1.getUsedResource().getMemorySize());
-    Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+    Assert.assertEquals(16 * GB,
+        report_nm1.getGuaranteedResourceUsed().getMemorySize());
+    Assert.assertEquals(0 * GB,
+        report_nm1.getAvailableGuaranteedResource().getMemorySize());
 
     rm.stop();
   }
@@ -261,8 +269,10 @@ public class TestApplicationPriority {
     // check node report, 8 GB used (1 AM and 7 containers) and 0 GB available
     SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport(
         nm1.getNodeId());
-    Assert.assertEquals(8 * GB, report_nm1.getUsedResource().getMemorySize());
-    Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+    Assert.assertEquals(8 * GB,
+        report_nm1.getGuaranteedResourceUsed().getMemorySize());
+    Assert.assertEquals(0 * GB,
+        report_nm1.getAvailableGuaranteedResource().getMemorySize());
 
     // Submit the second app App2 with priority 7
     Priority appPriority2 = Priority.newInstance(7);
@@ -288,8 +298,10 @@ public class TestApplicationPriority {
 
     // check node report, 1 GB used and 7 GB available
     report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
-    Assert.assertEquals(1 * GB, report_nm1.getUsedResource().getMemorySize());
-    Assert.assertEquals(7 * GB, report_nm1.getAvailableResource().getMemorySize());
+    Assert.assertEquals(1 * GB,
+        report_nm1.getGuaranteedResourceUsed().getMemorySize());
+    Assert.assertEquals(7 * GB,
+        report_nm1.getAvailableGuaranteedResource().getMemorySize());
 
     rm.stop();
   }
@@ -486,8 +498,10 @@ public class TestApplicationPriority {
     // check node report, 15 GB used (1 AM and 7 containers) and 1 GB available
     SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport(
         nm1.getNodeId());
-    Assert.assertEquals(15 * GB, report_nm1.getUsedResource().getMemorySize());
-    Assert.assertEquals(1 * GB, report_nm1.getAvailableResource().getMemorySize());
+    Assert.assertEquals(15 * GB,
+        report_nm1.getGuaranteedResourceUsed().getMemorySize());
+    Assert.assertEquals(1 * GB,
+        report_nm1.getAvailableGuaranteedResource().getMemorySize());
 
     // Submit the second app App2 with priority 8 (Higher than App1)
     Priority appPriority2 = Priority.newInstance(8);
@@ -499,8 +513,10 @@ public class TestApplicationPriority {
 
     // check node report, 16 GB used and 0 GB available
     report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
-    Assert.assertEquals(16 * GB, report_nm1.getUsedResource().getMemorySize());
-    Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+    Assert.assertEquals(16 * GB,
+        report_nm1.getGuaranteedResourceUsed().getMemorySize());
+    Assert.assertEquals(0 * GB,
+        report_nm1.getAvailableGuaranteedResource().getMemorySize());
 
     // get scheduler
     CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
@@ -522,8 +538,10 @@ public class TestApplicationPriority {
 
     // check node report, 12 GB used and 4 GB available
     report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
-    Assert.assertEquals(12 * GB, report_nm1.getUsedResource().getMemorySize());
-    Assert.assertEquals(4 * GB, report_nm1.getAvailableResource().getMemorySize());
+    Assert.assertEquals(12 * GB,
+        report_nm1.getGuaranteedResourceUsed().getMemorySize());
+    Assert.assertEquals(4 * GB,
+        report_nm1.getAvailableGuaranteedResource().getMemorySize());
 
     // add request for containers App1
     am1.allocate("127.0.0.1", 2 * GB, 10, new ArrayList<ContainerId>());
@@ -535,8 +553,10 @@ public class TestApplicationPriority {
     Assert.assertEquals(2, allocated2.size());
     // check node report, 16 GB used and 0 GB available
     report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
-    Assert.assertEquals(16 * GB, report_nm1.getUsedResource().getMemorySize());
-    Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+    Assert.assertEquals(16 * GB,
+        report_nm1.getGuaranteedResourceUsed().getMemorySize());
+    Assert.assertEquals(0 * GB,
+        report_nm1.getAvailableGuaranteedResource().getMemorySize());
 
     // kill 1 more
     counter = 0;
@@ -552,8 +572,10 @@ public class TestApplicationPriority {
 
     // check node report, 14 GB used and 2 GB available
     report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
-    Assert.assertEquals(14 * GB, report_nm1.getUsedResource().getMemorySize());
-    Assert.assertEquals(2 * GB, report_nm1.getAvailableResource().getMemorySize());
+    Assert.assertEquals(14 * GB,
+        report_nm1.getGuaranteedResourceUsed().getMemorySize());
+    Assert.assertEquals(2 * GB,
+        report_nm1.getAvailableGuaranteedResource().getMemorySize());
 
     // Change the priority of App1 to 3 (lowest)
     Priority appPriority3 = Priority.newInstance(3);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java

@@ -1246,8 +1246,10 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
     SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport(
         nm1.getNodeId());
     // check node report, 2 GB used and 2 GB available
-    Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize());
-    Assert.assertEquals(2 * GB, report_nm1.getAvailableResource().getMemorySize());
+    Assert.assertEquals(2 * GB,
+        report_nm1.getGuaranteedResourceUsed().getMemorySize());
+    Assert.assertEquals(2 * GB,
+        report_nm1.getAvailableGuaranteedResource().getMemorySize());
 
     // add request for containers
     am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 2 * GB, 1, 1);
@@ -1268,8 +1270,10 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
 
     report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
     // check node report, 4 GB used and 0 GB available
-    Assert.assertEquals(0, report_nm1.getAvailableResource().getMemorySize());
-    Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemorySize());
+    Assert.assertEquals(0,
+        report_nm1.getAvailableGuaranteedResource().getMemorySize());
+    Assert.assertEquals(4 * GB,
+        report_nm1.getGuaranteedResourceUsed().getMemorySize());
 
     // check container is assigned with 2 GB.
     Container c1 = allocated1.get(0);
@@ -1288,7 +1292,7 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
     waitCount = 0;
     while (waitCount++ != 20) {
       report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
-      if (report_nm1.getAvailableResource().getMemorySize() != 0) {
+      if (report_nm1.getAvailableGuaranteedResource().getMemorySize() != 0) {
         break;
       }
       LOG.info("Waiting for RMNodeResourceUpdateEvent to be handled... Tried "
@@ -1297,8 +1301,10 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
     }
     // Now, the used resource is still 4 GB, and available resource is minus value.
     report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
-    Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemorySize());
-    Assert.assertEquals(-2 * GB, report_nm1.getAvailableResource().getMemorySize());
+    Assert.assertEquals(4 * GB,
+        report_nm1.getGuaranteedResourceUsed().getMemorySize());
+    Assert.assertEquals(-2 * GB,
+        report_nm1.getAvailableGuaranteedResource().getMemorySize());
 
     // Check container can complete successfully in case of resource over-commitment.
     ContainerStatus containerStatus = BuilderUtils.newContainerStatus(
@@ -1314,9 +1320,11 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
     Assert.assertEquals(1, attempt1.getJustFinishedContainers().size());
     Assert.assertEquals(1, am1.schedule().getCompletedContainersStatuses().size());
     report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
-    Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize());
+    Assert.assertEquals(2 * GB,
+        report_nm1.getGuaranteedResourceUsed().getMemorySize());
     // As container return 2 GB back, the available resource becomes 0 again.
-    Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+    Assert.assertEquals(0 * GB,
+        report_nm1.getAvailableGuaranteedResource().getMemorySize());
 
     // Verify no NPE is trigger in schedule after resource is updated.
     am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 3 * GB, 1, 1);
@@ -2816,8 +2824,10 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
         rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
 
     // check node report
-    Assert.assertEquals(1 * GB, report_nm1.getUsedResource().getMemorySize());
-    Assert.assertEquals(9 * GB, report_nm1.getAvailableResource().getMemorySize());
+    Assert.assertEquals(1 * GB,
+        report_nm1.getGuaranteedResourceUsed().getMemorySize());
+    Assert.assertEquals(9 * GB,
+        report_nm1.getAvailableGuaranteedResource().getMemorySize());
 
     // add request for containers
     am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 1 * GB, 1, 1);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java

@@ -123,9 +123,10 @@ public class TestCapacitySchedulerMultiNodes extends CapacitySchedulerTestBase {
         rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
 
     // check node report
-    Assert.assertEquals(2 * GB, reportNm1.getUsedResource().getMemorySize());
+    Assert.assertEquals(2 * GB,
+        reportNm1.getGuaranteedResourceUsed().getMemorySize());
     Assert.assertEquals(8 * GB,
-        reportNm1.getAvailableResource().getMemorySize());
+        reportNm1.getAvailableGuaranteedResource().getMemorySize());
 
     // Ideally thread will invoke this, but thread operates every 1sec.
     // Hence forcefully recompute nodes.
@@ -137,9 +138,10 @@ public class TestCapacitySchedulerMultiNodes extends CapacitySchedulerTestBase {
         rm.getResourceScheduler().getNodeReport(nm2.getNodeId());
 
     // check node report
-    Assert.assertEquals(1 * GB, reportNm2.getUsedResource().getMemorySize());
+    Assert.assertEquals(1 * GB,
+        reportNm2.getGuaranteedResourceUsed().getMemorySize());
     Assert.assertEquals(9 * GB,
-        reportNm2.getAvailableResource().getMemorySize());
+        reportNm2.getAvailableGuaranteedResource().getMemorySize());
 
     // Ideally thread will invoke this, but thread operates every 1sec.
     // Hence forcefully recompute nodes.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java

@@ -1947,15 +1947,17 @@ public class TestNodeLabelContainerAllocation {
 
     SchedulerNodeReport reportNm1 = rm1.getResourceScheduler()
         .getNodeReport(nm1.getNodeId());
-    Assert.assertEquals(5 * GB, reportNm1.getUsedResource().getMemorySize());
     Assert.assertEquals(5 * GB,
-        reportNm1.getAvailableResource().getMemorySize());
+        reportNm1.getGuaranteedResourceUsed().getMemorySize());
+    Assert.assertEquals(5 * GB,
+        reportNm1.getAvailableGuaranteedResource().getMemorySize());
 
     SchedulerNodeReport reportNm2 = rm1.getResourceScheduler()
         .getNodeReport(nm2.getNodeId());
-    Assert.assertEquals(0 * GB, reportNm2.getUsedResource().getMemorySize());
+    Assert.assertEquals(0 * GB,
+        reportNm2.getGuaranteedResourceUsed().getMemorySize());
     Assert.assertEquals(10 * GB,
-        reportNm2.getAvailableResource().getMemorySize());
+        reportNm2.getAvailableGuaranteedResource().getMemorySize());
 
     LeafQueue leafQueue = (LeafQueue) cs.getQueue("a");
     assertEquals(5 * GB, leafQueue.getMetrics().getAvailableMB());
@@ -2047,15 +2049,17 @@ public class TestNodeLabelContainerAllocation {
 
     SchedulerNodeReport reportNm1 = rm1.getResourceScheduler()
         .getNodeReport(nm1.getNodeId());
-    Assert.assertEquals(3 * GB, reportNm1.getUsedResource().getMemorySize());
+    Assert.assertEquals(3 * GB,
+        reportNm1.getGuaranteedResourceUsed().getMemorySize());
     Assert.assertEquals(7 * GB,
-        reportNm1.getAvailableResource().getMemorySize());
+        reportNm1.getAvailableGuaranteedResource().getMemorySize());
 
     SchedulerNodeReport reportNm2 = rm1.getResourceScheduler()
         .getNodeReport(nm2.getNodeId());
-    Assert.assertEquals(1 * GB, reportNm2.getUsedResource().getMemorySize());
+    Assert.assertEquals(1 * GB,
+        reportNm2.getGuaranteedResourceUsed().getMemorySize());
     Assert.assertEquals(9 * GB,
-        reportNm2.getAvailableResource().getMemorySize());
+        reportNm2.getAvailableGuaranteedResource().getMemorySize());
 
     LeafQueue leafQueue = (LeafQueue) cs.getQueue("a");
     double delta = 0.0001;
@@ -2177,9 +2181,10 @@ public class TestNodeLabelContainerAllocation {
 
     SchedulerNodeReport reportNm1 = rm1.getResourceScheduler()
         .getNodeReport(nm1.getNodeId());
-    Assert.assertEquals(6 * GB, reportNm1.getUsedResource().getMemorySize());
+    Assert.assertEquals(6 * GB,
+        reportNm1.getGuaranteedResourceUsed().getMemorySize());
     Assert.assertEquals(14 * GB,
-        reportNm1.getAvailableResource().getMemorySize());
+        reportNm1.getAvailableGuaranteedResource().getMemorySize());
 
     // Try to launch app2 in a2, asked 2GB, should success
     // app2 -> a2
@@ -2196,9 +2201,10 @@ public class TestNodeLabelContainerAllocation {
 
     reportNm1 = rm1.getResourceScheduler()
         .getNodeReport(nm1.getNodeId());
-    Assert.assertEquals(10 * GB, reportNm1.getUsedResource().getMemorySize());
     Assert.assertEquals(10 * GB,
-        reportNm1.getAvailableResource().getMemorySize());
+        reportNm1.getGuaranteedResourceUsed().getMemorySize());
+    Assert.assertEquals(10 * GB,
+        reportNm1.getAvailableGuaranteedResource().getMemorySize());
 
     // Kill all apps in queue a2
     cs.killAllAppsInQueue("a2");
@@ -2217,9 +2223,10 @@ public class TestNodeLabelContainerAllocation {
     checkNumOfContainersInAnAppOnGivenNode(4, nm1.getNodeId(),
         cs.getApplicationAttempt(am3.getApplicationAttemptId()));
 
-    Assert.assertEquals(10 * GB, reportNm1.getUsedResource().getMemorySize());
     Assert.assertEquals(10 * GB,
-        reportNm1.getAvailableResource().getMemorySize());
+        reportNm1.getGuaranteedResourceUsed().getMemorySize());
+    Assert.assertEquals(10 * GB,
+        reportNm1.getAvailableGuaranteedResource().getMemorySize());
 
     // Kill all apps in queue a1
     cs.killAllAppsInQueue("a1");
@@ -2240,9 +2247,10 @@ public class TestNodeLabelContainerAllocation {
     checkNumOfContainersInAnAppOnGivenNode(6, nm1.getNodeId(),
         cs.getApplicationAttempt(am4.getApplicationAttemptId()));
 
-    Assert.assertEquals(10 * GB, reportNm1.getUsedResource().getMemorySize());
     Assert.assertEquals(10 * GB,
-        reportNm1.getAvailableResource().getMemorySize());
+        reportNm1.getGuaranteedResourceUsed().getMemorySize());
+    Assert.assertEquals(10 * GB,
+        reportNm1.getAvailableGuaranteedResource().getMemorySize());
 
     rm1.close();
   }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java

@@ -701,7 +701,8 @@ public class TestFifoScheduler {
     am1.registerAppAttempt();
     SchedulerNodeReport report_nm1 =
         rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
-    Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize());
+    Assert.assertEquals(2 * GB,
+        report_nm1.getGuaranteedResourceUsed().getMemorySize());
 
     RMApp app2 = rm.submitApp(2048);
     // kick the scheduling, 2GB given to AM, remaining 2 GB on nm2
@@ -711,7 +712,8 @@ public class TestFifoScheduler {
     am2.registerAppAttempt();
     SchedulerNodeReport report_nm2 =
         rm.getResourceScheduler().getNodeReport(nm2.getNodeId());
-    Assert.assertEquals(2 * GB, report_nm2.getUsedResource().getMemorySize());
+    Assert.assertEquals(2 * GB,
+        report_nm2.getGuaranteedResourceUsed().getMemorySize());
 
     // add request for containers
     am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, GB, 1, 1);
@@ -747,11 +749,15 @@ public class TestFifoScheduler {
 
     report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
     report_nm2 = rm.getResourceScheduler().getNodeReport(nm2.getNodeId());
-    Assert.assertEquals(0, report_nm1.getAvailableResource().getMemorySize());
-    Assert.assertEquals(2 * GB, report_nm2.getAvailableResource().getMemorySize());
+    Assert.assertEquals(0,
+        report_nm1.getAvailableGuaranteedResource().getMemorySize());
+    Assert.assertEquals(2 * GB,
+        report_nm2.getAvailableGuaranteedResource().getMemorySize());
 
-    Assert.assertEquals(6 * GB, report_nm1.getUsedResource().getMemorySize());
-    Assert.assertEquals(2 * GB, report_nm2.getUsedResource().getMemorySize());
+    Assert.assertEquals(6 * GB,
+        report_nm1.getGuaranteedResourceUsed().getMemorySize());
+    Assert.assertEquals(2 * GB,
+        report_nm2.getGuaranteedResourceUsed().getMemorySize());
 
     Container c1 = allocated1.get(0);
     Assert.assertEquals(GB, c1.getResource().getMemorySize());
@@ -769,7 +775,8 @@ public class TestFifoScheduler {
     Assert.assertEquals(1, am1.schedule().getCompletedContainersStatuses()
         .size());
     report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
-    Assert.assertEquals(5 * GB, report_nm1.getUsedResource().getMemorySize());
+    Assert.assertEquals(
+        5 * GB, report_nm1.getGuaranteedResourceUsed().getMemorySize());
 
     rm.stop();
   }
@@ -826,7 +833,8 @@ public class TestFifoScheduler {
     int checkAlloc =
         conf.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
             YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
-    Assert.assertEquals(checkAlloc, report_nm1.getUsedResource().getMemorySize());
+    Assert.assertEquals(
+        checkAlloc, report_nm1.getGuaranteedResourceUsed().getMemorySize());
 
     rm.stop();
   }
@@ -1117,8 +1125,10 @@ public class TestFifoScheduler {
     SchedulerNodeReport report_nm1 =
         rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
     // check node report, 2 GB used and 2 GB available
-    Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize());
-    Assert.assertEquals(2 * GB, report_nm1.getAvailableResource().getMemorySize());
+    Assert.assertEquals(2 * GB,
+        report_nm1.getGuaranteedResourceUsed().getMemorySize());
+    Assert.assertEquals(2 * GB,
+        report_nm1.getAvailableGuaranteedResource().getMemorySize());
 
     // add request for containers
     am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 2 * GB, 1, 1);
@@ -1139,8 +1149,10 @@ public class TestFifoScheduler {
 
     report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
     // check node report, 4 GB used and 0 GB available
-    Assert.assertEquals(0, report_nm1.getAvailableResource().getMemorySize());
-    Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemorySize());
+    Assert.assertEquals(0,
+        report_nm1.getAvailableGuaranteedResource().getMemorySize());
+    Assert.assertEquals(4 * GB,
+        report_nm1.getGuaranteedResourceUsed().getMemorySize());
 
     // check container is assigned with 2 GB.
     Container c1 = allocated1.get(0);
@@ -1159,7 +1171,7 @@ public class TestFifoScheduler {
     while (waitCount++ != 20) {
       report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
       if (null != report_nm1 &&
-          report_nm1.getAvailableResource().getMemorySize() != 0) {
+          report_nm1.getAvailableGuaranteedResource().getMemorySize() != 0) {
         break;
       }
       LOG.info("Waiting for RMNodeResourceUpdateEvent to be handled... Tried "
@@ -1169,8 +1181,10 @@ public class TestFifoScheduler {
     // Now, the used resource is still 4 GB, and available resource is minus
     // value.
     report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
-    Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemorySize());
-    Assert.assertEquals(-2 * GB, report_nm1.getAvailableResource().getMemorySize());
+    Assert.assertEquals(4 * GB,
+        report_nm1.getGuaranteedResourceUsed().getMemorySize());
+    Assert.assertEquals(-2 * GB,
+        report_nm1.getAvailableGuaranteedResource().getMemorySize());
 
     // Check container can complete successfully in case of resource
     // over-commitment.
@@ -1188,9 +1202,11 @@ public class TestFifoScheduler {
     Assert.assertEquals(1, am1.schedule().getCompletedContainersStatuses()
         .size());
     report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
-    Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize());
+    Assert.assertEquals(2 * GB,
+        report_nm1.getGuaranteedResourceUsed().getMemorySize());
     // As container return 2 GB back, the available resource becomes 0 again.
-    Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+    Assert.assertEquals(0 * GB,
+        report_nm1.getAvailableGuaranteedResource().getMemorySize());
     rm.stop();
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java

@@ -817,15 +817,16 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
 
     if (report != null) {
       assertEquals("numContainers doesn't match: " + numContainers,
-          report.getNumContainers(), numContainers);
+          report.getNumGuaranteedContainers(), numContainers);
       assertEquals("usedMemoryMB doesn't match: " + usedMemoryMB, report
-          .getUsedResource().getMemorySize(), usedMemoryMB);
+          .getGuaranteedResourceUsed().getMemorySize(), usedMemoryMB);
       assertEquals("availMemoryMB doesn't match: " + availMemoryMB, report
-          .getAvailableResource().getMemorySize(), availMemoryMB);
+          .getAvailableGuaranteedResource().getMemorySize(), availMemoryMB);
       assertEquals("usedVirtualCores doesn't match: " + usedVirtualCores, report
-          .getUsedResource().getVirtualCores(), usedVirtualCores);
-      assertEquals("availVirtualCores doesn't match: " + availVirtualCores, report
-          .getAvailableResource().getVirtualCores(), availVirtualCores);
+          .getGuaranteedResourceUsed().getVirtualCores(), usedVirtualCores);
+      assertEquals("availVirtualCores doesn't match: " + availVirtualCores,
+          report.getAvailableGuaranteedResource().getVirtualCores(),
+          availVirtualCores);
     }
 
     if (opportunisticStatus != null) {