YARN-2072. RM/NM UIs and webservices are missing vcore information. (Nathan Roberts via tgraves)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1605162 13f79535-47bb-0310-9956-ffa450edef68

hadoop-yarn-project/CHANGES.txt

@@ -183,6 +183,9 @@ Release 2.5.0 - UNRELEASED
 
     YARN-2109. Fix TestRM to work with both schedulers. (Anubhav Dhoot via kasha)
 
+    YARN-2072. RM/NM UIs and webservices are missing vcore information.
+    (Nathan Roberts via tgraves)
+
   OPTIMIZATIONS
 
   BUG FIXES 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ResourceView.java

@@ -27,4 +27,6 @@ public interface ResourceView {
   long getPmemAllocatedForContainers();
 
   boolean isPmemCheckEnabled();
+
+  long getVCoresAllocatedForContainers();
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java

@@ -70,6 +70,8 @@ public class ContainersMonitorImpl extends AbstractService implements
   private boolean pmemCheckEnabled;
   private boolean vmemCheckEnabled;
 
+  private long maxVCoresAllottedForContainers;
+
   private static final long UNKNOWN_MEMORY_LIMIT = -1L;
 
   public ContainersMonitorImpl(ContainerExecutor exec,
@@ -108,10 +110,16 @@ public class ContainersMonitorImpl extends AbstractService implements
         YarnConfiguration.NM_PMEM_MB,
         YarnConfiguration.DEFAULT_NM_PMEM_MB) * 1024 * 1024l;
 
+    long configuredVCoresForContainers = conf.getLong(
+        YarnConfiguration.NM_VCORES,
+        YarnConfiguration.DEFAULT_NM_VCORES);
+
+
     // Setting these irrespective of whether checks are enabled. Required in
     // the UI.
     // ///////// Physical memory configuration //////
     this.maxPmemAllottedForContainers = configuredPMemForContainers;
+    this.maxVCoresAllottedForContainers = configuredVCoresForContainers;
 
     // ///////// Virtual memory configuration //////
     float vmemRatio = conf.getFloat(YarnConfiguration.NM_VMEM_PMEM_RATIO,
@@ -518,6 +526,11 @@ public class ContainersMonitorImpl extends AbstractService implements
     return this.maxPmemAllottedForContainers;
   }
 
+  @Override
+  public long getVCoresAllocatedForContainers() {
+    return this.maxVCoresAllottedForContainers;
+  }
+
   /**
    * Is the total virtual memory check enabled?
    *

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java

@@ -40,6 +40,9 @@ public class NodeManagerMetrics {
   @Metric("Current # of allocated containers")
       MutableGaugeInt allocatedContainers;
   @Metric MutableGaugeInt availableGB;
+  @Metric("Current allocated Virtual Cores")
+      MutableGaugeInt allocatedVCores;
+  @Metric MutableGaugeInt availableVCores;
 
   public static NodeManagerMetrics create() {
     return create(DefaultMetricsSystem.instance());
@@ -88,16 +91,21 @@ public class NodeManagerMetrics {
     allocatedContainers.incr();
     allocatedGB.incr(res.getMemory() / 1024);
     availableGB.decr(res.getMemory() / 1024);
+    allocatedVCores.incr(res.getVirtualCores());
+    availableVCores.decr(res.getVirtualCores());
   }
 
   public void releaseContainer(Resource res) {
     allocatedContainers.decr();
     allocatedGB.decr(res.getMemory() / 1024);
     availableGB.incr(res.getMemory() / 1024);
+    allocatedVCores.decr(res.getVirtualCores());
+    availableVCores.incr(res.getVirtualCores());
   }
 
   public void addResource(Resource res) {
     availableGB.incr(res.getMemory() / 1024);
+    availableVCores.incr(res.getVirtualCores());
   }
   
   public int getRunningContainers() {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java

@@ -85,6 +85,7 @@ public class ContainerPage extends NMView implements YarnWebParams {
         ._("Diagnostics", info.getDiagnostics())
         ._("User", info.getUser())
         ._("TotalMemoryNeeded", info.getMemoryNeeded())
+        ._("TotalVCoresNeeded", info.getVCoresNeeded())
         ._("logs", info.getShortLogLink(), "Link to logs");
       html._(InfoBlock.class);
     }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java

@@ -73,6 +73,8 @@ public class NodePage extends NMView {
               StringUtils.byteDesc(info.getTotalPmemAllocated() * BYTES_IN_MB))
           ._("Pmem enforcement enabled",
               info.isVmemCheckEnabled())
+           ._("Total VCores allocated for Containers",
+              String.valueOf(info.getTotalVCoresAllocated())) 
           ._("NodeHealthyStatus",
               info.getHealthStatus())
           ._("LastNodeHealthTime", new Date(

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java

@@ -42,6 +42,7 @@ public class ContainerInfo {
   protected String diagnostics;
   protected String user;
   protected long totalMemoryNeededMB;
+  protected long totalVCoresNeeded;
   protected String containerLogsLink;
   protected String nodeId;
   @XmlTransient
@@ -76,6 +77,7 @@ public class ContainerInfo {
     Resource res = container.getResource();
     if (res != null) {
       this.totalMemoryNeededMB = res.getMemory();
+      this.totalVCoresNeeded = res.getVirtualCores();
     }
     this.containerLogsShortLink = ujoin("containerlogs", this.id,
         container.getUser());
@@ -130,4 +132,8 @@ public class ContainerInfo {
     return this.totalMemoryNeededMB;
   }
 
+  public long getVCoresNeeded() {
+    return this.totalVCoresNeeded;
+  }
+
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NodeInfo.java

@@ -36,6 +36,7 @@ public class NodeInfo {
   protected String healthReport;
   protected long totalVmemAllocatedContainersMB;
   protected long totalPmemAllocatedContainersMB;
+  protected long totalVCoresAllocatedContainers;
   protected boolean vmemCheckEnabled;
   protected boolean pmemCheckEnabled;
   protected long lastNodeUpdateTime;
@@ -62,6 +63,8 @@ public class NodeInfo {
     this.totalPmemAllocatedContainersMB = resourceView
         .getPmemAllocatedForContainers() / BYTES_IN_MB;
     this.pmemCheckEnabled = resourceView.isPmemCheckEnabled();
+    this.totalVCoresAllocatedContainers = resourceView
+        .getVCoresAllocatedForContainers();
     this.nodeHealthy = context.getNodeHealthStatus().getIsNodeHealthy();
     this.lastNodeUpdateTime = context.getNodeHealthStatus()
         .getLastHealthReportTime();
@@ -124,6 +127,10 @@ public class NodeInfo {
     return this.totalVmemAllocatedContainersMB;
   }
 
+  public long getTotalVCoresAllocated() {
+    return this.totalVCoresAllocatedContainers;
+  }
+
   public boolean isVmemCheckEnabled() {
     return this.vmemCheckEnabled;
   }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java

@@ -31,8 +31,11 @@ public class TestNodeManagerMetrics {
     NodeManagerMetrics metrics = NodeManagerMetrics.create();
     Resource total = Records.newRecord(Resource.class);
     total.setMemory(8*GiB);
+    total.setVirtualCores(16);
     Resource resource = Records.newRecord(Resource.class);
     resource.setMemory(1*GiB);
+    resource.setVirtualCores(2);
+
 
     metrics.addResource(total);
 
@@ -57,12 +60,12 @@ public class TestNodeManagerMetrics {
     metrics.initingContainer();
     metrics.runningContainer();
 
-    checkMetrics(5, 1, 1, 1, 1, 1, 2, 2, 6);
+    checkMetrics(5, 1, 1, 1, 1, 1, 2, 2, 6, 4, 12);
   }
 
   private void checkMetrics(int launched, int completed, int failed, int killed,
                             int initing, int running, int allocatedGB,
-                            int allocatedContainers, int availableGB) {
+                            int allocatedContainers, int availableGB, int allocatedVCores, int availableVCores) {
     MetricsRecordBuilder rb = getMetrics("NodeManagerMetrics");
     assertCounter("ContainersLaunched", launched, rb);
     assertCounter("ContainersCompleted", completed, rb);
@@ -71,7 +74,10 @@ public class TestNodeManagerMetrics {
     assertGauge("ContainersIniting", initing, rb);
     assertGauge("ContainersRunning", running, rb);
     assertGauge("AllocatedGB", allocatedGB, rb);
+    assertGauge("AllocatedVCores", allocatedVCores, rb);
     assertGauge("AllocatedContainers", allocatedContainers, rb);
     assertGauge("AvailableGB", availableGB, rb);
+    assertGauge("AvailableVCores",availableVCores, rb);
+
   }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java

@@ -91,6 +91,10 @@ public class TestNMWebServer {
         return 0;
       }
       @Override
+      public long getVCoresAllocatedForContainers() {
+        return 0;
+      }
+      @Override
       public boolean isVmemCheckEnabled() {
         return true;
       }
@@ -150,6 +154,10 @@ public class TestNMWebServer {
         return 0;
       }
       @Override
+      public long getVCoresAllocatedForContainers() {
+        return 0;
+      }
+      @Override
       public boolean isVmemCheckEnabled() {
         return true;
       }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java

@@ -124,6 +124,10 @@ public class TestNMWebServices extends JerseyTest {
           return new Long("17179869184");
         }
         @Override
+        public long getVCoresAllocatedForContainers() {
+          return new Long("4000");
+        }
+        @Override
         public boolean isVmemCheckEnabled() {
           return true;
         }
@@ -375,6 +379,8 @@ public class TestNMWebServices extends JerseyTest {
               "totalVmemAllocatedContainersMB"),
           WebServicesTestUtils.getXmlLong(element,
               "totalPmemAllocatedContainersMB"),
+          WebServicesTestUtils.getXmlLong(element,
+              "totalVCoresAllocatedContainers"),
           WebServicesTestUtils.getXmlBoolean(element, "vmemCheckEnabled"),
           WebServicesTestUtils.getXmlBoolean(element, "pmemCheckEnabled"),
           WebServicesTestUtils.getXmlLong(element, "lastNodeUpdateTime"),
@@ -393,10 +399,11 @@ public class TestNMWebServices extends JerseyTest {
   public void verifyNodeInfo(JSONObject json) throws JSONException, Exception {
     assertEquals("incorrect number of elements", 1, json.length());
     JSONObject info = json.getJSONObject("nodeInfo");
-    assertEquals("incorrect number of elements", 15, info.length());
+    assertEquals("incorrect number of elements", 16, info.length());
     verifyNodeInfoGeneric(info.getString("id"), info.getString("healthReport"),
         info.getLong("totalVmemAllocatedContainersMB"),
         info.getLong("totalPmemAllocatedContainersMB"),
+        info.getLong("totalVCoresAllocatedContainers"),
         info.getBoolean("vmemCheckEnabled"),
         info.getBoolean("pmemCheckEnabled"),
         info.getLong("lastNodeUpdateTime"), info.getBoolean("nodeHealthy"),
@@ -410,6 +417,7 @@ public class TestNMWebServices extends JerseyTest {
 
   public void verifyNodeInfoGeneric(String id, String healthReport,
       long totalVmemAllocatedContainersMB, long totalPmemAllocatedContainersMB,
+      long totalVCoresAllocatedContainers,
       boolean vmemCheckEnabled, boolean pmemCheckEnabled,
       long lastNodeUpdateTime, Boolean nodeHealthy, String nodeHostName,
       String hadoopVersionBuiltOn, String hadoopBuildVersion,
@@ -423,6 +431,8 @@ public class TestNMWebServices extends JerseyTest {
         totalVmemAllocatedContainersMB);
     assertEquals("totalPmemAllocatedContainersMB incorrect", 16384,
         totalPmemAllocatedContainersMB);
+    assertEquals("totalVCoresAllocatedContainers incorrect", 4000,
+        totalVCoresAllocatedContainers);
     assertEquals("vmemCheckEnabled incorrect",  true, vmemCheckEnabled);
     assertEquals("pmemCheckEnabled incorrect",  true, pmemCheckEnabled);
     assertTrue("lastNodeUpdateTime incorrect", lastNodeUpdateTime == nmContext

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java

@@ -116,6 +116,12 @@ public class TestNMWebServicesApps extends JerseyTest {
           return new Long("17179869184");
         }
 
+        @Override
+        public long getVCoresAllocatedForContainers() {
+          return new Long("4000");
+        }
+
+
         @Override
         public boolean isVmemCheckEnabled() {
           return true;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java

@@ -106,6 +106,11 @@ public class TestNMWebServicesContainers extends JerseyTest {
           return new Long("17179869184");
         }
 
+        @Override
+        public long getVCoresAllocatedForContainers() {
+          return new Long("4000");
+        }
+
         @Override
         public boolean isVmemCheckEnabled() {
           return true;
@@ -461,24 +466,27 @@ public class TestNMWebServicesContainers extends JerseyTest {
           WebServicesTestUtils.getXmlString(element, "diagnostics"),
           WebServicesTestUtils.getXmlString(element, "nodeId"),
           WebServicesTestUtils.getXmlInt(element, "totalMemoryNeededMB"),
+          WebServicesTestUtils.getXmlInt(element, "totalVCoresNeeded"),
           WebServicesTestUtils.getXmlString(element, "containerLogsLink"));
     }
   }
 
   public void verifyNodeContainerInfo(JSONObject info, Container cont)
       throws JSONException, Exception {
-    assertEquals("incorrect number of elements", 8, info.length());
+    assertEquals("incorrect number of elements", 9, info.length());
 
     verifyNodeContainerInfoGeneric(cont, info.getString("id"),
         info.getString("state"), info.getString("user"),
         info.getInt("exitCode"), info.getString("diagnostics"),
         info.getString("nodeId"), info.getInt("totalMemoryNeededMB"),
+        info.getInt("totalVCoresNeeded"),
         info.getString("containerLogsLink"));
   }
 
   public void verifyNodeContainerInfoGeneric(Container cont, String id,
       String state, String user, int exitCode, String diagnostics,
-      String nodeId, int totalMemoryNeededMB, String logsLink)
+      String nodeId, int totalMemoryNeededMB, int totalVCoresNeeded,
+      String logsLink)
       throws JSONException, Exception {
     WebServicesTestUtils.checkStringMatch("id", cont.getContainerId()
         .toString(), id);
@@ -495,6 +503,9 @@ public class TestNMWebServicesContainers extends JerseyTest {
     assertEquals("totalMemoryNeededMB wrong",
       YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
       totalMemoryNeededMB);
+    assertEquals("totalVCoresNeeded wrong",
+      YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
+      totalVCoresNeeded);
     String shortLink =
         ujoin("containerlogs", cont.getContainerId().toString(),
             cont.getUser());

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java

@@ -72,6 +72,9 @@ public class MetricsOverviewTable extends HtmlBlock {
         th().$class("ui-state-default")._("Memory Used")._().
         th().$class("ui-state-default")._("Memory Total")._().
         th().$class("ui-state-default")._("Memory Reserved")._().
+        th().$class("ui-state-default")._("VCores Used")._().
+        th().$class("ui-state-default")._("VCores Total")._().
+        th().$class("ui-state-default")._("VCores Reserved")._().
         th().$class("ui-state-default")._("Active Nodes")._().
         th().$class("ui-state-default")._("Decommissioned Nodes")._().
         th().$class("ui-state-default")._("Lost Nodes")._().
@@ -94,6 +97,9 @@ public class MetricsOverviewTable extends HtmlBlock {
         td(StringUtils.byteDesc(clusterMetrics.getAllocatedMB() * BYTES_IN_MB)).
         td(StringUtils.byteDesc(clusterMetrics.getTotalMB() * BYTES_IN_MB)).
         td(StringUtils.byteDesc(clusterMetrics.getReservedMB() * BYTES_IN_MB)).
+        td(String.valueOf(clusterMetrics.getAllocatedVirtualCores())).
+        td(String.valueOf(clusterMetrics.getTotalVirtualCores())).
+        td(String.valueOf(clusterMetrics.getReservedVirtualCores())).
         td().a(url("nodes"),String.valueOf(clusterMetrics.getActiveNodes()))._().
         td().a(url("nodes/decommissioned"),String.valueOf(clusterMetrics.getDecommissionedNodes()))._().
         td().a(url("nodes/lost"),String.valueOf(clusterMetrics.getLostNodes()))._().
@@ -120,6 +126,9 @@ public class MetricsOverviewTable extends HtmlBlock {
             th().$class("ui-state-default")._("Memory Used")._().
             th().$class("ui-state-default")._("Memory Pending")._().
             th().$class("ui-state-default")._("Memory Reserved")._().
+            th().$class("ui-state-default")._("VCores Used")._().
+            th().$class("ui-state-default")._("VCores Pending")._().
+            th().$class("ui-state-default")._("VCores Reserved")._().
           _().
         _().
         tbody().$class("ui-widget-content").
@@ -139,6 +148,9 @@ public class MetricsOverviewTable extends HtmlBlock {
             td(StringUtils.byteDesc(userMetrics.getAllocatedMB() * BYTES_IN_MB)).
             td(StringUtils.byteDesc(userMetrics.getPendingMB() * BYTES_IN_MB)).
             td(StringUtils.byteDesc(userMetrics.getReservedMB() * BYTES_IN_MB)).
+            td(String.valueOf(userMetrics.getAllocatedVirtualCores())).
+            td(String.valueOf(userMetrics.getPendingVirtualCores())).
+            td(String.valueOf(userMetrics.getReservedVirtualCores())).
           _().
         _()._();
         

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java

@@ -75,6 +75,8 @@ class NodesPage extends RmView {
           th(".containers", "Containers").
           th(".mem", "Mem Used").
           th(".mem", "Mem Avail").
+          th(".vcores", "VCores Used").
+          th(".vcores", "VCores Avail").
           th(".nodeManagerVersion", "Version").
           _()._().
           tbody();
@@ -127,8 +129,10 @@ class NodesPage extends RmView {
             td(String.valueOf(info.getNumContainers())).
             td().br().$title(String.valueOf(usedMemory))._().
               _(StringUtils.byteDesc(usedMemory * BYTES_IN_MB))._().
-            td().br().$title(String.valueOf(usedMemory))._().
+            td().br().$title(String.valueOf(availableMemory))._().
               _(StringUtils.byteDesc(availableMemory * BYTES_IN_MB))._().
+              td(String.valueOf(info.getUsedVirtualCores())).
+              td(String.valueOf(info.getAvailableVirtualCores())).
             td(ni.getNodeManagerVersion()).
             _();
       }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java

@@ -37,16 +37,21 @@ public class ClusterMetricsInfo {
   protected int appsRunning;
   protected int appsFailed;
   protected int appsKilled;
-  
+
   protected long reservedMB;
   protected long availableMB;
   protected long allocatedMB;
-  
+
+  protected long reservedVirtualCores;
+  protected long availableVirtualCores;
+  protected long allocatedVirtualCores;
+
   protected int containersAllocated;
   protected int containersReserved;
   protected int containersPending;
-  
+
   protected long totalMB;
+  protected long totalVirtualCores;
   protected int totalNodes;
   protected int lostNodes;
   protected int unhealthyNodes;
@@ -68,16 +73,21 @@ public class ClusterMetricsInfo {
     this.appsRunning = metrics.getAppsRunning();
     this.appsFailed = metrics.getAppsFailed();
     this.appsKilled = metrics.getAppsKilled();
-    
+
     this.reservedMB = metrics.getReservedMB();
     this.availableMB = metrics.getAvailableMB();
     this.allocatedMB = metrics.getAllocatedMB();
-    
+
+    this.reservedVirtualCores = metrics.getReservedVirtualCores();
+    this.availableVirtualCores = metrics.getAvailableVirtualCores();
+    this.allocatedVirtualCores = metrics.getAllocatedVirtualCores();
+
     this.containersAllocated = metrics.getAllocatedContainers();
     this.containersPending = metrics.getPendingContainers();
     this.containersReserved = metrics.getReservedContainers();
-    
+
     this.totalMB = availableMB + allocatedMB;
+    this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores;
     this.activeNodes = clusterMetrics.getNumActiveNMs();
     this.lostNodes = clusterMetrics.getNumLostNMs();
     this.unhealthyNodes = clusterMetrics.getUnhealthyNMs();
@@ -123,6 +133,18 @@ public class ClusterMetricsInfo {
     return this.allocatedMB;
   }
 
+  public long getReservedVirtualCores() {
+    return this.reservedVirtualCores;
+  }
+
+  public long getAvailableVirtualCores() {
+    return this.availableVirtualCores;
+  }
+
+  public long getAllocatedVirtualCores() {
+    return this.allocatedVirtualCores;
+  }
+
   public int getContainersAllocated() {
     return this.containersAllocated;
   }
@@ -134,15 +156,19 @@ public class ClusterMetricsInfo {
   public int getPendingContainers() {
     return this.containersPending;
   }
-  
+
   public long getTotalMB() {
     return this.totalMB;
   }
 
+  public long getTotalVirtualCores() {
+    return this.totalVirtualCores;
+  }
+
   public int getTotalNodes() {
     return this.totalNodes;
   }
-  
+
   public int getActiveNodes() {
     return this.activeNodes;
   }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java

@@ -43,6 +43,8 @@ public class NodeInfo {
   protected int numContainers;
   protected long usedMemoryMB;
   protected long availMemoryMB;
+  protected long usedVirtualCores;
+  protected long availableVirtualCores;
 
   public NodeInfo() {
   } // JAXB needs this
@@ -57,6 +59,8 @@ public class NodeInfo {
       this.numContainers = report.getNumContainers();
       this.usedMemoryMB = report.getUsedResource().getMemory();
       this.availMemoryMB = report.getAvailableResource().getMemory();
+      this.usedVirtualCores = report.getUsedResource().getVirtualCores();
+      this.availableVirtualCores = report.getAvailableResource().getVirtualCores();
     }
     this.id = id.toString();
     this.rack = ni.getRackName();
@@ -83,7 +87,7 @@ public class NodeInfo {
   public String getNodeHTTPAddress() {
     return this.nodeHTTPAddress;
   }
-  
+
   public void setNodeHTTPAddress(String nodeHTTPAddress) {
     this.nodeHTTPAddress = nodeHTTPAddress;
   }
@@ -112,4 +116,12 @@ public class NodeInfo {
     return this.availMemoryMB;
   }
 
+  public long getUsedVirtualCores() {
+    return this.usedVirtualCores;
+  }
+
+  public long getAvailableVirtualCores() {
+    return this.availableVirtualCores;
+  }
+
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UserMetricsInfo.java

@@ -43,6 +43,9 @@ public class UserMetricsInfo {
   protected long reservedMB;
   protected long pendingMB;
   protected long allocatedMB;
+  protected long reservedVirtualCores;
+  protected long pendingVirtualCores;
+  protected long allocatedVirtualCores;
 
   @XmlTransient
   protected boolean userMetricsAvailable;
@@ -59,7 +62,7 @@ public class UserMetricsInfo {
 
     if (userMetrics != null) {
       this.userMetricsAvailable = true;
-      
+
       this.appsSubmitted = userMetrics.getAppsSubmitted();
       this.appsCompleted = metrics.getAppsCompleted();
       this.appsPending = metrics.getAppsPending();
@@ -70,10 +73,14 @@ public class UserMetricsInfo {
       this.runningContainers = userMetrics.getAllocatedContainers();
       this.pendingContainers = userMetrics.getPendingContainers();
       this.reservedContainers = userMetrics.getReservedContainers();
-      
+
       this.reservedMB = userMetrics.getReservedMB();
       this.pendingMB = userMetrics.getPendingMB();
       this.allocatedMB = userMetrics.getAllocatedMB();
+
+      this.reservedVirtualCores = userMetrics.getReservedVirtualCores();
+      this.pendingVirtualCores = userMetrics.getPendingVirtualCores();
+      this.allocatedVirtualCores = userMetrics.getAllocatedVirtualCores();
     }
   }
 
@@ -117,6 +124,18 @@ public class UserMetricsInfo {
     return this.pendingMB;
   }
 
+  public long getReservedVirtualCores() {
+    return this.reservedVirtualCores;
+  }
+
+  public long getAllocatedVirtualCores() {
+    return this.allocatedVirtualCores;
+  }
+
+  public long getPendingVirtualCores() {
+    return this.pendingVirtualCores;
+  }
+
   public int getReservedContainers() {
     return this.reservedContainers;
   }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java

@@ -48,8 +48,8 @@ public class TestNodesPage {
 
   // Number of Actual Table Headers for NodesPage.NodesBlock might change in
   // future. In that case this value should be adjusted to the new value.
-  final int numberOfThInMetricsTable = 13;
-  final int numberOfActualTableHeaders = 10;
+  final int numberOfThInMetricsTable = 16;
+  final int numberOfActualTableHeaders = 12;
 
   private Injector injector;
   

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java

@@ -389,6 +389,10 @@ public class TestRMWebServices extends JerseyTest {
           WebServicesTestUtils.getXmlInt(element, "reservedMB"),
           WebServicesTestUtils.getXmlInt(element, "availableMB"),
           WebServicesTestUtils.getXmlInt(element, "allocatedMB"),
+          WebServicesTestUtils.getXmlInt(element, "reservedVirtualCores"),
+          WebServicesTestUtils.getXmlInt(element, "availableVirtualCores"),
+          WebServicesTestUtils.getXmlInt(element, "allocatedVirtualCores"),
+          WebServicesTestUtils.getXmlInt(element, "totalVirtualCores"),
           WebServicesTestUtils.getXmlInt(element, "containersAllocated"),
           WebServicesTestUtils.getXmlInt(element, "totalMB"),
           WebServicesTestUtils.getXmlInt(element, "totalNodes"),
@@ -404,11 +408,13 @@ public class TestRMWebServices extends JerseyTest {
       Exception {
     assertEquals("incorrect number of elements", 1, json.length());
     JSONObject clusterinfo = json.getJSONObject("clusterMetrics");
-    assertEquals("incorrect number of elements", 19, clusterinfo.length());
+    assertEquals("incorrect number of elements", 23, clusterinfo.length());
     verifyClusterMetrics(
         clusterinfo.getInt("appsSubmitted"), clusterinfo.getInt("appsCompleted"),
         clusterinfo.getInt("reservedMB"), clusterinfo.getInt("availableMB"),
         clusterinfo.getInt("allocatedMB"),
+        clusterinfo.getInt("reservedVirtualCores"), clusterinfo.getInt("availableVirtualCores"),
+        clusterinfo.getInt("allocatedVirtualCores"), clusterinfo.getInt("totalVirtualCores"),
         clusterinfo.getInt("containersAllocated"),
         clusterinfo.getInt("totalMB"), clusterinfo.getInt("totalNodes"),
         clusterinfo.getInt("lostNodes"), clusterinfo.getInt("unhealthyNodes"),
@@ -418,7 +424,9 @@ public class TestRMWebServices extends JerseyTest {
 
   public void verifyClusterMetrics(int submittedApps, int completedApps,
       int reservedMB, int availableMB,
-      int allocMB, int containersAlloc, int totalMB, int totalNodes,
+      int allocMB, int reservedVirtualCores, int availableVirtualCores, 
+      int allocVirtualCores, int totalVirtualCores,
+      int containersAlloc, int totalMB, int totalNodes,
       int lostNodes, int unhealthyNodes, int decommissionedNodes,
       int rebootedNodes, int activeNodes) throws JSONException, Exception {
 
@@ -428,7 +436,8 @@ public class TestRMWebServices extends JerseyTest {
 
     long totalMBExpect = 
         metrics.getAvailableMB() + metrics.getAllocatedMB();
-
+    long totalVirtualCoresExpect = 
+        metrics.getAvailableVirtualCores() + metrics.getAllocatedVirtualCores();
     assertEquals("appsSubmitted doesn't match", 
         metrics.getAppsSubmitted(), submittedApps);
     assertEquals("appsCompleted doesn't match", 
@@ -439,6 +448,12 @@ public class TestRMWebServices extends JerseyTest {
         metrics.getAvailableMB(), availableMB);
     assertEquals("allocatedMB doesn't match", 
         metrics.getAllocatedMB(), allocMB);
+    assertEquals("reservedVirtualCores doesn't match",
+        metrics.getReservedVirtualCores(), reservedVirtualCores);
+    assertEquals("availableVirtualCores doesn't match",
+        metrics.getAvailableVirtualCores(), availableVirtualCores);
+    assertEquals("allocatedVirtualCores doesn't match",
+        totalVirtualCoresExpect, allocVirtualCores);
     assertEquals("containersAllocated doesn't match", 0, containersAlloc);
     assertEquals("totalMB doesn't match", totalMBExpect, totalMB);
     assertEquals(

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java

@@ -656,13 +656,15 @@ public class TestRMWebServicesNodes extends JerseyTest {
           WebServicesTestUtils.getXmlInt(element, "numContainers"),
           WebServicesTestUtils.getXmlLong(element, "usedMemoryMB"),
           WebServicesTestUtils.getXmlLong(element, "availMemoryMB"),
+          WebServicesTestUtils.getXmlLong(element, "usedVirtualCores"),
+          WebServicesTestUtils.getXmlLong(element,  "availableVirtualCores"),
           WebServicesTestUtils.getXmlString(element, "version"));
     }
   }
 
   public void verifyNodeInfo(JSONObject nodeInfo, MockNM nm)
       throws JSONException, Exception {
-    assertEquals("incorrect number of elements", 11, nodeInfo.length());
+    assertEquals("incorrect number of elements", 13, nodeInfo.length());
 
     verifyNodeInfoGeneric(nm, nodeInfo.getString("state"),
         nodeInfo.getString("rack"),
@@ -671,6 +673,7 @@ public class TestRMWebServicesNodes extends JerseyTest {
         nodeInfo.getLong("lastHealthUpdate"),
         nodeInfo.getString("healthReport"), nodeInfo.getInt("numContainers"),
         nodeInfo.getLong("usedMemoryMB"), nodeInfo.getLong("availMemoryMB"),
+        nodeInfo.getLong("usedVirtualCores"), nodeInfo.getLong("availableVirtualCores"),
         nodeInfo.getString("version"));
 
   }
@@ -678,7 +681,8 @@ public class TestRMWebServicesNodes extends JerseyTest {
   public void verifyNodeInfoGeneric(MockNM nm, String state, String rack,
       String id, String nodeHostName,
       String nodeHTTPAddress, long lastHealthUpdate, String healthReport,
-      int numContainers, long usedMemoryMB, long availMemoryMB, String version)
+      int numContainers, long usedMemoryMB, long availMemoryMB, long usedVirtualCores, 
+      long availVirtualCores, String version)
       throws JSONException, Exception {
 
     RMNode node = rm.getRMContext().getRMNodes().get(nm.getNodeId());
@@ -712,6 +716,10 @@ public class TestRMWebServicesNodes extends JerseyTest {
           .getUsedResource().getMemory(), usedMemoryMB);
       assertEquals("availMemoryMB doesn't match: " + availMemoryMB, report
           .getAvailableResource().getMemory(), availMemoryMB);
+      assertEquals("usedVirtualCores doesn't match: " + usedVirtualCores, report
+          .getUsedResource().getVirtualCores(), usedVirtualCores);
+      assertEquals("availVirtualCores doesn't match: " + availVirtualCores, report
+          .getAvailableResource().getVirtualCores(), availVirtualCores);
     }
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerRest.apt.vm

@@ -62,6 +62,8 @@ NodeManager REST API's.
 *---------------+--------------+-------------------------------+
 | totalVmemAllocatedContainersMB | long         | The amount of virtual memory allocated for use by containers in MB |
 *---------------+--------------+-------------------------------+
+| totalVCoresAllocatedContainers | long         | The number of virtual cores allocated for use by containers |
+*---------------+--------------+-------------------------------+
 | lastNodeUpdateTime | long         | The last timestamp at which the health report was received (in ms since epoch)|
 *---------------+--------------+-------------------------------+
 | healthReport | string  | The diagnostic health report of the node |
@@ -109,6 +111,7 @@ NodeManager REST API's.
       "nodeManagerBuildVersion" : "0.23.1-SNAPSHOT from 1228355 by user1 source checksum 20647f76c36430e888cc7204826a445c",
       "lastNodeUpdateTime" : 1326222266126,
       "totalVmemAllocatedContainersMB" : 17203,
+      "totalVCoresAllocatedContainers" : 8,
       "nodeHealthy" : true,
       "healthReport" : "",
       "totalPmemAllocatedContainersMB" : 8192,
@@ -148,6 +151,7 @@ NodeManager REST API's.
   <healthReport/>
   <totalVmemAllocatedContainersMB>17203</totalVmemAllocatedContainersMB>
   <totalPmemAllocatedContainersMB>8192</totalPmemAllocatedContainersMB>
+  <totalVCoresAllocatedContainers>8</totalVCoresAllocatedContainers>
   <lastNodeUpdateTime>1326222386134</lastNodeUpdateTime>
   <nodeHealthy>true</nodeHealthy>
   <nodeManagerVersion>0.23.1-SNAPSHOT</nodeManagerVersion>
@@ -446,6 +450,7 @@ NodeManager REST API's.
          {
             "nodeId" : "host.domain.com:8041",
             "totalMemoryNeededMB" : 2048,
+            "totalVCoresNeeded" : 1,
             "state" : "RUNNING",
             "diagnostics" : "",
             "containerLogsLink" : "http://host.domain.com:8042/node/containerlogs/container_1326121700862_0006_01_000001/user1",
@@ -456,6 +461,7 @@ NodeManager REST API's.
          {
             "nodeId" : "host.domain.com:8041",
             "totalMemoryNeededMB" : 2048,
+            "totalVCoresNeeded" : 2,
             "state" : "RUNNING",
             "diagnostics" : "",
             "containerLogsLink" : "http://host.domain.com:8042/node/containerlogs/container_1326121700862_0006_01_000003/user1",
@@ -498,6 +504,7 @@ NodeManager REST API's.
     <diagnostics/>
     <user>user1</user>
     <totalMemoryNeededMB>2048</totalMemoryNeededMB>
+    <totalVCoresNeeded>1</totalVCoresNeeded>
     <containerLogsLink>http://host.domain.com:8042/node/containerlogs/container_1326121700862_0006_01_000001/user1</containerLogsLink>
     <nodeId>host.domain.com:8041</nodeId>
   </container>
@@ -508,6 +515,7 @@ NodeManager REST API's.
     <diagnostics>Container killed by the ApplicationMaster.</diagnostics>
     <user>user1</user>
     <totalMemoryNeededMB>2048</totalMemoryNeededMB>
+    <totalVCoresNeeded>2</totalVCoresNeeded>
     <containerLogsLink>http://host.domain.com:8042/node/containerlogs/container_1326121700862_0006_01_000003/user1</containerLogsLink>
     <nodeId>host.domain.com:8041</nodeId>
   </container>
@@ -560,6 +568,8 @@ NodeManager REST API's.
 *---------------+--------------+-------------------------------+
 | totalMemoryNeededMB | long | Total amout of memory needed by the container (in MB) |
 *---------------+--------------+-------------------------------+
+| totalVCoresNeeded | long | Total number of virtual cores needed by the container |
+*---------------+--------------+-------------------------------+
 
 ** Response Examples
 
@@ -587,6 +597,7 @@ NodeManager REST API's.
    "container" : {
       "nodeId" : "host.domain.com:8041",
       "totalMemoryNeededMB" : 2048,
+      "totalVCoresNeeded" : 1,
       "state" : "RUNNING",
       "diagnostics" : "",
       "containerLogsLink" : "http://host.domain.com:8042/node/containerlogs/container_1326121700862_0007_01_000001/user1",
@@ -626,6 +637,7 @@ NodeManager REST API's.
   <diagnostics/>
   <user>user1</user>
   <totalMemoryNeededMB>2048</totalMemoryNeededMB>
+  <totalVCoresNeeded>1</totalVCoresNeeded>
   <containerLogsLink>http://host.domain.com:8042/node/containerlogs/container_1326121700862_0007_01_000001/user1</containerLogsLink>
   <nodeId>host.domain.com:8041</nodeId>
 </container>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm

@@ -195,6 +195,14 @@ ResourceManager REST API's.
 *---------------+--------------+-------------------------------+
 | totalMB       | long         | The amount of total memory in MB |
 *---------------+--------------+-------------------------------+
+| reservedVirtualCores    | long         | The number of reserved virtual cores |
+*---------------+--------------+-------------------------------+
+| availableVirtualCores   | long         | The number of available virtual cores |
+*---------------+--------------+-------------------------------+
+| allocatedVirtualCores   | long         | The number of allocated virtual cores |
+*---------------+--------------+-------------------------------+
+| totalVirtualCores       | long         | The total number of virtual cores |
+*---------------+--------------+-------------------------------+
 | containersAllocated | int | The number of containers allocated |
 *---------------+--------------+-------------------------------+
 | containersReserved | int | The number of containers reserved |
@@ -247,10 +255,14 @@ ResourceManager REST API's.
       "reservedMB":0,
       "availableMB":17408,
       "allocatedMB":0,
+      "reservedVirtualCores":0,
+      "availableVirtualCores":7,
+      "allocatedVirtualCores":1,
       "containersAllocated":0,
       "containersReserved":0,
       "containersPending":0,
       "totalMB":17408,
+      "totalVirtualCores":8,
       "totalNodes":1,
       "lostNodes":0,
       "unhealthyNodes":0,
@@ -293,10 +305,14 @@ ResourceManager REST API's.
   <reservedMB>0</reservedMB>
   <availableMB>17408</availableMB>
   <allocatedMB>0</allocatedMB>
+  <reservedVirtualCores>0</reservedVirtualCores>
+  <availableVirtualCores>7</availableVirtualCores>
+  <allocatedVirtualCores>1</allocatedVirtualCores>
   <containersAllocated>0</containersAllocated>
   <containersReserved>0</containersReserved>
   <containersPending>0</containersPending>
   <totalMB>17408</totalMB>
+  <totalVirtualCores>8</totalVirtualCores>
   <totalNodes>1</totalNodes>
   <lostNodes>0</lostNodes>
   <unhealthyNodes>0</unhealthyNodes>
@@ -2040,8 +2056,10 @@ Server: Jetty(6.1.26)
         "lastHealthUpdate":1324056895432,
         "healthReport":"Healthy",
         "numContainers":0,
-        "usedMemoryMB":0
-        "availMemoryMB":8192
+        "usedMemoryMB":0,
+        "availMemoryMB":8192,
+        "usedVirtualCores":0,
+        "availableVirtualCores":8
       },
       {
         "rack":"\/default-rack",
@@ -2054,7 +2072,9 @@ Server: Jetty(6.1.26)
         "healthReport":"Healthy",
         "numContainers":0,
         "usedMemoryMB":0,
-        "availMemoryMB":8192
+        "availMemoryMB":8192,
+        "usedVirtualCores":0,
+        "availableVirtualCores":8
       }
     ]
   }
@@ -2096,6 +2116,8 @@ Server: Jetty(6.1.26)
     <numContainers>0</numContainers>
     <usedMemoryMB>0</usedMemoryMB>
     <availMemoryMB>5120</availMemoryMB>
+    <usedVirtualCores>0</usedVirtualCores>
+    <availableVirtualCores>8</availableVirtualCores>
   </node>
   <node>
     <rack>/default-rack</rack>
@@ -2109,6 +2131,8 @@ Server: Jetty(6.1.26)
     <numContainers>0</numContainers>
     <usedMemoryMB>0</usedMemoryMB>
     <availMemoryMB>5120</availMemoryMB>
+    <usedVirtualCores>0</usedVirtualCores>
+    <availableVirtualCores>8</availableVirtualCores>
   </node>
 </nodes>
 +---+
@@ -2159,10 +2183,14 @@ Server: Jetty(6.1.26)
 *---------------+--------------+-------------------------------+
 | lastHealthUpdate | long | The last time the node reported its health (in ms since epoch)|
 *---------------+--------------+-------------------------------+
-| usedMemoryMB | long | The total about of memory currently used on the node (in MB)|
+| usedMemoryMB | long | The total amount of memory currently used on the node (in MB)|
 *---------------+--------------+-------------------------------+
 | availMemoryMB | long | The total amount of memory currently available on the node (in MB)|
 *---------------+--------------+-------------------------------+
+| usedVirtualCores | long | The total number of vCores currently used on the node |
+*---------------+--------------+-------------------------------+
+| availableVirtualCores | long | The total number of vCores available on the node |
+*---------------+--------------+-------------------------------+
 | numContainers | int | The total number of containers currently running on the node|
 *---------------+--------------+-------------------------------+
 
@@ -2201,7 +2229,9 @@ Server: Jetty(6.1.26)
     "healthReport":"Healthy",
     "numContainers":0,
     "usedMemoryMB":0,
-    "availMemoryMB":5120
+    "availMemoryMB":5120,
+    "usedVirtualCores":0,
+    "availableVirtualCores":8
   }
 }
 +---+
@@ -2240,6 +2270,8 @@ Server: Jetty(6.1.26)
   <numContainers>0</numContainers>
   <usedMemoryMB>0</usedMemoryMB>
   <availMemoryMB>5120</availMemoryMB>
+  <usedVirtualCores>0</usedVirtualCores>
+  <availableVirtualCores>5120</availableVirtualCores>
 </node>
 +---+