YARN-7817. Add Resource reference to RM's NodeInfo object so REST API can get non memory/vcore resource usages. (Sunil G via wangda)

Change-Id: Ia7ceeabd82046645ddeaf487c763288f36cfbdee
(cherry picked from commit e0cfb0a31a1c6ad0f9dcf0705b44229593a0915f)

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java

@@ -55,7 +55,7 @@ public class JAXBContextResolver implements ContextResolver<JAXBContext> {
             UsersInfo.class, UserInfo.class, ApplicationStatisticsInfo.class,
             StatisticsItemInfo.class, CapacitySchedulerHealthInfo.class,
             FairSchedulerQueueInfoList.class, AppTimeoutsInfo.class,
-            AppTimeoutInfo.class };
+            AppTimeoutInfo.class, ResourceInformationsInfo.class };
     // these dao classes need root unwrapping
     final Class[] rootUnwrappedTypes =
         { NewApplication.class, ApplicationSubmissionContextInfo.class,

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java

@@ -61,6 +61,12 @@ public class ClusterMetricsInfo {
   private int activeNodes;
   private int shutdownNodes;
 
+  // Total used resource of the cluster, including all partitions
+  private ResourceInfo totalUsedResourcesAcrossPartition;
+
+  // Total registered resources of the cluster, including all partitions
+  private ResourceInfo totalClusterResourcesAcrossPartition;
+
   public ClusterMetricsInfo() {
   } // JAXB needs this
 
@@ -92,9 +98,20 @@ public class ClusterMetricsInfo {
     this.containersReserved = metrics.getReservedContainers();
 
     if (rs instanceof CapacityScheduler) {
+      CapacityScheduler cs = (CapacityScheduler) rs;
       this.totalMB = availableMB + allocatedMB + reservedMB;
       this.totalVirtualCores =
           availableVirtualCores + allocatedVirtualCores + containersReserved;
+      // TODO, add support of other schedulers to get total used resources
+      // across partition.
+      if (cs.getRootQueue() != null
+          && cs.getRootQueue().getQueueResourceUsage() != null
+          && cs.getRootQueue().getQueueResourceUsage().getAllUsed() != null) {
+        totalUsedResourcesAcrossPartition = new ResourceInfo(
+            cs.getRootQueue().getQueueResourceUsage().getAllUsed());
+        totalClusterResourcesAcrossPartition = new ResourceInfo(
+            cs.getClusterResource());
+      }
     } else {
       this.totalMB = availableMB + allocatedMB;
       this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores;
@@ -310,4 +327,11 @@ public class ClusterMetricsInfo {
     this.shutdownNodes = shutdownNodes;
   }
 
+  public ResourceInfo getTotalUsedResourcesAcrossPartition() {
+    return totalUsedResourcesAcrossPartition;
+  }
+
+  public ResourceInfo getTotalClusterResourcesAcrossPartition() {
+    return totalClusterResourcesAcrossPartition;
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java

@@ -58,6 +58,8 @@ public class NodeInfo {
   private int numQueuedContainers;
   protected ArrayList<String> nodeLabels = new ArrayList<String>();
   protected ResourceUtilizationInfo resourceUtilization;
+  protected ResourceInfo usedResource;
+  protected ResourceInfo availableResource;
 
   public NodeInfo() {
   } // JAXB needs this
@@ -75,6 +77,8 @@ public class NodeInfo {
       this.usedVirtualCores = report.getUsedResource().getVirtualCores();
       this.availableVirtualCores =
           report.getAvailableResource().getVirtualCores();
+      this.usedResource = new ResourceInfo(report.getUsedResource());
+      this.availableResource = new ResourceInfo(report.getAvailableResource());
     }
     this.id = id.toString();
     this.rack = ni.getRackName();
@@ -183,6 +187,22 @@ public class NodeInfo {
     return this.nodeLabels;
   }
 
+  public ResourceInfo getUsedResource() {
+    return usedResource;
+  }
+
+  public void setUsedResource(ResourceInfo used) {
+    this.usedResource = used;
+  }
+
+  public ResourceInfo getAvailableResource() {
+    return availableResource;
+  }
+
+  public void setAvailableResource(ResourceInfo avail) {
+    this.availableResource = avail;
+  }
+
   public ResourceUtilizationInfo getResourceUtilization() {
     return this.resourceUtilization;
   }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java

@@ -26,6 +26,7 @@ import javax.xml.bind.annotation.XmlRootElement;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.util.resource.Resources;
 
+
 @XmlRootElement
 @XmlAccessorType(XmlAccessType.NONE)
 public class ResourceInfo {
@@ -34,6 +35,9 @@ public class ResourceInfo {
   long memory;
   @XmlElement
   int vCores;
+  @XmlElement
+  ResourceInformationsInfo resourceInformations =
+      new ResourceInformationsInfo();
 
   private Resource resources;
 
@@ -41,9 +45,13 @@ public class ResourceInfo {
   }
 
   public ResourceInfo(Resource res) {
-    memory = res.getMemorySize();
-    vCores = res.getVirtualCores();
-    resources = Resources.clone(res);
+    // Make sure no NPE.
+    if (res != null) {
+      memory = res.getMemorySize();
+      vCores = res.getVirtualCores();
+      resources = Resources.clone(res);
+      resourceInformations.addAll(res.getAllResourcesListCopy());
+    }
   }
 
   public long getMemorySize() {
@@ -87,4 +95,8 @@ public class ResourceInfo {
     }
     return Resource.newInstance(resources);
   }
+
+  public ResourceInformationsInfo getResourcesInformations() {
+    return resourceInformations;
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInformationsInfo.java

@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+
+@XmlRootElement(name = "resourceInformations")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class ResourceInformationsInfo {
+
+  @XmlElement(name = "resourceInformation")
+  protected ArrayList<ResourceInformation> resourceInformation =
+      new ArrayList<ResourceInformation>();
+
+  public ResourceInformationsInfo() {
+  } // JAXB needs this
+
+  public ArrayList<ResourceInformation> getApps() {
+    return resourceInformation;
+  }
+
+  public void addAll(List<ResourceInformation> resourcesInformationsInfo) {
+    resourceInformation.addAll(resourcesInformationsInfo);
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js

@@ -169,8 +169,8 @@ export default DS.Model.extend({
       && this.get("totalUsedResourcesAcrossPartition")) {
       var usages = [];
 
-      var clusterResourceInformations = this.get("totalClusterResourcesAcrossPartition").resourcesInformations;
-      var usedResourceInformations = this.get("totalUsedResourcesAcrossPartition").resourcesInformations;
+      var clusterResourceInformations = this.get("totalClusterResourcesAcrossPartition").resourceInformations.resourceInformation;
+      var usedResourceInformations = this.get("totalUsedResourcesAcrossPartition").resourceInformations.resourceInformation;
 
       clusterResourceInformations.forEach(function(cluster) {
         var perResourceTypeUsage = {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js

@@ -99,7 +99,7 @@ export default DS.Model.extend({
 
     const usedResource = this.get("usedResource");
     const availableResource = this.get("availableResource");
-    var resourceInformations = usedResource ? usedResource.resourcesInformations : [];
+    var resourceInformations = usedResource ? usedResource.resourceInformations.resourceInformation : [];
     for (var i = 0; i < resourceInformations.length; i++) {
       ri = resourceInformations[i];
       if (ri.name === "yarn.io/gpu") {
@@ -108,7 +108,7 @@ export default DS.Model.extend({
     }
 
     var available = 0;
-    resourceInformations = availableResource ? availableResource.resourcesInformations : [];
+    resourceInformations = availableResource ? availableResource.resourceInformations.resourceInformation : [];
     for (i = 0; i < resourceInformations.length; i++) {
       ri = resourceInformations[i];
       if (ri.name === "yarn.io/gpu") {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js

@@ -42,8 +42,8 @@ export default DS.JSONAPISerializer.extend({
         availableVirtualCores: payload.availableVirtualCores,
         version: payload.version,
         nodeLabels: payload.nodeLabels,
-        usedResource: payload.used,
-        availableResource: payload.avail
+        usedResource: payload.usedResource,
+        availableResource: payload.availableResource
       }
     };
     return fixedPayload;