YARN-589. Expose a REST API for monitoring the fair scheduler (Sandy Ryza).

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1512111 13f79535-47bb-0310-9956-ffa450edef68

hadoop-yarn-project/CHANGES.txt

@@ -38,6 +38,8 @@ Release 2.1.1-beta - UNRELEASED
 
   IMPROVEMENTS
 
+    YARN-589. Expose a REST API for monitoring the fair scheduler (Sandy Ryza).
+
   OPTIMIZATIONS
 
   BUG FIXES

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java

@@ -52,6 +52,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo;
@@ -60,6 +61,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FifoSchedulerInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo;
@@ -145,6 +147,9 @@ public class RMWebServices {
       CapacityScheduler cs = (CapacityScheduler) rs;
       CSQueue root = cs.getRootQueue();
       sinfo = new CapacitySchedulerInfo(root);
+    } else if (rs instanceof FairScheduler) {
+      FairScheduler fs = (FairScheduler) rs;
+      sinfo = new FairSchedulerInfo(fs);
     } else if (rs instanceof FifoScheduler) {
       sinfo = new FifoSchedulerInfo(this.rm);
     } else {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java

@@ -18,14 +18,31 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
 
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+import javax.xml.bind.annotation.XmlTransient;
+import javax.xml.bind.annotation.XmlType;
+
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
 
-public class FairSchedulerInfo {
+@XmlRootElement(name = "fairScheduler")
+@XmlType(name = "fairScheduler")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class FairSchedulerInfo extends SchedulerInfo {
+  private FairSchedulerQueueInfo rootQueue;
+  
+  @XmlTransient
   private FairScheduler scheduler;
   
+  public FairSchedulerInfo() {
+  } // JAXB needs this
+  
   public FairSchedulerInfo(FairScheduler fs) {
     scheduler = fs;
+    rootQueue = new FairSchedulerQueueInfo(scheduler.getQueueManager().
+        getRootQueue(), scheduler);
   }
   
   public int getAppFairShare(ApplicationAttemptId appAttemptId) {
@@ -34,7 +51,6 @@ public class FairSchedulerInfo {
   }
   
   public FairSchedulerQueueInfo getRootQueueInfo() {
-    return new FairSchedulerQueueInfo(scheduler.getQueueManager().
-        getRootQueue(), scheduler);
+    return rootQueue;
   }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java

@@ -20,14 +20,23 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
 
 import java.util.Collection;
 
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AppSchedulable;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue;
 
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
 public class FairSchedulerLeafQueueInfo extends FairSchedulerQueueInfo {
   private int numPendingApps;
   private int numActiveApps;
   
+  public FairSchedulerLeafQueueInfo() {
+  }
+  
   public FairSchedulerLeafQueueInfo(FSLeafQueue queue, FairScheduler scheduler) {
     super(queue, scheduler);
     Collection<AppSchedulable> apps = queue.getAppSchedulables();

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java

@@ -22,6 +22,12 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
 import java.util.ArrayList;
 import java.util.Collection;
 
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+import javax.xml.bind.annotation.XmlSeeAlso;
+import javax.xml.bind.annotation.XmlTransient;
+
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
@@ -29,39 +35,53 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedule
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager;
 import org.apache.hadoop.yarn.util.resource.Resources;
 
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
+@XmlSeeAlso({FairSchedulerLeafQueueInfo.class})
 public class FairSchedulerQueueInfo {  
   private int maxApps;
   
+  @XmlTransient
   private float fractionMemUsed;
+  @XmlTransient
   private float fractionMemFairShare;
+  @XmlTransient
   private float fractionMemMinShare;
+  @XmlTransient
   private float fractionMemMaxShare;
   
-  private Resource minResources;
-  private Resource maxResources;
-  private Resource usedResources;
-  private Resource fairResources;
-  private Resource clusterResources;
+  private ResourceInfo minResources;
+  private ResourceInfo maxResources;
+  private ResourceInfo usedResources;
+  private ResourceInfo fairResources;
+  private ResourceInfo clusterResources;
   
   private String queueName;
+  private String schedulingPolicy;
+  
+  private Collection<FairSchedulerQueueInfo> childQueues;
   
-  private Collection<FairSchedulerQueueInfo> childInfos;
+  public FairSchedulerQueueInfo() {
+  }
   
   public FairSchedulerQueueInfo(FSQueue queue, FairScheduler scheduler) {
     QueueManager manager = scheduler.getQueueManager();
     
     queueName = queue.getName();
-        
-    clusterResources = scheduler.getClusterCapacity();
+    schedulingPolicy = queue.getPolicy().getName();
+    
+    clusterResources = new ResourceInfo(scheduler.getClusterCapacity());
     
-    usedResources = queue.getResourceUsage();
+    usedResources = new ResourceInfo(queue.getResourceUsage());
     fractionMemUsed = (float)usedResources.getMemory() /
         clusterResources.getMemory();
     
-    fairResources = queue.getFairShare();
-    minResources = queue.getMinShare();
-    maxResources = queue.getMaxShare();
-    maxResources = Resources.componentwiseMin(maxResources, clusterResources);
+    fairResources = new ResourceInfo(queue.getFairShare());
+    minResources = new ResourceInfo(queue.getMinShare());
+    maxResources = new ResourceInfo(queue.getMaxShare());
+    maxResources = new ResourceInfo(
+        Resources.componentwiseMin(queue.getMaxShare(),
+            scheduler.getClusterCapacity()));
     
     fractionMemFairShare = (float)fairResources.getMemory() / clusterResources.getMemory();
     fractionMemMinShare = (float)minResources.getMemory() / clusterResources.getMemory();
@@ -69,13 +89,13 @@ public class FairSchedulerQueueInfo {
     
     maxApps = manager.getQueueMaxApps(queueName);
     
-    Collection<FSQueue> childQueues = queue.getChildQueues();
-    childInfos = new ArrayList<FairSchedulerQueueInfo>();
-    for (FSQueue child : childQueues) {
+    Collection<FSQueue> children = queue.getChildQueues();
+    childQueues = new ArrayList<FairSchedulerQueueInfo>();
+    for (FSQueue child : children) {
       if (child instanceof FSLeafQueue) {
-        childInfos.add(new FairSchedulerLeafQueueInfo((FSLeafQueue)child, scheduler));
+        childQueues.add(new FairSchedulerLeafQueueInfo((FSLeafQueue)child, scheduler));
       } else {
-        childInfos.add(new FairSchedulerQueueInfo(child, scheduler));
+        childQueues.add(new FairSchedulerQueueInfo(child, scheduler));
       }
     }
   }
@@ -90,15 +110,15 @@ public class FairSchedulerQueueInfo {
   /**
    * Returns the fair share of this queue in megabytes.
    */
-  public Resource getFairShare() {
+  public ResourceInfo getFairShare() {
     return fairResources;
   }
     
-  public Resource getMinResources() {
+  public ResourceInfo getMinResources() {
     return minResources;
   }
   
-  public Resource getMaxResources() {
+  public ResourceInfo getMaxResources() {
     return maxResources;
   }
   
@@ -110,7 +130,7 @@ public class FairSchedulerQueueInfo {
     return queueName;
   }
   
-  public Resource getUsedResources() {
+  public ResourceInfo getUsedResources() {
     return usedResources;
   }
   
@@ -138,7 +158,14 @@ public class FairSchedulerQueueInfo {
     return fractionMemMaxShare;
   }
   
+  /**
+   * Returns the name of the scheduling policy used by this queue.
+   */
+  public String getSchedulingPolicy() {
+    return schedulingPolicy;
+  }
+  
   public Collection<FairSchedulerQueueInfo> getChildQueues() {
-    return childInfos;
+    return childQueues;
   }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java

@@ -45,4 +45,9 @@ public class ResourceInfo {
   public int getvCores() {
     return vCores;
   }
+  
+  @Override
+  public String toString() {
+    return "<memory:" + memory + ", vCores:" + vCores + ">";
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java

@@ -22,7 +22,8 @@ import javax.xml.bind.annotation.XmlRootElement;
 import javax.xml.bind.annotation.XmlSeeAlso;
 
 @XmlRootElement
-@XmlSeeAlso({ CapacitySchedulerInfo.class, FifoSchedulerInfo.class })
+@XmlSeeAlso({ CapacitySchedulerInfo.class, FairSchedulerInfo.class,
+  FifoSchedulerInfo.class })
 public class SchedulerInfo {
 
   public SchedulerInfo() {