Browse Source

YARN-10891. Extend QueueInfo with max-parallel-apps in CS. (#3314)

Co-authored-by: Tamas Domok <tdomok@cloudera.com>
Tamas Domok 3 years ago
parent
commit
16e6030e25
15 changed files with 88 additions and 16 deletions
  1. 19 5
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java
  2. 1 0
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
  3. 2 0
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/QueueCLI.java
  4. 1 1
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
  5. 5 3
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
  6. 12 0
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueInfoPBImpl.java
  7. 1 1
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
  8. 2 0
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
  9. 6 0
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
  10. 6 0
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
  11. 1 1
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java
  12. 1 1
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java
  13. 1 1
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
  14. 11 3
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
  15. 19 0
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md

+ 19 - 5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java

@@ -60,7 +60,8 @@ public abstract class QueueInfo {
       List<QueueInfo> childQueues, List<ApplicationReport> applications,
       List<QueueInfo> childQueues, List<ApplicationReport> applications,
       QueueState queueState, Set<String> accessibleNodeLabels,
       QueueState queueState, Set<String> accessibleNodeLabels,
       String defaultNodeLabelExpression, QueueStatistics queueStatistics,
       String defaultNodeLabelExpression, QueueStatistics queueStatistics,
-      boolean preemptionDisabled, float weight) {
+      boolean preemptionDisabled, float weight,
+      int maxParallelApps) {
     QueueInfo queueInfo = Records.newRecord(QueueInfo.class);
     QueueInfo queueInfo = Records.newRecord(QueueInfo.class);
     queueInfo.setQueueName(queueName);
     queueInfo.setQueueName(queueName);
     queueInfo.setQueuePath(queuePath);
     queueInfo.setQueuePath(queuePath);
@@ -75,6 +76,7 @@ public abstract class QueueInfo {
     queueInfo.setQueueStatistics(queueStatistics);
     queueInfo.setQueueStatistics(queueStatistics);
     queueInfo.setPreemptionDisabled(preemptionDisabled);
     queueInfo.setPreemptionDisabled(preemptionDisabled);
     queueInfo.setWeight(weight);
     queueInfo.setWeight(weight);
+    queueInfo.setMaxParallelApps(maxParallelApps);
     return queueInfo;
     return queueInfo;
   }
   }
 
 
@@ -86,14 +88,14 @@ public abstract class QueueInfo {
       List<QueueInfo> childQueues, List<ApplicationReport> applications,
       List<QueueInfo> childQueues, List<ApplicationReport> applications,
       QueueState queueState, Set<String> accessibleNodeLabels,
       QueueState queueState, Set<String> accessibleNodeLabels,
       String defaultNodeLabelExpression, QueueStatistics queueStatistics,
       String defaultNodeLabelExpression, QueueStatistics queueStatistics,
-      boolean preemptionDisabled, float weight,
+      boolean preemptionDisabled, float weight, int maxParallelApps,
       Map<String, QueueConfigurations> queueConfigurations) {
       Map<String, QueueConfigurations> queueConfigurations) {
     QueueInfo queueInfo = QueueInfo.newInstance(queueName, queuePath, capacity,
     QueueInfo queueInfo = QueueInfo.newInstance(queueName, queuePath, capacity,
         maximumCapacity, currentCapacity,
         maximumCapacity, currentCapacity,
         childQueues, applications,
         childQueues, applications,
         queueState, accessibleNodeLabels,
         queueState, accessibleNodeLabels,
         defaultNodeLabelExpression, queueStatistics,
         defaultNodeLabelExpression, queueStatistics,
-        preemptionDisabled, weight);
+        preemptionDisabled, weight, maxParallelApps);
     queueInfo.setQueueConfigurations(queueConfigurations);
     queueInfo.setQueueConfigurations(queueConfigurations);
     return queueInfo;
     return queueInfo;
   }
   }
@@ -106,7 +108,7 @@ public abstract class QueueInfo {
       List<QueueInfo> childQueues, List<ApplicationReport> applications,
       List<QueueInfo> childQueues, List<ApplicationReport> applications,
       QueueState queueState, Set<String> accessibleNodeLabels,
       QueueState queueState, Set<String> accessibleNodeLabels,
       String defaultNodeLabelExpression, QueueStatistics queueStatistics,
       String defaultNodeLabelExpression, QueueStatistics queueStatistics,
-      boolean preemptionDisabled, float weight,
+      boolean preemptionDisabled, float weight, int maxParallelApps,
       Map<String, QueueConfigurations> queueConfigurations,
       Map<String, QueueConfigurations> queueConfigurations,
       boolean intraQueuePreemptionDisabled) {
       boolean intraQueuePreemptionDisabled) {
     QueueInfo queueInfo = QueueInfo.newInstance(queueName, queuePath, capacity,
     QueueInfo queueInfo = QueueInfo.newInstance(queueName, queuePath, capacity,
@@ -114,7 +116,7 @@ public abstract class QueueInfo {
         childQueues, applications,
         childQueues, applications,
         queueState, accessibleNodeLabels,
         queueState, accessibleNodeLabels,
         defaultNodeLabelExpression, queueStatistics,
         defaultNodeLabelExpression, queueStatistics,
-        preemptionDisabled, weight, queueConfigurations);
+        preemptionDisabled, weight, maxParallelApps, queueConfigurations);
     queueInfo.setIntraQueuePreemptionDisabled(intraQueuePreemptionDisabled);
     queueInfo.setIntraQueuePreemptionDisabled(intraQueuePreemptionDisabled);
     return queueInfo;
     return queueInfo;
   }
   }
@@ -166,6 +168,18 @@ public abstract class QueueInfo {
   @Private
   @Private
   @Unstable
   @Unstable
   public abstract void setWeight(float weight);
   public abstract void setWeight(float weight);
+
+  /**
+   * Get the <em>configured max parallel apps</em> of the queue.
+   * @return <em>configured max parallel apps</em> of the queue
+   */
+  @Public
+  @Stable
+  public abstract int getMaxParallelApps();
+
+  @Private
+  @Unstable
+  public abstract void setMaxParallelApps(int maxParallelApps);
   
   
   /**
   /**
    * Get the <em>maximum capacity</em> of the queue.
    * Get the <em>maximum capacity</em> of the queue.

+ 1 - 0
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto

@@ -617,6 +617,7 @@ message QueueInfoProto {
   optional bool intraQueuePreemptionDisabled = 13;
   optional bool intraQueuePreemptionDisabled = 13;
   optional float weight = 14;
   optional float weight = 14;
   optional string queuePath = 15;
   optional string queuePath = 15;
+  optional int32 maxParallelApps = 16;
 }
 }
 
 
 message QueueConfigurationsProto {
 message QueueConfigurationsProto {

+ 2 - 0
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/QueueCLI.java

@@ -139,6 +139,8 @@ public class QueueCLI extends YarnCLI {
     writer.println(df.format(queueInfo.getMaximumCapacity() * 100) + "%");
     writer.println(df.format(queueInfo.getMaximumCapacity() * 100) + "%");
     writer.print("\tWeight : ");
     writer.print("\tWeight : ");
     writer.println(df.format(queueInfo.getWeight()));
     writer.println(df.format(queueInfo.getWeight()));
+    writer.print("\tMaximum Parallel Apps : ");
+    writer.println(queueInfo.getMaxParallelApps());
     writer.print("\tDefault Node Label expression : ");
     writer.print("\tDefault Node Label expression : ");
     String nodeLabelExpression = queueInfo.getDefaultNodeLabelExpression();
     String nodeLabelExpression = queueInfo.getDefaultNodeLabelExpression();
     nodeLabelExpression =
     nodeLabelExpression =

+ 1 - 1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java

@@ -670,7 +670,7 @@ public abstract class ProtocolHATestBase extends ClientBaseWithFixes {
     public QueueInfo createFakeQueueInfo() {
     public QueueInfo createFakeQueueInfo() {
       return QueueInfo.newInstance("root", "root", 100f, 100f, 50f, null,
       return QueueInfo.newInstance("root", "root", 100f, 100f, 50f, null,
           createFakeAppReports(), QueueState.RUNNING, null,
           createFakeAppReports(), QueueState.RUNNING, null,
-          null, null, false, -1.0f,
+          null, null, false, -1.0f, 10,
            null, false);
            null, false);
     }
     }
 
 

+ 5 - 3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java

@@ -1722,8 +1722,8 @@ public class TestYarnCLI {
         newInstance("queueA", "root.queueA",
         newInstance("queueA", "root.queueA",
             0.4f, 0.8f, 0.5f,
             0.4f, 0.8f, 0.5f,
         null, null, QueueState.RUNNING, nodeLabels,
         null, null, QueueState.RUNNING, nodeLabels,
-        "GPU", null, false, -1.0f, null,
-        false);
+        "GPU", null, false, -1.0f, 10,
+            null, false);
     when(client.getQueueInfo(any(String.class))).thenReturn(queueInfo);
     when(client.getQueueInfo(any(String.class))).thenReturn(queueInfo);
     int result = cli.run(new String[] { "-status", "queueA" });
     int result = cli.run(new String[] { "-status", "queueA" });
     assertEquals(0, result);
     assertEquals(0, result);
@@ -1738,6 +1738,7 @@ public class TestYarnCLI {
     pw.println("\tCurrent Capacity : " + "50.00%");
     pw.println("\tCurrent Capacity : " + "50.00%");
     pw.println("\tMaximum Capacity : " + "80.00%");
     pw.println("\tMaximum Capacity : " + "80.00%");
     pw.println("\tWeight : " + "-1.00");
     pw.println("\tWeight : " + "-1.00");
+    pw.println("\tMaximum Parallel Apps : " + "10");
     pw.println("\tDefault Node Label expression : " + "GPU");
     pw.println("\tDefault Node Label expression : " + "GPU");
     pw.println("\tAccessible Node Labels : " + "JDK_7,GPU");
     pw.println("\tAccessible Node Labels : " + "JDK_7,GPU");
     pw.println("\tPreemption : " + "enabled");
     pw.println("\tPreemption : " + "enabled");
@@ -1895,7 +1896,7 @@ public class TestYarnCLI {
         newInstance("queueA", "root.queueA",
         newInstance("queueA", "root.queueA",
             0.4f, 0.8f, 0.5f,
             0.4f, 0.8f, 0.5f,
         null, null, QueueState.RUNNING, null, null, null,
         null, null, QueueState.RUNNING, null, null, null,
-        true, -1.0f, null, true);
+        true, -1.0f, 10, null, true);
     when(client.getQueueInfo(any(String.class))).thenReturn(queueInfo);
     when(client.getQueueInfo(any(String.class))).thenReturn(queueInfo);
     int result = cli.run(new String[] { "-status", "queueA" });
     int result = cli.run(new String[] { "-status", "queueA" });
     assertEquals(0, result);
     assertEquals(0, result);
@@ -1910,6 +1911,7 @@ public class TestYarnCLI {
     pw.println("\tCurrent Capacity : " + "50.00%");
     pw.println("\tCurrent Capacity : " + "50.00%");
     pw.println("\tMaximum Capacity : " + "80.00%");
     pw.println("\tMaximum Capacity : " + "80.00%");
     pw.println("\tWeight : " + "-1.00");
     pw.println("\tWeight : " + "-1.00");
+    pw.println("\tMaximum Parallel Apps : " + "10");
     pw.println("\tDefault Node Label expression : "
     pw.println("\tDefault Node Label expression : "
         + NodeLabel.DEFAULT_NODE_LABEL_PARTITION);
         + NodeLabel.DEFAULT_NODE_LABEL_PARTITION);
     pw.println("\tAccessible Node Labels : ");
     pw.println("\tAccessible Node Labels : ");

+ 12 - 0
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueInfoPBImpl.java

@@ -142,6 +142,18 @@ public class QueueInfoPBImpl extends QueueInfo {
     builder.setWeight(weight);
     builder.setWeight(weight);
   }
   }
 
 
+  @Override
+  public int getMaxParallelApps() {
+    QueueInfoProtoOrBuilder p = viaProto ? proto : builder;
+    return (p.hasMaxParallelApps()) ? p.getMaxParallelApps() : -1;
+  }
+
+  @Override
+  public void setMaxParallelApps(int weight) {
+    maybeInitBuilder();
+    builder.setMaxParallelApps(weight);
+  }
+
   @Override
   @Override
   public void setChildQueues(List<QueueInfo> childQueues) {
   public void setChildQueues(List<QueueInfo> childQueues) {
     if (childQueues == null) {
     if (childQueues == null) {

+ 1 - 1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java

@@ -435,7 +435,7 @@ public class TestPBImplRecords extends BasePBImplRecordsTest {
     typeValueCache.put(QueueInfo.class, QueueInfo.
     typeValueCache.put(QueueInfo.class, QueueInfo.
         newInstance("root", "root", 1.0f,
         newInstance("root", "root", 1.0f,
         1.0f, 0.1f, null, null, QueueState.RUNNING, ImmutableSet.of("x", "y"),
         1.0f, 0.1f, null, null, QueueState.RUNNING, ImmutableSet.of("x", "y"),
-        "x && y", null, false, -1.0f, null, false));
+        "x && y", null, false, -1.0f, 10, null, false));
     generateByNewInstance(QueueStatistics.class);
     generateByNewInstance(QueueStatistics.class);
     generateByNewInstance(QueueUserACLInfo.class);
     generateByNewInstance(QueueUserACLInfo.class);
     generateByNewInstance(YarnClusterMetrics.class);
     generateByNewInstance(YarnClusterMetrics.class);

+ 2 - 0
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java

@@ -774,6 +774,7 @@ public abstract class AbstractCSQueue implements CSQueue {
         getIntraQueuePreemptionDisabled());
         getIntraQueuePreemptionDisabled());
     queueInfo.setQueueConfigurations(getQueueConfigurations());
     queueInfo.setQueueConfigurations(getQueueConfigurations());
     queueInfo.setWeight(queueCapacities.getWeight());
     queueInfo.setWeight(queueCapacities.getWeight());
+    queueInfo.setMaxParallelApps(maxParallelApps);
     return queueInfo;
     return queueInfo;
   }
   }
 
 
@@ -1494,6 +1495,7 @@ public abstract class AbstractCSQueue implements CSQueue {
     this.maxParallelApps = maxParallelApps;
     this.maxParallelApps = maxParallelApps;
   }
   }
 
 
+  @Override
   public int getMaxParallelApps() {
   public int getMaxParallelApps() {
     return maxParallelApps;
     return maxParallelApps;
   }
   }

+ 6 - 0
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java

@@ -151,6 +151,12 @@ public interface CSQueue extends SchedulerQueue<CSQueue> {
    * @return current run-state
    * @return current run-state
    */
    */
   public QueueState getState();
   public QueueState getState();
+
+  /**
+   * Get the max-parallel-applications property of the queue
+   * @return max-parallel-applications
+   */
+  public int getMaxParallelApps();
   
   
   /**
   /**
    * Get child queues
    * Get child queues

+ 6 - 0
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java

@@ -65,6 +65,7 @@ public class CapacitySchedulerQueueInfo {
   protected float weight;
   protected float weight;
   protected float normalizedWeight;
   protected float normalizedWeight;
   protected int numApplications;
   protected int numApplications;
+  protected int maxParallelApps;
   protected String queueName;
   protected String queueName;
   protected boolean isAbsoluteResource;
   protected boolean isAbsoluteResource;
   protected QueueState state;
   protected QueueState state;
@@ -120,6 +121,7 @@ public class CapacitySchedulerQueueInfo {
     weight = q.getQueueCapacities().getWeight();
     weight = q.getQueueCapacities().getWeight();
     normalizedWeight = q.getQueueCapacities().getNormalizedWeight();
     normalizedWeight = q.getQueueCapacities().getNormalizedWeight();
     numApplications = q.getNumApplications();
     numApplications = q.getNumApplications();
+    maxParallelApps = q.getMaxParallelApps();
     allocatedContainers = q.getMetrics().getAllocatedContainers();
     allocatedContainers = q.getMetrics().getAllocatedContainers();
     pendingContainers = q.getMetrics().getPendingContainers();
     pendingContainers = q.getMetrics().getPendingContainers();
     reservedContainers = q.getMetrics().getReservedContainers();
     reservedContainers = q.getMetrics().getReservedContainers();
@@ -352,6 +354,10 @@ public class CapacitySchedulerQueueInfo {
     return normalizedWeight;
     return normalizedWeight;
   }
   }
 
 
+  public int getMaxParallelApps() {
+    return maxParallelApps;
+  }
+
   public String getDefaultNodeLabelExpression() {
   public String getDefaultNodeLabelExpression() {
     return defaultNodeLabelExpression;
     return defaultNodeLabelExpression;
   }
   }

+ 1 - 1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java

@@ -70,7 +70,7 @@ public class TestConfigurationMutationACLPolicies {
     QueueInfo queueInfo = QueueInfo.
     QueueInfo queueInfo = QueueInfo.
         newInstance(queueName, queuePath, 0, 0,
         newInstance(queueName, queuePath, 0, 0,
             0, null, null,
             0, null, null,
-        null, null, null, null, false, -1.0f, null, false);
+        null, null, null, null, false, -1.0f, 10, null, false);
     when(confScheduler.getQueueInfo(eq(queueName), anyBoolean(), anyBoolean()))
     when(confScheduler.getQueueInfo(eq(queueName), anyBoolean(), anyBoolean()))
         .thenReturn(queueInfo);
         .thenReturn(queueInfo);
     Queue queue = mock(Queue.class);
     Queue queue = mock(Queue.class);

+ 1 - 1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java

@@ -219,7 +219,7 @@ public class TestSchedulerApplicationAttempt {
     QueueMetrics metrics = QueueMetrics.forQueue(name, parent, false, conf);
     QueueMetrics metrics = QueueMetrics.forQueue(name, parent, false, conf);
     QueueInfo queueInfo = QueueInfo.newInstance(name,
     QueueInfo queueInfo = QueueInfo.newInstance(name,
         "root." + name, capacity, 1.0f, 0, null,
         "root." + name, capacity, 1.0f, 0, null,
-        null, QueueState.RUNNING, null, "", null, false, -1.0f, null, false);
+        null, QueueState.RUNNING, null, "", null, false, -1.0f, 10, null, false);
     ActiveUsersManager activeUsersManager = new ActiveUsersManager(metrics);
     ActiveUsersManager activeUsersManager = new ActiveUsersManager(metrics);
     Queue queue = mock(Queue.class);
     Queue queue = mock(Queue.class);
     when(queue.getMetrics()).thenReturn(metrics);
     when(queue.getMetrics()).thenReturn(metrics);

+ 1 - 1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java

@@ -5101,7 +5101,7 @@ public class TestLeafQueue {
     CSQueueMetrics metrics = CSQueueMetrics.forQueue(name, parent, false, cs.getConf());
     CSQueueMetrics metrics = CSQueueMetrics.forQueue(name, parent, false, cs.getConf());
     QueueInfo queueInfo = QueueInfo.
     QueueInfo queueInfo = QueueInfo.
         newInstance(name, path, capacity, 1.0f, 0, null,
         newInstance(name, path, capacity, 1.0f, 0, null,
-        null, QueueState.RUNNING, null, "", null, false, -1.0f, null, false);
+        null, QueueState.RUNNING, null, "", null, false, -1.0f, 10, null, false);
     ActiveUsersManager activeUsersManager = new ActiveUsersManager(metrics);
     ActiveUsersManager activeUsersManager = new ActiveUsersManager(metrics);
     AbstractCSQueue queue = mock(AbstractCSQueue.class);
     AbstractCSQueue queue = mock(AbstractCSQueue.class);
     when(queue.getMetrics()).thenReturn(metrics);
     when(queue.getMetrics()).thenReturn(metrics);

+ 11 - 3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java

@@ -18,6 +18,7 @@
 
 
 package org.apache.hadoop.yarn.server.resourcemanager.webapp;
 package org.apache.hadoop.yarn.server.resourcemanager.webapp;
 
 
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAX_PARALLEL_APPLICATIONS;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertFalse;
@@ -83,6 +84,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
     float absoluteMaxCapacity;
     float absoluteMaxCapacity;
     float absoluteUsedCapacity;
     float absoluteUsedCapacity;
     int numApplications;
     int numApplications;
+    int maxParallelApps;
     String queueName;
     String queueName;
     private String queuePath;
     private String queuePath;
     String state;
     String state;
@@ -140,6 +142,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
     final String A = CapacitySchedulerConfiguration.ROOT + ".a";
     final String A = CapacitySchedulerConfiguration.ROOT + ".a";
     config.setCapacity(A, 10.5f);
     config.setCapacity(A, 10.5f);
     config.setMaximumCapacity(A, 50);
     config.setMaximumCapacity(A, 50);
+    config.setInt(CapacitySchedulerConfiguration.getQueuePrefix(A) + MAX_PARALLEL_APPLICATIONS, 42);
 
 
     final String B = CapacitySchedulerConfiguration.ROOT + ".b";
     final String B = CapacitySchedulerConfiguration.ROOT + ".b";
     config.setCapacity(B, 89.5f);
     config.setCapacity(B, 89.5f);
@@ -311,6 +314,8 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
       WebServicesTestUtils.getXmlFloat(qElem, "absoluteUsedCapacity");
       WebServicesTestUtils.getXmlFloat(qElem, "absoluteUsedCapacity");
     qi.numApplications =
     qi.numApplications =
         WebServicesTestUtils.getXmlInt(qElem, "numApplications");
         WebServicesTestUtils.getXmlInt(qElem, "numApplications");
+    qi.maxParallelApps =
+        WebServicesTestUtils.getXmlInt(qElem, "maxParallelApps");
     qi.queueName = WebServicesTestUtils.getXmlString(qElem, "queueName");
     qi.queueName = WebServicesTestUtils.getXmlString(qElem, "queueName");
     qi.queuePath = WebServicesTestUtils.getXmlString(qElem, "queuePath");
     qi.queuePath = WebServicesTestUtils.getXmlString(qElem, "queuePath");
     qi.state = WebServicesTestUtils.getXmlString(qElem, "state");
     qi.state = WebServicesTestUtils.getXmlString(qElem, "state");
@@ -424,10 +429,10 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
   private void verifySubQueue(JSONObject info, String q,
   private void verifySubQueue(JSONObject info, String q,
       float parentAbsCapacity, float parentAbsMaxCapacity)
       float parentAbsCapacity, float parentAbsMaxCapacity)
       throws JSONException, Exception {
       throws JSONException, Exception {
-    int numExpectedElements = 37;
+    int numExpectedElements = 38;
     boolean isParentQueue = true;
     boolean isParentQueue = true;
     if (!info.has("queues")) {
     if (!info.has("queues")) {
-      numExpectedElements = 55;
+      numExpectedElements = 56;
       isParentQueue = false;
       isParentQueue = false;
     }
     }
     assertEquals("incorrect number of elements", numExpectedElements, info.length());
     assertEquals("incorrect number of elements", numExpectedElements, info.length());
@@ -440,6 +445,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
     qi.absoluteMaxCapacity = (float) info.getDouble("absoluteMaxCapacity");
     qi.absoluteMaxCapacity = (float) info.getDouble("absoluteMaxCapacity");
     qi.absoluteUsedCapacity = (float) info.getDouble("absoluteUsedCapacity");
     qi.absoluteUsedCapacity = (float) info.getDouble("absoluteUsedCapacity");
     qi.numApplications = info.getInt("numApplications");
     qi.numApplications = info.getInt("numApplications");
+    qi.maxParallelApps = info.getInt("maxParallelApps");
     qi.queueName = info.getString("queueName");
     qi.queueName = info.getString("queueName");
     qi.queuePath = info.getString("queuePath");
     qi.queuePath = info.getString("queuePath");
     qi.state = info.getString("state");
     qi.state = info.getString("state");
@@ -526,7 +532,9 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
           + " queue is not configured in Absolute resource",
           + " queue is not configured in Absolute resource",
           info.isAbsoluteResource);
           info.isAbsoluteResource);
     }
     }
-
+    assertEquals("maxParallelApps doesn't match " + q,
+        (q.equals("root.a") ? 42 : Integer.MAX_VALUE),
+        info.maxParallelApps);
   }
   }
 
 
   private void verifyLeafQueueGeneric(String q, LeafQueueInfo info)
   private void verifyLeafQueueGeneric(String q, LeafQueueInfo info)

+ 19 - 0
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md

@@ -324,6 +324,7 @@ The capacity scheduler supports hierarchical queues. This one request will print
 | absoluteMaxCapacity | float | Absolute maximum capacity percentage this queue can use of the entire cluster |
 | absoluteMaxCapacity | float | Absolute maximum capacity percentage this queue can use of the entire cluster |
 | absoluteUsedCapacity | float | Absolute used capacity percentage this queue is using of the entire cluster |
 | absoluteUsedCapacity | float | Absolute used capacity percentage this queue is using of the entire cluster |
 | numApplications | int | The number of applications currently in the queue |
 | numApplications | int | The number of applications currently in the queue |
+| maxParallelApps | int | Maximum number of applications that can run at the same time |
 | usedResources | string | A string describing the current resources used by the queue |
 | usedResources | string | A string describing the current resources used by the queue |
 | queueName | string | The name of the queue |
 | queueName | string | The name of the queue |
 | state | string of QueueState | The state of the queue |
 | state | string of QueueState | The state of the queue |
@@ -423,6 +424,7 @@ Response Body:
                         "capacity": 10.5,
                         "capacity": 10.5,
                         "maxCapacity": 50.0,
                         "maxCapacity": 50.0,
                         "numApplications": 0,
                         "numApplications": 0,
+                        "maxParallelApps": 2147483647,
                         "queueName": "a",
                         "queueName": "a",
                         "queues": {
                         "queues": {
                             "queue": [
                             "queue": [
@@ -433,6 +435,7 @@ Response Body:
                                     "capacity": 30.000002,
                                     "capacity": 30.000002,
                                     "maxCapacity": 50.0,
                                     "maxCapacity": 50.0,
                                     "numApplications": 0,
                                     "numApplications": 0,
+                                    "maxParallelApps": 2147483647,
                                     "queueName": "a1",
                                     "queueName": "a1",
                                     "queues": {
                                     "queues": {
                                         "queue": [
                                         "queue": [
@@ -448,6 +451,7 @@ Response Body:
                                                 "maxCapacity": 100.0,
                                                 "maxCapacity": 100.0,
                                                 "numActiveApplications": 0,
                                                 "numActiveApplications": 0,
                                                 "numApplications": 0,
                                                 "numApplications": 0,
+                                                "maxParallelApps": 2147483647,
                                                 "numContainers": 0,
                                                 "numContainers": 0,
                                                 "numPendingApplications": 0,
                                                 "numPendingApplications": 0,
                                                 "queueName": "a1a",
                                                 "queueName": "a1a",
@@ -475,6 +479,7 @@ Response Body:
                                                 "maxCapacity": 100.0,
                                                 "maxCapacity": 100.0,
                                                 "numActiveApplications": 0,
                                                 "numActiveApplications": 0,
                                                 "numApplications": 0,
                                                 "numApplications": 0,
+                                                "maxParallelApps": 2147483647,
                                                 "numContainers": 0,
                                                 "numContainers": 0,
                                                 "numPendingApplications": 0,
                                                 "numPendingApplications": 0,
                                                 "queueName": "a1b",
                                                 "queueName": "a1b",
@@ -512,6 +517,7 @@ Response Body:
                                     "maxCapacity": 100.0,
                                     "maxCapacity": 100.0,
                                     "numActiveApplications": 0,
                                     "numActiveApplications": 0,
                                     "numApplications": 0,
                                     "numApplications": 0,
+                                    "maxParallelApps": 2147483647,
                                     "numContainers": 0,
                                     "numContainers": 0,
                                     "numPendingApplications": 0,
                                     "numPendingApplications": 0,
                                     "queueName": "a2",
                                     "queueName": "a2",
@@ -544,6 +550,7 @@ Response Body:
                         "capacity": 89.5,
                         "capacity": 89.5,
                         "maxCapacity": 100.0,
                         "maxCapacity": 100.0,
                         "numApplications": 2,
                         "numApplications": 2,
+                        "maxParallelApps": 2147483647,
                         "queueName": "b",
                         "queueName": "b",
                         "queues": {
                         "queues": {
                             "queue": [
                             "queue": [
@@ -559,6 +566,7 @@ Response Body:
                                     "maxCapacity": 100.0,
                                     "maxCapacity": 100.0,
                                     "numActiveApplications": 1,
                                     "numActiveApplications": 1,
                                     "numApplications": 2,
                                     "numApplications": 2,
+                                    "maxParallelApps": 2147483647,
                                     "numContainers": 0,
                                     "numContainers": 0,
                                     "numPendingApplications": 1,
                                     "numPendingApplications": 1,
                                     "queueName": "b1",
                                     "queueName": "b1",
@@ -607,6 +615,7 @@ Response Body:
                                     "maxCapacity": 100.0,
                                     "maxCapacity": 100.0,
                                     "numActiveApplications": 0,
                                     "numActiveApplications": 0,
                                     "numApplications": 0,
                                     "numApplications": 0,
+                                    "maxParallelApps": 2147483647,
                                     "numContainers": 0,
                                     "numContainers": 0,
                                     "numPendingApplications": 0,
                                     "numPendingApplications": 0,
                                     "queueName": "b2",
                                     "queueName": "b2",
@@ -634,6 +643,7 @@ Response Body:
                                     "maxCapacity": 100.0,
                                     "maxCapacity": 100.0,
                                     "numActiveApplications": 0,
                                     "numActiveApplications": 0,
                                     "numApplications": 0,
                                     "numApplications": 0,
+                                    "maxParallelApps": 2147483647,
                                     "numContainers": 0,
                                     "numContainers": 0,
                                     "numPendingApplications": 0,
                                     "numPendingApplications": 0,
                                     "queueName": "b3",
                                     "queueName": "b3",
@@ -756,6 +766,7 @@ Response Body:
         <absoluteMaxCapacity>50.0</absoluteMaxCapacity>
         <absoluteMaxCapacity>50.0</absoluteMaxCapacity>
         <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
         <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
         <numApplications>0</numApplications>
         <numApplications>0</numApplications>
+        <maxParallelApps>2147483647</maxParallelApps>
         <usedResources>&lt;memory:0, vCores:0&gt;</usedResources>
         <usedResources>&lt;memory:0, vCores:0&gt;</usedResources>
         <queueName>a</queueName>
         <queueName>a</queueName>
         <state>RUNNING</state>
         <state>RUNNING</state>
@@ -768,6 +779,7 @@ Response Body:
             <absoluteMaxCapacity>25.0</absoluteMaxCapacity>
             <absoluteMaxCapacity>25.0</absoluteMaxCapacity>
             <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
             <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
             <numApplications>0</numApplications>
             <numApplications>0</numApplications>
+            <maxParallelApps>2147483647</maxParallelApps>
             <usedResources>&lt;memory:0, vCores:0&gt;</usedResources>
             <usedResources>&lt;memory:0, vCores:0&gt;</usedResources>
             <queueName>a1</queueName>
             <queueName>a1</queueName>
             <state>RUNNING</state>
             <state>RUNNING</state>
@@ -780,6 +792,7 @@ Response Body:
                 <absoluteMaxCapacity>25.0</absoluteMaxCapacity>
                 <absoluteMaxCapacity>25.0</absoluteMaxCapacity>
                 <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
                 <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
                 <numApplications>0</numApplications>
                 <numApplications>0</numApplications>
+                <maxParallelApps>2147483647</maxParallelApps>
                 <usedResources>&lt;memory:0, vCores:0&gt;</usedResources>
                 <usedResources>&lt;memory:0, vCores:0&gt;</usedResources>
                 <queueName>a1a</queueName>
                 <queueName>a1a</queueName>
                 <state>RUNNING</state>
                 <state>RUNNING</state>
@@ -806,6 +819,7 @@ Response Body:
                 <absoluteMaxCapacity>25.0</absoluteMaxCapacity>
                 <absoluteMaxCapacity>25.0</absoluteMaxCapacity>
                 <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
                 <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
                 <numApplications>0</numApplications>
                 <numApplications>0</numApplications>
+                <maxParallelApps>2147483647</maxParallelApps>
                 <usedResources>&lt;memory:0, vCores:0&gt;</usedResources>
                 <usedResources>&lt;memory:0, vCores:0&gt;</usedResources>
                 <queueName>a1b</queueName>
                 <queueName>a1b</queueName>
                 <state>RUNNING</state>
                 <state>RUNNING</state>
@@ -838,6 +852,7 @@ Response Body:
             <absoluteMaxCapacity>50.0</absoluteMaxCapacity>
             <absoluteMaxCapacity>50.0</absoluteMaxCapacity>
             <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
             <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
             <numApplications>0</numApplications>
             <numApplications>0</numApplications>
+            <maxParallelApps>2147483647</maxParallelApps>
             <usedResources>&lt;memory:0, vCores:0&gt;</usedResources>
             <usedResources>&lt;memory:0, vCores:0&gt;</usedResources>
             <queueName>a2</queueName>
             <queueName>a2</queueName>
             <state>RUNNING</state>
             <state>RUNNING</state>
@@ -870,6 +885,7 @@ Response Body:
         <absoluteMaxCapacity>100.0</absoluteMaxCapacity>
         <absoluteMaxCapacity>100.0</absoluteMaxCapacity>
         <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
         <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
         <numApplications>2</numApplications>
         <numApplications>2</numApplications>
+        <maxParallelApps>2147483647</maxParallelApps>
         <usedResources>&lt;memory:0, vCores:0&gt;</usedResources>
         <usedResources>&lt;memory:0, vCores:0&gt;</usedResources>
         <queueName>b</queueName>
         <queueName>b</queueName>
         <state>RUNNING</state>
         <state>RUNNING</state>
@@ -882,6 +898,7 @@ Response Body:
             <absoluteMaxCapacity>100.0</absoluteMaxCapacity>
             <absoluteMaxCapacity>100.0</absoluteMaxCapacity>
             <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
             <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
             <numApplications>2</numApplications>
             <numApplications>2</numApplications>
+            <maxParallelApps>2147483647</maxParallelApps>
             <usedResources>&lt;memory:0, vCores:0&gt;</usedResources>
             <usedResources>&lt;memory:0, vCores:0&gt;</usedResources>
             <queueName>b1</queueName>
             <queueName>b1</queueName>
             <state>RUNNING</state>
             <state>RUNNING</state>
@@ -927,6 +944,7 @@ Response Body:
             <absoluteMaxCapacity>100.0</absoluteMaxCapacity>
             <absoluteMaxCapacity>100.0</absoluteMaxCapacity>
             <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
             <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
             <numApplications>0</numApplications>
             <numApplications>0</numApplications>
+            <maxParallelApps>2147483647</maxParallelApps>
             <usedResources>&lt;memory:0, vCores:0&gt;</usedResources>
             <usedResources>&lt;memory:0, vCores:0&gt;</usedResources>
             <queueName>b2</queueName>
             <queueName>b2</queueName>
             <state>RUNNING</state>
             <state>RUNNING</state>
@@ -953,6 +971,7 @@ Response Body:
             <absoluteMaxCapacity>100.0</absoluteMaxCapacity>
             <absoluteMaxCapacity>100.0</absoluteMaxCapacity>
             <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
             <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
             <numApplications>0</numApplications>
             <numApplications>0</numApplications>
+            <maxParallelApps>2147483647</maxParallelApps>
             <usedResources>&lt;memory:0, vCores:0&gt;</usedResources>
             <usedResources>&lt;memory:0, vCores:0&gt;</usedResources>
             <queueName>b3</queueName>
             <queueName>b3</queueName>
             <state>RUNNING</state>
             <state>RUNNING</state>