Support max queues limit configuration in new auto created queue, consistent with old auto created. Contributed by Qi Zhu

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java

@@ -2010,11 +2010,19 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
 
   @Private
   private static final String AUTO_QUEUE_CREATION_V2_PREFIX =
-      "auto-queue-creation-v2";
+      "auto-queue-creation-v2.";
 
   @Private
   public static final String AUTO_QUEUE_CREATION_V2_ENABLED =
-      AUTO_QUEUE_CREATION_V2_PREFIX + ".enabled";
+      AUTO_QUEUE_CREATION_V2_PREFIX + "enabled";
+
+  @Private
+  public static final String AUTO_QUEUE_CREATION_V2_MAX_QUEUES =
+      AUTO_QUEUE_CREATION_V2_PREFIX + "max-queues";
+
+  @Private
+  public static final int
+      DEFAULT_AUTO_QUEUE_CREATION_V2_MAX_QUEUES = 1000;
 
   @Private
   public static final boolean DEFAULT_AUTO_QUEUE_CREATION_ENABLED = false;
@@ -2130,6 +2138,28 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
         DEFAULT_AUTO_CREATE_QUEUE_MAX_QUEUES);
   }
 
+  /**
+   * Get the max number of queues that are allowed to be created under
+   * a parent queue which allowed auto creation v2.
+   *
+   * @param queuePath the parent queue's path
+   * @return the max number of queues allowed to be auto created,
+   * in new auto created.
+   */
+  @Private
+  public int getAutoCreatedQueuesV2MaxChildQueuesLimit(String queuePath) {
+    return getInt(getQueuePrefix(queuePath) +
+            AUTO_QUEUE_CREATION_V2_MAX_QUEUES,
+        DEFAULT_AUTO_QUEUE_CREATION_V2_MAX_QUEUES);
+  }
+
+  @VisibleForTesting
+  public void setAutoCreatedQueuesV2MaxChildQueuesLimit(String queuePath,
+      int maxQueues) {
+    setInt(getQueuePrefix(queuePath) +
+        AUTO_QUEUE_CREATION_V2_MAX_QUEUES, maxQueues);
+  }
+
   @Private
   public static final String AUTO_CREATED_QUEUE_MANAGEMENT_POLICY =
       AUTO_CREATE_CHILD_QUEUE_PREFIX + "management-policy";

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java

@@ -543,6 +543,16 @@ public class ParentQueue extends AbstractCSQueue {
         return queue;
       }
 
+      // Check if the max queue limit is exceeded.
+      int maxQueues = csContext.getConfiguration().
+          getAutoCreatedQueuesV2MaxChildQueuesLimit(getQueuePath());
+      if (childQueues.size() >= maxQueues) {
+        throw new SchedulerDynamicEditException(
+            "Cannot auto create queue " + childQueuePath + ". Max Child "
+                + "Queue limit exceeded which is configured as: " + maxQueues
+                + " and number of child queues is: " + childQueues.size());
+      }
+
       // First, check if we allow creation or not
       boolean weightsAreUsed = false;
       try {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java

@@ -544,6 +544,27 @@ public class TestCapacitySchedulerNewQueueAutoCreation
     Assert.assertFalse(bAutoLeafQueue.hasChildQueues());
   }
 
+  @Test
+  public void testAutoCreateQueueMaxQueuesLimit() throws Exception {
+    startScheduler();
+
+    csConf.setAutoCreatedQueuesV2MaxChildQueuesLimit("root.e", 5);
+    cs.reinitialize(csConf, mockRM.getRMContext());
+
+    for (int i = 0; i < 5; ++i) {
+      createQueue("root.e.q_" + i);
+    }
+
+    // Check if max queue limit can't be exceeded
+    try {
+      createQueue("root.e.q_6");
+      Assert.fail("Can't exceed max queue limit.");
+    } catch (Exception ex) {
+      Assert.assertTrue(ex
+          instanceof SchedulerDynamicEditException);
+    }
+  }
+
   private LeafQueue createQueue(String queuePath) throws YarnException {
     return autoQueueHandler.autoCreateQueue(
         CSQueueUtils.extractQueuePath(queuePath));