YARN-7769. FS QueueManager should not create default queue at init. Contributed by Benjamin Teke

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java

@@ -107,10 +107,6 @@ public class QueueManager {
     rootQueue.setDynamic(false);
     queues.put(rootQueue.getName(), rootQueue);
 
-    // Create the default queue
-    FSLeafQueue defaultQueue =
-        getLeafQueue(YarnConfiguration.DEFAULT_QUEUE_NAME, true);
-    defaultQueue.setDynamic(false);
     // Recursively reinitialize to propagate queue properties
     rootQueue.reinit(true);
   }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java

@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -1313,8 +1314,13 @@ public class TestAMRestart extends ParameterizedSchedulerTestBase {
       Assert.assertEquals(0,
           queue.getQueueResourceUsage().getUsed().getVirtualCores());
     } else if (getSchedulerType() == SchedulerType.FAIR) {
-      FSLeafQueue queue = ((FairScheduler) scheduler).getQueueManager()
-          .getLeafQueue("root.default", false);
+      // The default queue is not auto created after YARN-7769 so
+      // user-as-default-queue option is used
+      Collection<FSLeafQueue> queues = ((FairScheduler) scheduler)
+          .getQueueManager().getLeafQueues();
+      Assert.assertEquals(1, queues.size());
+
+      FSLeafQueue queue = queues.iterator().next();
       Assert.assertEquals(0, queue.getResourceUsage().getMemorySize());
       Assert.assertEquals(0, queue.getResourceUsage().getVirtualCores());
     }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java

@@ -125,9 +125,8 @@ public class TestAppRunnability extends FairSchedulerTestBase {
         new AppAttemptAddedSchedulerEvent(createAppAttemptId(1, 1), false);
     scheduler.handle(attemptAddedEvent);
 
-    // Scheduler should have two queues (the default and the one created for
-    // user1)
-    assertEquals(2, scheduler.getQueueManager().getLeafQueues().size());
+    // Scheduler should have one queue (the one created for user1)
+    assertEquals(1, scheduler.getQueueManager().getLeafQueues().size());
 
     // That queue should have one app
     assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1", true)
@@ -174,6 +173,7 @@ public class TestAppRunnability extends FairSchedulerTestBase {
 
     AllocationFileWriter.create()
         .addQueue(new AllocationFileQueue.Builder("jerry").build())
+        .addQueue(new AllocationFileQueue.Builder("default").build())
         .writeToFile(ALLOC_FILE);
 
     // Restarting resource manager since the file location and content is

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java

@@ -147,7 +147,7 @@ public class TestFSLeafQueue extends FairSchedulerTestBase {
     createSchedulingRequest(1 * 1024, "queueB", "user1");
     scheduler.update();
     Collection<FSLeafQueue> queues = scheduler.getQueueManager().getLeafQueues();
-    assertEquals(3, queues.size());
+    assertEquals(2, queues.size());
   }
 
   @Test

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java

@@ -1074,22 +1074,22 @@ public class TestFairScheduler extends FairSchedulerTestBase {
 
     QueueManager queueManager = scheduler.getQueueManager();
     FSLeafQueue leafQueue = queueManager.getLeafQueue("parent.child", true);
-    Assert.assertEquals(2, queueManager.getLeafQueues().size());
+    Assert.assertEquals(1, queueManager.getLeafQueues().size());
     Assert.assertNotNull(leafQueue);
     Assert.assertEquals("root.parent.child", leafQueue.getName());
 
     FSLeafQueue leafQueue2 = queueManager.getLeafQueue("parent", true);
     Assert.assertNull(leafQueue2);
-    Assert.assertEquals(2, queueManager.getLeafQueues().size());
+    Assert.assertEquals(1, queueManager.getLeafQueues().size());
     
     FSLeafQueue leafQueue3 = queueManager.getLeafQueue("parent.child.grandchild", true);
     Assert.assertNull(leafQueue3);
-    Assert.assertEquals(2, queueManager.getLeafQueues().size());
+    Assert.assertEquals(1, queueManager.getLeafQueues().size());
     
     FSLeafQueue leafQueue4 = queueManager.getLeafQueue("parent.sister", true);
     Assert.assertNotNull(leafQueue4);
     Assert.assertEquals("root.parent.sister", leafQueue4.getName());
-    Assert.assertEquals(3, queueManager.getLeafQueues().size());
+    Assert.assertEquals(2, queueManager.getLeafQueues().size());
   }
 
   @Test
@@ -1771,8 +1771,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
     scheduler.start();
     scheduler.reinitialize(conf, resourceManager.getRMContext());
 
-    // only default queue
-    assertEquals(1, scheduler.getQueueManager().getLeafQueues().size());
+    // no queue by default
+    assertEquals(0, scheduler.getQueueManager().getLeafQueues().size());
 
     // Submit app with empty queue
     // Submit fails before we reach the placement check.
@@ -1783,7 +1783,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
     scheduler.handle(appAddedEvent);
 
     // submission rejected
-    assertEquals(1, scheduler.getQueueManager().getLeafQueues().size());
+    assertEquals(0, scheduler.getQueueManager().getLeafQueues().size());
     assertNull(scheduler.getSchedulerApp(appAttemptId));
     assertEquals(0, resourceManager.getRMContext().getRMApps().size());
   }
@@ -1794,8 +1794,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
     scheduler.start();
     scheduler.reinitialize(conf, resourceManager.getRMContext());
 
-    // only default queue
-    assertEquals(1, scheduler.getQueueManager().getLeafQueues().size());
+    // no queue by default
+    assertEquals(0, scheduler.getQueueManager().getLeafQueues().size());
 
     // Submit app with queue name (.A)
     // Submit fails before we reach the placement check.
@@ -1805,7 +1805,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
             "user1");
     scheduler.handle(appAddedEvent1);
     // submission rejected
-    assertEquals(1, scheduler.getQueueManager().getLeafQueues().size());
+    assertEquals(0, scheduler.getQueueManager().getLeafQueues().size());
     assertNull(scheduler.getSchedulerApp(appAttemptId1));
     assertEquals(0, resourceManager.getRMContext().getRMApps().size());
 
@@ -1817,7 +1817,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
             "user1");
     scheduler.handle(appAddedEvent2);
     // submission rejected
-    assertEquals(1, scheduler.getQueueManager().getLeafQueues().size());
+    assertEquals(0, scheduler.getQueueManager().getLeafQueues().size());
     assertNull(scheduler.getSchedulerApp(appAttemptId2));
     assertEquals(0, resourceManager.getRMContext().getRMApps().size());
 
@@ -1829,7 +1829,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
             "user1", new ApplicationPlacementContext("A.B"));
     scheduler.handle(appAddedEvent3);
     // submission accepted
-    assertEquals(2, scheduler.getQueueManager().getLeafQueues().size());
+    assertEquals(1, scheduler.getQueueManager().getLeafQueues().size());
     assertNull(scheduler.getSchedulerApp(appAttemptId3));
     assertEquals(0, resourceManager.getRMContext().getRMApps().size());
   }
@@ -1863,7 +1863,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
     scheduler.update();
 
     Collection<FSLeafQueue> queues = scheduler.getQueueManager().getLeafQueues();
-    assertEquals(3, queues.size());
+    assertEquals(2, queues.size());
 
     for (FSLeafQueue p : queues) {
       if (p.getName().equals("root.queueA")) {
@@ -1944,6 +1944,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
         .fairDefaultQueueSchedulingPolicy()
         .addQueue(new AllocationFileQueue.Builder("root")
             .schedulingPolicy("drf")
+            .subQueue(new AllocationFileQueue.Builder("default")
+                .weight(1).build())
             .subQueue(new AllocationFileQueue.Builder("child1")
                 .weight(1).build())
             .subQueue(new AllocationFileQueue.Builder("child2")
@@ -1981,6 +1983,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
         .fairDefaultQueueSchedulingPolicy()
         .addQueue(new AllocationFileQueue.Builder("root")
             .schedulingPolicy("drf")
+            .subQueue(new AllocationFileQueue.Builder("default")
+                .weight(1).build())
             .subQueue(new AllocationFileQueue.Builder("child1")
                 .weight(1).build())
             .subQueue(new AllocationFileQueue.Builder("child2")
@@ -2015,6 +2019,17 @@ public class TestFairScheduler extends FairSchedulerTestBase {
     conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
         SimpleGroupsMapping.class, GroupMappingServiceProvider.class);
     conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true");
+    conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+
+    AllocationFileWriter.create()
+        .fairDefaultQueueSchedulingPolicy()
+        .addQueue(new AllocationFileQueue.Builder("root")
+            .schedulingPolicy("drf")
+            .subQueue(new AllocationFileQueue.Builder("default")
+                .weight(1).build())
+            .build())
+        .writeToFile(ALLOC_FILE);
+
     scheduler.init(conf);
     scheduler.start();
     scheduler.reinitialize(conf, resourceManager.getRMContext());
@@ -2129,13 +2144,12 @@ public class TestFairScheduler extends FairSchedulerTestBase {
 
     QueueManager queueManager = scheduler.getQueueManager();
     Collection<FSLeafQueue> leafQueues = queueManager.getLeafQueues();
-    Assert.assertEquals(4, leafQueues.size());
+    Assert.assertEquals(3, leafQueues.size());
     Assert.assertNotNull(queueManager.getLeafQueue("queueA", false));
     Assert.assertNotNull(queueManager.getLeafQueue("queueB.queueC", false));
     Assert.assertNotNull(queueManager.getLeafQueue("queueB.queueD", false));
-    Assert.assertNotNull(queueManager.getLeafQueue("default", false));
     // Make sure querying for queues didn't create any new ones:
-    Assert.assertEquals(4, leafQueues.size());
+    Assert.assertEquals(3, leafQueues.size());
   }
   
   @Test
@@ -4736,8 +4750,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
     scheduler.start();
     scheduler.reinitialize(conf, resourceManager.getRMContext());
 
-    // only default queue
-    assertEquals(1, scheduler.getQueueManager().getLeafQueues().size());
+    // no queue by default
+    assertEquals(0, scheduler.getQueueManager().getLeafQueues().size());
 
     // submit app with queue name "A"
     ApplicationAttemptId appAttemptId1 = createAppAttemptId(1, 1);
@@ -4747,7 +4761,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
         appAttemptId1.getApplicationId(), "A", "user1", apc);
     scheduler.handle(appAddedEvent1);
     // submission accepted
-    assertEquals(2, scheduler.getQueueManager().getLeafQueues().size());
+    assertEquals(1, scheduler.getQueueManager().getLeafQueues().size());
     assertNotNull(scheduler.getSchedulerApplications().get(appAttemptId1.
         getApplicationId()));
 
@@ -4771,7 +4785,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
       // expected ignore: rules should have filtered this out
     }
     // submission rejected
-    assertEquals(2, scheduler.getQueueManager().getLeafQueues().size());
+    assertEquals(1, scheduler.getQueueManager().getLeafQueues().size());
     assertNull(scheduler.getSchedulerApplications().get(appAttemptId2.
         getApplicationId()));
     assertNull(scheduler.getSchedulerApp(appAttemptId2));
@@ -4783,7 +4797,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
         appAttemptId3.getApplicationId(), "B.C", "user1", apc);
     scheduler.handle(appAddedEvent3);
     // submission accepted
-    assertEquals(3, scheduler.getQueueManager().getLeafQueues().size());
+    assertEquals(2, scheduler.getQueueManager().getLeafQueues().size());
     assertNotNull(scheduler.getSchedulerApplications().get(appAttemptId3.
         getApplicationId()));
 
@@ -4806,7 +4820,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
       // expected ignore: rules should have filtered this out
     }
     // submission rejected
-    assertEquals(3, scheduler.getQueueManager().getLeafQueues().size());
+    assertEquals(2, scheduler.getQueueManager().getLeafQueues().size());
     assertNull(scheduler.getSchedulerApplications().get(appAttemptId4.
         getApplicationId()));
     assertNull(scheduler.getSchedulerApp(appAttemptId4));

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerFairShare.java

@@ -68,6 +68,9 @@ public class TestFairSchedulerFairShare extends FairSchedulerTestBase {
       String policy) {
     AllocationFileWriter allocationFileWriter = AllocationFileWriter.create()
         .addQueue(new AllocationFileQueue.Builder("root")
+            .subQueue(new AllocationFileQueue.Builder("default")
+                .weight(1)
+                .build())
             .subQueue(new AllocationFileQueue.Builder("parentA")
                 .weight(8)
                 .subQueue(new AllocationFileQueue.Builder("childA1").build())

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java

@@ -116,10 +116,11 @@ public class TestQueueManager {
     assertNotNull(queueManager.getLeafQueue("queue1.queue2", false));
     assertNull(queueManager.getLeafQueue("queue1", false));
     
-    // Should never to be able to create a queue under the default queue
+    // Since YARN-7769 FS doesn't create the default queue during init, so
+    // it should be possible to create a queue under the root.default queue
     updateConfiguredLeafQueues(queueManager, "default.queue3");
-    assertNull(queueManager.getLeafQueue("default.queue3", false));
-    assertNotNull(queueManager.getLeafQueue("default", false));
+    assertNotNull(queueManager.getLeafQueue("default.queue3", false));
+    assertNull(queueManager.getLeafQueue("default", false));
   }
 
   /**

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java

@@ -151,7 +151,7 @@ public class TestFSQueueConverter {
     converter.convertQueueHierarchy(rootQueue);
 
     // root children
-    assertEquals("root children", "default,admins,users,misc",
+    assertEquals("root children", "admins,users,misc,default",
         csConfig.get(PREFIX + "root.queues"));
 
     // root.admins children

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/TestRMWebServicesFairScheduler.java

@@ -132,7 +132,7 @@ public class TestRMWebServicesFairScheduler extends JerseyTestBase {
     JSONObject json = response.getEntity(JSONObject.class);
     JSONArray subQueueInfo = json.getJSONObject("scheduler")
         .getJSONObject("schedulerInfo").getJSONObject("rootQueue")
-        .getJSONObject("childQueues").getJSONArray("queue").getJSONObject(1)
+        .getJSONObject("childQueues").getJSONArray("queue").getJSONObject(0)
         .getJSONObject("childQueues").getJSONArray("queue");
     // subQueueInfo is consist of subqueue1 and subqueue2 info
     assertEquals(2, subQueueInfo.length());

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/TestRMWebServicesFairSchedulerCustomResourceTypes.java

@@ -220,8 +220,9 @@ public class TestRMWebServicesFairSchedulerCustomResourceTypes
             .getJSONObject("schedulerInfo").getJSONObject("rootQueue")
             .getJSONObject("childQueues").getJSONArray("queue");
 
-        // childQueueInfo consists of subqueue1 and subqueue2 info
-        assertEquals(2, queues.length());
+        assertEquals(1, queues.length());
+
+        // firstChildQueue info contains subqueue1 and subqueue2 info
         JSONObject firstChildQueue = queues.getJSONObject(0);
         new FairSchedulerJsonVerifications(customResourceTypes)
             .verify(firstChildQueue);