YARN-10599. fs2cs should generate new 'auto-queue-creation-v2.enabled' properties for all parents. Contributed by Peter Bacsko

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightToWeightConverter.java

@@ -23,6 +23,7 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.C
 import java.util.List;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSParentQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
 
 public class WeightToWeightConverter
@@ -33,14 +34,21 @@ public class WeightToWeightConverter
       Configuration csConfig) {
     List<FSQueue> children = queue.getChildQueues();
 
-    children.forEach(fsQueue -> csConfig.set(
-        getProperty(fsQueue), getWeightString(fsQueue)));
+    if (queue instanceof FSParentQueue || !children.isEmpty()) {
+      children.forEach(fsQueue -> csConfig.set(
+          getProperty(fsQueue), getWeightString(fsQueue)));
+      csConfig.setBoolean(getAutoCreateV2EnabledProperty(queue), true);
+    }
   }
 
   private String getProperty(FSQueue queue) {
     return PREFIX + queue.getName() + ".capacity";
   }
 
+  private String getAutoCreateV2EnabledProperty(FSQueue queue) {
+    return PREFIX + queue.getName() + ".auto-queue-creation-v2.enabled";
+  }
+
   private String getWeightString(FSQueue queue) {
     return Float.toString(queue.getWeight()) + "w";
   }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java

@@ -334,6 +334,35 @@ public class TestFSQueueConverter {
         csConfig.get(PREFIX + "root.misc.b.capacity"));
   }
 
+  @Test
+  public void testAutoCreateV2FlagsInWeightMode() {
+    converter = builder.withPercentages(false).build();
+
+    converter.convertQueueHierarchy(rootQueue);
+
+    assertTrue("root autocreate v2 flag",
+        csConfig.getBoolean(
+            PREFIX + "root.auto-queue-creation-v2.enabled", false));
+    assertTrue("root.admins autocreate v2 flag",
+        csConfig.getBoolean(
+            PREFIX + "root.admins.auto-queue-creation-v2.enabled", false));
+    assertTrue("root.users autocreate v2 flag",
+        csConfig.getBoolean(
+            PREFIX + "root.users.auto-queue-creation-v2.enabled", false));
+    assertTrue("root.misc autocreate v2 flag",
+        csConfig.getBoolean(
+            PREFIX + "root.misc.auto-queue-creation-v2.enabled", false));
+
+    Set<String> leafs = Sets.difference(ALL_QUEUES,
+        Sets.newHashSet("root",
+            "root.default",
+            "root.admins",
+            "root.users",
+            "root.misc"));
+    assertNoValueForQueues(leafs, "auto-queue-creation-v2.enabled",
+        csConfig);
+  }
+
   @Test
   public void testZeroSumCapacityValidation() {
     converter = builder.withPercentages(true).build();

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/TestWeightToWeightConverter.java

@@ -20,6 +20,9 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.w
 
 import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
@@ -41,7 +44,7 @@ public class TestWeightToWeightConverter extends WeightConverterTestBase {
     FSQueue root = createFSQueues();
     converter.convertWeightsForChildQueues(root, config);
 
-    assertEquals("Converted items", 0,
+    assertEquals("Converted items", 1,
         config.getPropsWithPrefix(PREFIX).size());
   }
 
@@ -52,6 +55,8 @@ public class TestWeightToWeightConverter extends WeightConverterTestBase {
 
     assertEquals("root.a weight", "1.0w",
         config.get(PREFIX + "root.a.capacity"));
+    assertEquals("Number of properties", 2,
+        config.getPropsWithPrefix(PREFIX).size());
   }
 
   @Test
@@ -60,7 +65,7 @@ public class TestWeightToWeightConverter extends WeightConverterTestBase {
 
     converter.convertWeightsForChildQueues(root, config);
 
-    assertEquals("Number of properties", 3,
+    assertEquals("Number of properties", 4,
         config.getPropsWithPrefix(PREFIX).size());
     assertEquals("root.a weight", "1.0w",
         config.get(PREFIX + "root.a.capacity"));
@@ -69,4 +74,26 @@ public class TestWeightToWeightConverter extends WeightConverterTestBase {
     assertEquals("root.c weight", "3.0w",
         config.get(PREFIX + "root.c.capacity"));
   }
+
+  @Test
+  public void testAutoCreateV2FlagOnParent() {
+    FSQueue root = createFSQueues(1);
+    converter.convertWeightsForChildQueues(root, config);
+
+    assertTrue("root autocreate v2 enabled",
+        config.getBoolean(PREFIX + "root.auto-queue-creation-v2.enabled",
+            false));
+  }
+
+  @Test
+  public void testAutoCreateV2FlagOnParentWithoutChildren() {
+    FSQueue root = createParent(new ArrayList<>());
+    converter.convertWeightsForChildQueues(root, config);
+
+    assertEquals("Number of properties", 1,
+        config.getPropsWithPrefix(PREFIX).size());
+    assertTrue("root autocreate v2 enabled",
+        config.getBoolean(PREFIX + "root.auto-queue-creation-v2.enabled",
+            false));
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightConverterTestBase.java

@@ -24,6 +24,7 @@ import static org.mockito.Mockito.when;
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSParentQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
 import org.apache.hadoop.yarn.util.resource.Resources;
 
@@ -44,12 +45,15 @@ public abstract class WeightConverterTestBase {
       queues.add(queue);
     }
 
-    FSQueue root = mock(FSQueue.class);
+    return createParent(queues);
+  }
+
+  protected FSParentQueue createParent(List<FSQueue> children) {
+    FSParentQueue root = mock(FSParentQueue.class);
     when(root.getWeight()).thenReturn(1.0f);
     when(root.getName()).thenReturn("root");
     when(root.getMinShare()).thenReturn(Resources.none());
-    when(root.getChildQueues()).thenReturn(queues);
-
+    when(root.getChildQueues()).thenReturn(children);
     return root;
   }
 }