YARN-10175. FS-CS converter: only convert placement rules if a cmd line switch is defined. Contributed by Peter Bacsko

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java

@@ -98,6 +98,10 @@ public class FSConfigToCSConfigArgumentHandler {
         "Disables checking whether a placement rule is terminal to maintain" +
         " backward compatibility with configs that were made before YARN-8967.",
         false),
+    CONVERT_PLACEMENT_RULES("convert placement rules",
+        "m", "convert-placement-rules",
+        "Convert Fair Scheduler placement rules to Capacity" +
+        " Scheduler mapping rules", false),
     HELP("help", "h", "help", "Displays the list of options", false);
 
     private final String name;
@@ -216,6 +220,8 @@ public class FSConfigToCSConfigArgumentHandler {
         cliParser.getOptionValue(CliOption.CONVERSION_RULES.shortSwitch);
     String outputDir =
         cliParser.getOptionValue(CliOption.OUTPUT_DIR.shortSwitch);
+    boolean convertPlacementRules =
+        cliParser.hasOption(CliOption.CONVERT_PLACEMENT_RULES.shortSwitch);
 
     checkFile(CliOption.YARN_SITE, yarnSiteXmlFile);
     checkFile(CliOption.FAIR_SCHEDULER, fairSchedulerXmlFile);
@@ -231,6 +237,7 @@ public class FSConfigToCSConfigArgumentHandler {
             cliParser.getOptionValue(CliOption.CLUSTER_RESOURCE.shortSwitch))
         .withConsole(cliParser.hasOption(CliOption.CONSOLE_MODE.shortSwitch))
         .withOutputDirectory(outputDir)
+        .withConvertPlacementRules(convertPlacementRules)
         .build();
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java

@@ -91,10 +91,12 @@ public class FSConfigToCSConfigConverter {
   private Configuration convertedYarnSiteConfig;
   private Configuration capacitySchedulerConfig;
   private FSConfigToCSConfigRuleHandler ruleHandler;
+  private QueuePlacementConverter placementConverter;
 
   private OutputStream yarnSiteOutputStream;
   private OutputStream capacitySchedulerOutputStream;
   private boolean consoleMode = false;
+  private boolean convertPlacementRules = false;
 
   public FSConfigToCSConfigConverter(FSConfigToCSConfigRuleHandler
       ruleHandler, ConversionOptions conversionOptions) {
@@ -102,6 +104,7 @@ public class FSConfigToCSConfigConverter {
     this.conversionOptions = conversionOptions;
     this.yarnSiteOutputStream = System.out;
     this.capacitySchedulerOutputStream = System.out;
+    this.placementConverter = new QueuePlacementConverter();
   }
 
   public void convert(FSConfigToCSConfigConverterParams params)
@@ -113,6 +116,8 @@ public class FSConfigToCSConfigConverter {
     handleFairSchedulerConfig(params, inputYarnSiteConfig);
 
     this.clusterResource = getClusterResource(params);
+    this.convertPlacementRules = params.isConvertPlacementRules();
+
     convert(inputYarnSiteConfig);
   }
 
@@ -309,16 +314,19 @@ public class FSConfigToCSConfigConverter {
     queueConverter.convertQueueHierarchy(rootQueue);
     emitACLs(fs);
 
-    PlacementManager placementManager =
-        fs.getRMContext().getQueuePlacementManager();
+    if (convertPlacementRules) {
+      LOG.info("Converting placement rules");
+      PlacementManager placementManager =
+          fs.getRMContext().getQueuePlacementManager();
 
-    if (placementManager.getPlacementRules().size() > 0) {
-      QueuePlacementConverter placementConverter =
-          new QueuePlacementConverter();
-      Map<String, String> properties =
-          placementConverter.convertPlacementPolicy(placementManager,
-              ruleHandler, userAsDefaultQueue);
-      properties.forEach((k, v) -> capacitySchedulerConfig.set(k, v));
+      if (placementManager.getPlacementRules().size() > 0) {
+        Map<String, String> properties =
+            placementConverter.convertPlacementPolicy(placementManager,
+                ruleHandler, userAsDefaultQueue);
+        properties.forEach((k, v) -> capacitySchedulerConfig.set(k, v));
+      }
+    } else {
+      LOG.info("Ignoring the conversion of placement rules");
     }
   }
 
@@ -432,6 +440,15 @@ public class FSConfigToCSConfigConverter {
     return convertedYarnSiteConfig;
   }
 
+  @VisibleForTesting
+  void setConvertPlacementRules(boolean convertPlacementRules) {
+    this.convertPlacementRules = convertPlacementRules;
+  }
+
+  @VisibleForTesting
+  void setPlacementConverter(QueuePlacementConverter converter) {
+    this.placementConverter = converter;
+  }
   /*
    * Determines whether <queuePlacementPolicy> is present
    * in the allocation file or not.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverterParams.java

@@ -27,6 +27,9 @@ public final class FSConfigToCSConfigConverterParams {
   private boolean console;
   private String clusterResource;
   private String outputDirectory;
+  private boolean convertPlacementRules;
+
+
 
   private FSConfigToCSConfigConverterParams() {
     //must use builder
@@ -56,6 +59,10 @@ public final class FSConfigToCSConfigConverterParams {
     return outputDirectory;
   }
 
+  public boolean isConvertPlacementRules() {
+    return convertPlacementRules;
+  }
+
   @Override
   public String toString() {
     return "FSConfigToCSConfigConverterParams{" +
@@ -63,7 +70,8 @@ public final class FSConfigToCSConfigConverterParams {
         ", fairSchedulerXmlConfig='" + fairSchedulerXmlConfig + '\'' +
         ", conversionRulesConfig='" + conversionRulesConfig + '\'' +
         ", clusterResource='" + clusterResource + '\'' +
-        ", console=" + console +
+        ", console=" + console + '\'' +
+        ", convertPlacementRules=" + convertPlacementRules +
         '}';
   }
 
@@ -78,6 +86,7 @@ public final class FSConfigToCSConfigConverterParams {
     private boolean console;
     private String clusterResource;
     private String outputDirectory;
+    private boolean convertPlacementRules;
 
     private Builder() {
     }
@@ -116,6 +125,11 @@ public final class FSConfigToCSConfigConverterParams {
       return this;
     }
 
+    public Builder withConvertPlacementRules(boolean convertPlacementRules) {
+      this.convertPlacementRules = convertPlacementRules;
+      return this;
+    }
+
     public FSConfigToCSConfigConverterParams build() {
       FSConfigToCSConfigConverterParams params =
           new FSConfigToCSConfigConverterParams();
@@ -125,6 +139,7 @@ public final class FSConfigToCSConfigConverterParams {
       params.yarnSiteXmlConfig = this.yarnSiteXmlConfig;
       params.conversionRulesConfig = this.conversionRulesConfig;
       params.outputDirectory = this.outputDirectory;
+      params.convertPlacementRules = this.convertPlacementRules;
       return params;
     }
   }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java

@@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.verify;
 
 import java.io.File;
 import java.io.IOException;
@@ -555,4 +556,46 @@ public class TestFSConfigToCSConfigArgumentHandler {
     testFileExistsInOutputFolder(
         YarnConfiguration.CS_CONFIGURATION_FILE);
   }
+
+  @Test
+  public void testPlacementRulesConversionEnabled() throws Exception {
+    testPlacementRuleConversion(true);
+  }
+
+  @Test
+  public void testPlacementRulesConversionDisabled() throws Exception {
+    testPlacementRuleConversion(false);
+  }
+
+  private void testPlacementRuleConversion(boolean enabled) throws Exception {
+    setupFSConfigConversionFiles(true);
+
+    String[] args = null;
+    if (enabled) {
+      args = getArgumentsAsArrayWithDefaults("-f",
+          FSConfigConverterTestCommons.FS_ALLOC_FILE,
+          "-p", "-m");
+    } else {
+      args = getArgumentsAsArrayWithDefaults("-f",
+          FSConfigConverterTestCommons.FS_ALLOC_FILE,
+          "-p");
+    }
+    FSConfigToCSConfigArgumentHandler argumentHandler =
+        new FSConfigToCSConfigArgumentHandler(conversionOptions);
+    argumentHandler.setConverterSupplier(this::getMockConverter);
+
+    argumentHandler.parseAndConvert(args);
+
+    ArgumentCaptor<FSConfigToCSConfigConverterParams> captor =
+        ArgumentCaptor.forClass(FSConfigToCSConfigConverterParams.class);
+    verify(mockConverter).convert(captor.capture());
+    FSConfigToCSConfigConverterParams params = captor.getValue();
+
+    if (enabled) {
+      assertTrue("-m switch had no effect", params.isConvertPlacementRules());
+    } else {
+      assertFalse("Placement rule conversion was enabled",
+          params.isConvertPlacementRules());
+    }
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java

@@ -31,6 +31,9 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyZeroInteractions;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
@@ -43,6 +46,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.service.ServiceStateException;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
 import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
@@ -86,6 +90,9 @@ public class TestFSConfigToCSConfigConverter {
   @Mock
   private DryRunResultHolder dryRunResultHolder;
 
+  @Mock
+  private QueuePlacementConverter placementConverter;
+
   private FSConfigToCSConfigConverter converter;
   private Configuration config;
 
@@ -583,6 +590,7 @@ public class TestFSConfigToCSConfigConverter {
     config.setBoolean(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE,
         userAsDefaultQueue);
 
+    converter.setConvertPlacementRules(true);
     converter.convert(config);
 
     Configuration convertedConf = getConvertedCSConfig();
@@ -650,6 +658,37 @@ public class TestFSConfigToCSConfigConverter {
     }
   }
 
+  @Test
+  public void testPlacementRulesConversionDisabled() throws Exception {
+    FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder()
+        .withClusterResource(CLUSTER_RESOURCE_STRING)
+        .withFairSchedulerXmlConfig(FAIR_SCHEDULER_XML)
+        .withConvertPlacementRules(false)
+        .build();
+
+    converter.setPlacementConverter(placementConverter);
+    converter.convert(params);
+
+    verifyZeroInteractions(placementConverter);
+  }
+
+  @Test
+  public void testPlacementRulesConversionEnabled() throws Exception {
+    FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder()
+        .withClusterResource(CLUSTER_RESOURCE_STRING)
+        .withFairSchedulerXmlConfig(FAIR_SCHEDULER_XML)
+        .withConvertPlacementRules(true)
+        .build();
+
+    converter.setPlacementConverter(placementConverter);
+    converter.convert(params);
+
+    verify(placementConverter).convertPlacementPolicy(
+        any(PlacementManager.class),
+        any(FSConfigToCSConfigRuleHandler.class),
+        any(Boolean.class));
+  }
+
   private Configuration getConvertedCSConfig() {
     ByteArrayInputStream input =
         new ByteArrayInputStream(csConfigOut.toByteArray());