YARN-10625. FairScheduler: add global flag to disable AM-preemption. Contributed by Peter Bacsko

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java

@@ -1442,6 +1442,13 @@ public class FairScheduler extends
             + " ms instead");
       }
 
+      boolean globalAmPreemption = conf.getBoolean(
+          FairSchedulerConfiguration.AM_PREEMPTION,
+          FairSchedulerConfiguration.DEFAULT_AM_PREEMPTION);
+      if (!globalAmPreemption) {
+        LOG.info("AM preemption is DISABLED globally");
+      }
+
       rootMetrics = FSQueueMetrics.forQueue("root", null, true, conf);
       fsOpDurations = FSOpDurations.getInstance(true);
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java

@@ -176,6 +176,8 @@ public class FairSchedulerConfiguration extends Configuration {
   public static final String  PREEMPTION = CONF_PREFIX + "preemption";
   public static final boolean DEFAULT_PREEMPTION = false;
 
+  protected static final String AM_PREEMPTION =
+      CONF_PREFIX + "am.preemption";
   protected static final String AM_PREEMPTION_PREFIX =
           CONF_PREFIX + "am.preemption.";
   protected static final boolean DEFAULT_AM_PREEMPTION = true;
@@ -407,7 +409,17 @@ public class FairSchedulerConfiguration extends Configuration {
   }
 
   public boolean getAMPreemptionEnabled(String queueName) {
-    return getBoolean(AM_PREEMPTION_PREFIX + queueName, DEFAULT_AM_PREEMPTION);
+    String propertyName = AM_PREEMPTION_PREFIX + queueName;
+
+    if (get(propertyName) != null) {
+      boolean amPreemptionEnabled =
+          getBoolean(propertyName, DEFAULT_AM_PREEMPTION);
+      LOG.debug("AM preemption enabled for queue {}: {}",
+          queueName, amPreemptionEnabled);
+      return amPreemptionEnabled;
+    }
+
+    return getBoolean(AM_PREEMPTION, DEFAULT_AM_PREEMPTION);
   }
 
   public float getPreemptionUtilizationThreshold() {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java

@@ -35,7 +35,9 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.TestName;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
@@ -53,6 +55,8 @@ import java.util.List;
 public class TestFairSchedulerPreemption extends FairSchedulerTestBase {
   private static final File ALLOC_FILE = new File(TEST_DIR, "test-queues");
   private static final int GB = 1024;
+  private static final String TC_DISABLE_AM_PREEMPTION_GLOBALLY =
+      "testDisableAMPreemptionGlobally";
 
   // Scheduler clock
   private final ControlledClock clock = new ControlledClock();
@@ -69,6 +73,9 @@ public class TestFairSchedulerPreemption extends FairSchedulerTestBase {
   // Starving app that is expected to instigate preemption
   private FSAppAttempt starvingApp;
 
+  @Rule
+  public TestName testName = new TestName();
+
   @Parameterized.Parameters(name = "{0}")
   public static Collection<Object[]> getParameters() {
     return Arrays.asList(new Object[][] {
@@ -95,6 +102,10 @@ public class TestFairSchedulerPreemption extends FairSchedulerTestBase {
     conf.setFloat(FairSchedulerConfiguration.PREEMPTION_THRESHOLD, 0f);
     conf.setInt(FairSchedulerConfiguration.WAIT_TIME_BEFORE_KILL, 0);
     conf.setLong(FairSchedulerConfiguration.UPDATE_INTERVAL_MS, 60_000L);
+    String testMethod = testName.getMethodName();
+    if (testMethod.startsWith(TC_DISABLE_AM_PREEMPTION_GLOBALLY)) {
+      conf.setBoolean(FairSchedulerConfiguration.AM_PREEMPTION, false);
+    }
     setupCluster();
   }
 
@@ -417,13 +428,24 @@ public class TestFairSchedulerPreemption extends FairSchedulerTestBase {
 
   @Test
   public void testDisableAMPreemption() {
+    testDisableAMPreemption(false);
+  }
+
+  @Test
+  public void testDisableAMPreemptionGlobally() {
+    testDisableAMPreemption(true);
+  }
+
+  private void testDisableAMPreemption(boolean global) {
     takeAllResources("root.preemptable.child-1");
     setNumAMContainersPerNode(2);
     RMContainer container = greedyApp.getLiveContainers().stream()
             .filter(rmContainer -> rmContainer.isAMContainer())
             .findFirst()
             .get();
-    greedyApp.setEnableAMPreemption(false);
+    if (!global) {
+      greedyApp.setEnableAMPreemption(false);
+    }
     assertFalse(greedyApp.canContainerBePreempted(container, null));
   }