YARN-6705 Add separate NM preemption thresholds for cpu and memory (Haibo Chen)

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java

@@ -2144,10 +2144,33 @@ public class YarnConfiguration extends Configuration {
   public static final String NM_OVERALLOCATION_MEMORY_UTILIZATION_THRESHOLD =
       NM_PREFIX + "overallocation.memory-utilization-threshold";
 
-  public static final String NM_OVERALLOCATION_PREEMPTION_THRESHOLD =
-      NM_PREFIX + "overallocation.preemption-threshold";
-  public static final float DEFAULT_NM_OVERALLOCATION_PREEMPTION_THRESHOLD
-      = 0.96f;
+  /**
+   * The CPU utilization threshold, if went beyond for a few times in a row,
+   * OPPORTUNISTIC containers started due to overallocation should start
+   * getting preempted.
+   */
+  public static final String NM_OVERALLOCATION_CPU_PREEMPTION_THRESHOLD =
+      NM_PREFIX + "overallocation.preemption-threshold.cpu";
+  public static final float
+      DEFAULT_NM_OVERALLOCATION_CPU_PREEMPTION_THRESHOLD = 0.99f;
+
+  /**
+   * The number of times that CPU utilization must go over the CPU preemption
+   * threshold consecutively before preemption starts to kick in.
+   */
+  public static final String NM_OVERALLOCATION_PREEMPTION_CPU_COUNT =
+      NM_PREFIX + "overallocation.preemption-threshold-count.cpu";
+  public static final int DEFAULT_NM_OVERALLOCATION_PREEMPTION_CPU_COUNT = 4;
+
+
+  /**
+   * The memory utilization threshold beyond which OPPORTUNISTIC containers
+   * started due to overallocation should start getting preempted.
+   */
+  public static final String NM_OVERALLOCATION_MEMORY_PREEMPTION_THRESHOLD =
+      NM_PREFIX + "overallocation.preemption-threshold.memory";
+  public static final float
+      DEFAULT_NM_OVERALLOCATION_MEMORY_PREEMPTION_THRESHOLD = 0.95f;
 
   /**
    * Interval of time the linux container executor should try cleaning up

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml

@@ -1769,11 +1769,37 @@
 
   <property>
     <description>When a node is over-allocated to improve utilization by
-      running OPPORTUNISTIC containers, this config captures the utilization
-      beyond which OPPORTUNISTIC containers should start getting preempted.
+      running OPPORTUNISTIC containers, this config captures the CPU
+      utilization beyond which OPPORTUNISTIC containers should start getting
+      preempted. This is used in combination with
+      yarn.nodemanager.overallocation.preemption-threshold-count.cpu, that is,
+      only when the CPU utilization goes over this threshold consecutively for
+      a few times will preemption kicks in.
     </description>
-    <name>yarn.nodemanager.overallocation.preemption-threshold</name>
-    <value>0.96</value>
+    <name>yarn.nodemanager.overallocation.preemption-threshold.cpu</name>
+    <value>0.99</value>
+  </property>
+
+  <property>
+    <description>When a node is over-allocated to improve utilization by
+      running OPPORTUNISTIC containers, this config captures the number of
+      times that CPU utilization has to go above
+      ${yarn.nodemanager.overallocation.preemption-threshold.cpu}
+      consecutively for NM to start preempting OPPORTUNISTIC containers
+      started due to overallocation.
+    </description>
+    <name>yarn.nodemanager.overallocation.preemption-threshold-count.cpu</name>
+    <value>4</value>
+  </property>
+
+  <property>
+    <description>When a node is over-allocated to improve utilization by
+      running OPPORTUNISTIC containers, this config captures the CPU
+      utilization beyond which OPPORTUNISTIC containers should start getting
+      preempted.
+    </description>
+    <name>yarn.nodemanager.overallocation.preemption-threshold.memory</name>
+    <value>0.95</value>
   </property>
 
   <property>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java

@@ -114,6 +114,7 @@ public class ContainersMonitorImpl extends AbstractService implements
   private ResourceUtilization containersUtilization;
 
   private ResourceThresholds overAllocationPreemptionThresholds;
+  private int overAlloctionPreemptionCpuCount = -1;
 
   private volatile boolean stopped = false;
 
@@ -289,7 +290,7 @@ public class ContainersMonitorImpl extends AbstractService implements
         YarnConfiguration.MAX_NM_OVERALLOCATION_THRESHOLD);
     if (overAllocationMemoryUtilizationThreshold <= 0) {
       LOG.info("NodeManager oversubscription is disabled because the memory " +
-          "utilization threshold is no larger than zero.");
+          "overallocation threshold is no larger than zero.");
       return;
     }
 
@@ -301,36 +302,49 @@ public class ContainersMonitorImpl extends AbstractService implements
         YarnConfiguration.MAX_NM_OVERALLOCATION_THRESHOLD);
     if (overAllocationCpuUtilizationThreshold <= 0) {
       LOG.info("NodeManager oversubscription is disabled because the CPU " +
-          "utilization threshold is no larger than zero.");
+          "overallocation threshold is no larger than zero.");
       return;
     }
 
-    float preemptionThreshold = conf.getFloat(
-        YarnConfiguration.NM_OVERALLOCATION_PREEMPTION_THRESHOLD,
-        YarnConfiguration.DEFAULT_NM_OVERALLOCATION_PREEMPTION_THRESHOLD);
-    if (preemptionThreshold <= overAllocationCpuUtilizationThreshold) {
-      LOG.info("NodeManager oversubscription is disabled because preemption" +
-          "threshold is no larger than the cpu utilization threshold.");
+    float cpuPreemptionThreshold = conf.getFloat(
+        YarnConfiguration.NM_OVERALLOCATION_CPU_PREEMPTION_THRESHOLD,
+        YarnConfiguration.
+            DEFAULT_NM_OVERALLOCATION_CPU_PREEMPTION_THRESHOLD);
+    if (cpuPreemptionThreshold <= overAllocationCpuUtilizationThreshold) {
+      LOG.info("NodeManager oversubscription is disabled because the cpu " +
+          " preemption threshold is no larger than the cpu overallocation" +
+          " threshold.");
       return;
     }
-    if (preemptionThreshold <= overAllocationMemoryUtilizationThreshold) {
-      LOG.info("NodeManager oversubscription is disabled because preemption" +
-          "threshold is no larger than the memory utilization threshold.");
+
+    float memoryPreemptionThreshold = conf.getFloat(
+        YarnConfiguration.NM_OVERALLOCATION_MEMORY_PREEMPTION_THRESHOLD,
+        YarnConfiguration.
+            DEFAULT_NM_OVERALLOCATION_MEMORY_PREEMPTION_THRESHOLD);
+    if (memoryPreemptionThreshold <= overAllocationMemoryUtilizationThreshold) {
+      LOG.info("NodeManager oversubscription is disabled because the memory" +
+          " preemption threshold is no larger than the memory overallocation" +
+          " threshold.");
       return;
     }
 
+    this.overAlloctionPreemptionCpuCount = conf.getInt(
+        YarnConfiguration.NM_OVERALLOCATION_PREEMPTION_CPU_COUNT,
+        YarnConfiguration.DEFAULT_NM_OVERALLOCATION_PREEMPTION_CPU_COUNT);
+
     ResourceThresholds resourceThresholds = ResourceThresholds.newInstance(
         overAllocationCpuUtilizationThreshold,
         overAllocationMemoryUtilizationThreshold);
     ((NodeManager.NMContext) context).setOverAllocationInfo(
         OverAllocationInfo.newInstance(resourceThresholds));
-    this.overAllocationPreemptionThresholds =
-        ResourceThresholds.newInstance(preemptionThreshold);
+    this.overAllocationPreemptionThresholds = ResourceThresholds.newInstance(
+        cpuPreemptionThreshold, memoryPreemptionThreshold);
 
     LOG.info("NodeManager oversubscription enabled with overallocation " +
         "thresholds (memory:" + overAllocationMemoryUtilizationThreshold +
         ", CPU:" + overAllocationCpuUtilizationThreshold + ") and preemption" +
-        " threshold: " + preemptionThreshold);
+        " threshold (memory:" + memoryPreemptionThreshold + ", CPU:" +
+        cpuPreemptionThreshold + ")");
   }
 
   private boolean isResourceCalculatorAvailable() {