Browse Source

YARN-10983. Follow-up changes for YARN-10904. Contributed by Benjamin Teke

9uapaw 3 năm trước cách đây
mục cha
commit
9e475aede6

+ 2 - 2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java

@@ -331,7 +331,7 @@ public abstract class AbstractCSQueue implements CSQueue {
 
 
       // Collect and set the Node label configuration
       // Collect and set the Node label configuration
       this.queueNodeLabelsSettings = new QueueNodeLabelsSettings(configuration, parent,
       this.queueNodeLabelsSettings = new QueueNodeLabelsSettings(configuration, parent,
-          getQueuePath(), queueContext.getQueueManager().getConfiguredNodeLabelsForAllQueues());
+          queuePath, queueContext.getQueueManager().getConfiguredNodeLabelsForAllQueues());
 
 
       // Initialize the queue capacities
       // Initialize the queue capacities
       setupConfigurableCapacities();
       setupConfigurableCapacities();
@@ -379,7 +379,7 @@ public abstract class AbstractCSQueue implements CSQueue {
 
 
       // Setup application related limits
       // Setup application related limits
       this.queueAppLifetimeSettings = new QueueAppLifetimeAndLimitSettings(configuration,
       this.queueAppLifetimeSettings = new QueueAppLifetimeAndLimitSettings(configuration,
-          this, getQueuePath());
+          this, queuePath);
     } finally {
     } finally {
       writeLock.unlock();
       writeLock.unlock();
     }
     }

+ 7 - 7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAppLifetimeAndLimitSettings.java

@@ -35,9 +35,9 @@ public class QueueAppLifetimeAndLimitSettings {
   private int maxParallelApps;
   private int maxParallelApps;
 
 
   public QueueAppLifetimeAndLimitSettings(CapacitySchedulerConfiguration configuration,
   public QueueAppLifetimeAndLimitSettings(CapacitySchedulerConfiguration configuration,
-      AbstractCSQueue q, String queuePath) {
+      AbstractCSQueue q, QueuePath queuePath) {
     // Store max parallel apps property
     // Store max parallel apps property
-    this.maxParallelApps = configuration.getMaxParallelAppsForQueue(queuePath);
+    this.maxParallelApps = configuration.getMaxParallelAppsForQueue(queuePath.getFullPath());
     this.maxApplicationLifetime = getInheritedMaxAppLifetime(q, configuration);
     this.maxApplicationLifetime = getInheritedMaxAppLifetime(q, configuration);
     this.defaultApplicationLifetime = setupInheritedDefaultAppLifetime(q, queuePath, configuration,
     this.defaultApplicationLifetime = setupInheritedDefaultAppLifetime(q, queuePath, configuration,
         maxApplicationLifetime);
         maxApplicationLifetime);
@@ -48,7 +48,7 @@ public class QueueAppLifetimeAndLimitSettings {
     long maxAppLifetime = conf.getMaximumLifetimePerQueue(q.getQueuePath());
     long maxAppLifetime = conf.getMaximumLifetimePerQueue(q.getQueuePath());
 
 
     // If q is the root queue, then get max app lifetime from conf.
     // If q is the root queue, then get max app lifetime from conf.
-    if (parentQ == null) {
+    if (q.getQueuePathObject().isRoot()) {
       return maxAppLifetime;
       return maxAppLifetime;
     }
     }
 
 
@@ -62,16 +62,16 @@ public class QueueAppLifetimeAndLimitSettings {
   }
   }
 
 
   private long setupInheritedDefaultAppLifetime(CSQueue q,
   private long setupInheritedDefaultAppLifetime(CSQueue q,
-      String queuePath, CapacitySchedulerConfiguration conf, long myMaxAppLifetime) {
+      QueuePath queuePath, CapacitySchedulerConfiguration conf, long myMaxAppLifetime) {
     CSQueue parentQ = q.getParent();
     CSQueue parentQ = q.getParent();
-    long defaultAppLifetime = conf.getDefaultLifetimePerQueue(queuePath);
+    long defaultAppLifetime = conf.getDefaultLifetimePerQueue(queuePath.getFullPath());
     defaultAppLifetimeWasSpecifiedInConfig =
     defaultAppLifetimeWasSpecifiedInConfig =
         (defaultAppLifetime >= 0
         (defaultAppLifetime >= 0
-            || (parentQ != null &&
+            || (!queuePath.isRoot() &&
             parentQ.getDefaultAppLifetimeWasSpecifiedInConfig()));
             parentQ.getDefaultAppLifetimeWasSpecifiedInConfig()));
 
 
     // If q is the root queue, then get default app lifetime from conf.
     // If q is the root queue, then get default app lifetime from conf.
-    if (parentQ == null) {
+    if (queuePath.isRoot()) {
       return defaultAppLifetime;
       return defaultAppLifetime;
     }
     }
 
 

+ 10 - 13
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueNodeLabelsSettings.java

@@ -19,7 +19,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.util.Sets;
 import org.apache.hadoop.util.Sets;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ROOT;
 import java.io.IOException;
 import java.io.IOException;
 import java.util.Set;
 import java.util.Set;
 
 
@@ -30,14 +29,14 @@ import java.util.Set;
  */
  */
 public class QueueNodeLabelsSettings {
 public class QueueNodeLabelsSettings {
   private final CSQueue parent;
   private final CSQueue parent;
-  private final String queuePath;
+  private final QueuePath queuePath;
   private Set<String> accessibleLabels;
   private Set<String> accessibleLabels;
   private Set<String> configuredNodeLabels;
   private Set<String> configuredNodeLabels;
   private String defaultLabelExpression;
   private String defaultLabelExpression;
 
 
   public QueueNodeLabelsSettings(CapacitySchedulerConfiguration configuration,
   public QueueNodeLabelsSettings(CapacitySchedulerConfiguration configuration,
       CSQueue parent,
       CSQueue parent,
-      String queuePath,
+      QueuePath queuePath,
       ConfiguredNodeLabels configuredNodeLabels) throws IOException {
       ConfiguredNodeLabels configuredNodeLabels) throws IOException {
     this.parent = parent;
     this.parent = parent;
     this.queuePath = queuePath;
     this.queuePath = queuePath;
@@ -54,7 +53,7 @@ public class QueueNodeLabelsSettings {
   }
   }
 
 
   private void initializeAccessibleLabels(CapacitySchedulerConfiguration configuration) {
   private void initializeAccessibleLabels(CapacitySchedulerConfiguration configuration) {
-    this.accessibleLabels = configuration.getAccessibleNodeLabels(queuePath);
+    this.accessibleLabels = configuration.getAccessibleNodeLabels(queuePath.getFullPath());
     // Inherit labels from parent if not set
     // Inherit labels from parent if not set
     if (this.accessibleLabels == null && parent != null) {
     if (this.accessibleLabels == null && parent != null) {
       this.accessibleLabels = parent.getAccessibleNodeLabels();
       this.accessibleLabels = parent.getAccessibleNodeLabels();
@@ -62,7 +61,8 @@ public class QueueNodeLabelsSettings {
   }
   }
 
 
   private void initializeDefaultLabelExpression(CapacitySchedulerConfiguration configuration) {
   private void initializeDefaultLabelExpression(CapacitySchedulerConfiguration configuration) {
-    this.defaultLabelExpression = configuration.getDefaultNodeLabelExpression(queuePath);
+    this.defaultLabelExpression = configuration.getDefaultNodeLabelExpression(
+        queuePath.getFullPath());
     // If the accessible labels is not null and the queue has a parent with a
     // If the accessible labels is not null and the queue has a parent with a
     // similar set of labels copy the defaultNodeLabelExpression from the parent
     // similar set of labels copy the defaultNodeLabelExpression from the parent
     if (this.accessibleLabels != null && parent != null
     if (this.accessibleLabels != null && parent != null
@@ -75,21 +75,22 @@ public class QueueNodeLabelsSettings {
   private void initializeConfiguredNodeLabels(CapacitySchedulerConfiguration configuration,
   private void initializeConfiguredNodeLabels(CapacitySchedulerConfiguration configuration,
       ConfiguredNodeLabels configuredNodeLabelsParam) {
       ConfiguredNodeLabels configuredNodeLabelsParam) {
     if (configuredNodeLabelsParam != null) {
     if (configuredNodeLabelsParam != null) {
-      if (queuePath.equals(ROOT)) {
+      if (queuePath.isRoot()) {
         this.configuredNodeLabels = configuredNodeLabelsParam.getAllConfiguredLabels();
         this.configuredNodeLabels = configuredNodeLabelsParam.getAllConfiguredLabels();
       } else {
       } else {
-        this.configuredNodeLabels = configuredNodeLabelsParam.getLabelsByQueue(queuePath);
+        this.configuredNodeLabels = configuredNodeLabelsParam.getLabelsByQueue(
+            queuePath.getFullPath());
       }
       }
     } else {
     } else {
       // Fallback to suboptimal but correct logic
       // Fallback to suboptimal but correct logic
-      this.configuredNodeLabels = configuration.getConfiguredNodeLabels(queuePath);
+      this.configuredNodeLabels = configuration.getConfiguredNodeLabels(queuePath.getFullPath());
     }
     }
   }
   }
 
 
   private void validateNodeLabels() throws IOException {
   private void validateNodeLabels() throws IOException {
     // Check if labels of this queue is a subset of parent queue, only do this
     // Check if labels of this queue is a subset of parent queue, only do this
     // when the queue in question is not root
     // when the queue in question is not root
-    if (isNotRoot()) {
+    if (!queuePath.isRoot()) {
       if (parent.getAccessibleNodeLabels() != null && !parent
       if (parent.getAccessibleNodeLabels() != null && !parent
           .getAccessibleNodeLabels().contains(RMNodeLabelsManager.ANY)) {
           .getAccessibleNodeLabels().contains(RMNodeLabelsManager.ANY)) {
         // If parent isn't "*", child shouldn't be "*" too
         // If parent isn't "*", child shouldn't be "*" too
@@ -109,10 +110,6 @@ public class QueueNodeLabelsSettings {
     }
     }
   }
   }
 
 
-  private boolean isNotRoot() {
-    return parent != null && parent.getParent() != null;
-  }
-
   public boolean isAccessibleToPartition(String nodePartition) {
   public boolean isAccessibleToPartition(String nodePartition) {
     // if queue's label is *, it can access any node
     // if queue's label is *, it can access any node
     if (accessibleLabels != null && accessibleLabels.contains(RMNodeLabelsManager.ANY)) {
     if (accessibleLabels != null && accessibleLabels.contains(RMNodeLabelsManager.ANY)) {