Forráskód Böngészése

YARN-10807. Parents node labels are incorrectly added to child queues in weight mode. Contributed by Benjamin Teke.

zhuqi-lucas 3 éve
szülő
commit
ec16b1d3b9

+ 12 - 5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java

@@ -1250,16 +1250,23 @@ public class ParentQueue extends AbstractCSQueue {
           float sumOfWeight = 0;
 
           for (CSQueue queue : childQueues) {
-            float weight = Math.max(0,
-                queue.getQueueCapacities().getWeight(nodeLabel));
-            sumOfWeight += weight;
+            if (queue.getQueueCapacities().getExistingNodeLabels()
+                .contains(nodeLabel)) {
+              float weight = Math.max(0,
+                  queue.getQueueCapacities().getWeight(nodeLabel));
+              sumOfWeight += weight;
+            }
           }
           // When sum of weight == 0, skip setting normalized_weight (so
           // normalized weight will be 0).
           if (Math.abs(sumOfWeight) > 1e-6) {
             for (CSQueue queue : childQueues) {
-              queue.getQueueCapacities().setNormalizedWeight(nodeLabel,
-                  queue.getQueueCapacities().getWeight(nodeLabel) / sumOfWeight);
+              if (queue.getQueueCapacities().getExistingNodeLabels()
+                  .contains(nodeLabel)) {
+                queue.getQueueCapacities().setNormalizedWeight(nodeLabel,
+                    queue.getQueueCapacities().getWeight(nodeLabel) /
+                        sumOfWeight);
+              }
             }
           }
         }

+ 30 - 0
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java

@@ -338,6 +338,36 @@ public class TestCapacitySchedulerWeightMode {
     }
   }
 
+  /**
+   * This test ensures that while iterating through a parent's Node Labels
+   * (when calculating the normalized weights) the parent's Node Labels won't
+   * be added to the children with weight -1. If the parent
+   * has a node label that a specific child doesn't the normalized calling the
+   * normalized weight setter will be skipped. The queue root.b has access to
+   * the labels "x" and "y", but root.b.b1 won't. For more information see
+   * YARN-10807.
+   * @throws Exception
+   */
+  @Test
+  public void testChildAccessibleNodeLabelsWeightMode() throws Exception {
+    MockRM rm = new MockRM(getCSConfWithQueueLabelsWeightOnly(conf));
+    rm.start();
+
+    CapacityScheduler cs =
+        (CapacityScheduler) rm.getRMContext().getScheduler();
+    LeafQueue b1 = (LeafQueue) cs.getQueue(B1);
+
+    Assert.assertNotNull(b1);
+    Assert.assertTrue(b1.getAccessibleNodeLabels().isEmpty());
+
+    Set<String> b1ExistingNodeLabels = ((CSQueue) b1).getQueueCapacities()
+        .getExistingNodeLabels();
+    Assert.assertEquals(1, b1ExistingNodeLabels.size());
+    Assert.assertEquals("", b1ExistingNodeLabels.iterator().next());
+
+    rm.close();
+  }
+
   @Test
   public void testQueueInfoWeight() throws Exception {
     MockRM rm = new MockRM(conf);