|
|
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
|
|
|
import java.util.ArrayList;
|
|
|
import java.util.Arrays;
|
|
|
import java.util.HashSet;
|
|
|
+import java.util.List;
|
|
|
import java.util.Set;
|
|
|
|
|
|
import org.apache.hadoop.conf.Configuration;
|
|
|
@@ -81,18 +82,20 @@ public class TestNodeLabelContainerAllocation {
|
|
|
conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b", "c"});
|
|
|
conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100);
|
|
|
conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "y", 100);
|
|
|
+ conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "z", 100);
|
|
|
|
|
|
final String A = CapacitySchedulerConfiguration.ROOT + ".a";
|
|
|
conf.setCapacity(A, 10);
|
|
|
conf.setMaximumCapacity(A, 15);
|
|
|
conf.setAccessibleNodeLabels(A, toSet("x"));
|
|
|
conf.setCapacityByLabel(A, "x", 100);
|
|
|
-
|
|
|
+
|
|
|
final String B = CapacitySchedulerConfiguration.ROOT + ".b";
|
|
|
conf.setCapacity(B, 20);
|
|
|
- conf.setAccessibleNodeLabels(B, toSet("y"));
|
|
|
+ conf.setAccessibleNodeLabels(B, toSet("y", "z"));
|
|
|
conf.setCapacityByLabel(B, "y", 100);
|
|
|
-
|
|
|
+ conf.setCapacityByLabel(B, "z", 100);
|
|
|
+
|
|
|
final String C = CapacitySchedulerConfiguration.ROOT + ".c";
|
|
|
conf.setCapacity(C, 70);
|
|
|
conf.setMaximumCapacity(C, 70);
|
|
|
@@ -110,6 +113,7 @@ public class TestNodeLabelContainerAllocation {
|
|
|
conf.setCapacity(B1, 100);
|
|
|
conf.setMaximumCapacity(B1, 100);
|
|
|
conf.setCapacityByLabel(B1, "y", 100);
|
|
|
+ conf.setCapacityByLabel(B1, "z", 100);
|
|
|
|
|
|
final String C1 = C + ".c1";
|
|
|
conf.setQueues(C, new String[] {"c1"});
|
|
|
@@ -474,7 +478,111 @@ public class TestNodeLabelContainerAllocation {
|
|
|
SchedulerNode node = cs.getSchedulerNode(nodeId);
|
|
|
Assert.assertEquals(numContainers, node.getNumContainers());
|
|
|
}
|
|
|
-
|
|
|
+
|
|
|
+ /**
|
|
|
+ * JIRA YARN-4140, In Resource request set node label will be set only on ANY
|
|
|
+ * reqest. RACK/NODE local and default requests label expression need to be
|
|
|
+ * updated. This testcase is to verify the label expression is getting changed
|
|
|
+ * based on ANY requests.
|
|
|
+ *
|
|
|
+ * @throws Exception
|
|
|
+ */
|
|
|
+ @Test
|
|
|
+ public void testResourceRequestUpdateNodePartitions() throws Exception {
|
|
|
+ // set node -> label
|
|
|
+ mgr.addToCluserNodeLabels(ImmutableSet.of(NodeLabel.newInstance("x"),
|
|
|
+ NodeLabel.newInstance("y", false), NodeLabel.newInstance("z", false)));
|
|
|
+ mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("y")));
|
|
|
+ // inject node label manager
|
|
|
+ MockRM rm1 = new MockRM(getConfigurationWithQueueLabels(conf)) {
|
|
|
+ @Override
|
|
|
+ public RMNodeLabelsManager createNodeLabelManager() {
|
|
|
+ return mgr;
|
|
|
+ }
|
|
|
+ };
|
|
|
+ rm1.getRMContext().setNodeLabelManager(mgr);
|
|
|
+ rm1.start();
|
|
|
+ MockNM nm1 = rm1.registerNode("h1:1234", 8 * GB); // no label
|
|
|
+ MockNM nm2 = rm1.registerNode("h2:1234", 40 * GB); // label = y
|
|
|
+ // launch an app to queue b1 (label = y), AM container should be launched in
|
|
|
+ // nm2
|
|
|
+ RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "b1");
|
|
|
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm2);
|
|
|
+ // Creating request set when request before ANY is not having label and any
|
|
|
+ // is having label
|
|
|
+ List<ResourceRequest> resourceRequest = new ArrayList<ResourceRequest>();
|
|
|
+ resourceRequest.add(am1.createResourceReq("/default-rack", 1024, 3, 1,
|
|
|
+ RMNodeLabelsManager.NO_LABEL));
|
|
|
+ resourceRequest.add(am1.createResourceReq("*", 1024, 3, 5, "y"));
|
|
|
+ resourceRequest.add(am1.createResourceReq("h1:1234", 1024, 3, 2,
|
|
|
+ RMNodeLabelsManager.NO_LABEL));
|
|
|
+ resourceRequest.add(am1.createResourceReq("*", 1024, 2, 3, "y"));
|
|
|
+ resourceRequest.add(am1.createResourceReq("h2:1234", 1024, 2, 4, null));
|
|
|
+ resourceRequest.add(am1.createResourceReq("*", 1024, 4, 3, null));
|
|
|
+ resourceRequest.add(am1.createResourceReq("h2:1234", 1024, 4, 4, null));
|
|
|
+ am1.allocate(resourceRequest, new ArrayList<ContainerId>());
|
|
|
+ CapacityScheduler cs =
|
|
|
+ (CapacityScheduler) rm1.getRMContext().getScheduler();
|
|
|
+ FiCaSchedulerApp app =
|
|
|
+ cs.getApplicationAttempt(am1.getApplicationAttemptId());
|
|
|
+ List<ResourceRequest> allResourceRequests =
|
|
|
+ app.getAppSchedulingInfo().getAllResourceRequests();
|
|
|
+ for (ResourceRequest changeReq : allResourceRequests) {
|
|
|
+ if (changeReq.getPriority().getPriority() == 2
|
|
|
+ || changeReq.getPriority().getPriority() == 3) {
|
|
|
+ Assert.assertEquals("Expected label y", "y",
|
|
|
+ changeReq.getNodeLabelExpression());
|
|
|
+ } else if (changeReq.getPriority().getPriority() == 4) {
|
|
|
+ Assert.assertEquals("Expected label EMPTY",
|
|
|
+ RMNodeLabelsManager.NO_LABEL, changeReq.getNodeLabelExpression());
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ // Previous any request was Y trying to update with z and the
|
|
|
+ // request before ANY label is null
|
|
|
+ List<ResourceRequest> newReq = new ArrayList<ResourceRequest>();
|
|
|
+ newReq.add(am1.createResourceReq("h2:1234", 1024, 3, 4, null));
|
|
|
+ newReq.add(am1.createResourceReq("*", 1024, 3, 5, "z"));
|
|
|
+ newReq.add(am1.createResourceReq("h1:1234", 1024, 3, 4, null));
|
|
|
+ newReq.add(am1.createResourceReq("*", 1024, 4, 5, "z"));
|
|
|
+ am1.allocate(newReq, new ArrayList<ContainerId>());
|
|
|
+ allResourceRequests = app.getAppSchedulingInfo().getAllResourceRequests();
|
|
|
+ for (ResourceRequest changeReq : allResourceRequests) {
|
|
|
+ if (changeReq.getPriority().getPriority() == 3
|
|
|
+ || changeReq.getPriority().getPriority() == 4) {
|
|
|
+ Assert.assertEquals("Expected label z", "z",
|
|
|
+ changeReq.getNodeLabelExpression());
|
|
|
+ } else if (changeReq.getPriority().getPriority() == 2) {
|
|
|
+ Assert.assertEquals("Expected label y", "y",
|
|
|
+ changeReq.getNodeLabelExpression());
|
|
|
+ }
|
|
|
+ }
|
|
|
+ // Request before ANY and ANY request is set as NULL. Request should be set
|
|
|
+ // with Empty Label
|
|
|
+ List<ResourceRequest> resourceRequest1 = new ArrayList<ResourceRequest>();
|
|
|
+ resourceRequest1.add(am1.createResourceReq("/default-rack", 1024, 3, 1,
|
|
|
+ null));
|
|
|
+ resourceRequest1.add(am1.createResourceReq("*", 1024, 3, 5, null));
|
|
|
+ resourceRequest1.add(am1.createResourceReq("h1:1234", 1024, 3, 2,
|
|
|
+ RMNodeLabelsManager.NO_LABEL));
|
|
|
+ resourceRequest1.add(am1.createResourceReq("/default-rack", 1024, 2, 1,
|
|
|
+ null));
|
|
|
+ resourceRequest1.add(am1.createResourceReq("*", 1024, 2, 3,
|
|
|
+ RMNodeLabelsManager.NO_LABEL));
|
|
|
+ resourceRequest1.add(am1.createResourceReq("h2:1234", 1024, 2, 4, null));
|
|
|
+ am1.allocate(resourceRequest1, new ArrayList<ContainerId>());
|
|
|
+ allResourceRequests = app.getAppSchedulingInfo().getAllResourceRequests();
|
|
|
+ for (ResourceRequest changeReq : allResourceRequests) {
|
|
|
+ if (changeReq.getPriority().getPriority() == 3) {
|
|
|
+ Assert.assertEquals("Expected label Empty",
|
|
|
+ RMNodeLabelsManager.NO_LABEL, changeReq.getNodeLabelExpression());
|
|
|
+ } else if (changeReq.getPriority().getPriority() == 2) {
|
|
|
+ Assert.assertEquals("Expected label y", RMNodeLabelsManager.NO_LABEL,
|
|
|
+ changeReq.getNodeLabelExpression());
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
@Test
|
|
|
public void testPreferenceOfNeedyAppsTowardsNodePartitions() throws Exception {
|
|
|
/**
|
Wangda Tan