YARN-3647. RMWebServices api's should use updated api from CommonNodeLabelsManager to get NodeLabel object. (Sunil G via wangda)

(cherry picked from commit ec0a852a37d5c91a62d3d0ff3ddbd9d58235b312)

hadoop-yarn-project/CHANGES.txt

@@ -220,6 +220,9 @@ Release 2.8.0 - UNRELEASED
     YARN-3594. WintuilsProcessStubExecutor.startStreamReader leaks streams.
     (Lars Francke via junping_du)
 
+    YARN-3647. RMWebServices api's should use updated api from CommonNodeLabelsManager 
+    to get NodeLabel object. (Sunil G via wangda)
+
   OPTIMIZATIONS
 
     YARN-3339. TestDockerContainerExecutor should pull a single image and not

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java

@@ -993,13 +993,18 @@ public class CommonNodeLabelsManager extends AbstractService {
     }
   }
   
-  private Set<NodeLabel> getLabelsInfoByNode(NodeId nodeId) {
-    Set<String> labels = getLabelsByNode(nodeId, nodeCollections);
-    if (labels.isEmpty()) {
-      return EMPTY_NODELABEL_SET;
+  public Set<NodeLabel> getLabelsInfoByNode(NodeId nodeId) {
+    try {
+      readLock.lock();
+      Set<String> labels = getLabelsByNode(nodeId, nodeCollections);
+      if (labels.isEmpty()) {
+        return EMPTY_NODELABEL_SET;
+      }
+      Set<NodeLabel> nodeLabels = createNodeLabelFromLabelNames(labels);
+      return nodeLabels;
+    } finally {
+      readLock.unlock();
     }
-    Set<NodeLabel> nodeLabels = createNodeLabelFromLabelNames(labels);
-    return nodeLabels;
   }
 
   private Set<NodeLabel> createNodeLabelFromLabelNames(Set<String> labels) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java

@@ -34,6 +34,7 @@ import java.util.concurrent.ConcurrentMap;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
 import org.apache.hadoop.yarn.nodelabels.RMNodeLabel;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java

@@ -810,12 +810,12 @@ public class RMWebServices {
 
     NodeToLabelsInfo ntl = new NodeToLabelsInfo();
     HashMap<String, NodeLabelsInfo> ntlMap = ntl.getNodeToLabels();
-    Map<NodeId, Set<String>> nodeIdToLabels = rm.getRMContext()
-        .getNodeLabelManager().getNodeLabels();
+    Map<NodeId, Set<NodeLabel>> nodeIdToLabels = rm.getRMContext()
+        .getNodeLabelManager().getNodeLabelsInfo();
 
-    for (Map.Entry<NodeId, Set<String>> nitle : nodeIdToLabels.entrySet()) {
-      ntlMap.put(nitle.getKey().toString(),
-          new NodeLabelsInfo(nitle.getValue()));
+    for (Map.Entry<NodeId, Set<NodeLabel>> nitle : nodeIdToLabels.entrySet()) {
+      List<NodeLabel> labels = new ArrayList<NodeLabel>(nitle.getValue());
+      ntlMap.put(nitle.getKey().toString(), new NodeLabelsInfo(labels));
     }
 
     return ntl;
@@ -830,16 +830,16 @@ public class RMWebServices {
 
     LabelsToNodesInfo lts = new LabelsToNodesInfo();
     Map<NodeLabelInfo, NodeIDsInfo> ltsMap = lts.getLabelsToNodes();
-    Map<String, Set<NodeId>> labelsToNodeId = null;
+    Map<NodeLabel, Set<NodeId>> labelsToNodeId = null;
     if (labels == null || labels.size() == 0) {
       labelsToNodeId =
-          rm.getRMContext().getNodeLabelManager().getLabelsToNodes();
+          rm.getRMContext().getNodeLabelManager().getLabelsInfoToNodes();
     } else {
       labelsToNodeId =
-          rm.getRMContext().getNodeLabelManager().getLabelsToNodes(labels);
+          rm.getRMContext().getNodeLabelManager().getLabelsInfoToNodes(labels);
     }
 
-    for (Entry<String, Set<NodeId>> entry : labelsToNodeId.entrySet()) {
+    for (Entry<NodeLabel, Set<NodeId>> entry : labelsToNodeId.entrySet()) {
       List<String> nodeIdStrList = new ArrayList<String>();
       for (NodeId nodeId : entry.getValue()) {
         nodeIdStrList.add(nodeId.toString());
@@ -985,14 +985,13 @@ public class RMWebServices {
   @Path("/nodes/{nodeId}/get-labels")
   @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
   public NodeLabelsInfo getLabelsOnNode(@Context HttpServletRequest hsr,
-                                  @PathParam("nodeId") String nodeId) 
-    throws IOException {
+      @PathParam("nodeId") String nodeId) throws IOException {
     init();
 
     NodeId nid = ConverterUtils.toNodeIdWithDefaultPort(nodeId);
-    return new NodeLabelsInfo(
-      rm.getRMContext().getNodeLabelManager().getLabelsOnNode(nid));
-
+    List<NodeLabel> labels = new ArrayList<NodeLabel>(rm.getRMContext()
+        .getNodeLabelManager().getLabelsInfoByNode(nid));
+    return new NodeLabelsInfo(labels);
   }
 
   protected Response killApp(RMApp app, UserGroupInformation callerUGI,

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java

@@ -133,12 +133,15 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
             .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
     assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
     nlsifo = response.getEntity(NodeLabelsInfo.class);
-    assertEquals("a", nlsifo.getNodeLabelsInfo().get(0).getName());
     assertEquals(1, nlsifo.getNodeLabels().size());
+    for (NodeLabelInfo nl : nlsifo.getNodeLabelsInfo()) {
+      assertEquals("a", nl.getName());
+      assertTrue(nl.getExclusivity());
+    }
     
     // Add another
     nlsifo = new NodeLabelsInfo();
-    nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("b"));
+    nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("b", false));
     response =
         r.path("ws").path("v1").path("cluster")
             .path("add-node-labels").queryParam("user.name", userName)
@@ -154,6 +157,12 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
     assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
     nlsifo = response.getEntity(NodeLabelsInfo.class);
     assertEquals(2, nlsifo.getNodeLabels().size());
+    // Verify exclusivity for 'y' as false
+    for (NodeLabelInfo nl : nlsifo.getNodeLabelsInfo()) {
+      if (nl.getName().equals("b")) {
+        assertFalse(nl.getExclusivity());
+      }
+    }
     
     // Add labels to a node
     MultivaluedMapImpl params = new MultivaluedMapImpl();
@@ -202,7 +211,8 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
     assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
     LabelsToNodesInfo ltni = response.getEntity(LabelsToNodesInfo.class);
     assertEquals(2, ltni.getLabelsToNodes().size());
-    NodeIDsInfo nodes = ltni.getLabelsToNodes().get(new NodeLabelInfo("b"));
+    NodeIDsInfo nodes = ltni.getLabelsToNodes().get(
+        new NodeLabelInfo("b", false));
     assertTrue(nodes.getNodeIDs().contains("nid2:0"));
     assertTrue(nodes.getNodeIDs().contains("nid1:0"));
     nodes = ltni.getLabelsToNodes().get(new NodeLabelInfo("a"));
@@ -231,7 +241,7 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
             .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
     assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
     nlsifo = response.getEntity(NodeLabelsInfo.class);
-    assertTrue(nlsifo.getNodeLabelsName().contains("a"));
+    assertTrue(nlsifo.getNodeLabelsInfo().contains(new NodeLabelInfo("a")));
 
     
     // Replace
@@ -255,7 +265,8 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
             .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
     assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
     nlsifo = response.getEntity(NodeLabelsInfo.class);
-    assertTrue(nlsifo.getNodeLabelsName().contains("b"));
+    assertTrue(nlsifo.getNodeLabelsInfo().contains(
+        new NodeLabelInfo("b", false)));
             
     // Replace labels using node-to-labels
     NodeToLabelsEntryList ntli = new NodeToLabelsEntryList();
@@ -281,7 +292,7 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
     NodeToLabelsInfo ntlinfo = response.getEntity(NodeToLabelsInfo.class);
     NodeLabelsInfo nlinfo = ntlinfo.getNodeToLabels().get("nid:0");
     assertEquals(1, nlinfo.getNodeLabels().size());
-    assertTrue(nlinfo.getNodeLabelsName().contains("a"));
+    assertTrue(nlinfo.getNodeLabelsInfo().contains(new NodeLabelInfo("a")));
     
     // Remove all
     params = new MultivaluedMapImpl();
@@ -303,7 +314,7 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
             .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
     assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
     nlsifo = response.getEntity(NodeLabelsInfo.class);
-    assertTrue(nlsifo.getNodeLabelsName().contains(""));
+    assertTrue(nlsifo.getNodeLabelsInfo().isEmpty());
     
     // Add a label back for auth tests
     params = new MultivaluedMapImpl();
@@ -326,7 +337,7 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
             .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
     assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
     nlsifo = response.getEntity(NodeLabelsInfo.class);
-    assertTrue(nlsifo.getNodeLabelsName().contains("a"));
+    assertTrue(nlsifo.getNodeLabelsInfo().contains(new NodeLabelInfo("a")));
     
     // Auth fail replace labels on node
     params = new MultivaluedMapImpl();
@@ -347,7 +358,7 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
             .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
     assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
     nlsifo = response.getEntity(NodeLabelsInfo.class);
-    assertTrue(nlsifo.getNodeLabelsName().contains("a"));
+    assertTrue(nlsifo.getNodeLabelsInfo().contains(new NodeLabelInfo("a")));
     
     // Fail to add a label with post
     response =
@@ -383,8 +394,11 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
             .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
     assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
     nlsifo = response.getEntity(NodeLabelsInfo.class);
-    assertEquals("a", nlsifo.getNodeLabelsInfo().get(0).getName());
     assertEquals(1, nlsifo.getNodeLabels().size());
+    for (NodeLabelInfo nl : nlsifo.getNodeLabelsInfo()) {
+      assertEquals("a", nl.getName());
+      assertTrue(nl.getExclusivity());
+    }
     
     // Remove cluster label with post
     params = new MultivaluedMapImpl();
@@ -409,8 +423,8 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
     // configuration is on
     // Reset for testing : add cluster labels
     nlsifo = new NodeLabelsInfo();
-    nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("x"));
-    nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("y"));
+    nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("x", false));
+    nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("y", false));
     response =
         r.path("ws")
             .path("v1")
@@ -421,8 +435,6 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
             .entity(toJson(nlsifo, NodeLabelsInfo.class),
                 MediaType.APPLICATION_JSON).post(ClientResponse.class);
     // Reset for testing : Add labels to a node
-    nlsifo = new NodeLabelsInfo();
-    nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("y"));
     params = new MultivaluedMapImpl();
     params.add("labels", "y");
     response =
@@ -461,7 +473,8 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
     ntlinfo = response.getEntity(NodeToLabelsInfo.class);
     nlinfo = ntlinfo.getNodeToLabels().get("nid:0");
     assertEquals(1, nlinfo.getNodeLabels().size());
-    assertFalse(nlinfo.getNodeLabels().contains("x"));
+    assertFalse(nlinfo.getNodeLabelsInfo().contains(
+        new NodeLabelInfo("x", false)));
 
     // Case2 : failure to Replace labels using replace-labels
     response =
@@ -481,7 +494,8 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
     ntlinfo = response.getEntity(NodeToLabelsInfo.class);
     nlinfo = ntlinfo.getNodeToLabels().get("nid:0");
     assertEquals(1, nlinfo.getNodeLabels().size());
-    assertFalse(nlinfo.getNodeLabels().contains("x"));
+    assertFalse(nlinfo.getNodeLabelsInfo().contains(
+        new NodeLabelInfo("x", false)));
 
     //  Case3 : Remove cluster label should be successful
     params = new MultivaluedMapImpl();
@@ -500,7 +514,10 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
             .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
     assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
     nlsifo = response.getEntity(NodeLabelsInfo.class);
+    assertEquals(new NodeLabelInfo("y", false),
+        nlsifo.getNodeLabelsInfo().get(0));
     assertEquals("y", nlsifo.getNodeLabelsInfo().get(0).getName());
+    assertFalse(nlsifo.getNodeLabelsInfo().get(0).getExclusivity());
 
     // Remove y
     params = new MultivaluedMapImpl();
@@ -543,7 +560,7 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
     assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
     nlsifo = response.getEntity(NodeLabelsInfo.class);
     assertEquals("z", nlsifo.getNodeLabelsInfo().get(0).getName());
-    assertEquals(false, nlsifo.getNodeLabelsInfo().get(0).getExclusivity());
+    assertFalse(nlsifo.getNodeLabelsInfo().get(0).getExclusivity());
     assertEquals(1, nlsifo.getNodeLabels().size());
   }