YARN-7041. Nodemanager NPE running jobs with security off. Contributed by Varun Saxena.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/CollectorInfoPBImpl.java

@@ -114,9 +114,13 @@ public class CollectorInfoPBImpl extends CollectorInfo {
   @Override
   public Token getCollectorToken() {
     CollectorInfoProtoOrBuilder p = viaProto ? proto : builder;
-    if (this.collectorToken == null && p.hasCollectorToken()) {
-      this.collectorToken = convertFromProtoFormat(p.getCollectorToken());
+    if (this.collectorToken != null) {
+      return this.collectorToken;
+    }
+    if (!p.hasCollectorToken()) {
+      return null;
     }
+    this.collectorToken = convertFromProtoFormat(p.getCollectorToken());
     return this.collectorToken;
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java

@@ -168,12 +168,17 @@ public class NodeHeartbeatRequestPBImpl extends NodeHeartbeatRequest {
     for (Map.Entry<ApplicationId, AppCollectorData> entry :
         registeringCollectors.entrySet()) {
       AppCollectorData data = entry.getValue();
-      builder.addRegisteringCollectors(AppCollectorDataProto.newBuilder()
-          .setAppId(convertToProtoFormat(entry.getKey()))
-          .setAppCollectorAddr(data.getCollectorAddr())
-          .setAppCollectorToken(convertToProtoFormat(data.getCollectorToken()))
-          .setRmIdentifier(data.getRMIdentifier())
-          .setVersion(data.getVersion()));
+      AppCollectorDataProto.Builder appCollectorDataBuilder =
+          AppCollectorDataProto.newBuilder()
+              .setAppId(convertToProtoFormat(entry.getKey()))
+              .setAppCollectorAddr(data.getCollectorAddr())
+              .setRmIdentifier(data.getRMIdentifier())
+              .setVersion(data.getVersion());
+      if (data.getCollectorToken() != null) {
+        appCollectorDataBuilder.setAppCollectorToken(
+            convertToProtoFormat(data.getCollectorToken()));
+      }
+      builder.addRegisteringCollectors(appCollectorDataBuilder);
     }
   }
 
@@ -274,7 +279,10 @@ public class NodeHeartbeatRequestPBImpl extends NodeHeartbeatRequest {
       this.registeringCollectors = new HashMap<>();
       for (AppCollectorDataProto c : list) {
         ApplicationId appId = convertFromProtoFormat(c.getAppId());
-        Token collectorToken = convertFromProtoFormat(c.getAppCollectorToken());
+        Token collectorToken = null;
+        if (c.hasAppCollectorToken()){
+          collectorToken = convertFromProtoFormat(c.getAppCollectorToken());
+        }
         AppCollectorData data = AppCollectorData.newInstance(appId,
             c.getAppCollectorAddr(), c.getRmIdentifier(), c.getVersion(),
             collectorToken);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java

@@ -154,13 +154,17 @@ public class NodeHeartbeatResponsePBImpl extends NodeHeartbeatResponse {
     for (Map.Entry<ApplicationId, AppCollectorData> entry
         : appCollectorsMap.entrySet()) {
       AppCollectorData data = entry.getValue();
-      builder.addAppCollectors(AppCollectorDataProto.newBuilder()
-          .setAppId(convertToProtoFormat(entry.getKey()))
-          .setAppCollectorAddr(data.getCollectorAddr())
-          .setAppCollectorToken(
-              convertToProtoFormat(entry.getValue().getCollectorToken()))
-          .setRmIdentifier(data.getRMIdentifier())
-          .setVersion(data.getVersion()));
+      AppCollectorDataProto.Builder appCollectorDataBuilder =
+          AppCollectorDataProto.newBuilder()
+              .setAppId(convertToProtoFormat(entry.getKey()))
+              .setAppCollectorAddr(data.getCollectorAddr())
+              .setRmIdentifier(data.getRMIdentifier())
+              .setVersion(data.getVersion());
+      if (data.getCollectorToken() != null) {
+        appCollectorDataBuilder.setAppCollectorToken(
+            convertToProtoFormat(data.getCollectorToken()));
+      }
+      builder.addAppCollectors(appCollectorDataBuilder);
     }
   }
 
@@ -604,7 +608,10 @@ public class NodeHeartbeatResponsePBImpl extends NodeHeartbeatResponse {
       this.appCollectorsMap = new HashMap<>();
       for (AppCollectorDataProto c : list) {
         ApplicationId appId = convertFromProtoFormat(c.getAppId());
-        Token collectorToken = convertFromProtoFormat(c.getAppCollectorToken());
+        Token collectorToken = null;
+        if (c.hasAppCollectorToken()){
+          collectorToken = convertFromProtoFormat(c.getAppCollectorToken());
+        }
         AppCollectorData data = AppCollectorData.newInstance(appId,
             c.getAppCollectorAddr(), c.getRmIdentifier(), c.getVersion(),
             collectorToken);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/AppCollectorDataPBImpl.java

@@ -163,9 +163,13 @@ public class AppCollectorDataPBImpl extends AppCollectorData {
   @Override
   public Token getCollectorToken() {
     AppCollectorDataProtoOrBuilder p = viaProto ? proto : builder;
-    if (this.collectorToken == null && p.hasAppCollectorToken()) {
-      this.collectorToken = new TokenPBImpl(p.getAppCollectorToken());
+    if (this.collectorToken != null) {
+      return this.collectorToken;
+    }
+    if (!p.hasAppCollectorToken()) {
+      return null;
     }
+    this.collectorToken = new TokenPBImpl(p.getAppCollectorToken());
     return this.collectorToken;
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java

@@ -111,7 +111,7 @@ public class TestYarnServerApiClasses {
     original.setLastKnownNMTokenMasterKey(getMasterKey());
     original.setNodeStatus(getNodeStatus());
     original.setNodeLabels(getValidNodeLabels());
-    Map<ApplicationId, AppCollectorData> collectors = getCollectors();
+    Map<ApplicationId, AppCollectorData> collectors = getCollectors(false);
     original.setRegisteringCollectors(collectors);
     NodeHeartbeatRequestPBImpl copy = new NodeHeartbeatRequestPBImpl(
         original.getProto());
@@ -130,6 +130,16 @@ public class TestYarnServerApiClasses {
     Assert.assertEquals(0, copy.getNodeLabels().size());
   }
 
+  @Test
+  public void testNodeHBRequestPBImplWithNullCollectorToken() {
+    NodeHeartbeatRequestPBImpl original = new NodeHeartbeatRequestPBImpl();
+    Map<ApplicationId, AppCollectorData> collectors = getCollectors(true);
+    original.setRegisteringCollectors(collectors);
+    NodeHeartbeatRequestPBImpl copy = new NodeHeartbeatRequestPBImpl(
+        original.getProto());
+    assertEquals(collectors, copy.getRegisteringCollectors());
+  }
+
   /**
    * Test NodeHeartbeatRequestPBImpl.
    */
@@ -155,7 +165,7 @@ public class TestYarnServerApiClasses {
     original.setNextHeartBeatInterval(1000);
     original.setNodeAction(NodeAction.NORMAL);
     original.setResponseId(100);
-    Map<ApplicationId, AppCollectorData> collectors = getCollectors();
+    Map<ApplicationId, AppCollectorData> collectors = getCollectors(false);
     original.setAppCollectors(collectors);
 
     NodeHeartbeatResponsePBImpl copy = new NodeHeartbeatResponsePBImpl(
@@ -179,6 +189,16 @@ public class TestYarnServerApiClasses {
     assertTrue(copy.getAreNodeLabelsAcceptedByRM());
   }
 
+  @Test
+  public void testNodeHBResponsePBImplWithNullCollectorToken() {
+    NodeHeartbeatResponsePBImpl original = new NodeHeartbeatResponsePBImpl();
+    Map<ApplicationId, AppCollectorData> collectors = getCollectors(true);
+    original.setAppCollectors(collectors);
+    NodeHeartbeatResponsePBImpl copy = new NodeHeartbeatResponsePBImpl(
+        original.getProto());
+    assertEquals(collectors, copy.getAppCollectors());
+  }
+
   @Test
   public void testNodeHeartbeatResponsePBImplWithDecreasedContainers() {
     NodeHeartbeatResponsePBImpl original = new NodeHeartbeatResponsePBImpl();
@@ -349,11 +369,15 @@ public class TestYarnServerApiClasses {
     return nodeLabels;
   }
 
-  private Map<ApplicationId, AppCollectorData> getCollectors() {
+  private Map<ApplicationId, AppCollectorData> getCollectors(
+      boolean hasNullCollectorToken) {
     ApplicationId appID = ApplicationId.newInstance(1L, 1);
     String collectorAddr = "localhost:0";
-    AppCollectorData data = AppCollectorData.newInstance(appID, collectorAddr,
-        Token.newInstance(new byte[0], "kind", new byte[0], "s"));
+    AppCollectorData data = AppCollectorData.newInstance(appID, collectorAddr);
+    if (!hasNullCollectorToken) {
+      data.setCollectorToken(
+          Token.newInstance(new byte[0], "kind", new byte[0], "s"));
+    }
     Map<ApplicationId, AppCollectorData> collectorMap =
         new HashMap<>();
     collectorMap.put(appID, data);