YARN-854. Fixing YARN bugs that are failing applications in secure environment. Contributed by Omkar Vinit Joshi.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1494845 13f79535-47bb-0310-9956-ffa450edef68

hadoop-yarn-project/CHANGES.txt

@@ -621,6 +621,9 @@ Release 2.1.0-beta - UNRELEASED
     YARN-848. Nodemanager does not register with RM using the fully qualified
     hostname. (Hitesh Shah via sseth)
 
+    YARN-854. Fixing YARN bugs that are failing applications in secure
+    environment. (Omkar Vinit Joshi via vinodkv)
+
   BREAKDOWN OF HADOOP-8562 SUBTASKS AND RELATED JIRAS
 
     YARN-158. Yarn creating package-info.java must not depend on sh.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java

@@ -211,6 +211,7 @@ public class RegisterApplicationMasterResponsePBImpl extends
   @Override
   public void setClientToAMTokenMasterKey(ByteBuffer key) {
     if (key == null) {
+      builder.clearClientToAmTokenMasterKey();
       return;
     }
     maybeInitBuilder();
@@ -219,6 +220,7 @@ public class RegisterApplicationMasterResponsePBImpl extends
   
   @Override
   public ByteBuffer getClientToAMTokenMasterKey() {
+    maybeInitBuilder();
     ByteBuffer key =
         ByteBuffer.wrap(builder.getClientToAmTokenMasterKey().toByteArray());
     return key;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java

@@ -64,7 +64,6 @@ import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerHe
 import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerStatus;
 import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.ResourceStatusType;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.security.LocalizerTokenIdentifier;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.security.LocalizerTokenSecretManager;
 import org.apache.hadoop.yarn.server.utils.YarnServerBuilderUtils;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.util.FSDownload;
@@ -141,12 +140,7 @@ public class ContainerLocalizer {
     // create localizer context
     UserGroupInformation remoteUser =
       UserGroupInformation.createRemoteUser(user);
-    LocalizerTokenSecretManager secretManager =
-      new LocalizerTokenSecretManager();
-    LocalizerTokenIdentifier id = secretManager.createIdentifier();
-    Token<LocalizerTokenIdentifier> localizerToken =
-      new Token<LocalizerTokenIdentifier>(id, secretManager);
-    remoteUser.addToken(localizerToken);
+    remoteUser.addToken(creds.getToken(LocalizerTokenIdentifier.KIND));
     final LocalizationProtocol nodeManager =
         remoteUser.doAs(new PrivilegedAction<LocalizationProtocol>() {
           @Override

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java

@@ -108,6 +108,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.even
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceLocalizedEvent;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceReleaseEvent;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceRequestEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.security.LocalizerTokenIdentifier;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.security.LocalizerTokenSecretManager;
 import org.apache.hadoop.yarn.server.nodemanager.security.authorize.NMPolicyProvider;
 import org.apache.hadoop.yarn.server.nodemanager.util.NodeManagerBuilderUtils;
@@ -135,6 +136,7 @@ public class ResourceLocalizationService extends CompositeService
   private LocalizerTracker localizerTracker;
   private RecordFactory recordFactory;
   private final ScheduledExecutorService cacheCleanup;
+  private LocalizerTokenSecretManager secretManager;
 
   private LocalResourcesTracker publicRsrc;
 
@@ -267,9 +269,8 @@ public class ResourceLocalizationService extends CompositeService
   Server createServer() {
     Configuration conf = getConfig();
     YarnRPC rpc = YarnRPC.create(conf);
-    LocalizerTokenSecretManager secretManager = null;
     if (UserGroupInformation.isSecurityEnabled()) {
-      secretManager = new LocalizerTokenSecretManager();
+      secretManager = new LocalizerTokenSecretManager();      
     }
     
     Server server = rpc.getServer(LocalizationProtocol.class, this,
@@ -1017,6 +1018,12 @@ public class ResourceLocalizationService extends CompositeService
             LOG.debug(tk.getService() + " : " + tk.encodeToUrlString());
           }
         }
+        if (UserGroupInformation.isSecurityEnabled()) {
+          LocalizerTokenIdentifier id = secretManager.createIdentifier();
+          Token<LocalizerTokenIdentifier> localizerToken =
+              new Token<LocalizerTokenIdentifier>(id, secretManager);
+          credentials.addToken(id.getKind(), localizerToken);
+        }
         credentials.writeTokenStorageToStream(tokenOut);
       } finally {
         if (tokenOut != null) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java

@@ -168,6 +168,9 @@ public class TestAMAuthorization {
     request.setApplicationAttemptId(applicationAttemptId);
     RegisterApplicationMasterResponse response =
         client.registerApplicationMaster(request);
+    Assert.assertNotNull(response.getClientToAMTokenMasterKey());
+    Assert
+        .assertTrue(response.getClientToAMTokenMasterKey().array().length > 0);
     Assert.assertEquals("Register response has bad ACLs", "*",
         response.getApplicationACLs().get(ApplicationAccessType.VIEW_APP));
 
@@ -216,6 +219,7 @@ public class TestAMAuthorization {
                 serviceAddr, conf);
           }
         });
+    
     RegisterApplicationMasterRequest request = Records
         .newRecord(RegisterApplicationMasterRequest.class);
     request.setApplicationAttemptId(applicationAttemptId);