YARN-874. Making common RPC to switch to not switch to simple when other mechanisms are enabled and thus fix YARN/MR test failures after HADOOP-9421. Contributed by Daryn Sharp and Vinod Kumar Vavilapalli.

svn merge --ignore-ancestry -c 1496692 ../../trunk/


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2.1-beta@1496694 13f79535-47bb-0310-9956-ffa450edef68

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java

@@ -71,17 +71,20 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.io.DataOutputBuffer;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableUtils;
-import org.apache.hadoop.ipc.ProtobufRpcEngine.RpcResponseWrapper;
 import org.apache.hadoop.ipc.ProtobufRpcEngine.RpcRequestMessageWrapper;
+import org.apache.hadoop.ipc.ProtobufRpcEngine.RpcResponseWrapper;
 import org.apache.hadoop.ipc.RPC.RpcInvoker;
 import org.apache.hadoop.ipc.RPC.VersionMismatch;
 import org.apache.hadoop.ipc.metrics.RpcDetailedMetrics;
 import org.apache.hadoop.ipc.metrics.RpcMetrics;
 import org.apache.hadoop.ipc.protobuf.IpcConnectionContextProtos.IpcConnectionContextProto;
-import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcResponseHeaderProto.RpcStatusProto;
+import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcKindProto;
+import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcRequestHeaderProto;
+import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcResponseHeaderProto;
 import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcResponseHeaderProto.RpcErrorCodeProto;
-import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcSaslProto.*;
-import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.*;
+import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcResponseHeaderProto.RpcStatusProto;
+import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcSaslProto;
+import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcSaslProto.SaslState;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.SaslRpcServer;
@@ -1491,7 +1494,9 @@ public abstract class Server {
           break;
         }
         case SASL: {
-          if (isSimpleEnabled) { // switch to simple hack
+          // switch to simple hack, but don't switch if other auths are
+          // supported, ex. tokens
+          if (isSimpleEnabled && enabledAuthMethods.size() == 1) {
             skipInitialSaslHandshake = true;
             doSaslReply(buildSaslResponse(SaslState.SUCCESS, null));
           }

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java

@@ -18,8 +18,14 @@
 
 package org.apache.hadoop.ipc;
 
-import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.*;
-import static org.junit.Assert.*;
+import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS;
+import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.SIMPLE;
+import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.TOKEN;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
 
 import java.io.DataInput;
 import java.io.DataOutput;
@@ -32,8 +38,17 @@ import java.util.Collection;
 import java.util.Set;
 import java.util.regex.Pattern;
 
-import javax.security.auth.callback.*;
-import javax.security.sasl.*;
+import javax.security.auth.callback.Callback;
+import javax.security.auth.callback.CallbackHandler;
+import javax.security.auth.callback.NameCallback;
+import javax.security.auth.callback.PasswordCallback;
+import javax.security.auth.callback.UnsupportedCallbackException;
+import javax.security.sasl.AuthorizeCallback;
+import javax.security.sasl.Sasl;
+import javax.security.sasl.SaslClient;
+import javax.security.sasl.SaslException;
+import javax.security.sasl.SaslServer;
+
 import junit.framework.Assert;
 
 import org.apache.commons.logging.Log;
@@ -45,16 +60,23 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.ipc.Client.ConnectionId;
 import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.*;
+import org.apache.hadoop.security.KerberosInfo;
+import org.apache.hadoop.security.SaslInputStream;
+import org.apache.hadoop.security.SaslPlainServer;
+import org.apache.hadoop.security.SaslRpcClient;
+import org.apache.hadoop.security.SaslRpcServer;
 import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
+import org.apache.hadoop.security.SecurityInfo;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.TestUserGroupInformation;
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
 import org.apache.hadoop.security.token.SecretManager;
+import org.apache.hadoop.security.token.SecretManager.InvalidToken;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenIdentifier;
 import org.apache.hadoop.security.token.TokenInfo;
 import org.apache.hadoop.security.token.TokenSelector;
-import org.apache.hadoop.security.token.SecretManager.InvalidToken;
-
 import org.apache.log4j.Level;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -73,7 +95,13 @@ public class TestSaslRPC {
   static final String SERVER_PRINCIPAL_1 = "p1/foo@BAR";
   static final String SERVER_PRINCIPAL_2 = "p2/foo@BAR";
   private static Configuration conf;
+  // If this is set to true AND the auth-method is not simple, secretManager
+  // will be enabled.
+  static Boolean enableSecretManager = null;
+  // If this is set to true, secretManager will be forecefully enabled
+  // irrespective of auth-method.
   static Boolean forceSecretManager = null;
+  static Boolean clientFallBackToSimpleAllowed = true;
   
   @BeforeClass
   public static void setupKerb() {
@@ -87,7 +115,7 @@ public class TestSaslRPC {
     conf = new Configuration();
     SecurityUtil.setAuthenticationMethod(KERBEROS, conf);
     UserGroupInformation.setConfiguration(conf);
-    forceSecretManager = null;
+    enableSecretManager = null;
   }
 
   static {
@@ -589,22 +617,55 @@ public class TestSaslRPC {
     assertAuthEquals(KrbFailed, getAuthMethod(KERBEROS, SIMPLE));
   }
 
+  @Test
+  public void testSimpleServerWithTokensWithNoClientFallbackToSimple()
+      throws Exception {
+
+    clientFallBackToSimpleAllowed = false;
+
+    try{
+      // Client has a token even though its configs says simple auth. Server
+      // is configured for simple auth, but as client sends the token, and
+      // server asks to switch to simple, this should fail.
+      getAuthMethod(SIMPLE,   SIMPLE, true);
+    } catch (IOException ioe) {
+      Assert
+        .assertTrue(ioe.getMessage().contains("Failed on local exception: " +
+        		"java.io.IOException: java.io.IOException: " +
+        		"Server asks us to fall back to SIMPLE auth, " +
+        		"but this client is configured to only allow secure connections"
+          ));
+    }
+
+    // Now set server to simple and also force the secret-manager. Now server
+    // should have both simple and token enabled.
+    forceSecretManager = true;
+    assertAuthEquals(TOKEN, getAuthMethod(SIMPLE,   SIMPLE, true));
+    forceSecretManager = false;
+    clientFallBackToSimpleAllowed = true;
+  }
+
   @Test
   public void testSimpleServerWithTokens() throws Exception {
-    // Tokens are ignored because client is reverted to simple
-    assertAuthEquals(SIMPLE, getAuthMethod(SIMPLE,   SIMPLE, true));
+    // Client not using tokens
+    assertAuthEquals(SIMPLE, getAuthMethod(SIMPLE,   SIMPLE));
+    // SASL methods are reverted to SIMPLE, but test setup fails
+    assertAuthEquals(KrbFailed, getAuthMethod(KERBEROS, SIMPLE));
+
+    // Use tokens. But tokens are ignored because client is reverted to simple
     assertAuthEquals(SIMPLE, getAuthMethod(KERBEROS, SIMPLE, true));
-    forceSecretManager = true;
+
+    enableSecretManager = true;
     assertAuthEquals(SIMPLE, getAuthMethod(SIMPLE,   SIMPLE, true));
     assertAuthEquals(SIMPLE, getAuthMethod(KERBEROS, SIMPLE, true));
   }
-    
+
   @Test
   public void testSimpleServerWithInvalidTokens() throws Exception {
     // Tokens are ignored because client is reverted to simple
     assertAuthEquals(SIMPLE, getAuthMethod(SIMPLE,   SIMPLE, false));
     assertAuthEquals(SIMPLE, getAuthMethod(KERBEROS, SIMPLE, false));
-    forceSecretManager = true;
+    enableSecretManager = true;
     assertAuthEquals(SIMPLE, getAuthMethod(SIMPLE,   SIMPLE, false));
     assertAuthEquals(SIMPLE, getAuthMethod(KERBEROS, SIMPLE, false));
   }
@@ -622,7 +683,7 @@ public class TestSaslRPC {
   public void testTokenOnlyServerWithTokens() throws Exception {
     assertAuthEquals(TOKEN, getAuthMethod(SIMPLE,   TOKEN, true));
     assertAuthEquals(TOKEN, getAuthMethod(KERBEROS, TOKEN, true));
-    forceSecretManager = false;
+    enableSecretManager = false;
     assertAuthEquals(NoTokenAuth, getAuthMethod(SIMPLE,   TOKEN, true));
     assertAuthEquals(NoTokenAuth, getAuthMethod(KERBEROS, TOKEN, true));
   }
@@ -631,7 +692,7 @@ public class TestSaslRPC {
   public void testTokenOnlyServerWithInvalidTokens() throws Exception {
     assertAuthEquals(BadToken, getAuthMethod(SIMPLE,   TOKEN, false));
     assertAuthEquals(BadToken, getAuthMethod(KERBEROS, TOKEN, false));
-    forceSecretManager = false;
+    enableSecretManager = false;
     assertAuthEquals(NoTokenAuth, getAuthMethod(SIMPLE,   TOKEN, false));
     assertAuthEquals(NoTokenAuth, getAuthMethod(KERBEROS, TOKEN, false));
   }
@@ -651,7 +712,7 @@ public class TestSaslRPC {
     assertAuthEquals(TOKEN, getAuthMethod(SIMPLE,   KERBEROS, true));
     assertAuthEquals(TOKEN, getAuthMethod(KERBEROS, KERBEROS, true));
     // can't fallback to simple when using kerberos w/o tokens
-    forceSecretManager = false;
+    enableSecretManager = false;
     assertAuthEquals(Denied(TOKEN), getAuthMethod(SIMPLE,   KERBEROS, true));
     assertAuthEquals(Denied(TOKEN), getAuthMethod(KERBEROS, KERBEROS, true));
   }
@@ -660,7 +721,7 @@ public class TestSaslRPC {
   public void testKerberosServerWithInvalidTokens() throws Exception {
     assertAuthEquals(BadToken, getAuthMethod(SIMPLE,   KERBEROS, false));
     assertAuthEquals(BadToken, getAuthMethod(KERBEROS, KERBEROS, false));
-    forceSecretManager = false;
+    enableSecretManager = false;
     assertAuthEquals(Denied(TOKEN), getAuthMethod(SIMPLE,   KERBEROS, false));
     assertAuthEquals(Denied(TOKEN), getAuthMethod(KERBEROS, KERBEROS, false));
   }
@@ -709,8 +770,11 @@ public class TestSaslRPC {
 
     final TestTokenSecretManager sm = new TestTokenSecretManager();
     boolean useSecretManager = (serverAuth != SIMPLE);
+    if (enableSecretManager != null) {
+      useSecretManager &= enableSecretManager.booleanValue();
+    }
     if (forceSecretManager != null) {
-      useSecretManager &= forceSecretManager.booleanValue();
+      useSecretManager |= forceSecretManager.booleanValue();
     }
     final SecretManager<?> serverSm = useSecretManager ? sm : null;
     
@@ -730,10 +794,10 @@ public class TestSaslRPC {
 
     final Configuration clientConf = new Configuration(conf);
     SecurityUtil.setAuthenticationMethod(clientAuth, clientConf);
-    UserGroupInformation.setConfiguration(clientConf);
     clientConf.setBoolean(
         CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_KEY,
-        true);
+        clientFallBackToSimpleAllowed);
+    UserGroupInformation.setConfiguration(clientConf);
     
     final UserGroupInformation clientUgi =
         UserGroupInformation.createRemoteUser(currentUser + "-CLIENT");

hadoop-yarn-project/CHANGES.txt

@@ -593,6 +593,10 @@ Release 2.1.0-beta - UNRELEASED
     YARN-861. TestContainerManager is failing. (Vinod Kumar Vavilapalli via
     hitesh)
 
+    YARN-874. Making common RPC to switch to not switch to simple when other
+    mechanisms are enabled and thus fix YARN/MR test failures after HADOOP-9421.
+    (Daryn Sharp and Vinod Kumar Vavilapalli via vinodkv)
+
   BREAKDOWN OF HADOOP-8562 SUBTASKS AND RELATED JIRAS
 
     YARN-158. Yarn creating package-info.java must not depend on sh.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java

@@ -231,7 +231,7 @@ public class TestAMAuthorization {
       // server side will assume we are trying simple auth.
       Assert.assertTrue(e.getCause().getMessage().contains(
         "SIMPLE authentication is not enabled.  "
-            + "Available:[KERBEROS, DIGEST]"));
+            + "Available:[TOKEN, KERBEROS]"));
     }
 
     // Now try to validate invalid authorization.