HADOOP-13228. Add delegation token to the connection in DelegationTokenAuthenticator. Contributed by Xiao Chen.

(cherry picked from commit 35356de1ba1cad0fa469ff546263290109c61b77)

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java

@@ -51,6 +51,8 @@ import org.apache.hadoop.util.StringUtils;
 import org.codehaus.jackson.map.ObjectMapper;
 
 import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * An {@link AuthenticationHandler} that implements Kerberos SPNEGO mechanism
@@ -78,6 +80,9 @@ import com.google.common.annotations.VisibleForTesting;
 public abstract class DelegationTokenAuthenticationHandler
     implements AuthenticationHandler {
 
+  private static final Logger LOG =
+      LoggerFactory.getLogger(DelegationTokenAuthenticationHandler.class);
+
   protected static final String TYPE_POSTFIX = "-dt";
 
   public static final String PREFIX = "delegation-token.";
@@ -327,6 +332,8 @@ public abstract class DelegationTokenAuthenticationHandler
       throws IOException, AuthenticationException {
     AuthenticationToken token;
     String delegationParam = getDelegationToken(request);
+    LOG.debug("Authenticating with delegationParam: {}, query string: {}",
+        delegationParam, request.getQueryString());
     if (delegationParam != null) {
       try {
         Token<AbstractDelegationTokenIdentifier> dt = new Token();

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java

@@ -121,6 +121,24 @@ public abstract class DelegationTokenAuthenticator implements Authenticator {
     return hasDt;
   }
 
+  /**
+   * Append the delegation token to the request header if needed.
+   */
+  private void appendDelegationToken(final AuthenticatedURL.Token token,
+      final Token<?> dToken, final HttpURLConnection conn) throws IOException {
+    if (token.isSet()) {
+      LOG.debug("Auth token is set, not appending delegation token.");
+      return;
+    }
+    if (dToken == null) {
+      LOG.warn("Delegation token is null, cannot set on request header.");
+      return;
+    }
+    conn.setRequestProperty(
+        DelegationTokenAuthenticator.DELEGATION_TOKEN_HEADER,
+        dToken.encodeToUrlString());
+  }
+
   @Override
   public void authenticate(URL url, AuthenticatedURL.Token token)
       throws IOException, AuthenticationException {
@@ -283,6 +301,7 @@ public abstract class DelegationTokenAuthenticator implements Authenticator {
     url = new URL(sb.toString());
     AuthenticatedURL aUrl = new AuthenticatedURL(this, connConfigurator);
     HttpURLConnection conn = aUrl.openConnection(url, token);
+    appendDelegationToken(token, dToken, conn);
     conn.setRequestMethod(operation.getHttpMethod());
     HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
     if (hasResponse) {

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestWebDelegationToken.java

@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.security.token.delegation.web;
 
+import static org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator.DelegationTokenOperation;
+
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.minikdc.MiniKdc;
@@ -30,6 +32,7 @@ import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHa
 import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler;
 import org.apache.hadoop.security.authentication.util.KerberosUtil;
 import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager;
+import org.apache.hadoop.test.GenericTestUtils;
 import org.codehaus.jackson.map.ObjectMapper;
 import org.junit.After;
 import org.junit.Assert;
@@ -41,6 +44,8 @@ import org.mortbay.jetty.Server;
 import org.mortbay.jetty.servlet.Context;
 import org.mortbay.jetty.servlet.FilterHolder;
 import org.mortbay.jetty.servlet.ServletHolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import javax.security.auth.Subject;
 import javax.security.auth.kerberos.KerberosPrincipal;
@@ -60,8 +65,6 @@ import java.io.File;
 import java.io.IOException;
 import java.io.Writer;
 import java.net.HttpURLConnection;
-import java.net.InetAddress;
-import java.net.ServerSocket;
 import java.net.URL;
 import java.security.Principal;
 import java.security.PrivilegedActionException;
@@ -76,6 +79,9 @@ import java.util.UUID;
 import java.util.concurrent.Callable;
 
 public class TestWebDelegationToken {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestWebDelegationToken.class);
   private static final String OK_USER = "ok-user";
   private static final String FAIL_USER = "fail-user";
   private static final String FOO_USER = "foo";
@@ -111,7 +117,7 @@ public class TestWebDelegationToken {
       AuthenticationToken token = null;
       if (request.getParameter("authenticated") != null) {
         token = new AuthenticationToken(request.getParameter("authenticated"),
-            "U", "test");
+            "U", "unsupported type");
       } else {
         response.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
         response.setHeader(KerberosAuthenticator.WWW_AUTHENTICATE, "dummy");
@@ -134,6 +140,32 @@ public class TestWebDelegationToken {
     }
   }
 
+  /**
+   * A dummy DelegationTokenAuthenticationHandler to verify that the request
+   * header contains delegation token.
+   */
+  public static class HeaderVerifyingDelegationTokenAuthenticationHandler
+      extends DummyDelegationTokenAuthenticationHandler {
+
+    @Override
+    public boolean managementOperation(AuthenticationToken token,
+        HttpServletRequest request, HttpServletResponse response)
+        throws IOException, AuthenticationException {
+      String op = ServletUtils.getParameter(request,
+          KerberosDelegationTokenAuthenticator.OP_PARAM);
+      if (op != null) {
+        DelegationTokenOperation dtOp = DelegationTokenOperation.valueOf(op);
+        if (dtOp == DelegationTokenOperation.RENEWDELEGATIONTOKEN
+            || dtOp == DelegationTokenOperation.CANCELDELEGATIONTOKEN) {
+          Assert.assertNotNull("Request header should have delegation token",
+              request.getHeader(
+                  DelegationTokenAuthenticator.DELEGATION_TOKEN_HEADER));
+        }
+      }
+      return super.managementOperation(token, request, response);
+    }
+  }
+
   public static class AFilter extends DelegationTokenAuthenticationFilter {
 
     @Override
@@ -146,6 +178,24 @@ public class TestWebDelegationToken {
     }
   }
 
+  /**
+   * A dummy DelegationTokenAuthenticationFilter that uses a
+   * {@link HeaderVerifyingDelegationTokenAuthenticationHandler} to verify that
+   * the request header contains delegation token.
+   */
+  public static class HeaderVerifyingFilter
+      extends DelegationTokenAuthenticationFilter {
+
+    @Override
+    protected Properties getConfiguration(String configPrefix,
+        FilterConfig filterConfig) {
+      Properties conf = new Properties();
+      conf.setProperty(AUTH_TYPE,
+          HeaderVerifyingDelegationTokenAuthenticationHandler.class.getName());
+      return conf;
+    }
+  }
+
   public static class PingServlet extends HttpServlet {
 
     @Override
@@ -203,6 +253,7 @@ public class TestWebDelegationToken {
   @After
   public void cleanUp() throws Exception {
     jetty.stop();
+    jetty = null;
 
     // resetting hadoop security to simple
     org.apache.hadoop.conf.Configuration conf =
@@ -427,6 +478,63 @@ public class TestWebDelegationToken {
     }
   }
 
+  @Test(timeout=120000)
+  public void testDelegationTokenAuthenticatorUsingDT() throws Exception {
+    Context context = new Context();
+    context.setContextPath("/foo");
+    jetty.setHandler(context);
+    context.addFilter(new FilterHolder(HeaderVerifyingFilter.class), "/*", 0);
+    context.addServlet(new ServletHolder(PingServlet.class), "/bar");
+
+    jetty.start();
+    final URL nonAuthURL = new URL(getJettyURL() + "/foo/bar");
+    URL authURL = new URL(getJettyURL() + "/foo/bar?authenticated=foo");
+    URL authURL2 = new URL(getJettyURL() + "/foo/bar?authenticated=bar");
+
+    DelegationTokenAuthenticatedURL.Token token =
+        new DelegationTokenAuthenticatedURL.Token();
+    final DelegationTokenAuthenticatedURL aUrl =
+        new DelegationTokenAuthenticatedURL();
+    aUrl.getDelegationToken(authURL, token, FOO_USER);
+    Assert.assertNotNull(token.getDelegationToken());
+    Assert.assertEquals(new Text("token-kind"),
+        token.getDelegationToken().getKind());
+
+    // Create a token that only has dt so that we can test ops when
+    // authenticating with a delegation token.
+    DelegationTokenAuthenticatedURL.Token dtOnlyToken =
+        new DelegationTokenAuthenticatedURL.Token();
+    dtOnlyToken.setDelegationToken(token.getDelegationToken());
+
+    /**
+     * We're using delegation token, so everything comes from that.
+     * {@link DelegationTokenAuthenticationHandler#authenticate}.
+     *
+     * This means that the special logic we injected at
+     * {@link DummyAuthenticationHandler#authenticate}
+     * (check "authenticated" and return 401) wouldn't work any more.
+     */
+
+    aUrl.getDelegationToken(authURL, dtOnlyToken, FOO_USER);
+    aUrl.renewDelegationToken(authURL, dtOnlyToken);
+    aUrl.renewDelegationToken(nonAuthURL, dtOnlyToken);
+    aUrl.renewDelegationToken(authURL2, dtOnlyToken);
+
+    // Verify that after cancelling, we can't renew.
+    // After cancelling, the dt on token will be set to null. Back it up here.
+    DelegationTokenAuthenticatedURL.Token cancelledToken =
+        new DelegationTokenAuthenticatedURL.Token();
+    cancelledToken.setDelegationToken(dtOnlyToken.getDelegationToken());
+    aUrl.cancelDelegationToken(authURL, dtOnlyToken);
+    try {
+      aUrl.renewDelegationToken(authURL, cancelledToken);
+      Assert.fail();
+    } catch (Exception ex) {
+      LOG.info("Intentional exception caught:", ex);
+      GenericTestUtils.assertExceptionContains("can't be found in cache", ex);
+    }
+  }
+
   private static class DummyDelegationTokenSecretManager
       extends AbstractDelegationTokenSecretManager<DelegationTokenIdentifier> {