Merged r1188836 from branch-0.20-security for HDFS-2416.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.20-security-205@1188851 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -27,6 +27,8 @@ Release 0.20.205.1 - unreleased
     getDefaultBlockSize() and getDefaultReplication() in WebHdfsFileSystem
     and cleared content type in ExceptionHandler. (szetszwo)
 
+    HDFS-2416. Distcp with a webhdfs uri on a secure cluster fails. (jitendra)
+
 Release 0.20.205.0 - 2011.10.06
 
   NEW FEATURES

src/core/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java

@@ -216,6 +216,21 @@ extends AbstractDelegationTokenIdentifier>
     return info.getPassword();
   }
 
+  /**
+   * Verifies that the given identifier and password are valid and match.
+   * @param identifier Token identifier.
+   * @param password Password in the token.
+   * @throws InvalidToken
+   */
+  public synchronized void verifyToken(TokenIdent identifier, byte[] password)
+      throws InvalidToken {
+    byte[] storedPassword = retrievePassword(identifier);
+    if (!Arrays.equals(password, storedPassword)) {
+      throw new InvalidToken("token (" + identifier
+          + ") is invalid, password doesn't match");
+    }
+  }
+  
   /**
    * Renew a delegation token.
    * @param token the token to renew

src/hdfs/org/apache/hadoop/hdfs/ByteRangeInputStream.java

@@ -40,13 +40,13 @@ import org.apache.hadoop.hdfs.web.resources.OffsetParam;
 public class ByteRangeInputStream extends FSInputStream {
   
   /**
-   * This class wraps a URL to allow easy mocking when testing. The URL class
-   * cannot be easily mocked because it is public.
+   * This class wraps a URL and provides method to open connection.
+   * It can be overridden to change how a connection is opened.
    */
-  static class URLOpener {
+  public static class URLOpener {
     protected URL url;
     /** The url with offset parameter */
-    private URL offsetUrl;
+    protected URL offsetUrl;
   
     public URLOpener(URL u) {
       url = u;
@@ -60,7 +60,7 @@ public class ByteRangeInputStream extends FSInputStream {
       return url;
     }
 
-    HttpURLConnection openConnection() throws IOException {
+    protected HttpURLConnection openConnection() throws IOException {
       return (HttpURLConnection)offsetUrl.openConnection();
     }
 
@@ -125,7 +125,13 @@ public class ByteRangeInputStream extends FSInputStream {
     this(new URLOpener(url), new URLOpener(null));
   }
   
-  ByteRangeInputStream(URLOpener o, URLOpener r) {
+  /**
+   * Create with the specified URLOpeners. Original url is used to open the 
+   * stream for the first time. Resolved url is used in subsequent requests.
+   * @param o Original url
+   * @param r Resolved url
+   */
+  public ByteRangeInputStream(URLOpener o, URLOpener r) {
     this.originalURL = o;
     this.resolvedURL = r;
   }

src/hdfs/org/apache/hadoop/hdfs/server/namenode/DfsServlet.java

@@ -71,7 +71,7 @@ abstract class DfsServlet extends HttpServlet {
   protected UserGroupInformation getUGI(HttpServletRequest request,
                                         Configuration conf
 					) throws IOException {
-    return JspHelper.getUGI(request, conf);
+    return JspHelper.getUGI(getServletContext(), request, conf);
   }
 
   /**

src/hdfs/org/apache/hadoop/hdfs/server/namenode/JspHelper.java

@@ -31,6 +31,7 @@ import java.util.Comparator;
 import java.util.Random;
 import java.util.TreeSet;
 
+import javax.servlet.ServletContext;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.jsp.JspWriter;
 
@@ -427,15 +428,30 @@ public class JspHelper {
     return UserGroupInformation.createRemoteUser(strings[0]);
   }
 
-  /** Same as getUGI(request, conf, KERBEROS_SSL, true). */
+  /** Same as getUGI(null, request, conf). */
   public static UserGroupInformation getUGI(final HttpServletRequest request,
       final Configuration conf) throws IOException {
-    return getUGI(request, conf, AuthenticationMethod.KERBEROS_SSL, true);
+    return getUGI(null, request, conf);
+  }
+  
+  /**
+   * Get {@link UserGroupInformation} and possibly the delegation token out of
+   * the request.
+   * @param context the Servlet context
+   * @param request the http request
+   * @param conf configuration
+   * @throws AccessControlException if the request has no token
+   */
+  public static UserGroupInformation getUGI(ServletContext context,
+      HttpServletRequest request, Configuration conf) throws IOException {
+    return getUGI(context, request, conf, AuthenticationMethod.KERBEROS_SSL,
+        true);
   }
 
   /**
    * Get {@link UserGroupInformation} and possibly the delegation token out of
    * the request.
+   * @param context the Servlet context
    * @param request the http request
    * @param conf configuration
    * @param secureAuthMethod the AuthenticationMethod used in secure mode.
@@ -443,8 +459,8 @@ public class JspHelper {
    * @return a new user from the request
    * @throws AccessControlException if the request has no token
    */
-  public static UserGroupInformation getUGI(HttpServletRequest request,
-      Configuration conf,
+  public static UserGroupInformation getUGI(ServletContext context,
+      HttpServletRequest request, Configuration conf,
       final AuthenticationMethod secureAuthMethod,
       final boolean tryUgiParameter) throws IOException {
     final UserGroupInformation ugi;
@@ -464,6 +480,14 @@ public class JspHelper {
         DataInputStream in = new DataInputStream(buf);
         DelegationTokenIdentifier id = new DelegationTokenIdentifier();
         id.readFields(in);
+        if (context != null) {
+          NameNode nn = (NameNode) context.getAttribute("name.node");
+          if (nn != null) {
+            //Verify the token.
+            nn.getNamesystem().getDelegationTokenSecretManager()
+                .verifyToken(id, token.getPassword());
+          }
+        }
         ugi = id.getUser();
         checkUsername(ugi.getShortUserName(), usernameFromQuery);
         checkUsername(ugi.getShortUserName(), user);

src/hdfs/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java

@@ -62,6 +62,7 @@ import org.apache.hadoop.hdfs.web.resources.AccessTimeParam;
 import org.apache.hadoop.hdfs.web.resources.BlockSizeParam;
 import org.apache.hadoop.hdfs.web.resources.BufferSizeParam;
 import org.apache.hadoop.hdfs.web.resources.DelegationParam;
+import org.apache.hadoop.hdfs.web.resources.TokenArgumentParam;
 import org.apache.hadoop.hdfs.web.resources.DeleteOpParam;
 import org.apache.hadoop.hdfs.web.resources.DestinationParam;
 import org.apache.hadoop.hdfs.web.resources.GetOpParam;
@@ -211,7 +212,9 @@ public class NamenodeWebHdfsMethods {
       @QueryParam(ModificationTimeParam.NAME) @DefaultValue(ModificationTimeParam.DEFAULT)
           final ModificationTimeParam modificationTime,
       @QueryParam(AccessTimeParam.NAME) @DefaultValue(AccessTimeParam.DEFAULT)
-          final AccessTimeParam accessTime
+          final AccessTimeParam accessTime,
+      @QueryParam(TokenArgumentParam.NAME) @DefaultValue(TokenArgumentParam.DEFAULT)
+          final TokenArgumentParam delegationTokenArgument
       ) throws IOException, InterruptedException {
 
     if (LOG.isTraceEnabled()) {
@@ -278,7 +281,7 @@ public class NamenodeWebHdfsMethods {
     case RENEWDELEGATIONTOKEN:
     {
       final Token<DelegationTokenIdentifier> token = new Token<DelegationTokenIdentifier>();
-      token.decodeFromUrlString(delegation.getValue());
+      token.decodeFromUrlString(delegationTokenArgument.getValue());
       final long expiryTime = namenode.renewDelegationToken(token);
       final String js = JsonUtil.toJsonString("long", expiryTime);
       return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
@@ -286,7 +289,7 @@ public class NamenodeWebHdfsMethods {
     case CANCELDELEGATIONTOKEN:
     {
       final Token<DelegationTokenIdentifier> token = new Token<DelegationTokenIdentifier>();
-      token.decodeFromUrlString(delegation.getValue());
+      token.decodeFromUrlString(delegationTokenArgument.getValue());
       namenode.cancelDelegationToken(token);
       return Response.ok().type(MediaType.APPLICATION_JSON).build();
     }
@@ -353,6 +356,45 @@ public class NamenodeWebHdfsMethods {
   }
 
   private static final UriFsPathParam ROOT = new UriFsPathParam("");
+  
+  /** Handle HTTP PUT request for root. */
+  @PUT
+  @Path("/")
+  @Consumes({"*/*"})
+  @Produces({MediaType.APPLICATION_JSON})
+  public Response putRoot(
+      @Context final UserGroupInformation ugi,
+      @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
+          final DelegationParam delegation,
+      @QueryParam(PutOpParam.NAME) @DefaultValue(PutOpParam.DEFAULT)
+          final PutOpParam op,
+      @QueryParam(DestinationParam.NAME) @DefaultValue(DestinationParam.DEFAULT)
+          final DestinationParam destination,
+      @QueryParam(OwnerParam.NAME) @DefaultValue(OwnerParam.DEFAULT)
+          final OwnerParam owner,
+      @QueryParam(GroupParam.NAME) @DefaultValue(GroupParam.DEFAULT)
+          final GroupParam group,
+      @QueryParam(PermissionParam.NAME) @DefaultValue(PermissionParam.DEFAULT)
+          final PermissionParam permission,
+      @QueryParam(OverwriteParam.NAME) @DefaultValue(OverwriteParam.DEFAULT)
+          final OverwriteParam overwrite,
+      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
+          final BufferSizeParam bufferSize,
+      @QueryParam(ReplicationParam.NAME) @DefaultValue(ReplicationParam.DEFAULT)
+          final ReplicationParam replication,
+      @QueryParam(BlockSizeParam.NAME) @DefaultValue(BlockSizeParam.DEFAULT)
+          final BlockSizeParam blockSize,
+      @QueryParam(ModificationTimeParam.NAME) @DefaultValue(ModificationTimeParam.DEFAULT)
+          final ModificationTimeParam modificationTime,
+      @QueryParam(AccessTimeParam.NAME) @DefaultValue(AccessTimeParam.DEFAULT)
+          final AccessTimeParam accessTime,
+      @QueryParam(TokenArgumentParam.NAME) @DefaultValue(TokenArgumentParam.DEFAULT)
+          final TokenArgumentParam delegationTokenArgument
+      ) throws IOException, InterruptedException {
+    return put(ugi, delegation, ROOT, op, destination, owner, group,
+        permission, overwrite, bufferSize, replication, blockSize,
+        modificationTime, accessTime, delegationTokenArgument);
+  }
 
   /** Handle HTTP GET request for the root. */
   @GET

src/hdfs/org/apache/hadoop/hdfs/web/AuthFilter.java

@@ -17,11 +17,17 @@
  */
 package org.apache.hadoop.hdfs.web;
 
+import java.io.IOException;
 import java.util.Properties;
 
+import javax.servlet.FilterChain;
 import javax.servlet.FilterConfig;
 import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletRequest;
 
+import org.apache.hadoop.hdfs.web.resources.DelegationParam;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
 import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
@@ -55,6 +61,21 @@ public class AuthFilter extends AuthenticationFilter {
     p.setProperty(PseudoAuthenticationHandler.ANONYMOUS_ALLOWED, "true");
     //set cookie path
     p.setProperty(COOKIE_PATH, "/");
-   return p;
+    return p;
+  }
+
+  @Override
+  public void doFilter(ServletRequest request, ServletResponse response,
+      FilterChain filterChain) throws IOException, ServletException {
+    HttpServletRequest httpRequest = (HttpServletRequest) request;
+    String tokenString = httpRequest
+        .getParameter(DelegationParam.NAME);
+    if (tokenString != null) {
+      //Token is present in the url, therefore token will be used for
+      //authentication, bypass kerberos authentication.
+      filterChain.doFilter(httpRequest, response);
+      return;
+    }
+    super.doFilter(request, response, filterChain);
   }
 }

src/hdfs/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java

@@ -58,6 +58,7 @@ import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
 import org.apache.hadoop.hdfs.web.resources.AccessTimeParam;
 import org.apache.hadoop.hdfs.web.resources.BlockSizeParam;
 import org.apache.hadoop.hdfs.web.resources.BufferSizeParam;
+import org.apache.hadoop.hdfs.web.resources.TokenArgumentParam;
 import org.apache.hadoop.hdfs.web.resources.DeleteOpParam;
 import org.apache.hadoop.hdfs.web.resources.DestinationParam;
 import org.apache.hadoop.hdfs.web.resources.GetOpParam;
@@ -255,7 +256,7 @@ public class WebHdfsFileSystem extends FileSystem
     }
     return url;
   }
-
+  
   private String addDt2Query(String query) throws IOException {
     if (UserGroupInformation.isSecurityEnabled()) {
       synchronized (this) {
@@ -276,24 +277,42 @@ public class WebHdfsFileSystem extends FileSystem
     final String query = op.toQueryString()
         + '&' + new UserParam(ugi)
         + Param.toSortedString("&", parameters);
-    final URL url = getNamenodeURL(path, addDt2Query(query));
+    final URL url;
+    if (op.equals(PutOpParam.Op.RENEWDELEGATIONTOKEN)
+        || op.equals(GetOpParam.Op.GETDELEGATIONTOKEN)) {
+      // Skip adding delegation token for getting or renewing delegation token,
+      // because these operations require kerberos authentication.
+      url = getNamenodeURL(path, query);
+    } else {
+      url = getNamenodeURL(path, addDt2Query(query));
+    }
     if (LOG.isTraceEnabled()) {
       LOG.trace("url=" + url);
     }
     return url;
   }
 
+  private HttpURLConnection getHttpUrlConnection(URL url)
+      throws IOException {
+    final HttpURLConnection conn;
+    try {
+      if (ugi.hasKerberosCredentials()) { 
+        conn = new AuthenticatedURL(AUTH).openConnection(url, authToken);
+      } else {
+        conn = (HttpURLConnection)url.openConnection();
+      }
+    } catch (AuthenticationException e) {
+      throw new IOException("Authentication failed, url=" + url, e);
+    }
+    return conn;
+  }
+  
   private HttpURLConnection httpConnect(final HttpOpParam.Op op, final Path fspath,
       final Param<?,?>... parameters) throws IOException {
     final URL url = toUrl(op, fspath, parameters);
 
     //connect and get response
-    final HttpURLConnection conn;
-    try {
-      conn = new AuthenticatedURL(AUTH).openConnection(url, authToken);
-    } catch(AuthenticationException e) {
-      throw new IOException("Authentication failed, url=" + url, e);
-    }
+    final HttpURLConnection conn = getHttpUrlConnection(url);
     try {
       conn.setRequestMethod(op.getType().toString());
       conn.setDoOutput(op.getDoOutput());
@@ -303,7 +322,7 @@ public class WebHdfsFileSystem extends FileSystem
       }
       conn.connect();
       return conn;
-    } catch(IOException e) {
+    } catch (IOException e) {
       conn.disconnect();
       throw e;
     }
@@ -488,7 +507,24 @@ public class WebHdfsFileSystem extends FileSystem
     statistics.incrementReadOps(1);
     final HttpOpParam.Op op = GetOpParam.Op.OPEN;
     final URL url = toUrl(op, f, new BufferSizeParam(buffersize));
-    return new FSDataInputStream(new ByteRangeInputStream(url));
+    ByteRangeInputStream str = getByteRangeInputStream(url);
+    return new FSDataInputStream(str);
+  }
+
+  private class URLOpener extends ByteRangeInputStream.URLOpener {
+
+    public URLOpener(URL u) {
+      super(u);
+    }
+
+    @Override
+    public HttpURLConnection openConnection() throws IOException {
+      return getHttpUrlConnection(offsetUrl);
+    }
+  }
+  
+  private ByteRangeInputStream getByteRangeInputStream(URL url) {
+    return new ByteRangeInputStream(new URLOpener(url), new URLOpener(null));
   }
 
   @Override
@@ -542,17 +578,19 @@ public class WebHdfsFileSystem extends FileSystem
 
   private synchronized long renewDelegationToken(final Token<?> token
       ) throws IOException {
-    delegationToken = token;
     final HttpOpParam.Op op = PutOpParam.Op.RENEWDELEGATIONTOKEN;
-    final Map<String, Object> m = run(op, null);
-    return (Long)m.get("long");
+    TokenArgumentParam dtargParam = new TokenArgumentParam(
+        token.encodeToUrlString());
+    final Map<String, Object> m = run(op, null, dtargParam);
+    return (Long) m.get("long");
   }
 
   private synchronized void cancelDelegationToken(final Token<?> token
       ) throws IOException {
-    delegationToken = token;
     final HttpOpParam.Op op = PutOpParam.Op.CANCELDELEGATIONTOKEN;
-    run(op, null);
+    TokenArgumentParam dtargParam = new TokenArgumentParam(
+        token.encodeToUrlString());
+    run(op, null, dtargParam);
   }
 
   @Override
@@ -627,13 +665,12 @@ public class WebHdfsFileSystem extends FileSystem
       // update the kerberos credentials, if they are coming from a keytab
       ugi.checkTGTAndReloginFromKeytab();
 
-      return ugi.doAs(new PrivilegedExceptionAction<Long>() {
-        @Override
-        public Long run() throws Exception {
-          final WebHdfsFileSystem webhdfs = getWebHdfs(token, conf);
-          return webhdfs.renewDelegationToken(token);
-        }
-      });
+      try {
+        WebHdfsFileSystem webhdfs = getWebHdfs(token, conf);
+        return webhdfs.renewDelegationToken(token);
+      } catch (URISyntaxException e) {
+        throw new IOException(e);
+      }
     }
   
     @Override
@@ -643,14 +680,12 @@ public class WebHdfsFileSystem extends FileSystem
       // update the kerberos credentials, if they are coming from a keytab
       ugi.checkTGTAndReloginFromKeytab();
 
-      ugi.doAs(new PrivilegedExceptionAction<Void>() {
-        @Override
-        public Void run() throws Exception {
-          final WebHdfsFileSystem webhdfs = getWebHdfs(token, conf);
-          webhdfs.cancelDelegationToken(token);
-          return null;
-        }
-      });
+      try {
+        final WebHdfsFileSystem webhdfs = getWebHdfs(token, conf);
+        webhdfs.cancelDelegationToken(token);
+      } catch (URISyntaxException e) {
+        throw new IOException(e);
+      }
     }
   }
 }

src/hdfs/org/apache/hadoop/hdfs/web/resources/DelegationParam.java

@@ -19,7 +19,7 @@ package org.apache.hadoop.hdfs.web.resources;
 
 import org.apache.hadoop.security.UserGroupInformation;
 
-/** Delegation token parameter. */
+/** Represents delegation token used for authentication. */
 public class DelegationParam extends StringParam {
   /** Parameter name. */
   public static final String NAME = "delegation";

src/hdfs/org/apache/hadoop/hdfs/web/resources/TokenArgumentParam.java

@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+/**
+ * Represents delegation token parameter as method arguments. This is
+ * different from {@link DelegationParam}.
+ */
+public class TokenArgumentParam extends StringParam {
+  /** Parameter name. */
+  public static final String NAME = "token";
+  /** Default parameter value. */
+  public static final String DEFAULT = "";
+
+  private static final Domain DOMAIN = new Domain(NAME, null);
+
+  /**
+   * Constructor.
+   * @param str A string representation of the parameter value.
+   */
+  public TokenArgumentParam(final String str) {
+    super(DOMAIN, str != null && !str.equals(DEFAULT) ? str : null);
+  }
+
+  @Override
+  public String getName() {
+    return NAME;
+  }
+}

src/hdfs/org/apache/hadoop/hdfs/web/resources/UserProvider.java

@@ -50,7 +50,7 @@ public class UserProvider
     final Configuration conf = (Configuration) servletcontext
         .getAttribute(JspHelper.CURRENT_CONF);
     try {
-      return JspHelper.getUGI(request, conf,
+      return JspHelper.getUGI(servletcontext, request, conf,
           AuthenticationMethod.KERBEROS, false);
     } catch (IOException e) {
       throw new RuntimeException(e);

src/test/org/apache/hadoop/hdfs/security/TestDelegationToken.java

@@ -154,7 +154,7 @@ public class TestDelegationToken {
     DelegationTokenSecretManager dtSecretManager = cluster.getNameNode()
         .getNamesystem().getDelegationTokenSecretManager();
     DistributedFileSystem dfs = (DistributedFileSystem) cluster.getFileSystem();
-    Token<DelegationTokenIdentifier> token = dfs.getDelegationToken(new Text("JobTracker"));
+    final Token<DelegationTokenIdentifier> token = dfs.getDelegationToken(new Text("JobTracker"));
     DelegationTokenIdentifier identifier = new DelegationTokenIdentifier();
     byte[] tokenId = token.getIdentifier();
     identifier.readFields(new DataInputStream(
@@ -162,6 +162,15 @@ public class TestDelegationToken {
     LOG.info("A valid token should have non-null password, and should be renewed successfully");
     Assert.assertTrue(null != dtSecretManager.retrievePassword(identifier));
     dtSecretManager.renewToken(token, "JobTracker");
+    UserGroupInformation.createRemoteUser("JobTracker").doAs(
+        new PrivilegedExceptionAction<Object>() {
+          @Override
+          public Object run() throws Exception {
+            token.renew(config);
+            token.cancel(config);
+            return null;
+          }
+        });
   }
   
   @Test
@@ -182,13 +191,23 @@ public class TestDelegationToken {
       }
     });
 
-    final Token<DelegationTokenIdentifier> token = webhdfs.getDelegationToken("JobTracker");
+    final Token<DelegationTokenIdentifier> token = webhdfs
+        .getDelegationToken("JobTracker");
     DelegationTokenIdentifier identifier = new DelegationTokenIdentifier();
     byte[] tokenId = token.getIdentifier();
-    identifier.readFields(new DataInputStream(new ByteArrayInputStream(tokenId)));
+    identifier
+        .readFields(new DataInputStream(new ByteArrayInputStream(tokenId)));
     LOG.info("A valid token should have non-null password, and should be renewed successfully");
     Assert.assertTrue(null != dtSecretManager.retrievePassword(identifier));
     dtSecretManager.renewToken(token, "JobTracker");
+    ugi.doAs(new PrivilegedExceptionAction<Object>() {
+      @Override
+      public Object run() throws Exception {
+        token.renew(config);
+        token.cancel(config);
+        return null;
+      }
+    });
   }
 
   @Test

src/test/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java

@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.web;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URL;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager;
+import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.apache.hadoop.hdfs.web.resources.DelegationParam;
+import org.apache.hadoop.hdfs.web.resources.TokenArgumentParam;
+import org.apache.hadoop.hdfs.web.resources.HttpOpParam;
+import org.apache.hadoop.hdfs.web.resources.PutOpParam;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.junit.Assert;
+import org.junit.Test;
+import static org.mockito.Mockito.mock;
+
+public class TestWebHdfsUrl {
+
+  @Test
+  public void testDelegationTokenInUrl() throws IOException {
+    Configuration conf = new Configuration();
+    final String uri = WebHdfsFileSystem.SCHEME + "://" + "127.0.0.1:9071";
+    // Turn on security
+    conf.set(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
+    UserGroupInformation.setConfiguration(conf);
+    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+    DelegationTokenIdentifier dtId = new DelegationTokenIdentifier(new Text(
+        ugi.getUserName()), null, null);
+    FSNamesystem namesystem = mock(FSNamesystem.class);
+    DelegationTokenSecretManager dtSecretManager = new DelegationTokenSecretManager(
+        86400000, 86400000, 86400000, 86400000, namesystem);
+    dtSecretManager.startThreads();
+    Token<DelegationTokenIdentifier> token = new Token<DelegationTokenIdentifier>(
+        dtId, dtSecretManager);
+    token.setService(new Text("127.0.0.1:9071"));
+    token.setKind(WebHdfsFileSystem.TOKEN_KIND);
+    ugi.addToken(token);
+    final WebHdfsFileSystem webhdfs = (WebHdfsFileSystem) FileSystem.get(
+        URI.create(uri), conf);
+    String tokenString = token.encodeToUrlString();
+    Path fsPath = new Path("/");
+    URL renewTokenUrl = webhdfs.toUrl(PutOpParam.Op.RENEWDELEGATIONTOKEN,
+        fsPath, new TokenArgumentParam(tokenString));
+    URL cancelTokenUrl = webhdfs.toUrl(PutOpParam.Op.CANCELDELEGATIONTOKEN,
+        fsPath, new TokenArgumentParam(tokenString));
+    Assert.assertEquals(
+        generateUrlQueryPrefix(PutOpParam.Op.RENEWDELEGATIONTOKEN,
+            ugi.getUserName())
+            + "&token=" + tokenString, renewTokenUrl.getQuery());
+    Token<DelegationTokenIdentifier> delegationToken = new Token<DelegationTokenIdentifier>(
+        token);
+    delegationToken.setKind(DelegationTokenIdentifier.HDFS_DELEGATION_KIND);
+    Assert.assertEquals(
+        generateUrlQueryPrefix(PutOpParam.Op.CANCELDELEGATIONTOKEN,
+            ugi.getUserName())
+            + "&token="
+            + tokenString
+            + "&"
+            + DelegationParam.NAME
+            + "="
+            + delegationToken.encodeToUrlString(), cancelTokenUrl.getQuery());
+  }
+
+  private String generateUrlQueryPrefix(HttpOpParam.Op op, String username) {
+    return "op=" + op.toString() + "&user.name=" + username;
+  }
+}

src/test/org/apache/hadoop/security/token/delegation/TestDelegationToken.java

@@ -360,6 +360,8 @@ public class TestDelegationToken {
         byte[] storedPassword = dtSecretManager.retrievePassword(id);
         byte[] password = dtSecretManager.createPassword(id, key);
         Assert.assertTrue(Arrays.equals(password, storedPassword));
+        //verify by secret manager api
+        dtSecretManager.verifyToken(id, password);
       }
     } finally {
       dtSecretManager.stopThreads();

src/webapps/hdfs/nn_browsedfscontent.jsp

@@ -21,9 +21,9 @@
 %>
 <%!
   static String getDelegationToken(final NameNode nn,
+                                   final UserGroupInformation ugi,
                                    HttpServletRequest request, Configuration conf) 
                                    throws IOException, InterruptedException {
-    final UserGroupInformation ugi = JspHelper.getUGI(request, conf);
     Token<DelegationTokenIdentifier> token =
       ugi.doAs(
               new PrivilegedExceptionAction<Token<DelegationTokenIdentifier>>()
@@ -36,14 +36,16 @@
   }
 
   public void redirectToRandomDataNode(
-                            NameNode nn, 
+                            ServletContext context, 
                             HttpServletRequest request,
-                            HttpServletResponse resp,
-                            Configuration conf
+                            HttpServletResponse resp
                            ) throws IOException, InterruptedException {
+    Configuration conf = (Configuration) context.getAttribute(JspHelper.CURRENT_CONF);
+    NameNode nn = (NameNode)context.getAttribute("name.node");
+    final UserGroupInformation ugi = JspHelper.getUGI(context, request, conf);
     String tokenString = null;
     if (UserGroupInformation.isSecurityEnabled()) {
-      tokenString = getDelegationToken(nn, request, conf);
+      tokenString = getDelegationToken(nn, ugi, request, conf);
     }
     FSNamesystem fsn = nn.getNamesystem();
     String datanode = fsn.randomDataNode();
@@ -76,9 +78,7 @@
 
 <body>
 <% 
-  NameNode nn = (NameNode)application.getAttribute("name.node");
-  Configuration conf = (Configuration) application.getAttribute(JspHelper.CURRENT_CONF);
-  redirectToRandomDataNode(nn, request, response, conf); 
+  redirectToRandomDataNode(application, request, response); 
 %>
 <hr>