瀏覽代碼

commit 4cd1cdf6b35ba5c8f50c0614c335076cae814f49
Author: Jitendra Nath Pandey <jitendra@yahoo-inc.com>
Date: Tue Feb 9 01:34:27 2010 -0800

HADOOP-6547, HDFS-949, MAPREDUCE-1470 from https://issues.apache.org/jira/secure/attachment/12435271/6547-949-1470-0_20.1.patch

+++ b/YAHOO-CHANGES.txt
+ HADOOP-6547, HDFS-949, MAPREDUCE-1470. Move Delegation token into Common so that we
+ can use it for MapReduce also. It is a combined patch for common, hdfs and mr.
+ (jitendra)
+


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.20-security-patches@1077158 13f79535-47bb-0310-9956-ffa450edef68

Owen O'Malley 14 年之前
父節點
當前提交
6aff56e2cd
共有 18 個文件被更改,包括 489 次插入73 次删除
  1. 13 10
      src/core/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java
  2. 21 26
      src/core/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
  3. 17 6
      src/core/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSelector.java
  4. 6 2
      src/core/org/apache/hadoop/security/token/delegation/DelegationKey.java
  5. 1 1
      src/hdfs/org/apache/hadoop/hdfs/DFSClient.java
  6. 1 2
      src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java
  7. 2 2
      src/hdfs/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
  8. 54 0
      src/hdfs/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java
  9. 56 0
      src/hdfs/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
  10. 33 0
      src/hdfs/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSelector.java
  11. 2 2
      src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
  12. 1 2
      src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java
  13. 1 1
      src/mapred/org/apache/hadoop/mapreduce/security/TokenCache.java
  14. 1 1
      src/test/org/apache/hadoop/hdfs/TestDFSClientRetries.java
  15. 2 2
      src/test/org/apache/hadoop/hdfs/security/TestClientProtocolWithDelegationToken.java
  16. 15 15
      src/test/org/apache/hadoop/hdfs/security/TestDelegationToken.java
  17. 1 1
      src/test/org/apache/hadoop/mapreduce/security/TestTokenCache.java
  18. 262 0
      src/test/org/apache/hadoop/security/token/delegation/TestDelegationToken.java

+ 13 - 10
src/hdfs/org/apache/hadoop/hdfs/security/token/DelegationTokenIdentifier.java → src/core/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java

@@ -16,7 +16,11 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.hdfs.security.token;
+package org.apache.hadoop.security.token.delegation;
+
+//import org.apache.hadoop.classification.InterfaceAudience;
+//import static org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate.Project.HDFS;
+//import static org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate.Project.MAPREDUCE;
 
 import java.io.DataInput;
 import java.io.DataOutput;
@@ -27,8 +31,9 @@ import org.apache.hadoop.io.WritableUtils;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.TokenIdentifier;
 
-public class DelegationTokenIdentifier extends TokenIdentifier {
-  static final Text KIND_NAME = new Text("HDFS_DELEGATION_TOKEN");
+//@InterfaceAudience.LimitedPrivate({HDFS, MAPREDUCE})
+public abstract class AbstractDelegationTokenIdentifier 
+extends TokenIdentifier {
 
   private Text owner;
   private Text renewer;
@@ -38,11 +43,11 @@ public class DelegationTokenIdentifier extends TokenIdentifier {
   private int sequenceNumber;
   private int masterKeyId = 0;
   
-  public DelegationTokenIdentifier() {
+  public AbstractDelegationTokenIdentifier() {
     this(new Text(), new Text(), new Text());
   }
   
-  public DelegationTokenIdentifier(Text owner, Text renewer, Text realUser) {
+  public AbstractDelegationTokenIdentifier(Text owner, Text renewer, Text realUser) {
     this.owner = owner;
     this.renewer = renewer;
     if (realUser == null) {
@@ -55,9 +60,7 @@ public class DelegationTokenIdentifier extends TokenIdentifier {
   }
 
   @Override
-  public Text getKind() {
-    return KIND_NAME;
-  }
+  public abstract Text getKind();
   
   /**
    * Get the username encoded in the token identifier
@@ -123,8 +126,8 @@ public class DelegationTokenIdentifier extends TokenIdentifier {
     if (obj == this) {
       return true;
     }
-    if (obj instanceof DelegationTokenIdentifier) {
-      DelegationTokenIdentifier that = (DelegationTokenIdentifier) obj;
+    if (obj instanceof AbstractDelegationTokenIdentifier) {
+      AbstractDelegationTokenIdentifier that = (AbstractDelegationTokenIdentifier) obj;
       return this.sequenceNumber == that.sequenceNumber 
           && this.issueDate == that.issueDate 
           && this.maxDate == that.maxDate

+ 21 - 26
src/hdfs/org/apache/hadoop/hdfs/security/token/DelegationTokenSecretManager.java → src/core/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java

@@ -16,7 +16,11 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.hdfs.security.token;
+package org.apache.hadoop.security.token.delegation;
+
+//import org.apache.hadoop.classification.InterfaceAudience;
+//import static org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate.Project.HDFS;
+//import static org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate.Project.MAPREDUCE;
 
 import java.io.ByteArrayInputStream;
 import java.io.DataInputStream;
@@ -30,23 +34,25 @@ import javax.crypto.SecretKey;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.io.Text;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.SecretManager;
 import org.apache.hadoop.util.Daemon;
 import org.apache.hadoop.util.StringUtils;
 
-public class DelegationTokenSecretManager 
-   extends SecretManager<DelegationTokenIdentifier> {
+//@InterfaceAudience.LimitedPrivate({HDFS, MAPREDUCE})
+public abstract 
+class AbstractDelegationTokenSecretManager<TokenIdent 
+extends AbstractDelegationTokenIdentifier> 
+   extends SecretManager<TokenIdent> {
   private static final Log LOG = LogFactory
-      .getLog(DelegationTokenSecretManager.class);
+      .getLog(AbstractDelegationTokenSecretManager.class);
 
   /** 
    * Cache of currently valid tokens, mapping from DelegationTokenIdentifier 
    * to DelegationTokenInformation. Protected by its own lock.
    */
-  private final Map<DelegationTokenIdentifier, DelegationTokenInformation> currentTokens 
-      = new HashMap<DelegationTokenIdentifier, DelegationTokenInformation>();
+  private final Map<TokenIdent, DelegationTokenInformation> currentTokens 
+      = new HashMap<TokenIdent, DelegationTokenInformation>();
   
   /**
    * Sequence number to create DelegationTokenIdentifier
@@ -68,9 +74,8 @@ public class DelegationTokenSecretManager
   private long tokenRenewInterval;
   private Thread tokenRemoverThread;
   private volatile boolean running;
-  
 
-  public DelegationTokenSecretManager(long delegationKeyUpdateInterval,
+  public AbstractDelegationTokenSecretManager(long delegationKeyUpdateInterval,
       long delegationTokenMaxLifetime, long delegationTokenRenewInterval,
       long delegationTokenRemoverScanInterval) {
     this.keyUpdateInterval = delegationKeyUpdateInterval;
@@ -78,7 +83,7 @@ public class DelegationTokenSecretManager
     this.tokenRenewInterval = delegationTokenRenewInterval;
     this.tokenRemoverScanInterval = delegationTokenRemoverScanInterval;
   }
-  
+
   /** should be called before this object is used */
   public synchronized void startThreads() throws IOException {
     updateCurrentKey();
@@ -140,7 +145,7 @@ public class DelegationTokenSecretManager
   }
   
   @Override
-  protected byte[] createPassword(DelegationTokenIdentifier identifier) {
+  protected byte[] createPassword(TokenIdent identifier) {
     int sequenceNum;
     int id;
     DelegationKey key;
@@ -163,7 +168,7 @@ public class DelegationTokenSecretManager
   }
 
   @Override
-  public byte[] retrievePassword(DelegationTokenIdentifier identifier
+  public byte[] retrievePassword(TokenIdent identifier
                                  ) throws InvalidToken {
     DelegationTokenInformation info = null;
     synchronized (currentTokens) {
@@ -183,12 +188,12 @@ public class DelegationTokenSecretManager
    * Renew a delegation token. Canceled tokens are not renewed. Return true if
    * the token is successfully renewed; false otherwise.
    */
-  public Boolean renewToken(Token<DelegationTokenIdentifier> token,
+  public Boolean renewToken(Token<TokenIdent> token,
       String renewer) throws InvalidToken, IOException {
     long now = System.currentTimeMillis();
     ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier());
     DataInputStream in = new DataInputStream(buf);
-    DelegationTokenIdentifier id = new DelegationTokenIdentifier();
+    TokenIdent id = createIdentifier();
     id.readFields(in);
     synchronized (currentTokens) {
       if (currentTokens.get(id) == null) {
@@ -232,11 +237,11 @@ public class DelegationTokenSecretManager
    * Cancel a token by removing it from cache. Return true if 
    * token exists in cache; false otherwise.
    */
-  public Boolean cancelToken(Token<DelegationTokenIdentifier> token,
+  public Boolean cancelToken(Token<TokenIdent> token,
       String canceller) throws IOException {
     ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier());
     DataInputStream in = new DataInputStream(buf);
-    DelegationTokenIdentifier id = new DelegationTokenIdentifier();
+    TokenIdent id = createIdentifier();
     id.readFields(in);
     if (id.getRenewer() == null) {
       LOG.warn("Renewer is null: Invalid Identifier");
@@ -267,16 +272,6 @@ public class DelegationTokenSecretManager
   public static SecretKey createSecretKey(byte[] key) {
     return SecretManager.createSecretKey(key);
   }
-  
-  /**
-   * Create an empty delegation token identifier
-   * @return a newly created empty delegation token identifier
-   */
-  @Override
-  public DelegationTokenIdentifier createIdentifier() {
-    return new DelegationTokenIdentifier();
-  }
-
 
   /** Utility class to encapsulate a token's renew date and password. */
   private static class DelegationTokenInformation {

+ 17 - 6
src/hdfs/org/apache/hadoop/hdfs/security/token/DelegationTokenSelector.java → src/core/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSelector.java

@@ -16,33 +16,44 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.hdfs.security.token;
+package org.apache.hadoop.security.token.delegation;
 
 import java.util.Collection;
 
+//import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenIdentifier;
 import org.apache.hadoop.security.token.TokenSelector;
+//import static org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate.Project.HDFS;
+//import static org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate.Project.MAPREDUCE;
 
 /**
  * Look through tokens to find the first delegation token that matches the
  * service and return it.
  */
-public class DelegationTokenSelector implements
-    TokenSelector<DelegationTokenIdentifier> {
+//@InterfaceAudience.LimitedPrivate({HDFS, MAPREDUCE})
+public 
+class AbstractDelegationTokenSelector<TokenIdent 
+extends AbstractDelegationTokenIdentifier> 
+    implements TokenSelector<TokenIdent> {
+  private Text kindName;
+  
+  protected AbstractDelegationTokenSelector(Text kindName) {
+    this.kindName = kindName;
+  }
 
   @SuppressWarnings("unchecked")
   @Override
-  public Token<DelegationTokenIdentifier> selectToken(Text service,
+  public Token<TokenIdent> selectToken(Text service,
       Collection<Token<? extends TokenIdentifier>> tokens) {
     if (service == null) {
       return null;
     }
     for (Token<? extends TokenIdentifier> token : tokens) {
-      if (DelegationTokenIdentifier.KIND_NAME.equals(token.getKind())
+      if (kindName.equals(token.getKind())
           && service.equals(token.getService())) {
-        return (Token<DelegationTokenIdentifier>) token;
+        return (Token<TokenIdent>) token;
       }
     }
     return null;

+ 6 - 2
src/hdfs/org/apache/hadoop/hdfs/security/token/DelegationKey.java → src/core/org/apache/hadoop/security/token/delegation/DelegationKey.java

@@ -16,8 +16,9 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.hdfs.security.token;
+package org.apache.hadoop.security.token.delegation;
 
+//import org.apache.hadoop.classification.InterfaceAudience;
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
@@ -26,10 +27,13 @@ import javax.crypto.SecretKey;
 
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableUtils;
+//import static org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate.Project.HDFS;
+//import static org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate.Project.MAPREDUCE;
 
 /**
  * Key used for generating and verifying delegation tokens
  */
+//@InterfaceAudience.LimitedPrivate({HDFS, MAPREDUCE})
 public class DelegationKey implements Writable {
   private int keyId;
   private long expiryDate;
@@ -79,6 +83,6 @@ public class DelegationKey implements Writable {
     int len = WritableUtils.readVInt(in);
     byte[] keyBytes = new byte[len];
     in.readFully(keyBytes);
-    key = DelegationTokenSecretManager.createSecretKey(keyBytes);
+    key = AbstractDelegationTokenSecretManager.createSecretKey(keyBytes);
   }
 }

+ 1 - 1
src/hdfs/org/apache/hadoop/hdfs/DFSClient.java

@@ -31,7 +31,7 @@ import org.apache.hadoop.hdfs.DistributedFileSystem.DiskStatus;
 import org.apache.hadoop.hdfs.protocol.*;
 import org.apache.hadoop.hdfs.security.BlockAccessToken;
 import org.apache.hadoop.hdfs.security.InvalidAccessTokenException;
-import org.apache.hadoop.hdfs.security.token.DelegationTokenIdentifier;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
 import org.apache.hadoop.hdfs.server.common.HdfsConstants;
 import org.apache.hadoop.hdfs.server.common.UpgradeStatusReport;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;

+ 1 - 2
src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java

@@ -30,12 +30,11 @@ import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.FSConstants.DatanodeReportType;
 import org.apache.hadoop.hdfs.protocol.FSConstants.UpgradeAction;
-import org.apache.hadoop.hdfs.security.token.DelegationTokenIdentifier;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
 import org.apache.hadoop.hdfs.server.common.UpgradeStatusReport;
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.hdfs.DFSClient.DFSOutputStream;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.ipc.RemoteException;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.SecretManager.InvalidToken;

+ 2 - 2
src/hdfs/org/apache/hadoop/hdfs/protocol/ClientProtocol.java

@@ -22,8 +22,6 @@ import java.io.*;
 import org.apache.hadoop.ipc.VersionedProtocol;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.hdfs.protocol.FSConstants.UpgradeAction;
-import org.apache.hadoop.hdfs.security.token.DelegationTokenIdentifier;
-import org.apache.hadoop.hdfs.security.token.DelegationTokenSelector;
 import org.apache.hadoop.hdfs.server.common.UpgradeStatusReport;
 import org.apache.hadoop.fs.permission.*;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -33,6 +31,8 @@ import org.apache.hadoop.io.Text;
 import org.apache.hadoop.security.KerberosInfo;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenInfo;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSelector;
 
 /**********************************************************************
  * ClientProtocol is used by user code via 

+ 54 - 0
src/hdfs/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java

@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.security.token.delegation;
+
+//import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
+
+/**
+ * A delegation token identifier that is specific to HDFS.
+ */
+//@InterfaceAudience.Private
+public class DelegationTokenIdentifier 
+    extends AbstractDelegationTokenIdentifier {
+  static final Text HDFS_DELEGATION_KIND = new Text("HDFS_DELEGATION_TOKEN");
+
+  /**
+   * Create an empty delegation token identifier for reading into.
+   */
+  public DelegationTokenIdentifier() {
+  }
+
+  /**
+   * Create a new delegation token identifier
+   * @param owner the effective username of the token owner
+   * @param renewer the username of the renewer
+   * @param realUser the real username of the token owner
+   */
+  public DelegationTokenIdentifier(Text owner, Text renewer, Text realUser) {
+    super(owner, renewer, realUser);
+  }
+
+  @Override
+  public Text getKind() {
+    return HDFS_DELEGATION_KIND;
+  }
+
+}

+ 56 - 0
src/hdfs/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java

@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.security.token.delegation;
+
+//import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager;
+
+/**
+ * A HDFS specific delegation token secret manager.
+ * The secret manager is responsible for generating and accepting the password
+ * for each token.
+ */
+//@InterfaceAudience.Private
+public class DelegationTokenSecretManager
+    extends AbstractDelegationTokenSecretManager<DelegationTokenIdentifier> {
+
+  /**
+   * Create a secret manager
+   * @param delegationKeyUpdateInterval the number of seconds for rolling new
+   *        secret keys.
+   * @param delegationTokenMaxLifetime the maximum lifetime of the delegation
+   *        tokens
+   * @param delegationTokenRenewInterval how often the tokens must be renewed
+   * @param delegationTokenRemoverScanInterval how often the tokens are scanned
+   *        for expired tokens
+   */
+  public DelegationTokenSecretManager(long delegationKeyUpdateInterval,
+                                      long delegationTokenMaxLifetime, 
+                                      long delegationTokenRenewInterval,
+                                      long delegationTokenRemoverScanInterval) {
+    super(delegationKeyUpdateInterval, delegationTokenMaxLifetime,
+          delegationTokenRenewInterval, delegationTokenRemoverScanInterval);
+  }
+
+  @Override
+  public DelegationTokenIdentifier createIdentifier() {
+    return new DelegationTokenIdentifier();
+  }
+
+}

+ 33 - 0
src/hdfs/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSelector.java

@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.security.token.delegation;
+
+//import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSelector;
+
+/**
+ * A delegation token that is specialized for HDFS
+ */
+//@InterfaceAudience.Private
+public class DelegationTokenSelector
+    extends AbstractDelegationTokenSelector<DelegationTokenIdentifier>{
+
+  public DelegationTokenSelector() {
+    super(DelegationTokenIdentifier.HDFS_DELEGATION_KIND);
+  }
+}

+ 2 - 2
src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java

@@ -35,8 +35,8 @@ import org.apache.hadoop.hdfs.security.ExportedAccessKeys;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.SecretManager.InvalidToken;
-import org.apache.hadoop.hdfs.security.token.DelegationTokenIdentifier;
-import org.apache.hadoop.hdfs.security.token.DelegationTokenSecretManager;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager;
 import org.apache.hadoop.util.*;
 import org.apache.hadoop.metrics.util.MBeanUtil;
 import org.apache.hadoop.net.CachedDNSToSwitchMapping;

+ 1 - 2
src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java

@@ -46,7 +46,6 @@ import org.apache.hadoop.http.HttpServer;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.ipc.*;
 import org.apache.hadoop.conf.*;
-import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.net.NetworkTopology;
@@ -59,7 +58,7 @@ import org.apache.hadoop.security.authorize.RefreshAuthorizationPolicyProtocol;
 import org.apache.hadoop.security.authorize.ServiceAuthorizationManager;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.SecretManager.InvalidToken;
-import org.apache.hadoop.hdfs.security.token.DelegationTokenIdentifier;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
 import org.apache.hadoop.security.Groups;
 import org.apache.hadoop.security.RefreshUserToGroupMappingsProtocol;
 

+ 1 - 1
src/mapred/org/apache/hadoop/mapreduce/security/TokenCache.java

@@ -29,7 +29,7 @@ import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.security.token.DelegationTokenIdentifier;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapred.JobConf;

+ 1 - 1
src/test/org/apache/hadoop/hdfs/TestDFSClientRetries.java

@@ -28,7 +28,6 @@ import org.apache.hadoop.fs.*;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.protocol.*;
 import org.apache.hadoop.hdfs.protocol.FSConstants.UpgradeAction;
-import org.apache.hadoop.hdfs.security.token.DelegationTokenIdentifier;
 import org.apache.hadoop.hdfs.server.common.*;
 import org.apache.hadoop.hdfs.server.namenode.NotReplicatedYetException;
 import org.apache.hadoop.io.*;
@@ -37,6 +36,7 @@ import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.SecretManager.InvalidToken;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
 
 import junit.framework.TestCase;
 

+ 2 - 2
src/test/org/apache/hadoop/hdfs/security/TestClientProtocolWithDelegationToken.java

@@ -38,10 +38,10 @@ import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.ipc.Server;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.protocol.ClientProtocol;
-import org.apache.hadoop.hdfs.security.token.DelegationTokenIdentifier;
-import org.apache.hadoop.hdfs.security.token.DelegationTokenSecretManager;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager;
 import org.apache.hadoop.security.SaslInputStream;
 import org.apache.hadoop.security.SaslRpcClient;
 import org.apache.hadoop.security.SaslRpcServer;

+ 15 - 15
src/test/org/apache/hadoop/security/TestDelegationToken.java → src/test/org/apache/hadoop/hdfs/security/TestDelegationToken.java

@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.security;
+package org.apache.hadoop.hdfs.security;
 
 
 
@@ -29,14 +29,15 @@ import junit.framework.Assert;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.SecretManager.InvalidToken;
-import org.apache.hadoop.hdfs.security.token.DelegationTokenIdentifier;
-import org.apache.hadoop.hdfs.security.token.DelegationTokenSecretManager;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -44,7 +45,6 @@ import org.mortbay.log.Log;
 
 public class TestDelegationToken {
   private MiniDFSCluster cluster;
-  FSNamesystem nameSystem;
   Configuration config;
   final private static String GROUP1_NAME = "group1";
   final private static String GROUP2_NAME = "group2";
@@ -54,12 +54,11 @@ public class TestDelegationToken {
   @Before
   public void setUp() throws Exception {
     config = new Configuration();
-    config.setLong("dfs.namenode.delegation.token.max-lifetime", 10000);
-    config.setLong("dfs.namenode.delegation.token.renew-interval", 5000);
+    config.setLong(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY, 10000);
+    config.setLong(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY, 5000);
     FileSystem.setDefaultUri(config, "hdfs://localhost:" + "0");
     cluster = new MiniDFSCluster(0, config, 1, true, true, true,  null, null, null, null);
     cluster.waitActive();
-    nameSystem = cluster.getNameNode().namesystem;
   }
 
   @After
@@ -71,8 +70,8 @@ public class TestDelegationToken {
 
   private Token<DelegationTokenIdentifier> generateDelegationToken(
       String owner, String renewer) {
-    DelegationTokenSecretManager dtSecretManager = 
-        nameSystem.getDelegationTokenSecretManager();
+    DelegationTokenSecretManager dtSecretManager = cluster.getNameNode()
+        .getNamesystem().getDelegationTokenSecretManager();
     DelegationTokenIdentifier dtId = new DelegationTokenIdentifier(new Text(
         owner), new Text(renewer), null);
     return new Token<DelegationTokenIdentifier>(dtId, dtSecretManager);
@@ -80,8 +79,8 @@ public class TestDelegationToken {
   
   @Test
   public void testDelegationTokenSecretManager() throws Exception {
-    DelegationTokenSecretManager dtSecretManager = 
-        nameSystem.getDelegationTokenSecretManager();
+    DelegationTokenSecretManager dtSecretManager = cluster.getNameNode()
+        .getNamesystem().getDelegationTokenSecretManager();
     Token<DelegationTokenIdentifier> token = generateDelegationToken(
         "SomeUser", "JobTracker");
     // Fake renewer should not be able to renew
@@ -110,8 +109,8 @@ public class TestDelegationToken {
   
   @Test 
   public void testCancelDelegationToken() throws Exception {
-    DelegationTokenSecretManager dtSecretManager = 
-        nameSystem.getDelegationTokenSecretManager();
+    DelegationTokenSecretManager dtSecretManager = cluster.getNameNode()
+        .getNamesystem().getDelegationTokenSecretManager();
     Token<DelegationTokenIdentifier> token = generateDelegationToken(
         "SomeUser", "JobTracker");
     //Fake renewer should not be able to renew
@@ -122,7 +121,8 @@ public class TestDelegationToken {
   
   @Test
   public void testDelegationTokenDFSApi() throws Exception {
-    DelegationTokenSecretManager dtSecretManager = nameSystem.getDelegationTokenSecretManager();
+    DelegationTokenSecretManager dtSecretManager = cluster.getNameNode()
+        .getNamesystem().getDelegationTokenSecretManager();
     DistributedFileSystem dfs = (DistributedFileSystem) cluster.getFileSystem();
     Token<DelegationTokenIdentifier> token = dfs.getDelegationToken(new Text("JobTracker"));
     DelegationTokenIdentifier identifier = new DelegationTokenIdentifier();

+ 1 - 1
src/test/org/apache/hadoop/mapreduce/security/TestTokenCache.java

@@ -38,7 +38,7 @@ import org.apache.hadoop.examples.SleepJob;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.security.token.DelegationTokenIdentifier;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.NullWritable;

+ 262 - 0
src/test/org/apache/hadoop/security/token/delegation/TestDelegationToken.java

@@ -0,0 +1,262 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.security.token.delegation;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+import junit.framework.Assert;
+
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.security.token.SecretManager.InvalidToken;
+import org.junit.Test;
+import org.mortbay.log.Log;
+
+import static org.junit.Assert.*;
+
+public class TestDelegationToken {
+  private static final Text KIND = new Text("MY KIND");
+
+  public static class TestDelegationTokenIdentifier 
+  extends AbstractDelegationTokenIdentifier
+  implements Writable {
+
+    public TestDelegationTokenIdentifier() {
+    }
+
+    public TestDelegationTokenIdentifier(Text owner, Text renewer, Text realUser) {
+      super(owner, renewer, realUser);
+    }
+
+    @Override
+    public Text getKind() {
+      return KIND;
+    }
+    
+    public void write(DataOutput out) throws IOException {
+      super.write(out); 
+    }
+    public void readFields(DataInput in) throws IOException {
+      super.readFields(in);
+    }
+  }
+  
+  public static class TestDelegationTokenSecretManager 
+  extends AbstractDelegationTokenSecretManager<TestDelegationTokenIdentifier> {
+
+    public TestDelegationTokenSecretManager(long delegationKeyUpdateInterval,
+                         long delegationTokenMaxLifetime,
+                         long delegationTokenRenewInterval,
+                         long delegationTokenRemoverScanInterval) {
+      super(delegationKeyUpdateInterval, delegationTokenMaxLifetime,
+            delegationTokenRenewInterval, delegationTokenRemoverScanInterval);
+    }
+
+    @Override
+    public TestDelegationTokenIdentifier createIdentifier() {
+      return new TestDelegationTokenIdentifier();
+    }
+    
+    @Override
+    protected byte[] createPassword(TestDelegationTokenIdentifier t) {
+      return super.createPassword(t);
+    }
+  }
+  
+  public static class TokenSelector extends 
+  AbstractDelegationTokenSelector<TestDelegationTokenIdentifier>{
+
+    protected TokenSelector() {
+      super(KIND);
+    }    
+  }
+  
+  @Test
+  public void testSerialization() throws Exception {
+    TestDelegationTokenIdentifier origToken = new 
+                        TestDelegationTokenIdentifier(new Text("alice"), 
+                                          new Text("bob"), 
+                                          new Text("colin"));
+    TestDelegationTokenIdentifier newToken = new TestDelegationTokenIdentifier();
+    origToken.setIssueDate(123);
+    origToken.setMasterKeyId(321);
+    origToken.setMaxDate(314);
+    origToken.setSequenceNumber(12345);
+    
+    // clone origToken into newToken
+    DataInputBuffer inBuf = new DataInputBuffer();
+    DataOutputBuffer outBuf = new DataOutputBuffer();
+    origToken.write(outBuf);
+    inBuf.reset(outBuf.getData(), 0, outBuf.getLength());
+    newToken.readFields(inBuf);
+    
+    // now test the fields
+    assertEquals("alice", newToken.getUser().getUserName());
+    assertEquals(new Text("bob"), newToken.getRenewer());
+    assertEquals("colin", newToken.getUser().getRealUser().getUserName());
+    assertEquals(123, newToken.getIssueDate());
+    assertEquals(321, newToken.getMasterKeyId());
+    assertEquals(314, newToken.getMaxDate());
+    assertEquals(12345, newToken.getSequenceNumber());
+    assertEquals(origToken, newToken);
+  }
+  
+  private Token<TestDelegationTokenIdentifier> generateDelegationToken(
+      TestDelegationTokenSecretManager dtSecretManager,
+      String owner, String renewer) {
+    TestDelegationTokenIdentifier dtId = 
+      new TestDelegationTokenIdentifier(new Text(
+        owner), new Text(renewer), null);
+    return new Token<TestDelegationTokenIdentifier>(dtId, dtSecretManager);
+  }
+  @Test
+  public void testDelegationTokenSecretManager() throws Exception {
+    TestDelegationTokenSecretManager dtSecretManager = 
+      new TestDelegationTokenSecretManager(24*60*60*1000,
+          3*1000,1*1000,3600000);
+    try {
+      dtSecretManager.startThreads();
+      Token<TestDelegationTokenIdentifier> token = generateDelegationToken(
+          dtSecretManager, "SomeUser", "JobTracker");
+      // Fake renewer should not be able to renew
+      Assert.assertFalse(dtSecretManager.renewToken(token, "FakeRenewer"));
+      Assert.assertTrue(dtSecretManager.renewToken(token, "JobTracker"));
+      TestDelegationTokenIdentifier identifier = 
+        new TestDelegationTokenIdentifier();
+      byte[] tokenId = token.getIdentifier();
+      identifier.readFields(new DataInputStream(
+          new ByteArrayInputStream(tokenId)));
+      Assert.assertTrue(null != dtSecretManager.retrievePassword(identifier));
+      Log.info("Sleep to expire the token");
+      Thread.sleep(2000);
+      //Token should be expired
+      try {
+        dtSecretManager.retrievePassword(identifier);
+        //Should not come here
+        Assert.fail("Token should have expired");
+      } catch (InvalidToken e) {
+        //Success
+      }
+      Assert.assertTrue(dtSecretManager.renewToken(token, "JobTracker"));
+      Log.info("Sleep beyond the max lifetime");
+      Thread.sleep(2000);
+      Assert.assertFalse(dtSecretManager.renewToken(token, "JobTracker"));
+    } finally {
+      dtSecretManager.stopThreads();
+    }
+  }
+  @Test 
+  public void testCancelDelegationToken() throws Exception {
+    TestDelegationTokenSecretManager dtSecretManager = 
+      new TestDelegationTokenSecretManager(24*60*60*1000,
+        10*1000,1*1000,3600000);
+    try {
+      dtSecretManager.startThreads();
+      Token<TestDelegationTokenIdentifier> token = generateDelegationToken(
+          dtSecretManager, "SomeUser", "JobTracker");
+      //Fake renewer should not be able to renew
+      Assert.assertFalse(dtSecretManager.cancelToken(token, "FakeCanceller"));
+      Assert.assertTrue(dtSecretManager.cancelToken(token, "JobTracker"));
+      Assert.assertFalse(dtSecretManager.renewToken(token, "JobTracker"));
+    } finally {
+      dtSecretManager.stopThreads();
+    }
+  }
+  @Test
+  public void testRollMasterKey() throws Exception {
+    TestDelegationTokenSecretManager dtSecretManager = 
+      new TestDelegationTokenSecretManager(24*60*60*1000,
+        10*1000,1*1000,3600000);
+    try {
+      dtSecretManager.startThreads();
+      //generate a token and store the password
+      Token<TestDelegationTokenIdentifier> token = generateDelegationToken(
+          dtSecretManager, "SomeUser", "JobTracker");
+      byte[] oldPasswd = token.getPassword();
+      //store the length of the keys list
+      int prevNumKeys = dtSecretManager.getAllKeys().length;
+      
+      dtSecretManager.rollMasterKey();
+      
+      //after rolling, the length of the keys list must increase
+      int currNumKeys = dtSecretManager.getAllKeys().length;
+      Assert.assertEquals((currNumKeys - prevNumKeys) >= 1, true);
+      
+      //after rolling, the token that was generated earlier must
+      //still be valid (retrievePassword will fail if the token
+      //is not valid)
+      ByteArrayInputStream bi = 
+        new ByteArrayInputStream(token.getIdentifier());
+      TestDelegationTokenIdentifier identifier = 
+        dtSecretManager.createIdentifier();
+      identifier.readFields(new DataInputStream(bi));
+      byte[] newPasswd = 
+        dtSecretManager.retrievePassword(identifier);
+      //compare the passwords
+      Assert.assertEquals(oldPasswd, newPasswd);
+    } finally {
+      dtSecretManager.stopThreads();
+    }
+  }
+  @Test
+  @SuppressWarnings("unchecked")
+  public void testDelegationTokenSelector() throws Exception {
+    TestDelegationTokenSecretManager dtSecretManager = 
+      new TestDelegationTokenSecretManager(24*60*60*1000,
+        10*1000,1*1000,3600000);
+    try {
+      dtSecretManager.startThreads();
+      AbstractDelegationTokenSelector ds = 
+      new AbstractDelegationTokenSelector<TestDelegationTokenIdentifier>(KIND);
+      
+      //Creates a collection of tokens
+      Token<TestDelegationTokenIdentifier> token1 = generateDelegationToken(
+          dtSecretManager, "SomeUser1", "JobTracker");
+      token1.setService(new Text("MY-SERVICE1"));
+      
+      Token<TestDelegationTokenIdentifier> token2 = generateDelegationToken(
+          dtSecretManager, "SomeUser2", "JobTracker");
+      token2.setService(new Text("MY-SERVICE2"));
+      
+      List<Token<TestDelegationTokenIdentifier>> tokens =
+        new ArrayList<Token<TestDelegationTokenIdentifier>>();
+      tokens.add(token1);
+      tokens.add(token2);
+      
+      //try to select a token with a given service name (created earlier)
+      Token<TestDelegationTokenIdentifier> t = 
+        ds.selectToken(new Text("MY-SERVICE1"), tokens);
+      Assert.assertEquals(t, token1);
+    } finally {
+      dtSecretManager.stopThreads();
+    }
+  }
+}