commit 3bd8328161d6b16b48d4abf29914e8289ae8e375

Author: Jakob Homan <jhoman@yahoo-inc.com>
Date:   Wed Sep 22 14:55:04 2010 -0700

    Modulate HDFS-1353 to not bump the RPC protocol versions.


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.20-security-patches@1077725 13f79535-47bb-0310-9956-ffa450edef68

src/hdfs/org/apache/hadoop/hdfs/protocol/ClientProtocol.java

@@ -50,9 +50,12 @@ public interface ClientProtocol extends VersionedProtocol {
    * Compared to the previous version the following changes have been introduced:
    * (Only the latest change is reflected.
    * The log of historical changes can be retrieved from the svn).
-   * 62: Remove getBlockLocations optimization
+   * 61: Serialized format of BlockTokenIdentifier changed to contain
+   *     multiple blocks within a single BlockTokenIdentifier 
+   *     
+   *     (bumped to 61 to bring in line with trunk)
    */
-  public static final long versionID = 62L;
+  public static final long versionID = 61L;
   
   ///////////////////////////////////////
   // File contents

src/hdfs/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java

@@ -21,6 +21,7 @@ package org.apache.hadoop.hdfs.security.token.block;
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.EnumSet;
 
 import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager.AccessMode;
@@ -35,20 +36,23 @@ public class BlockTokenIdentifier extends TokenIdentifier {
   private long expiryDate;
   private int keyId;
   private String userId;
-  private long blockId;
+  private long [] blockIds;
   private EnumSet<AccessMode> modes;
 
   private byte [] cache;
   
   public BlockTokenIdentifier() {
-    this(null, 0l, EnumSet.noneOf(AccessMode.class));
+    this(null, new long [] {}, EnumSet.noneOf(AccessMode.class));
   }
 
-  public BlockTokenIdentifier(String userId, long blockId,
+  public BlockTokenIdentifier(String userId, long [] blockIds,
       EnumSet<AccessMode> modes) {
+    if(blockIds == null) 
+      throw new IllegalArgumentException("blockIds can't be null");
     this.cache = null;
     this.userId = userId;
-    this.blockId  = blockId;
+    this.blockIds = Arrays.copyOf(blockIds, blockIds.length);
+    Arrays.sort(this.blockIds);
     this.modes = modes == null ? EnumSet.noneOf(AccessMode.class) : modes;
   }
 
@@ -60,7 +64,7 @@ public class BlockTokenIdentifier extends TokenIdentifier {
   @Override
   public UserGroupInformation getUser() {
     if (userId == null || "".equals(userId)) {
-      return UserGroupInformation.createRemoteUser(Long.toString(blockId));
+      return UserGroupInformation.createRemoteUser(Arrays.toString(blockIds));
     }
     return UserGroupInformation.createRemoteUser(userId);
   }
@@ -87,8 +91,25 @@ public class BlockTokenIdentifier extends TokenIdentifier {
     return userId;
   }
 
-  public long getBlockId() {
-    return blockId;
+  /**
+   * Return sorted array of blockIds this {@link BlockTokenIdentifier} includes
+   */
+  public long [] getBlockIds() {
+    return blockIds;
+  }
+  
+  /**
+   * Is specified blockId included in this BlockTokenIdentifier?
+   */
+  public boolean isBlockIncluded(long blockId) {
+    switch(blockIds.length) {
+    case 1:
+      return blockIds[0] == blockId;
+    case 2:
+      return (blockIds[0] == blockId) || (blockIds[1] == blockId);
+    default:
+      return Arrays.binarySearch(blockIds, blockId) >= 0;  
+    }
   }
 
   public EnumSet<AccessMode> getAccessModes() {
@@ -99,7 +120,7 @@ public class BlockTokenIdentifier extends TokenIdentifier {
   public String toString() {
     return "block_token_identifier (expiryDate=" + this.getExpiryDate()
         + ", keyId=" + this.getKeyId() + ", userId=" + this.getUserId()
-        + ", blockIds=" + blockId + ", access modes="
+        + ", blockIds=" + Arrays.toString(blockIds) + ", access modes="
         + this.getAccessModes() + ")";
   }
 
@@ -116,7 +137,7 @@ public class BlockTokenIdentifier extends TokenIdentifier {
       BlockTokenIdentifier that = (BlockTokenIdentifier) obj;
       return this.expiryDate == that.expiryDate && this.keyId == that.keyId
           && isEqual(this.userId, that.userId) 
-          && this.blockId == that.blockId
+          && Arrays.equals(this.blockIds, that.blockIds)
           && isEqual(this.modes, that.modes);
     }
     return false;
@@ -124,7 +145,7 @@ public class BlockTokenIdentifier extends TokenIdentifier {
 
   /** {@inheritDoc} */
   public int hashCode() {
-    return (int) expiryDate ^ keyId ^ (int)blockId ^ modes.hashCode()
+    return (int) expiryDate ^ keyId ^ Arrays.hashCode(blockIds) ^ modes.hashCode()
         ^ (userId == null ? 0 : userId.hashCode());
   }
 
@@ -133,7 +154,9 @@ public class BlockTokenIdentifier extends TokenIdentifier {
     expiryDate = WritableUtils.readVLong(in);
     keyId = WritableUtils.readVInt(in);
     userId = WritableUtils.readString(in);
-    blockId = WritableUtils.readVLong(in);
+    blockIds = new long[WritableUtils.readVInt(in)];
+    for(int i = 0; i < blockIds.length; i++)
+      blockIds[i] = WritableUtils.readVLong(in);
     int length = WritableUtils.readVInt(in);
     for (int i = 0; i < length; i++) {
       modes.add(WritableUtils.readEnum(in, AccessMode.class));
@@ -144,7 +167,9 @@ public class BlockTokenIdentifier extends TokenIdentifier {
     WritableUtils.writeVLong(out, expiryDate);
     WritableUtils.writeVInt(out, keyId);
     WritableUtils.writeString(out, userId);
-    WritableUtils.writeVLong(out, blockId);
+    WritableUtils.writeVInt(out, blockIds.length);
+    for(int i = 0; i < blockIds.length; i++)
+      WritableUtils.writeVLong(out, blockIds[i]);
     WritableUtils.writeVInt(out, modes.size());
     for (AccessMode aMode : modes) {
       WritableUtils.writeEnum(out, aMode);

src/hdfs/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java

@@ -174,16 +174,29 @@ public class BlockTokenSecretManager extends
   /** Generate an block token for current user */
   public Token<BlockTokenIdentifier> generateToken(Block block,
       EnumSet<AccessMode> modes) throws IOException {
-    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
-    String userID = (ugi == null ? null : ugi.getShortUserName());
-    return generateToken(userID, block, modes);
+    return generateToken(new long [] { block.getBlockId() }, modes);
   }
 
   /** Generate a block token for a specified user */
   public Token<BlockTokenIdentifier> generateToken(String userId, Block block,
       EnumSet<AccessMode> modes) throws IOException {
-    BlockTokenIdentifier id = new BlockTokenIdentifier(userId, 
-        block.getBlockId(), modes);
+    return generateToken(userId, new long [] { block.getBlockId() }, modes);
+  }
+
+  /** Generate a block token for the current user based on a collection
+   * of blockIds
+   */
+  public Token<BlockTokenIdentifier> generateToken(long[] blockIds,
+      EnumSet<AccessMode> modes) throws IOException {
+    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+    String userID = (ugi == null ? null : ugi.getShortUserName());
+    return generateToken(userID, blockIds, modes);
+  }
+  
+  /** Generate a block token based on a collection of blockIds */
+  public Token<BlockTokenIdentifier> generateToken(String userID,
+      long[] blockIds, EnumSet<AccessMode> modes) {
+    BlockTokenIdentifier id = new BlockTokenIdentifier(userID, blockIds, modes);
     return new Token<BlockTokenIdentifier>(id, this);
   }
 
@@ -202,7 +215,7 @@ public class BlockTokenSecretManager extends
       throw new InvalidToken("Block token with " + id.toString()
           + " doesn't belong to user " + userId);
     }
-    if (id.getBlockId() != block.getBlockId()) {
+    if (!id.isBlockIncluded(block.getBlockId())) {
       throw new InvalidToken("Block token with " + id.toString()
           + " doesn't apply to block " + block);
     }

src/hdfs/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java

@@ -41,9 +41,12 @@ import org.apache.hadoop.security.KerberosInfo;
     clientPrincipal = DFSConfigKeys.DFS_DATANODE_USER_NAME_KEY)
 public interface DatanodeProtocol extends VersionedProtocol {
   /**
-   * 26: Remove getBlockLocations optimization
+   * 25: Serialized format of BlockTokenIdentifier changed to contain
+   *     multiple blocks within a single BlockTokenIdentifier
+   *     
+   *     (bumped to 25 to bring in line with trunk)
    */
-  public static final long versionID = 26L;
+  public static final long versionID = 25L;
   
   // error code
   final static int NOTIFY = 0;

src/test/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java

@@ -18,21 +18,11 @@
 
 package org.apache.hadoop.hdfs.security.token.block;
 
-import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyBoolean;
-import static org.mockito.Matchers.anyLong;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
 import java.io.ByteArrayInputStream;
 import java.io.DataInputStream;
 import java.io.IOException;
 import java.net.InetSocketAddress;
+import java.util.Arrays;
 import java.util.EnumSet;
 import java.util.Set;
 
@@ -40,8 +30,8 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.commons.logging.impl.Log4JLogger;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
+import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.io.TestWritable;
@@ -56,7 +46,18 @@ import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenIdentifier;
 import org.apache.log4j.Level;
+
 import org.junit.Test;
+
+import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION;
+import static org.junit.Assert.*;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyBoolean;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 import org.mockito.invocation.InvocationOnMock;
 import org.mockito.stubbing.Answer;
 
@@ -111,7 +112,7 @@ public class TestBlockToken {
         LOG.info("Got: " + id.toString());
         assertTrue("Received BlockTokenIdentifier is wrong", ident.equals(id));
         sm.checkAccess(id, null, block, BlockTokenSecretManager.AccessMode.WRITE);
-        result = new LocatedBlock(new Block(id.getBlockId()), null);
+        result = new LocatedBlock(new Block(id.getBlockIds()[0]), null);
       }
       return result;
     }
@@ -224,4 +225,29 @@ public class TestBlockToken {
       }
     }
   }
+
+  @Test
+  public void collectionOfBlocksActsSanely() {
+    final long[][] testBlockIds = new long [][] {{99l, 7l, -32l, 0l},
+                                                 {},
+                                                 {42l},
+                                                 {-5235l, 2352}};
+    final long [] notBlockIds = new long [] { 32l, 1l, -23423423l};
+    
+    for(long [] bids : testBlockIds) {
+      BlockTokenIdentifier bti = new BlockTokenIdentifier("Madame Butterfly", 
+          bids, EnumSet.noneOf(BlockTokenSecretManager.AccessMode.class));
+      
+      for(long bid : bids) assertTrue(bti.isBlockIncluded(bid));
+      
+      for(long nbid : notBlockIds) assertFalse(bti.isBlockIncluded(nbid));
+      
+      // BlockTokenIdentifiers maintain a sorted array of the block Ids.
+      long[] sorted = Arrays.copyOf(bids, bids.length);
+      Arrays.sort(sorted);
+      
+      assertTrue(Arrays.toString(bids)+" doesn't equal "+Arrays.toString(sorted), 
+          Arrays.equals(bti.getBlockIds(), sorted));
+    }
+  }
 }