HDFS-2320. Make 0.20-append protocol changes compatible with 0.20-secuirty. Contributed by Suresh Srinivas.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.20-security@1166946 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -161,6 +161,9 @@ Release 0.20.205.0 - unreleased
     HDFS-1242. Add test for appendFile() race solved in HDFS-142.
     (Todd Lipcon via jitendra)
 
+    HDFS-2320. Make 0.20-append protocol changes compatible with
+    0.20-secuirty. (suresh)
+
 Release 0.20.204.0 - 2011-8-25
 
   NEW FEATURES

src/hdfs/org/apache/hadoop/hdfs/DFSClient.java

@@ -96,6 +96,7 @@ public class DFSClient implements FSConstants, java.io.Closeable {
    * it doesn't, we'll set this false and stop trying.
    */
   private volatile boolean serverSupportsHdfs630 = true;
+  private volatile boolean serverSupportsHdfs200 = true;
  
   public static ClientProtocol createNamenode(Configuration conf) throws IOException {
     return createNamenode(NameNode.getAddress(conf), conf);
@@ -1641,41 +1642,56 @@ public class DFSClient implements FSConstants, java.io.Closeable {
           }
         }
       }
+      updateBlockInfo(newInfo);
+      this.locatedBlocks = newInfo;
+      this.currentNode = null;
+    }
+    
+    /** 
+     * For files under construction, update the last block size based
+     * on the length of the block from the datanode.
+     */
+    private void updateBlockInfo(LocatedBlocks newInfo) {
+      if (!serverSupportsHdfs200 || !newInfo.isUnderConstruction()
+          || !(newInfo.locatedBlockCount() > 0)) {
+        return;
+      }
 
-      // if the file is under construction, then fetch size of last block
-      // from datanode.
-      if (newInfo.isUnderConstruction() && newInfo.locatedBlockCount() > 0) {
-        LocatedBlock last = newInfo.get(newInfo.locatedBlockCount()-1);
-        boolean lastBlockInFile = (last.getStartOffset() + 
-                                   last.getBlockSize() == 
-                                   newInfo.getFileLength()); 
-        if (lastBlockInFile && last.getLocations().length > 0) {
-          ClientDatanodeProtocol primary =  null;
-          DatanodeInfo primaryNode = last.getLocations()[0];
-          try {
-            primary = createClientDatanodeProtocolProxy(primaryNode, conf,
-                last.getBlock(), last.getBlockToken(), socketTimeout);
-            Block newBlock = primary.getBlockInfo(last.getBlock());
-            long newBlockSize = newBlock.getNumBytes();
-            long delta = newBlockSize - last.getBlockSize();
-            // if the size of the block on the datanode is different
-            // from what the NN knows about, the datanode wins!
-            last.getBlock().setNumBytes(newBlockSize);
-            long newlength = newInfo.getFileLength() + delta;
-            newInfo.setFileLength(newlength);
-            LOG.debug("DFSClient setting last block " + last + 
-                      " to length " + newBlockSize +
-                      " filesize is now " + newInfo.getFileLength());
-          } catch (IOException e) {
-            LOG.debug("DFSClient file " + src + 
-                      " is being concurrently append to" +
-                      " but datanode " + primaryNode.getHostName() +
-                      " probably does not have block " + last.getBlock());
-          }
+      LocatedBlock last = newInfo.get(newInfo.locatedBlockCount() - 1);
+      boolean lastBlockInFile = (last.getStartOffset() + last.getBlockSize() == newInfo
+          .getFileLength());
+      if (!lastBlockInFile || last.getLocations().length <= 0) {
+        return;
+      }
+      ClientDatanodeProtocol primary = null;
+      DatanodeInfo primaryNode = last.getLocations()[0];
+      try {
+        primary = createClientDatanodeProtocolProxy(primaryNode, conf,
+            last.getBlock(), last.getBlockToken(), socketTimeout);
+        Block newBlock = primary.getBlockInfo(last.getBlock());
+        long newBlockSize = newBlock.getNumBytes();
+        long delta = newBlockSize - last.getBlockSize();
+        // if the size of the block on the datanode is different
+        // from what the NN knows about, the datanode wins!
+        last.getBlock().setNumBytes(newBlockSize);
+        long newlength = newInfo.getFileLength() + delta;
+        newInfo.setFileLength(newlength);
+        LOG.debug("DFSClient setting last block " + last + " to length "
+            + newBlockSize + " filesize is now " + newInfo.getFileLength());
+      } catch (IOException e) {
+        if (e.getMessage().startsWith(
+            "java.io.IOException: java.lang.NoSuchMethodException: "
+                + "org.apache.hadoop.hdfs.protocol"
+                + ".ClientDatanodeProtocol.getBlockInfo")) {
+          // We're talking to a server that doesn't implement HDFS-200.
+          serverSupportsHdfs200 = false;
+        } else {
+          LOG.debug("DFSClient file " + src
+              + " is being concurrently append to" + " but datanode "
+              + primaryNode.getHostName() + " probably does not have block "
+              + last.getBlock());
         }
       }
-      this.locatedBlocks = newInfo;
-      this.currentNode = null;
     }
     
     public synchronized long getFileLength() {

src/hdfs/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java

@@ -32,9 +32,9 @@ public interface ClientDatanodeProtocol extends VersionedProtocol {
   public static final Log LOG = LogFactory.getLog(ClientDatanodeProtocol.class);
 
   /**
-   * 5: added getBlockInfo
+   * 4: never return null and always return a newly generated access token
    */
-  public static final long versionID = 5L;
+  public static final long versionID = 4L;
 
   /** Start generation-stamp recovery for specified block
    * @param block the specified block

src/hdfs/org/apache/hadoop/hdfs/protocol/ClientProtocol.java

@@ -50,9 +50,12 @@ public interface ClientProtocol extends VersionedProtocol {
    * Compared to the previous version the following changes have been introduced:
    * (Only the latest change is reflected.
    * The log of historical changes can be retrieved from the svn).
-   * 63: recoverLease return if the file is closed or not
+   * 61: Serialized format of BlockTokenIdentifier changed to contain
+   *     multiple blocks within a single BlockTokenIdentifier 
+   *     
+   *     (bumped to 61 to bring in line with trunk)
    */
-  public static final long versionID = 63L;
+  public static final long versionID = 61L;
   
   ///////////////////////////////////////
   // File contents

src/hdfs/org/apache/hadoop/hdfs/protocol/DataTransferProtocol.java

@@ -36,10 +36,11 @@ public interface DataTransferProtocol {
    * when protocol changes. It is not very obvious. 
    */
   /*
-   * Version 19:
-   * A heartbeat is sent from the client to pipeline and then acked back
+   * Version 18:
+   *    Change the block packet ack protocol to include seqno,
+   *    numberOfReplies, reply0, reply1, ...
    */
-  public static final int DATA_TRANSFER_VERSION = 19;
+  public static final int DATA_TRANSFER_VERSION = 17;
 
   // Processed at datanode stream-handler
   public static final byte OP_WRITE_BLOCK = (byte) 80;

src/hdfs/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java

@@ -41,11 +41,13 @@ import org.apache.hadoop.security.KerberosInfo;
     clientPrincipal = DFSConfigKeys.DFS_DATANODE_USER_NAME_KEY)
 public interface DatanodeProtocol extends VersionedProtocol {
   /**
-   * 26: nextGenerationStamp has a new parameter indicating if it is for
-   * NameNode initiated lease recovery or not
+   * 25: Serialized format of BlockTokenIdentifier changed to contain
+   *     multiple blocks within a single BlockTokenIdentifier
+   *     
+   *     (bumped to 25 to bring in line with trunk)
    */
-  public static final long versionID = 26L;
-
+  public static final long versionID = 25L;
+  
   // error code
   final static int NOTIFY = 0;
   final static int DISK_ERROR = 1; // there are still valid volumes on DN