HDFS-1449. Fix test failures - ExtendedBlock must return block file name in #getBlockName(). Contributed by Suresh Srinivas.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/hdfs/branches/HDFS-1052@1024336 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -309,6 +309,9 @@ Trunk (unreleased changes)
 
     HDFS-1440. Fix TestComputeInvalidateWork failure. (suresh)
 
+    HDFS-1449. Fix test failures - ExtendedBlock must return 
+    block file name in #getBlockName(). (suresh)
+
 Release 0.21.0 - Unreleased
 
   INCOMPATIBLE CHANGES

src/java/org/apache/hadoop/hdfs/protocol/ExtendedBlock.java

@@ -21,6 +21,8 @@ import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
 
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.hdfs.DeprecatedUTF8;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableFactories;
@@ -29,6 +31,8 @@ import org.apache.hadoop.io.WritableFactory;
 /**
  * Identifies a Block uniquely across the block pools
  */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
 public class ExtendedBlock implements Writable {
   private String poolId;
   private Block block;
@@ -95,8 +99,9 @@ public class ExtendedBlock implements Writable {
     return poolId;
   }
 
+  /** Returns the block file name for the block */
   public String getBlockName() {
-    return poolId + ":" + block;
+    return block.getBlockName();
   }
 
   public long getNumBytes() {
@@ -155,6 +160,6 @@ public class ExtendedBlock implements Writable {
   
   @Override // Object
   public String toString() {
-    return getBlockName();
+    return poolId + ":" + block;
   }
 }

src/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java

@@ -311,7 +311,8 @@ public class NamenodeFsck {
       if (isCorrupt) {
         corrupt++;
         res.corruptBlocks++;
-        out.print("\n" + path + ": CORRUPT block " + block.getBlockName()+"\n");
+        out.print("\n" + path + ": CORRUPT blockpool " + block.getPoolId() + 
+            " block " + block.getBlockName()+"\n");
       }
       if (locs.length >= minReplication)
         res.numMinReplicatedBlocks++;

src/test/hdfs/org/apache/hadoop/hdfs/MiniDFSCluster.java

@@ -41,6 +41,7 @@ import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
 import org.apache.hadoop.hdfs.protocol.ClientProtocol;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.protocol.FSConstants.DatanodeReportType;
 import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
@@ -689,20 +690,23 @@ public class MiniDFSCluster {
   /*
    * Corrupt a block on all datanode
    */
-  void corruptBlockOnDataNodes(String blockName) throws Exception{
+  void corruptBlockOnDataNodes(ExtendedBlock block) throws Exception{
     for (int i=0; i < dataNodes.size(); i++)
-      corruptBlockOnDataNode(i,blockName);
+      corruptBlockOnDataNode(i, block);
   }
 
   /*
    * Corrupt a block on a particular datanode
    */
-  boolean corruptBlockOnDataNode(int i, String blockName) throws Exception {
+  boolean corruptBlockOnDataNode(int i, ExtendedBlock blk) throws Exception {
     Random random = new Random();
     boolean corrupted = false;
     File dataDir = new File(getBaseDirectory() + "data");
     if (i < 0 || i >= dataNodes.size())
       return false;
+    
+    // TODO:FEDERATION use blockPoolId
+    String blockName = blk.getBlockName();
     for (int dn = i*2; dn < i*2+2; dn++) {
       File blockFile = new File(dataDir, "data" + (dn+1) + FINALIZED_DIR_NAME +
                                 blockName);

src/test/hdfs/org/apache/hadoop/hdfs/TestCrcCorruption.java

@@ -31,6 +31,7 @@ import static org.junit.Assert.*;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.io.IOUtils;
 
 /**
@@ -256,7 +257,7 @@ public class TestCrcCorruption {
       DFSTestUtil.createFile(fs, file, fileSize, (short)numDataNodes, 12345L /*seed*/);
       DFSTestUtil.waitReplication(fs, file, (short)numDataNodes);
 
-      String block = DFSTestUtil.getFirstBlock(fs, file).getBlockName();
+      ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, file);
       cluster.corruptBlockOnDataNodes(block);
 
       try {

src/test/hdfs/org/apache/hadoop/hdfs/TestDatanodeBlockScanner.java

@@ -140,10 +140,12 @@ public class TestDatanodeBlockScanner extends TestCase {
     cluster.shutdown();
   }
 
-  public static boolean corruptReplica(String blockName, int replica) throws IOException {
+  public static boolean corruptReplica(ExtendedBlock blk, int replica) throws IOException {
+    String blockName = blk.getLocalBlock().getBlockName();
     Random random = new Random();
     File baseDir = new File(MiniDFSCluster.getBaseDirectory(), "data");
     boolean corrupted = false;
+    // TODO:FEDERATION use BlockPoolId
     for (int i=replica*2; i<replica*2+2; i++) {
       File blockFile = new File(baseDir, "data" + (i+1) + 
           MiniDFSCluster.FINALIZED_DIR_NAME + blockName);
@@ -177,7 +179,7 @@ public class TestDatanodeBlockScanner extends TestCase {
     fs = cluster.getFileSystem();
     Path file1 = new Path("/tmp/testBlockVerification/file1");
     DFSTestUtil.createFile(fs, file1, 1024, (short)3, 0);
-    String block = DFSTestUtil.getFirstBlock(fs, file1).getBlockName();
+    ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, file1);
     
     dfsClient = new DFSClient(new InetSocketAddress("localhost", 
                                         cluster.getNameNodePort()), conf);
@@ -289,7 +291,6 @@ public class TestDatanodeBlockScanner extends TestCase {
     Path file1 = new Path("/tmp/testBlockCorruptRecovery/file");
     DFSTestUtil.createFile(fs, file1, 1024, numReplicas, 0);
     ExtendedBlock blk = DFSTestUtil.getFirstBlock(fs, file1);
-    String block = blk.getBlockName();
     
     dfsClient = new DFSClient(new InetSocketAddress("localhost", 
                                         cluster.getNameNodePort()), conf);
@@ -314,7 +315,7 @@ public class TestDatanodeBlockScanner extends TestCase {
     // Corrupt numCorruptReplicas replicas of block 
     int[] corruptReplicasDNIDs = new int[numCorruptReplicas];
     for (int i=0, j=0; (j != numCorruptReplicas) && (i < numDataNodes); i++) {
-      if (corruptReplica(block, i)) 
+      if (corruptReplica(blk, i)) 
         corruptReplicasDNIDs[j++] = i;
     }
     
@@ -393,7 +394,7 @@ public class TestDatanodeBlockScanner extends TestCase {
       DFSTestUtil.createFile(fs, fileName, 1, REPLICATION_FACTOR, 0);
       DFSTestUtil.waitReplication(fs, fileName, REPLICATION_FACTOR);
 
-      String block = DFSTestUtil.getFirstBlock(fs, fileName).getBlockName();
+      ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, fileName);
 
       // Truncate replica of block
       changeReplicaLength(block, 0, -1);
@@ -420,7 +421,9 @@ public class TestDatanodeBlockScanner extends TestCase {
   /**
    * Change the length of a block at datanode dnIndex
    */
-  static boolean changeReplicaLength(String blockName, int dnIndex, int lenDelta) throws IOException {
+  static boolean changeReplicaLength(ExtendedBlock blk, int dnIndex,
+      int lenDelta) throws IOException {
+    String blockName = blk.getBlockName();
     File baseDir = new File(MiniDFSCluster.getBaseDirectory(), "data");
     for (int i=dnIndex*2; i<dnIndex*2+2; i++) {
       File blockFile = new File(baseDir, "data" + (i+1) + 
@@ -435,8 +438,9 @@ public class TestDatanodeBlockScanner extends TestCase {
     return false;
   }
   
-  private static void waitForBlockDeleted(String blockName, int dnIndex) 
+  private static void waitForBlockDeleted(ExtendedBlock blk, int dnIndex) 
   throws IOException, InterruptedException {
+    String blockName = blk.getBlockName();
     File baseDir = new File(MiniDFSCluster.getBaseDirectory(), "data");
     File blockFile1 = new File(baseDir, "data" + (2*dnIndex+1) + 
         MiniDFSCluster.FINALIZED_DIR_NAME + blockName);

src/test/hdfs/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java

@@ -28,6 +28,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.ChecksumException;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 
 /**
  * The test makes sure that NameNode detects presense blocks that do not have
@@ -66,7 +67,7 @@ public class TestMissingBlocksAlert extends TestCase {
 
 
       // Corrupt the block
-      String block = DFSTestUtil.getFirstBlock(dfs, corruptFile).getLocalBlock().getBlockName();
+      ExtendedBlock block = DFSTestUtil.getFirstBlock(dfs, corruptFile);
       assertTrue(TestDatanodeBlockScanner.corruptReplica(block, 0));
 
       // read the file so that the corrupt block is reported to NN

src/test/hdfs/org/apache/hadoop/hdfs/TestReplication.java

@@ -37,6 +37,7 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.protocol.ClientProtocol;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
 import org.apache.hadoop.hdfs.protocol.FSConstants.DatanodeReportType;
@@ -163,7 +164,7 @@ public class TestReplication extends TestCase {
     DFSTestUtil.waitReplication(fs, file1, (short)1);
   
     // Corrupt the block belonging to the created file
-    String block = DFSTestUtil.getFirstBlock(fs, file1).getBlockName();
+    ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, file1);
     cluster.corruptBlockOnDataNodes(block);
   
     // Increase replication factor, this should invoke transfer request
@@ -422,7 +423,7 @@ public class TestReplication extends TestCase {
     DFSTestUtil.createFile(fs, fileName, fileLen, REPLICATION_FACTOR, 0);
     DFSTestUtil.waitReplication(fs, fileName, REPLICATION_FACTOR);
 
-    String block = DFSTestUtil.getFirstBlock(fs, fileName).getBlockName();
+    ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, fileName);
 
     // Change the length of a replica
     for (int i=0; i<cluster.getDataNodes().size(); i++) {

src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java

@@ -17,7 +17,6 @@
  */
 package org.apache.hadoop.hdfs.server.datanode;
 
-import static org.apache.hadoop.hdfs.protocol.DataTransferProtocol.Op.REPLACE_BLOCK;
 import static org.apache.hadoop.hdfs.protocol.DataTransferProtocol.Status.*;
 
 import java.io.DataInputStream;
@@ -50,7 +49,6 @@ import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager;
 import org.apache.hadoop.hdfs.server.common.HdfsConstants;
 import org.apache.hadoop.hdfs.server.common.Util;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.io.Text;
 import org.apache.hadoop.net.NetUtils;
 /**
  * This class tests if block replacement request to data nodes work correctly.
@@ -159,11 +157,11 @@ public class TestBlockReplacement extends TestCase {
       // start to replace the block
       // case 1: proxySource does not contain the block
       LOG.info("Testcase 1: Proxy " + newNode.getName() 
-          + " does not contain the block " + b.getBlockName() );
+           + " does not contain the block " + b);
       assertFalse(replaceBlock(b, source, newNode, proxies.get(0)));
       // case 2: destination contains the block
       LOG.info("Testcase 2: Destination " + proxies.get(1).getName() 
-          + " contains the block " + b.getBlockName() );
+          + " contains the block " + b);
       assertFalse(replaceBlock(b, source, proxies.get(0), proxies.get(1)));
       // case 3: correct case
       LOG.info("Testcase 3: Proxy=" + source.getName() + " source=" + 

src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCorruptFilesJsp.java

@@ -32,6 +32,7 @@ import org.apache.hadoop.hdfs.DFSTestUtil;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.TestDatanodeBlockScanner;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.junit.Test;
 
 /** A JUnit test for corrupt_files.jsp */
@@ -80,9 +81,8 @@ public class TestCorruptFilesJsp  {
 
       // Now corrupt all the files except for the last one
       for (int idx = 0; idx < filepaths.length - 1; idx++) {
-        String blockName = DFSTestUtil.getFirstBlock(fs, filepaths[idx])
-            .getBlockName();
-        TestDatanodeBlockScanner.corruptReplica(blockName, 0);
+        ExtendedBlock blk = DFSTestUtil.getFirstBlock(fs, filepaths[idx]);
+        TestDatanodeBlockScanner.corruptReplica(blk, 0);
 
         // read the file so that the corrupt block is reported to NN
         FSDataInputStream in = fs.open(filepaths[idx]);

src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestOverReplicatedBlocks.java

@@ -54,8 +54,7 @@ public class TestOverReplicatedBlocks extends TestCase {
       
       // corrupt the block on datanode 0
       ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, fileName);
-      TestDatanodeBlockScanner.corruptReplica(block.getLocalBlock()
-          .getBlockName(), 0);
+      TestDatanodeBlockScanner.corruptReplica(block, 0);
       DataNodeProperties dnProps = cluster.stopDataNode(0);
       // remove block scanner log to trigger block scanning
       File scanLog = new File(System.getProperty("test.build.data"),