HADOOP-1564. Add unit tests for HDFS block-level checksums. Contributed by Dhruba.

git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk@557039 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -370,6 +370,9 @@ Trunk (unreleased changes)
 115. HADOOP-1400.  Make JobClient retry requests, so that clients can
      survive jobtracker problems.  (omalley via cutting)
 
+116. HADOOP-1564.  Add unit tests for HDFS block-level checksums.
+     (Dhruba Borthakur via cutting)
+
 
 Release 0.13.0 - 2007-06-08
 

src/test/org/apache/hadoop/dfs/DFSTestUtil.java

@@ -90,11 +90,16 @@ public class DFSTestUtil extends TestCase {
     int getSize() { return size; }
     long getSeed() { return seed; }
   }
+
+  void createFiles(FileSystem fs, String topdir) throws IOException {
+    createFiles(fs, topdir, (short)3);
+  }
   
   /** create nFiles with random names and directory hierarchies
-   * with random (but reproducible) data in them.
+   *  with random (but reproducible) data in them.
    */
-  void createFiles(FileSystem fs, String topdir) throws IOException {
+  void createFiles(FileSystem fs, String topdir,
+                   short replicationFactor) throws IOException {
     files = new MyFile[nFiles];
     
     for (int idx = 0; idx < nFiles; idx++) {
@@ -109,7 +114,7 @@ public class DFSTestUtil extends TestCase {
         throw new IOException("Mkdirs failed to create " + 
                               fPath.getParent().toString());
       }
-      FSDataOutputStream out = fs.create(fPath);
+      FSDataOutputStream out = fs.create(fPath, replicationFactor);
       byte[] toWrite = new byte[files[idx].getSize()];
       Random rb = new Random(files[idx].getSeed());
       rb.nextBytes(toWrite);
@@ -132,7 +137,7 @@ public class DFSTestUtil extends TestCase {
       byte[] toCompare = new byte[files[idx].getSize()];
       Random rb = new Random(files[idx].getSeed());
       rb.nextBytes(toCompare);
-      assertEquals("Cannnot read file.", toRead.length, in.read(toRead));
+      in.readFully(0, toRead);
       in.close();
       for (int i = 0; i < toRead.length; i++) {
         if (toRead[i] != toCompare[i]) {
@@ -145,6 +150,52 @@ public class DFSTestUtil extends TestCase {
     
     return true;
   }
+
+  void setReplication(FileSystem fs, String topdir, short value) 
+                                              throws IOException {
+    Path root = new Path(topdir);
+    for (int idx = 0; idx < nFiles; idx++) {
+      Path fPath = new Path(root, files[idx].getName());
+      fs.setReplication(fPath, value);
+    }
+  }
+
+  // waits for the replication factor of all files to reach the
+  // specified target
+  //
+  void waitReplication(FileSystem fs, String topdir, short value) 
+                                              throws IOException {
+    Path root = new Path(topdir);
+
+    /** wait for the replication factor to settle down */
+    while (true) {
+      boolean good = true;
+      for (int idx = 0; idx < nFiles; idx++) {
+        Path fPath = new Path(root, files[idx].getName());
+        String locs[][] = fs.getFileCacheHints(fPath, 0, Long.MAX_VALUE);
+        for (int j = 0; j < locs.length; j++) {
+          String[] loc = locs[j];
+          if (loc.length != value) {
+            System.out.println("File " + fPath + " has replication factor " +
+                               loc.length);
+            good = false;
+            break;
+          }
+        }
+        if (!good) {
+          break;
+        }
+      }
+      if (!good) {
+        try {
+          System.out.println("Waiting for replication factor to drain");
+          Thread.sleep(1000);
+        } catch (InterruptedException e) {} 
+        continue;
+      }
+      break;
+    }
+  }
   
   /** delete directory and everything underneath it.*/
   void cleanup(FileSystem fs, String topdir) throws IOException {

src/test/org/apache/hadoop/dfs/TestCrcCorruption.java

@@ -0,0 +1,225 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.dfs;
+
+import java.io.*;
+import java.nio.channels.FileChannel;
+import java.nio.ByteBuffer;
+import java.util.Random;
+import junit.framework.*;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.ChecksumException;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * A JUnit test for corrupted file handling.
+ * This test creates a bunch of files/directories with replication 
+ * factor of 2. Then verifies that a client can automatically 
+ * access the remaining valid replica inspite of the following 
+ * types of simulated errors:
+ *
+ *  1. Delete meta file on one replica
+ *  2. Truncates meta file on one replica
+ *  3. Corrupts the meta file header on one replica
+ *  4. Corrupts any random offset and portion of the meta file
+ *  5. Swaps two meta files, i.e the format of the meta files 
+ *     are valid but their CRCs do not match with their corresponding 
+ *     data blocks
+ * The above tests are run for varied values of io.bytes.per.checksum 
+ * and dfs.block.size. It tests for the case when the meta file is 
+ * multiple blocks.
+ *
+ * Another portion of the test is commented out till HADOOP-1557 
+ * is addressed:
+ *  1. Create file with 2 replica, corrupt the meta file of replica, 
+ *     decrease replication factor from 2 to 1. Validate that the 
+ *     remaining replica is the good one.
+ *  2. Create file with 2 replica, corrupt the meta file of one replica, 
+ *     increase replication factor of file to 3. verify that the new 
+ *     replica was created from the non-corrupted replica.
+ */
+public class TestCrcCorruption extends TestCase {
+  
+  public TestCrcCorruption(String testName) {
+    super(testName);
+  }
+
+  protected void setUp() throws Exception {
+  }
+
+  protected void tearDown() throws Exception {
+  }
+  
+  /** 
+   * check if DFS can handle corrupted CRC blocks
+   */
+  private void thistest(Configuration conf, DFSTestUtil util) throws Exception {
+    MiniDFSCluster cluster = null;
+    int numDataNodes = 2;
+    short replFactor = 2;
+    Random random = new Random();
+
+    try {
+      cluster = new MiniDFSCluster(conf, numDataNodes, true, null);
+      cluster.waitActive();
+      FileSystem fs = cluster.getFileSystem();
+      util.createFiles(fs, "/srcdat", replFactor);
+      util.waitReplication(fs, "/srcdat", (short)2);
+
+      // Now deliberately remove/truncate meta blocks from the first
+      // directory of the first datanode. The complete absense of a meta
+      // file disallows this Datanode to send data to another datanode.
+      // However, a client is alowed access to this block.
+      //
+      File data_dir = new File(System.getProperty("test.build.data"),
+                               "dfs/data/data1/current");
+      assertTrue("data directory does not exist", data_dir.exists());
+      File[] blocks = data_dir.listFiles();
+      assertTrue("Blocks do not exist in data-dir", (blocks != null) && (blocks.length > 0));
+      int num = 0;
+      for (int idx = 0; idx < blocks.length; idx++) {
+        if (blocks[idx].getName().startsWith("blk_") &&
+            blocks[idx].getName().endsWith(".meta")) {
+          num++;
+          if (num % 3 == 0) {
+            //
+            // remove .meta file
+            //
+            System.out.println("Deliberately removing file " + blocks[idx].getName());
+            assertTrue("Cannot remove file.", blocks[idx].delete());
+          } else if (num % 3 == 1) {
+            //
+            // shorten .meta file
+            //
+            RandomAccessFile file = new RandomAccessFile(blocks[idx], "rw");
+            FileChannel channel = file.getChannel();
+            int newsize = random.nextInt((int)channel.size()/2);
+            System.out.println("Deliberately truncating file " + 
+                               blocks[idx].getName() + 
+                               " to size " + newsize + " bytes.");
+            channel.truncate(newsize);
+            file.close();
+          } else {
+            //
+            // corrupt a few bytes of the metafile
+            //
+            RandomAccessFile file = new RandomAccessFile(blocks[idx], "rw");
+            FileChannel channel = file.getChannel();
+            long position = 0;
+            //
+            // The very first time, corrupt the meta header at offset 0
+            //
+            if (num != 2) {
+              position = (long)random.nextInt((int)channel.size());
+            }
+            int length = random.nextInt((int)(channel.size() - position + 1));
+            byte[] buffer = new byte[length];
+            random.nextBytes(buffer);
+            channel.write(ByteBuffer.wrap(buffer), position);
+            System.out.println("Deliberately corrupting file " + 
+                               blocks[idx].getName() + 
+                               " at offset " + position +
+                               " length " + length);
+            file.close();
+          }
+        }
+      }
+      //
+      // Now deliberately corrupt all meta blocks from the second
+      // directory of the first datanode
+      //
+      data_dir = new File(System.getProperty("test.build.data"),
+                               "dfs/data/data2/current");
+      assertTrue("data directory does not exist", data_dir.exists());
+      blocks = data_dir.listFiles();
+      assertTrue("Blocks do not exist in data-dir", (blocks != null) && (blocks.length > 0));
+
+      int count = 0;
+      File previous = null;
+      for (int idx = 0; idx < blocks.length; idx++) {
+        if (blocks[idx].getName().startsWith("blk_") &&
+            blocks[idx].getName().endsWith(".meta")) {
+          //
+          // Move the previous metafile into the current one.
+          //
+          count++;
+          if (count % 2 == 0) {
+            System.out.println("Deliberately insertimg bad crc into files " +
+                                blocks[idx].getName() + " " + previous.getName());
+            assertTrue("Cannot remove file.", blocks[idx].delete());
+            assertTrue("Cannot corrupt meta file.", previous.renameTo(blocks[idx]));
+            assertTrue("Cannot recreate empty meta file.", previous.createNewFile());
+            previous = null;
+          } else {
+            previous = blocks[idx];
+          }
+        }
+      }
+
+      //
+      // Only one replica is possibly corrupted. The other replica should still
+      // be good. Verify.
+      //
+      assertTrue("Corrupted replicas not handled properly.",
+                 util.checkFiles(fs, "/srcdat"));
+      System.out.println("All File still have a valid replica");
+
+      //
+      // set replication factor back to 1. This causes only one replica of
+      // of each block to remain in HDFS. The check is to make sure that 
+      // the corrupted replica generated above is the one that gets deleted.
+      // This test is currently disabled until HADOOP-1557 is solved.
+      //
+      util.setReplication(fs, "/srcdat", (short)1);
+      //util.waitReplication(fs, "/srcdat", (short)1);
+      //System.out.println("All Files done with removing replicas");
+      //assertTrue("Excess replicas deleted. Corrupted replicas found.",
+      //           util.checkFiles(fs, "/srcdat"));
+      System.out.println("The excess-corrupted-replica test is disabled " +
+                         " pending HADOOP-1557");
+
+      util.cleanup(fs, "/srcdat");
+    } finally {
+      if (cluster != null) { cluster.shutdown(); }
+    }
+  }
+
+  public void testCrcCorruption() throws Exception {
+    //
+    // default parameters
+    //
+    System.out.println("TestCrcCorruption with default parameters");
+    Configuration conf1 = new Configuration();
+    conf1.setInt("dfs.blockreport.intervalMsec", 3 * 1000);
+    DFSTestUtil util1 = new DFSTestUtil("TestCrcCorruption", 40, 3, 8*1024);
+    thistest(conf1, util1);
+
+    //
+    // specific parameters
+    //
+    System.out.println("TestCrcCorruption with specific parameters");
+    Configuration conf2 = new Configuration();
+    conf2.setInt("io.bytes.per.checksum", 17);
+    conf2.setInt("dfs.block.size", 34);
+    DFSTestUtil util2 = new DFSTestUtil("TestCrcCorruption", 40, 3, 400);
+    thistest(conf2, util2);
+  }
+}