HADOOP-10490. Merging change r1586572 from branch-2 to branch-2.4

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2.4@1586573 13f79535-47bb-0310-9956-ffa450edef68

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -24,6 +24,9 @@ Release 2.4.1 - UNRELEASED
     HADOOP-10473. TestCallQueueManager should interrupt before counting calls.
     (szetszwo)
 
+    HADOOP-10490. TestMapFile and TestBloomMapFile leak file descriptors.
+    (cnauroth)
+
 Release 2.4.0 - 2014-04-07 
 
   INCOMPATIBLE CHANGES

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapFile.java

@@ -30,6 +30,7 @@ import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.io.SequenceFile.CompressionType;
 import org.apache.hadoop.io.compress.CompressionCodec;
 import org.apache.hadoop.util.Options;
@@ -836,21 +837,24 @@ public class MapFile {
 
     Configuration conf = new Configuration();
     FileSystem fs = FileSystem.getLocal(conf);
-    MapFile.Reader reader = new MapFile.Reader(fs, in, conf);
-    MapFile.Writer writer =
-      new MapFile.Writer(conf, fs, out,
-          reader.getKeyClass().asSubclass(WritableComparable.class),
-          reader.getValueClass());
-
-    WritableComparable key =
-      ReflectionUtils.newInstance(reader.getKeyClass().asSubclass(WritableComparable.class), conf);
-    Writable value =
-      ReflectionUtils.newInstance(reader.getValueClass().asSubclass(Writable.class), conf);
-
-    while (reader.next(key, value))               // copy all entries
-      writer.append(key, value);
-
-    writer.close();
+    MapFile.Reader reader = null;
+    MapFile.Writer writer = null;
+    try {
+      reader = new MapFile.Reader(fs, in, conf);
+      writer =
+        new MapFile.Writer(conf, fs, out,
+            reader.getKeyClass().asSubclass(WritableComparable.class),
+            reader.getValueClass());
+
+      WritableComparable key = ReflectionUtils.newInstance(reader.getKeyClass()
+        .asSubclass(WritableComparable.class), conf);
+      Writable value = ReflectionUtils.newInstance(reader.getValueClass()
+        .asSubclass(Writable.class), conf);
+
+      while (reader.next(key, value))               // copy all entries
+        writer.append(key, value);
+    } finally {
+      IOUtils.cleanup(LOG, writer, reader);
+    }
   }
-
 }

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestBloomMapFile.java

@@ -18,6 +18,8 @@
 
 package org.apache.hadoop.io;
 
+import static org.mockito.Mockito.*;
+
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -31,6 +33,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.LocalFileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.io.SequenceFile.CompressionType;
 import org.apache.hadoop.io.compress.CompressionCodec;
 import org.apache.hadoop.io.compress.CompressionInputStream;
@@ -63,39 +66,44 @@ public class TestBloomMapFile extends TestCase {
     FileSystem fs = FileSystem.getLocal(conf);
     Path qualifiedDirName = fs.makeQualified(TEST_DIR);
     conf.setInt("io.mapfile.bloom.size", 2048);
-    BloomMapFile.Writer writer = new BloomMapFile.Writer(conf, fs,
-        qualifiedDirName.toString(), IntWritable.class, Text.class);
-    IntWritable key = new IntWritable();
-    Text value = new Text();
-    for (int i = 0; i < 2000; i += 2) {
-      key.set(i);
-      value.set("00" + i);
-      writer.append(key, value);
-    }
-    writer.close();
+    BloomMapFile.Writer writer = null;
+    BloomMapFile.Reader reader = null;
+    try {
+      writer = new BloomMapFile.Writer(conf, fs, qualifiedDirName.toString(),
+        IntWritable.class, Text.class);
+      IntWritable key = new IntWritable();
+      Text value = new Text();
+      for (int i = 0; i < 2000; i += 2) {
+        key.set(i);
+        value.set("00" + i);
+        writer.append(key, value);
+      }
+      writer.close();
 
-    BloomMapFile.Reader reader = new BloomMapFile.Reader(fs,
-        qualifiedDirName.toString(), conf);
-    // check false positives rate
-    int falsePos = 0;
-    int falseNeg = 0;
-    for (int i = 0; i < 2000; i++) {
-      key.set(i);
-      boolean exists = reader.probablyHasKey(key);
-      if (i % 2 == 0) {
-        if (!exists)
-          falseNeg++;
-      } else {
-        if (exists)
-          falsePos++;
+      reader = new BloomMapFile.Reader(fs, qualifiedDirName.toString(), conf);
+      // check false positives rate
+      int falsePos = 0;
+      int falseNeg = 0;
+      for (int i = 0; i < 2000; i++) {
+        key.set(i);
+        boolean exists = reader.probablyHasKey(key);
+        if (i % 2 == 0) {
+          if (!exists)
+            falseNeg++;
+        } else {
+          if (exists)
+            falsePos++;
+        }
       }
+      reader.close();
+      fs.delete(qualifiedDirName, true);
+      System.out.println("False negatives: " + falseNeg);
+      assertEquals(0, falseNeg);
+      System.out.println("False positives: " + falsePos);
+      assertTrue(falsePos < 2);
+    } finally {
+      IOUtils.cleanup(null, writer, reader);
     }
-    reader.close();
-    fs.delete(qualifiedDirName, true);
-    System.out.println("False negatives: " + falseNeg);
-    assertEquals(0, falseNeg);
-    System.out.println("False positives: " + falsePos);
-    assertTrue(falsePos < 2);
   }
 
   @SuppressWarnings("deprecation")
@@ -103,23 +111,28 @@ public class TestBloomMapFile extends TestCase {
       throws Exception {
     FileSystem fs = FileSystem.getLocal(conf);
     Path qualifiedDirName = fs.makeQualified(TEST_DIR);
-    BloomMapFile.Writer writer = new BloomMapFile.Writer(conf, fs,
-        qualifiedDirName.toString(), Text.class, NullWritable.class);
-    for (Text key : keys) {
-      writer.append(key, NullWritable.get());
-    }
-    writer.close();
+    BloomMapFile.Writer writer = null;
+    BloomMapFile.Reader reader = null;
+    try {
+      writer = new BloomMapFile.Writer(conf, fs, qualifiedDirName.toString(),
+        Text.class, NullWritable.class);
+      for (Text key : keys) {
+        writer.append(key, NullWritable.get());
+      }
+      writer.close();
 
-    // will check for membership in the opposite order of how keys were inserted
-    BloomMapFile.Reader reader = new BloomMapFile.Reader(fs,
-        qualifiedDirName.toString(), conf);
-    Collections.reverse(keys);
-    for (Text key : keys) {
-      assertTrue("False negative for existing key " + key,
+      // will check for membership in opposite order of how keys were inserted
+      reader = new BloomMapFile.Reader(fs, qualifiedDirName.toString(), conf);
+      Collections.reverse(keys);
+      for (Text key : keys) {
+        assertTrue("False negative for existing key " + key,
           reader.probablyHasKey(key));
+      }
+      reader.close();
+      fs.delete(qualifiedDirName, true);
+    } finally {
+      IOUtils.cleanup(null, writer, reader);
     }
-    reader.close();
-    fs.delete(qualifiedDirName, true);
   }
 
   public void testMembershipVaryingSizedKeysTest1() throws Exception {
@@ -140,15 +153,19 @@ public class TestBloomMapFile extends TestCase {
    * test {@code BloomMapFile.delete()} method
    */
   public void testDeleteFile() {
+    BloomMapFile.Writer writer = null;
     try {
       FileSystem fs = FileSystem.getLocal(conf);
-      BloomMapFile.Writer writer = new BloomMapFile.Writer(conf, TEST_FILE,
+      writer = new BloomMapFile.Writer(conf, TEST_FILE,
           MapFile.Writer.keyClass(IntWritable.class),
           MapFile.Writer.valueClass(Text.class));
       assertNotNull("testDeleteFile error !!!", writer);
-      BloomMapFile.delete(fs, "." + TEST_FILE);
+      writer.close();
+      BloomMapFile.delete(fs, TEST_FILE.toString());
     } catch (Exception ex) {
       fail("unexpect ex in testDeleteFile !!!");
+    } finally {
+      IOUtils.cleanup(null, writer);
     }
   }
   
@@ -157,24 +174,26 @@ public class TestBloomMapFile extends TestCase {
    * IOException
    */
   public void testIOExceptionInWriterConstructor() {
-    Path dirNameSpy = org.mockito.Mockito.spy(TEST_FILE);
+    Path dirNameSpy = spy(TEST_FILE);
+    BloomMapFile.Reader reader = null;
+    BloomMapFile.Writer writer = null;
     try {
-      BloomMapFile.Writer writer = new BloomMapFile.Writer(conf, TEST_FILE,
+      writer = new BloomMapFile.Writer(conf, TEST_FILE,
           MapFile.Writer.keyClass(IntWritable.class),
           MapFile.Writer.valueClass(Text.class));
       writer.append(new IntWritable(1), new Text("123124142"));
       writer.close();
 
-      org.mockito.Mockito.when(dirNameSpy.getFileSystem(conf)).thenThrow(
-          new IOException());
-      BloomMapFile.Reader reader = new BloomMapFile.Reader(dirNameSpy, conf,
+      when(dirNameSpy.getFileSystem(conf)).thenThrow(new IOException());
+      reader = new BloomMapFile.Reader(dirNameSpy, conf,
           MapFile.Reader.comparator(new WritableComparator(IntWritable.class)));
 
       assertNull("testIOExceptionInWriterConstructor error !!!",
           reader.getBloomFilter());
-      reader.close();
     } catch (Exception ex) {
       fail("unexpect ex in testIOExceptionInWriterConstructor !!!");
+    } finally {
+      IOUtils.cleanup(null, writer, reader);
     }
   }
 
@@ -183,8 +202,10 @@ public class TestBloomMapFile extends TestCase {
    */
   public void testGetBloomMapFile() {
     int SIZE = 10;
+    BloomMapFile.Reader reader = null;
+    BloomMapFile.Writer writer = null;
     try {
-      BloomMapFile.Writer writer = new BloomMapFile.Writer(conf, TEST_FILE,
+      writer = new BloomMapFile.Writer(conf, TEST_FILE,
           MapFile.Writer.keyClass(IntWritable.class),
           MapFile.Writer.valueClass(Text.class));
 
@@ -193,7 +214,7 @@ public class TestBloomMapFile extends TestCase {
       }
       writer.close();
 
-      BloomMapFile.Reader reader = new BloomMapFile.Reader(TEST_FILE, conf,
+      reader = new BloomMapFile.Reader(TEST_FILE, conf,
           MapFile.Reader.comparator(new WritableComparator(IntWritable.class)));
 
       for (int i = 0; i < SIZE; i++) {
@@ -203,9 +224,10 @@ public class TestBloomMapFile extends TestCase {
             
       assertNull("testGetBloomMapFile error !!!",
           reader.get(new IntWritable(SIZE + 5), new Text()));
-      reader.close();
     } catch (Exception ex) {
       fail("unexpect ex in testGetBloomMapFile !!!");
+    } finally {
+      IOUtils.cleanup(null, writer, reader);
     }
   }
 
@@ -214,36 +236,46 @@ public class TestBloomMapFile extends TestCase {
    */
   @SuppressWarnings("deprecation")
   public void testBloomMapFileConstructors() {
+    BloomMapFile.Writer writer = null;
     try {
       FileSystem ts = FileSystem.get(conf);
       String testFileName = TEST_FILE.toString();
-      BloomMapFile.Writer writer1 = new BloomMapFile.Writer(conf, ts,
+      writer = new BloomMapFile.Writer(conf, ts,
           testFileName, IntWritable.class, Text.class, CompressionType.BLOCK,
           defaultCodec, defaultProgress);
-      assertNotNull("testBloomMapFileConstructors error !!!", writer1);
-      BloomMapFile.Writer writer2 = new BloomMapFile.Writer(conf, ts,
+      assertNotNull("testBloomMapFileConstructors error !!!", writer);
+      writer.close();
+      writer = new BloomMapFile.Writer(conf, ts,
           testFileName, IntWritable.class, Text.class, CompressionType.BLOCK,
           defaultProgress);
-      assertNotNull("testBloomMapFileConstructors error !!!", writer2);
-      BloomMapFile.Writer writer3 = new BloomMapFile.Writer(conf, ts,
+      assertNotNull("testBloomMapFileConstructors error !!!", writer);
+      writer.close();
+      writer = new BloomMapFile.Writer(conf, ts,
           testFileName, IntWritable.class, Text.class, CompressionType.BLOCK);
-      assertNotNull("testBloomMapFileConstructors error !!!", writer3);
-      BloomMapFile.Writer writer4 = new BloomMapFile.Writer(conf, ts,
+      assertNotNull("testBloomMapFileConstructors error !!!", writer);
+      writer.close();
+      writer = new BloomMapFile.Writer(conf, ts,
           testFileName, IntWritable.class, Text.class, CompressionType.RECORD,
           defaultCodec, defaultProgress);
-      assertNotNull("testBloomMapFileConstructors error !!!", writer4);
-      BloomMapFile.Writer writer5 = new BloomMapFile.Writer(conf, ts,
+      assertNotNull("testBloomMapFileConstructors error !!!", writer);
+      writer.close();
+      writer = new BloomMapFile.Writer(conf, ts,
           testFileName, IntWritable.class, Text.class, CompressionType.RECORD,
           defaultProgress);
-      assertNotNull("testBloomMapFileConstructors error !!!", writer5);
-      BloomMapFile.Writer writer6 = new BloomMapFile.Writer(conf, ts,
+      assertNotNull("testBloomMapFileConstructors error !!!", writer);
+      writer.close();
+      writer = new BloomMapFile.Writer(conf, ts,
           testFileName, IntWritable.class, Text.class, CompressionType.RECORD);
-      assertNotNull("testBloomMapFileConstructors error !!!", writer6);
-      BloomMapFile.Writer writer7 = new BloomMapFile.Writer(conf, ts,
+      assertNotNull("testBloomMapFileConstructors error !!!", writer);
+      writer.close();
+      writer = new BloomMapFile.Writer(conf, ts,
           testFileName, WritableComparator.get(Text.class), Text.class);
-      assertNotNull("testBloomMapFileConstructors error !!!", writer7);
+      assertNotNull("testBloomMapFileConstructors error !!!", writer);
+      writer.close();
     } catch (Exception ex) {
       fail("testBloomMapFileConstructors error !!!");
+    } finally {
+      IOUtils.cleanup(null, writer);
     }
   }
 
@@ -272,13 +304,13 @@ public class TestBloomMapFile extends TestCase {
     @Override
     public CompressionOutputStream createOutputStream(OutputStream out,
         Compressor compressor) throws IOException {
-      return null;
+      return mock(CompressionOutputStream.class);
     }
 
     @Override
     public CompressionOutputStream createOutputStream(OutputStream out)
         throws IOException {
-      return null;
+      return mock(CompressionOutputStream.class);
     }
 
     @Override

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestMapFile.java

@@ -26,6 +26,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.LocalFileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.io.SequenceFile.CompressionType;
 import org.apache.hadoop.io.compress.CompressionCodec;
 import org.apache.hadoop.io.compress.CompressionInputStream;
@@ -68,13 +69,13 @@ public class TestMapFile {
     @Override
     public CompressionOutputStream createOutputStream(OutputStream out)
         throws IOException {
-      return null;
+      return mock(CompressionOutputStream.class);
     }
 
     @Override
     public CompressionOutputStream createOutputStream(OutputStream out,
         Compressor compressor) throws IOException {
-      return null;
+      return mock(CompressionOutputStream.class);
     }
 
     @Override
@@ -138,46 +139,52 @@ public class TestMapFile {
   @Test
   public void testGetClosestOnCurrentApi() throws Exception {
     final String TEST_PREFIX = "testGetClosestOnCurrentApi.mapfile";
-    MapFile.Writer writer = createWriter(TEST_PREFIX, Text.class, Text.class);
-    int FIRST_KEY = 1;
-    // Test keys: 11,21,31,...,91
-    for (int i = FIRST_KEY; i < 100; i += 10) {      
-      Text t = new Text(Integer.toString(i));
-      writer.append(t, t);
-    }
-    writer.close();
+    MapFile.Writer writer = null;
+    MapFile.Reader reader = null;
+    try {
+      writer = createWriter(TEST_PREFIX, Text.class, Text.class);
+      int FIRST_KEY = 1;
+      // Test keys: 11,21,31,...,91
+      for (int i = FIRST_KEY; i < 100; i += 10) {      
+        Text t = new Text(Integer.toString(i));
+        writer.append(t, t);
+      }
+      writer.close();
 
-    MapFile.Reader reader = createReader(TEST_PREFIX, Text.class);
-    Text key = new Text("55");
-    Text value = new Text();
-
-    // Test get closest with step forward
-    Text closest = (Text) reader.getClosest(key, value);
-    assertEquals(new Text("61"), closest);
-
-    // Test get closest with step back
-    closest = (Text) reader.getClosest(key, value, true);
-    assertEquals(new Text("51"), closest);
-
-    // Test get closest when we pass explicit key
-    final Text explicitKey = new Text("21");
-    closest = (Text) reader.getClosest(explicitKey, value);
-    assertEquals(new Text("21"), explicitKey);
-
-    // Test what happens at boundaries. Assert if searching a key that is
-    // less than first key in the mapfile, that the first key is returned.
-    key = new Text("00");
-    closest = (Text) reader.getClosest(key, value);
-    assertEquals(FIRST_KEY, Integer.parseInt(closest.toString()));
-
-    // Assert that null is returned if key is > last entry in mapfile.
-    key = new Text("92");
-    closest = (Text) reader.getClosest(key, value);
-    assertNull("Not null key in testGetClosestWithNewCode", closest);
-
-    // If we were looking for the key before, we should get the last key
-    closest = (Text) reader.getClosest(key, value, true);
-    assertEquals(new Text("91"), closest);
+      reader = createReader(TEST_PREFIX, Text.class);
+      Text key = new Text("55");
+      Text value = new Text();
+
+      // Test get closest with step forward
+      Text closest = (Text) reader.getClosest(key, value);
+      assertEquals(new Text("61"), closest);
+
+      // Test get closest with step back
+      closest = (Text) reader.getClosest(key, value, true);
+      assertEquals(new Text("51"), closest);
+
+      // Test get closest when we pass explicit key
+      final Text explicitKey = new Text("21");
+      closest = (Text) reader.getClosest(explicitKey, value);
+      assertEquals(new Text("21"), explicitKey);
+
+      // Test what happens at boundaries. Assert if searching a key that is
+      // less than first key in the mapfile, that the first key is returned.
+      key = new Text("00");
+      closest = (Text) reader.getClosest(key, value);
+      assertEquals(FIRST_KEY, Integer.parseInt(closest.toString()));
+
+      // Assert that null is returned if key is > last entry in mapfile.
+      key = new Text("92");
+      closest = (Text) reader.getClosest(key, value);
+      assertNull("Not null key in testGetClosestWithNewCode", closest);
+
+      // If we were looking for the key before, we should get the last key
+      closest = (Text) reader.getClosest(key, value, true);
+      assertEquals(new Text("91"), closest);
+    } finally {
+      IOUtils.cleanup(null, writer, reader);
+    }
   }
   
   /**
@@ -187,16 +194,21 @@ public class TestMapFile {
   public void testMidKeyOnCurrentApi() throws Exception {
     // Write a mapfile of simple data: keys are
     final String TEST_PREFIX = "testMidKeyOnCurrentApi.mapfile";
-    MapFile.Writer writer = createWriter(TEST_PREFIX, IntWritable.class,
-        IntWritable.class);
-    // 0,1,....9
-    int SIZE = 10;
-    for (int i = 0; i < SIZE; i++)
-      writer.append(new IntWritable(i), new IntWritable(i));
-    writer.close();
+    MapFile.Writer writer = null;
+    MapFile.Reader reader = null;
+    try {
+      writer = createWriter(TEST_PREFIX, IntWritable.class, IntWritable.class);
+      // 0,1,....9
+      int SIZE = 10;
+      for (int i = 0; i < SIZE; i++)
+        writer.append(new IntWritable(i), new IntWritable(i));
+      writer.close();
 
-    MapFile.Reader reader = createReader(TEST_PREFIX, IntWritable.class);
-    assertEquals(new IntWritable((SIZE - 1) / 2), reader.midKey());
+      reader = createReader(TEST_PREFIX, IntWritable.class);
+      assertEquals(new IntWritable((SIZE - 1) / 2), reader.midKey());
+    } finally {
+      IOUtils.cleanup(null, writer, reader);
+    }
   }
   
   /**
@@ -206,16 +218,18 @@ public class TestMapFile {
   public void testRename() {
     final String NEW_FILE_NAME = "test-new.mapfile";
     final String OLD_FILE_NAME = "test-old.mapfile";
+    MapFile.Writer writer = null;
     try {
       FileSystem fs = FileSystem.getLocal(conf);
-      MapFile.Writer writer = createWriter(OLD_FILE_NAME, IntWritable.class,
-          IntWritable.class);
+      writer = createWriter(OLD_FILE_NAME, IntWritable.class, IntWritable.class);
       writer.close();
       MapFile.rename(fs, new Path(TEST_DIR, OLD_FILE_NAME).toString(), 
           new Path(TEST_DIR, NEW_FILE_NAME).toString());
       MapFile.delete(fs, new Path(TEST_DIR, NEW_FILE_NAME).toString());
     } catch (IOException ex) {
       fail("testRename error " + ex);
+    } finally {
+      IOUtils.cleanup(null, writer);
     }
   }
   
@@ -228,12 +242,12 @@ public class TestMapFile {
     final String ERROR_MESSAGE = "Can't rename file";
     final String NEW_FILE_NAME = "test-new.mapfile";
     final String OLD_FILE_NAME = "test-old.mapfile";
+    MapFile.Writer writer = null;
     try {
       FileSystem fs = FileSystem.getLocal(conf);
       FileSystem spyFs = spy(fs);
 
-      MapFile.Writer writer = createWriter(OLD_FILE_NAME, IntWritable.class,
-          IntWritable.class);
+      writer = createWriter(OLD_FILE_NAME, IntWritable.class, IntWritable.class);
       writer.close();
 
       Path oldDir = new Path(TEST_DIR, OLD_FILE_NAME);
@@ -246,6 +260,8 @@ public class TestMapFile {
     } catch (IOException ex) {
       assertEquals("testRenameWithException invalid IOExceptionMessage !!!",
           ex.getMessage(), ERROR_MESSAGE);
+    } finally {
+      IOUtils.cleanup(null, writer);
     }
   }
 
@@ -254,12 +270,12 @@ public class TestMapFile {
     final String ERROR_MESSAGE = "Could not rename";
     final String NEW_FILE_NAME = "test-new.mapfile";
     final String OLD_FILE_NAME = "test-old.mapfile";
+    MapFile.Writer writer = null;
     try {
       FileSystem fs = FileSystem.getLocal(conf);
       FileSystem spyFs = spy(fs);
 
-      MapFile.Writer writer = createWriter(OLD_FILE_NAME, IntWritable.class,
-          IntWritable.class);
+      writer = createWriter(OLD_FILE_NAME, IntWritable.class, IntWritable.class);
       writer.close();
 
       Path oldDir = new Path(TEST_DIR, OLD_FILE_NAME);
@@ -271,6 +287,8 @@ public class TestMapFile {
     } catch (IOException ex) {
       assertTrue("testRenameWithFalse invalid IOExceptionMessage error !!!", ex
           .getMessage().startsWith(ERROR_MESSAGE));
+    } finally {
+      IOUtils.cleanup(null, writer);
     }
   }
   
@@ -297,11 +315,7 @@ public class TestMapFile {
       assertTrue("testWriteWithFailDirCreation ex error !!!", ex.getMessage()
           .startsWith(ERROR_MESSAGE));
     } finally {
-      if (writer != null)
-        try {
-          writer.close();
-        } catch (IOException e) {
-        }
+      IOUtils.cleanup(null, writer);
     }
   }
 
@@ -312,20 +326,24 @@ public class TestMapFile {
   public void testOnFinalKey() {
     final String TEST_METHOD_KEY = "testOnFinalKey.mapfile";
     int SIZE = 10;
+    MapFile.Writer writer = null;
+    MapFile.Reader reader = null;
     try {
-      MapFile.Writer writer = createWriter(TEST_METHOD_KEY, IntWritable.class,
-          IntWritable.class);
+      writer = createWriter(TEST_METHOD_KEY, IntWritable.class,
+        IntWritable.class);
       for (int i = 0; i < SIZE; i++)
         writer.append(new IntWritable(i), new IntWritable(i));
       writer.close();
 
-      MapFile.Reader reader = createReader(TEST_METHOD_KEY, IntWritable.class);
+      reader = createReader(TEST_METHOD_KEY, IntWritable.class);
       IntWritable expectedKey = new IntWritable(0);
       reader.finalKey(expectedKey);
       assertEquals("testOnFinalKey not same !!!", expectedKey, new IntWritable(
           9));
     } catch (IOException ex) {
       fail("testOnFinalKey error !!!");
+    } finally {
+      IOUtils.cleanup(null, writer, reader);
     }
   }
   
@@ -338,7 +356,8 @@ public class TestMapFile {
     Class<? extends WritableComparable<?>> keyClass = IntWritable.class;
     Class<?> valueClass = Text.class;
     try {
-      createWriter("testKeyValueClasses.mapfile", IntWritable.class, Text.class);
+      createWriter("testKeyValueClasses.mapfile", IntWritable.class, Text.class)
+        .close();
       assertNotNull("writer key class null error !!!",
           MapFile.Writer.keyClass(keyClass));
       assertNotNull("writer value class null error !!!",
@@ -354,19 +373,22 @@ public class TestMapFile {
   @Test
   public void testReaderGetClosest() throws Exception {
     final String TEST_METHOD_KEY = "testReaderWithWrongKeyClass.mapfile";
+    MapFile.Writer writer = null;
+    MapFile.Reader reader = null;
     try {
-      MapFile.Writer writer = createWriter(TEST_METHOD_KEY, IntWritable.class,
-          Text.class);
+      writer = createWriter(TEST_METHOD_KEY, IntWritable.class, Text.class);
 
       for (int i = 0; i < 10; i++)
         writer.append(new IntWritable(i), new Text("value" + i));
       writer.close();
 
-      MapFile.Reader reader = createReader(TEST_METHOD_KEY, Text.class);
+      reader = createReader(TEST_METHOD_KEY, Text.class);
       reader.getClosest(new Text("2"), new Text(""));
       fail("no excepted exception in testReaderWithWrongKeyClass !!!");
     } catch (IOException ex) {
       /* Should be thrown to pass the test */
+    } finally {
+      IOUtils.cleanup(null, writer, reader);
     }
   }
   
@@ -376,13 +398,15 @@ public class TestMapFile {
   @Test
   public void testReaderWithWrongValueClass() {
     final String TEST_METHOD_KEY = "testReaderWithWrongValueClass.mapfile";
+    MapFile.Writer writer = null;
     try {
-      MapFile.Writer writer = createWriter(TEST_METHOD_KEY, IntWritable.class,
-          Text.class);
+      writer = createWriter(TEST_METHOD_KEY, IntWritable.class, Text.class);
       writer.append(new IntWritable(0), new IntWritable(0));
       fail("no excepted exception in testReaderWithWrongKeyClass !!!");
     } catch (IOException ex) {
       /* Should be thrown to pass the test */
+    } finally {
+      IOUtils.cleanup(null, writer);
     }
   }
   
@@ -394,15 +418,16 @@ public class TestMapFile {
     final String TEST_METHOD_KEY = "testReaderKeyIteration.mapfile";
     int SIZE = 10;
     int ITERATIONS = 5;
+    MapFile.Writer writer = null;
+    MapFile.Reader reader = null;
     try {
-      MapFile.Writer writer = createWriter(TEST_METHOD_KEY, IntWritable.class,
-          Text.class);
+      writer = createWriter(TEST_METHOD_KEY, IntWritable.class, Text.class);
       int start = 0;
       for (int i = 0; i < SIZE; i++)
         writer.append(new IntWritable(i), new Text("Value:" + i));
       writer.close();
 
-      MapFile.Reader reader = createReader(TEST_METHOD_KEY, IntWritable.class);
+      reader = createReader(TEST_METHOD_KEY, IntWritable.class);
       // test iteration
       Writable startValue = new Text("Value:" + start);
       int i = 0;
@@ -421,6 +446,8 @@ public class TestMapFile {
           reader.seek(new IntWritable(SIZE * 2)));
     } catch (IOException ex) {
       fail("reader seek error !!!");
+    } finally {
+      IOUtils.cleanup(null, writer, reader);
     }
   }
 
@@ -431,11 +458,11 @@ public class TestMapFile {
   public void testFix() {
     final String INDEX_LESS_MAP_FILE = "testFix.mapfile";
     int PAIR_SIZE = 20;
+    MapFile.Writer writer = null;
     try {
       FileSystem fs = FileSystem.getLocal(conf);
       Path dir = new Path(TEST_DIR, INDEX_LESS_MAP_FILE);
-      MapFile.Writer writer = createWriter(INDEX_LESS_MAP_FILE,
-          IntWritable.class, Text.class);
+      writer = createWriter(INDEX_LESS_MAP_FILE, IntWritable.class, Text.class);
       for (int i = 0; i < PAIR_SIZE; i++)
         writer.append(new IntWritable(0), new Text("value"));
       writer.close();
@@ -450,6 +477,8 @@ public class TestMapFile {
             MapFile.fix(fs, dir, IntWritable.class, Text.class, true, conf) == PAIR_SIZE);
     } catch (Exception ex) {
       fail("testFix error !!!");
+    } finally {
+      IOUtils.cleanup(null, writer);
     }
   }
   /**
@@ -459,38 +488,46 @@ public class TestMapFile {
   @SuppressWarnings("deprecation")
   public void testDeprecatedConstructors() {
     String path = new Path(TEST_DIR, "writes.mapfile").toString();
+    MapFile.Writer writer = null;
+    MapFile.Reader reader = null;
     try {
       FileSystem fs = FileSystem.getLocal(conf);
-      MapFile.Writer writer = new MapFile.Writer(conf, fs, path,
+      writer = new MapFile.Writer(conf, fs, path,
           IntWritable.class, Text.class, CompressionType.RECORD);
       assertNotNull(writer);
+      writer.close();
       writer = new MapFile.Writer(conf, fs, path, IntWritable.class,
           Text.class, CompressionType.RECORD, defaultProgressable);
       assertNotNull(writer);
+      writer.close();
       writer = new MapFile.Writer(conf, fs, path, IntWritable.class,
           Text.class, CompressionType.RECORD, defaultCodec, defaultProgressable);
       assertNotNull(writer);
+      writer.close();
       writer = new MapFile.Writer(conf, fs, path,
           WritableComparator.get(Text.class), Text.class);
       assertNotNull(writer);
+      writer.close();
       writer = new MapFile.Writer(conf, fs, path,
           WritableComparator.get(Text.class), Text.class,
           SequenceFile.CompressionType.RECORD);
       assertNotNull(writer);
+      writer.close();
       writer = new MapFile.Writer(conf, fs, path,
           WritableComparator.get(Text.class), Text.class,
           CompressionType.RECORD, defaultProgressable);
       assertNotNull(writer);
       writer.close();
 
-      MapFile.Reader reader = new MapFile.Reader(fs, path,
+      reader = new MapFile.Reader(fs, path,
           WritableComparator.get(IntWritable.class), conf);
       assertNotNull(reader);
       assertNotNull("reader key is null !!!", reader.getKeyClass());
       assertNotNull("reader value in null", reader.getValueClass());
-
     } catch (IOException e) {
       fail(e.getMessage());
+    } finally {
+      IOUtils.cleanup(null, writer, reader);
     }
   }
   
@@ -509,11 +546,7 @@ public class TestMapFile {
     } catch (Exception e) {
       fail("fail in testKeyLessWriterCreation. Other ex !!!");
     } finally {
-      if (writer != null)
-        try {
-          writer.close();
-        } catch (IOException e) {
-        }
+      IOUtils.cleanup(null, writer);
     }
   }
   /**
@@ -542,11 +575,7 @@ public class TestMapFile {
     } catch (Exception e) {
       fail("fail in testPathExplosionWriterCreation. Other ex !!!");
     } finally {
-      if (writer != null)
-        try {
-          writer.close();
-        } catch (IOException e) {
-        }
+      IOUtils.cleanup(null, writer);
     }
   }
 
@@ -555,9 +584,9 @@ public class TestMapFile {
    */
   @Test
   public void testDescOrderWithThrowExceptionWriterAppend() {
+    MapFile.Writer writer = null;
     try {
-      MapFile.Writer writer = createWriter(".mapfile", IntWritable.class,
-          Text.class);
+      writer = createWriter(".mapfile", IntWritable.class, Text.class);
       writer.append(new IntWritable(2), new Text("value: " + 1));
       writer.append(new IntWritable(2), new Text("value: " + 2));
       writer.append(new IntWritable(2), new Text("value: " + 4));
@@ -566,6 +595,8 @@ public class TestMapFile {
     } catch (IOException ex) {
     } catch (Exception e) {
       fail("testDescOrderWithThrowExceptionWriterAppend other ex throw !!!");
+    } finally {
+      IOUtils.cleanup(null, writer);
     }
   }
 
@@ -575,15 +606,17 @@ public class TestMapFile {
     String inFile = "mainMethodMapFile.mapfile";
     String outFile = "mainMethodMapFile.mapfile";
     String[] args = { path, outFile };
+    MapFile.Writer writer = null;
     try {
-      MapFile.Writer writer = createWriter(inFile, IntWritable.class,
-          Text.class);
+      writer = createWriter(inFile, IntWritable.class, Text.class);
       writer.append(new IntWritable(1), new Text("test_text1"));
       writer.append(new IntWritable(2), new Text("test_text2"));
       writer.close();
       MapFile.main(args);
     } catch (Exception ex) {
       fail("testMainMethodMapFile error !!!");
+    } finally {
+      IOUtils.cleanup(null, writer);
     }
   }
 
@@ -601,56 +634,58 @@ public class TestMapFile {
     Path qualifiedDirName = fs.makeQualified(dirName);
     // Make an index entry for every third insertion.
     MapFile.Writer.setIndexInterval(conf, 3);
-    MapFile.Writer writer = new MapFile.Writer(conf, fs,
-        qualifiedDirName.toString(), Text.class, Text.class);
-    // Assert that the index interval is 1
-    assertEquals(3, writer.getIndexInterval());
-    // Add entries up to 100 in intervals of ten.
-    final int FIRST_KEY = 10;
-    for (int i = FIRST_KEY; i < 100; i += 10) {
-      String iStr = Integer.toString(i);
-      Text t = new Text("00".substring(iStr.length()) + iStr);
-      writer.append(t, t);
-    }
-    writer.close();
-    // Now do getClosest on created mapfile.
-    MapFile.Reader reader = new MapFile.Reader(qualifiedDirName, conf);
+    MapFile.Writer writer = null;
+    MapFile.Reader reader = null;
     try {
-    Text key = new Text("55");
-    Text value = new Text();
-    Text closest = (Text) reader.getClosest(key, value);
-    // Assert that closest after 55 is 60
-    assertEquals(new Text("60"), closest);
-    // Get closest that falls before the passed key: 50
-    closest = (Text) reader.getClosest(key, value, true);
-    assertEquals(new Text("50"), closest);
-    // Test get closest when we pass explicit key
-    final Text TWENTY = new Text("20");
-    closest = (Text) reader.getClosest(TWENTY, value);
-    assertEquals(TWENTY, closest);
-    closest = (Text) reader.getClosest(TWENTY, value, true);
-    assertEquals(TWENTY, closest);
-    // Test what happens at boundaries. Assert if searching a key that is
-    // less than first key in the mapfile, that the first key is returned.
-    key = new Text("00");
-    closest = (Text) reader.getClosest(key, value);
-    assertEquals(FIRST_KEY, Integer.parseInt(closest.toString()));
-
-    // If we're looking for the first key before, and we pass in a key before
-    // the first key in the file, we should get null
-    closest = (Text) reader.getClosest(key, value, true);
-    assertNull(closest);
-
-    // Assert that null is returned if key is > last entry in mapfile.
-    key = new Text("99");
-    closest = (Text) reader.getClosest(key, value);
-    assertNull(closest);
-
-    // If we were looking for the key before, we should get the last key
-    closest = (Text) reader.getClosest(key, value, true);
-    assertEquals(new Text("90"), closest);
+      writer = new MapFile.Writer(conf, fs, qualifiedDirName.toString(),
+        Text.class, Text.class);
+      // Assert that the index interval is 1
+      assertEquals(3, writer.getIndexInterval());
+      // Add entries up to 100 in intervals of ten.
+      final int FIRST_KEY = 10;
+      for (int i = FIRST_KEY; i < 100; i += 10) {
+        String iStr = Integer.toString(i);
+        Text t = new Text("00".substring(iStr.length()) + iStr);
+        writer.append(t, t);
+      }
+      writer.close();
+      // Now do getClosest on created mapfile.
+      reader = new MapFile.Reader(qualifiedDirName, conf);
+      Text key = new Text("55");
+      Text value = new Text();
+      Text closest = (Text) reader.getClosest(key, value);
+      // Assert that closest after 55 is 60
+      assertEquals(new Text("60"), closest);
+      // Get closest that falls before the passed key: 50
+      closest = (Text) reader.getClosest(key, value, true);
+      assertEquals(new Text("50"), closest);
+      // Test get closest when we pass explicit key
+      final Text TWENTY = new Text("20");
+      closest = (Text) reader.getClosest(TWENTY, value);
+      assertEquals(TWENTY, closest);
+      closest = (Text) reader.getClosest(TWENTY, value, true);
+      assertEquals(TWENTY, closest);
+      // Test what happens at boundaries. Assert if searching a key that is
+      // less than first key in the mapfile, that the first key is returned.
+      key = new Text("00");
+      closest = (Text) reader.getClosest(key, value);
+      assertEquals(FIRST_KEY, Integer.parseInt(closest.toString()));
+
+      // If we're looking for the first key before, and we pass in a key before
+      // the first key in the file, we should get null
+      closest = (Text) reader.getClosest(key, value, true);
+      assertNull(closest);
+
+      // Assert that null is returned if key is > last entry in mapfile.
+      key = new Text("99");
+      closest = (Text) reader.getClosest(key, value);
+      assertNull(closest);
+
+      // If we were looking for the key before, we should get the last key
+      closest = (Text) reader.getClosest(key, value, true);
+      assertEquals(new Text("90"), closest);
     } finally {
-      reader.close();
+      IOUtils.cleanup(null, writer, reader);
     }
   }
 
@@ -662,16 +697,18 @@ public class TestMapFile {
     FileSystem fs = FileSystem.getLocal(conf);
     Path qualifiedDirName = fs.makeQualified(dirName);
 
-    MapFile.Writer writer = new MapFile.Writer(conf, fs,
-        qualifiedDirName.toString(), IntWritable.class, IntWritable.class);
-    writer.append(new IntWritable(1), new IntWritable(1));
-    writer.close();
-    // Now do getClosest on created mapfile.
-    MapFile.Reader reader = new MapFile.Reader(qualifiedDirName, conf);
+    MapFile.Writer writer = null;
+    MapFile.Reader reader = null;
     try {
+      writer = new MapFile.Writer(conf, fs, qualifiedDirName.toString(),
+        IntWritable.class, IntWritable.class);
+      writer.append(new IntWritable(1), new IntWritable(1));
+      writer.close();
+      // Now do getClosest on created mapfile.
+      reader = new MapFile.Reader(qualifiedDirName, conf);
       assertEquals(new IntWritable(1), reader.midKey());
     } finally {
-      reader.close();
+      IOUtils.cleanup(null, writer, reader);
     }
   }