HADOOP-1978. Name-node removes edits.new after a successful startup.

(Konstantin Shvachko via dhruba)



git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk@581745 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -367,6 +367,9 @@ Release 0.14.2 - unreleased
     HADOOP-1970.  Fix deadlock in progress reporting in the task. (Vivek
     Ratan via omalley)
 
+    HADOOP-1978.  Name-node removes edits.new after a successful startup.
+    (Konstantin Shvachko via dhruba)
+
 Release 0.14.1 - 2007-09-04
 
   BUG FIXES

src/java/org/apache/hadoop/dfs/FSEditLog.java

@@ -22,7 +22,6 @@ import java.io.DataInputStream;
 import java.io.DataOutputStream;
 import java.io.EOFException;
 import java.io.File;
-import java.io.FileDescriptor;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
@@ -54,19 +53,16 @@ class FSEditLog {
   private long lastSyncTime;
   
   static class EditLogOutputStream extends DataOutputStream {
-    private FileDescriptor fd;
-
     EditLogOutputStream(File name) throws IOException {
       super(new FileOutputStream(name, true)); // open for append
-      this.fd = ((FileOutputStream)out).getFD();
     }
 
     void flushAndSync() throws IOException {
-      this.flush();
-      this.fd.sync();
+      ((FileOutputStream)out).getChannel().force(true);
     }
 
     void create() throws IOException {
+      ((FileOutputStream)out).getChannel().truncate(0);
       writeInt(FSConstants.LAYOUT_VERSION);
       flushAndSync();
     }
@@ -124,7 +120,6 @@ class FSEditLog {
   synchronized void createEditLogFile(File name) throws IOException {
     EditLogOutputStream eStream = new EditLogOutputStream(name);
     eStream.create();
-    eStream.flushAndSync();
     eStream.close();
   }
 

src/java/org/apache/hadoop/dfs/FSImage.java

@@ -762,6 +762,7 @@ class FSImage extends Storage {
 
   /**
    * Save the contents of the FS image
+   * and create empty edits.
    */
   void saveFSImage() throws IOException {
     editLog.createNewIfMissing();
@@ -769,6 +770,9 @@ class FSImage extends Storage {
       StorageDirectory sd = getStorageDir(idx);
       saveFSImage(getImageFile(sd, NameNodeFile.IMAGE_NEW));
       editLog.createEditLogFile(getImageFile(sd, NameNodeFile.EDITS));
+      File editsNew = getImageFile(sd, NameNodeFile.EDITS_NEW);
+      if (editsNew.exists()) 
+        editLog.createEditLogFile(editsNew);
     }
     rollFSImage();
   }

src/test/org/apache/hadoop/dfs/TestCheckpoint.java

@@ -22,6 +22,7 @@ import java.io.*;
 import java.util.Collection;
 import java.util.Random;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.dfs.FSImage.NameNodeFile;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -163,6 +164,19 @@ public class TestCheckpoint extends TestCase {
     System.out.println("Starting testSecondaryNamenodeError 2");
     cluster = new MiniDFSCluster(conf, numDatanodes, false, null);
     cluster.waitActive();
+    // Also check that the edits file is empty here
+    // and that temporary checkpoint files are gone.
+    FSImage image = cluster.getNameNode().getFSImage();
+    int nrDirs = image.getNumStorageDirs();
+    for(int idx = 0; idx < nrDirs; idx++) {
+      assertFalse(image.getImageFile(idx, NameNodeFile.IMAGE_NEW).exists());
+      assertFalse(image.getEditNewFile(idx).exists());
+      File edits = image.getEditFile(idx);
+      assertTrue(edits.exists()); // edits should exist and be empty
+      assertTrue(
+        (new RandomAccessFile(edits, "r")).length() == Integer.SIZE/Byte.SIZE);
+    }
+    
     fileSys = cluster.getFileSystem();
     try {
       checkFile(fileSys, file1, replication);

src/test/org/apache/hadoop/dfs/TestFsck.java

@@ -81,6 +81,7 @@ public class TestFsck extends TestCase {
       
       // bring up data nodes & cleanup cluster
       cluster.startDataNodes(conf, 4, true, null, null);
+      cluster.waitActive();
       util.cleanup(cluster.getFileSystem(), "/srcdat");
     } finally {
       if (cluster != null) { cluster.shutdown(); }