Fix for HADOOP-19. A namenode must now be formatted before it may be used. Attempts to start a namenode in an unformatted directory will fail, rather than automatically creating a new, empty filesystem, causing existing datanodes to delete all blocks. Thus a mis-configured dfs.data.dir should no longer cause data loss.

git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk@389566 13f79535-47bb-0310-9956-ffa450edef68

src/java/org/apache/hadoop/dfs/FSDirectory.java

@@ -245,14 +245,11 @@ class FSDirectory implements FSConstants {
     DataOutputStream editlog = null;
     boolean ready = false;
 
-    /**
-     * Create a FileSystem directory, and load its info
-     * from the indicated place.
-     */
+    /** Access an existing dfs name directory. */
     public FSDirectory(File dir) throws IOException {
         File fullimage = new File(dir, "image");
         if (! fullimage.exists()) {
-            fullimage.mkdirs();
+          throw new IOException("NameNode not formatted: " + dir);
         }
         File edits = new File(dir, "edits");
         if (loadFSImage(fullimage, edits)) {
@@ -266,6 +263,19 @@ class FSDirectory implements FSConstants {
         }
     }
 
+    /** Create a new dfs name directory.  Caution: this destroys all files
+     * in this filesystem. */
+    public static void format(File dir) throws IOException {
+        File image = new File(dir, "image");
+        File edits = new File(dir, "edits");
+
+        if (!((!image.exists() || image.delete()) &&
+              (!edits.exists() || edits.delete()) &&
+              image.mkdirs())) {
+          throw new IOException("Unable to format: "+dir);
+        }
+    }
+
     /**
      * Shutdown the filestore
      */

src/java/org/apache/hadoop/dfs/NameNode.java

@@ -65,12 +65,17 @@ public class NameNode implements ClientProtocol, DatanodeProtocol, FSConstants {
     /** only used for testing purposes  */
     private boolean stopRequested = false;
 
+    /** Format a new filesystem.  Destroys any filesystem that may already
+     * exist at this location.  **/
+    public static void format(Configuration conf) throws IOException {
+      FSDirectory.format(getDir(conf));
+    }
+
     /**
      * Create a NameNode at the default location
      */
     public NameNode(Configuration conf) throws IOException {
-        this(new File(conf.get("dfs.name.dir",
-                                          "/tmp/hadoop/dfs/name")),
+        this(getDir(conf),
              DataNode.createSocketAddr
              (conf.get("fs.default.name", "local")).getPort(), conf);
     }
@@ -85,6 +90,11 @@ public class NameNode implements ClientProtocol, DatanodeProtocol, FSConstants {
         this.server.start();
     }
 
+    /** Return the configured directory where name data is stored. */
+    private static File getDir(Configuration conf) {
+      return new File(conf.get("dfs.name.dir", "/tmp/hadoop/dfs/name"));
+    }
+
     /**
      * Wait for service to finish.
      * (Normally, it runs forever.)
@@ -364,8 +374,24 @@ public class NameNode implements ClientProtocol, DatanodeProtocol, FSConstants {
 
     /**
      */
-    public static void main(String argv[]) throws IOException, InterruptedException {
-        NameNode namenode = new NameNode(new Configuration());
+    public static void main(String argv[]) throws Exception {
+        Configuration conf = new Configuration();
+
+        if (argv.length == 1 && argv[0].equals("-format")) {
+          File dir = getDir(conf);
+          if (dir.exists()) {
+            System.err.print("Re-format filesystem in " + dir +" ? (Y or N) ");
+            if (!(System.in.read() == 'Y')) {
+              System.err.println("Format aborted.");
+              System.exit(1);
+            }
+          }
+          format(conf);
+          System.err.println("Formatted "+dir);
+          System.exit(0);
+        }
+
+        NameNode namenode = new NameNode(conf);
         namenode.join();
     }
 }

src/java/overview.html

@@ -134,6 +134,11 @@ cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys
 
 <h3>Bootstrapping</h3>
 
+<p>A new distributed filesystem must formatted with the following
+command, run on the master node:</p>
+
+<p><tt>bin/hadoop namenode -format</tt></p>
+
 <p>The Hadoop daemons are started with the following command:</p>
 
 <p><tt>bin/start-all.sh</tt></p>

src/test/org/apache/hadoop/dfs/MiniDFSCluster.java

@@ -83,7 +83,7 @@ public class MiniDFSCluster {
   /**
    * Create the config and start up the servers.
    */
-  public MiniDFSCluster(int namenodePort, Configuration conf) {
+  public MiniDFSCluster(int namenodePort, Configuration conf) throws IOException {
     this.conf = conf;
     conf.set("fs.default.name", 
              "localhost:"+ Integer.toString(namenodePort));
@@ -95,6 +95,7 @@ public class MiniDFSCluster {
     // this timeout seems to control the minimum time for the test, so
     // set it down at 5 seconds.
     conf.setInt("ipc.client.timeout", 5000);
+    NameNode.format(conf);
     nameNode = new NameNodeRunner();
     nameNodeThread = new Thread(nameNode);
     nameNodeThread.start();