HADOOP-8873. Port HADOOP-8175 (Add mkdir -p flag) to branch-1. Contributed by Akira Ajisaka.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-1@1504007 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -18,6 +18,9 @@ Release 1.3.0 - unreleased
     support an arbitrary filesystem URI. (Tom White, backported by
     Chelsey Chang via ivanmi)
 
+    HADOOP-8873. Port HADOOP-8175 (Add mkdir -p flag) to branch-1.
+    (Akira Ajisaka via suresh)
+
   BUG FIXES
 
     MAPREDUCE-5047. keep.failed.task.files=true causes job failure on 

src/core/org/apache/hadoop/fs/FsShell.java

@@ -709,26 +709,31 @@ public class FsShell extends Configured implements Tool {
   }
 
   /**
-   * Create the given dir
+   * Create the given dir.
+   * @param src a path to create dir
+   * @param createParents
+   *   if set to true, mkdir will not fail when the given dir already exists
+   * @throws IOException
    */
-  void mkdir(String src) throws IOException {
+  void mkdir(String src, boolean createParents) throws IOException {
     Path f = new Path(src);
     FileSystem srcFs = f.getFileSystem(getConf());
     FileStatus fstatus = null;
     try {
       fstatus = srcFs.getFileStatus(f);
       if (fstatus.isDir()) {
-        throw new IOException("cannot create directory " 
-            + src + ": File exists");
-      }
-      else {
+        if (!createParents) {
+          throw new IOException("cannot create directory "
+              + src + ": File exists");
+        }
+      } else {
         throw new IOException(src + " exists but " +
             "is not a directory");
       }
-    } catch(FileNotFoundException e) {
-        if (!srcFs.mkdirs(f)) {
-          throw new IOException("failed to create " + src);
-        }
+    } catch (FileNotFoundException e) {
+      if (!srcFs.mkdirs(f)) {
+        throw new IOException("failed to create " + src);
+      }
     }
   }
 
@@ -1294,7 +1299,7 @@ public class FsShell extends Configured implements Tool {
       GET_SHORT_USAGE + "\n\t" +
       "[-getmerge <src> <localdst> [addnl]] [-cat <src>]\n\t" +
       "[" + COPYTOLOCAL_SHORT_USAGE + "] [-moveToLocal <src> <localdst>]\n\t" +
-      "[-mkdir <path>] [-report] [" + SETREP_SHORT_USAGE + "]\n\t" +
+      "[-mkdir [-p] <path>] [-report] [" + SETREP_SHORT_USAGE + "]\n\t" +
       "[-touchz <path>] [-test -[ezd] <path>] [-stat [format] <path>]\n\t" +
       "[-tail [-f] <path>] [-text <path>]\n\t" +
       "[" + FsShellPermissions.CHMOD_USAGE + "]\n\t" +
@@ -1394,7 +1399,9 @@ public class FsShell extends Configured implements Tool {
 
     String moveToLocal = "-moveToLocal <src> <localdst>:  Not implemented yet \n";
         
-    String mkdir = "-mkdir <path>: \tCreate a directory in specified location. \n";
+    String mkdir = "-mkdir [-p] <path>: "
+      + "\tCreate a directory in specified location. \n"
+      + "\t\t-p Do not fail if the directory already exists.";
 
     String setrep = SETREP_SHORT_USAGE
       + ":  Set the replication level of a file. \n"
@@ -1556,8 +1563,15 @@ public class FsShell extends Configured implements Tool {
   private int doall(String cmd, String argv[], int startindex) {
     int exitCode = 0;
     int i = startindex;
+    boolean mkdirCreateParents = false;
     boolean rmSkipTrash = false;
     
+    // Check for -p option in mkdir
+    if("-mkdir".equals(cmd) && "-p".equals(argv[i])) {
+      mkdirCreateParents = true;
+      i++;
+    }
+    
     // Check for -skipTrash option in rm/rmr
     if(("-rm".equals(cmd) || "-rmr".equals(cmd)) 
         && "-skipTrash".equals(argv[i])) {
@@ -1576,7 +1590,7 @@ public class FsShell extends Configured implements Tool {
         if ("-cat".equals(cmd)) {
           cat(argv[i], true);
         } else if ("-mkdir".equals(cmd)) {
-          mkdir(argv[i]);
+          mkdir(argv[i], mkdirCreateParents);
         } else if ("-rm".equals(cmd)) {
           delete(argv[i], false, rmSkipTrash);
         } else if ("-rmr".equals(cmd)) {
@@ -1644,10 +1658,12 @@ public class FsShell extends Configured implements Tool {
                          " [-D <[property=value>]");
     } else if ("-ls".equals(cmd) || "-lsr".equals(cmd) ||
                "-du".equals(cmd) || "-dus".equals(cmd) ||
-               "-touchz".equals(cmd) || "-mkdir".equals(cmd) ||
-               "-text".equals(cmd)) {
+               "-touchz".equals(cmd) || "-text".equals(cmd)){
       System.err.println("Usage: java FsShell" + 
                          " [" + cmd + " <path>]");
+    } else if ("-mkdir".equals(cmd)) {
+      System.err.println("Usage: java FsShell" + 
+                         " [" + cmd + " [-p] <path>]");
     } else if (Count.matches(cmd)) {
       System.err.println(prefix + " [" + Count.USAGE + "]");
     } else if ("-rm".equals(cmd) || "-rmr".equals(cmd)) {
@@ -1701,7 +1717,7 @@ public class FsShell extends Configured implements Tool {
       System.err.println("           [-text <src>]");
       System.err.println("           [" + COPYTOLOCAL_SHORT_USAGE + "]");
       System.err.println("           [-moveToLocal [-crc] <src> <localdst>]");
-      System.err.println("           [-mkdir <path>]");
+      System.err.println("           [-mkdir [-p] <path>]");
       System.err.println("           [" + SETREP_SHORT_USAGE + "]");
       System.err.println("           [-touchz <path>]");
       System.err.println("           [-test -[ezd] <path>]");

src/docs/src/documentation/content/xdocs/file_system_shell.xml

@@ -307,12 +307,12 @@
 		<section>
 			<title> mkdir </title>
 			<p>
-				<code>Usage: hdfs dfs -mkdir &lt;paths&gt;</code>
-				<br/>
+				<code>Usage: hdfs dfs -mkdir [-p] &lt;paths&gt;</code><br/>
 			</p>
 			<p>
-	   Takes path uri's as argument and creates directories. The behavior is much like unix mkdir -p creating parent directories along the path.
-	  </p>
+	            Takes path uri's as argument and creates directories. The behavior is much like unix mkdir -p creating parent directories along the path.
+	            With <code>-p</code>, mkdir doesn't fail if the target already exists.
+	        </p>
 			<p>Example:</p>
 			<ul>
 				<li>

src/test/org/apache/hadoop/hdfs/TestDFSShell.java

@@ -1102,7 +1102,36 @@ public class TestDFSShell extends TestCase {
         }
         assertTrue(val == 0);
       }
-        
+
+      // Verify mkdir when the directory already exists.
+      {
+        // no option should fail.
+        String[] args = new String[2];
+        args[0] = "-mkdir";
+        args[1] = "/test";
+        int val = 0;
+        try {
+          val = shell.run(args);
+        } catch (Exception e) {
+          System.err.println("Exception raised from DFSShell.run " +
+                             e.getLocalizedMessage());
+        }
+        assertTrue(val == -1);
+
+        // with -p option, mkdir should succeed.
+        String[] args1 = new String[3];
+        args1[0] = "-mkdir";
+        args1[1] = "-p";
+        args1[2] = "/test";
+        val = -1;
+        try {
+          val = shell.run(args1);
+        } catch (Exception e) {
+          System.err.println("Exception raised from DFSShell.run " +
+                             e.getLocalizedMessage());
+        }
+        assertTrue(val == 0);
+      }
     } finally {
       try {
         fileSys.close();