HADOOP-6080. Introduce -skipTrash option to rm and rmr. Contributed by Jakob Homan.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@789875 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -154,6 +154,9 @@ Trunk (unreleased changes)
 
     HADOOP-3315. Add a new, binary file foramt, TFile. (Hong Tang via cdouglas)
 
+    HADOOP-6080. Introduce -skipTrash option to rm and rmr.
+    (Jakob Homan via shv)
+
   IMPROVEMENTS
 
     HADOOP-4565. Added CombineFileInputFormat to use data locality information

src/docs/src/documentation/content/xdocs/hdfs_shell.xml

@@ -325,10 +325,13 @@
 		<section>
 			<title> rm </title>
 			<p>
-				<code>Usage: hadoop fs -rm URI [URI &#x2026;] </code>
+				<code>Usage: hadoop fs -rm [-skipTrash] URI [URI &#x2026;] </code>
 			</p>
 			<p>
-	   Delete files specified as args. Only deletes non empty directory and files. Refer to rmr for recursive deletes.<br/>
+	   Delete files specified as args. Only deletes non empty directory and files. If the <code>-skipTrash</code> option
+	   is specified, the trash, if enabled, will be bypassed and the specified file(s) deleted immediately.  	This can be
+		   useful when it is necessary to delete files from an over-quota directory.
+	   Refer to rmr for recursive deletes.<br/>
 	   Example:
 	   </p>
 			<ul>
@@ -344,9 +347,11 @@
 		<section>
 			<title> rmr </title>
 			<p>
-				<code>Usage: hadoop fs -rmr URI [URI &#x2026;]</code>
+				<code>Usage: hadoop fs -rmr [-skipTrash] URI [URI &#x2026;]</code>
 			</p>
-			<p>Recursive version of delete.<br/>
+			<p>Recursive version of delete. If the <code>-skipTrash</code> option
+		   is specified, the trash, if enabled, will be bypassed and the specified file(s) deleted immediately. This can be
+		   useful when it is necessary to delete files from an over-quota directory.<br/>
 	   Example:
 	   </p>
 			<ul>

src/java/org/apache/hadoop/fs/FsShell.java

@@ -1086,10 +1086,12 @@ public class FsShell extends Configured implements Tool {
    * Delete all files that match the file pattern <i>srcf</i>.
    * @param srcf a file pattern specifying source files
    * @param recursive if need to delete subdirs
+   * @param skipTrash Should we skip the trash, if it's enabled?
    * @throws IOException  
    * @see org.apache.hadoop.fs.FileSystem#globStatus(Path)
    */
-  void delete(String srcf, final boolean recursive) throws IOException {
+  void delete(String srcf, final boolean recursive, final boolean skipTrash) 
+                                                            throws IOException {
     //rm behavior in Linux
     //  [~/1207]$ ls ?.txt
     //  x.txt  z.txt
@@ -1100,22 +1102,27 @@ public class FsShell extends Configured implements Tool {
     new DelayedExceptionThrowing() {
       @Override
       void process(Path p, FileSystem srcFs) throws IOException {
-        delete(p, srcFs, recursive);
+        delete(p, srcFs, recursive, skipTrash);
       }
     }.globAndProcess(srcPattern, srcPattern.getFileSystem(getConf()));
   }
     
   /* delete a file */
-  private void delete(Path src, FileSystem srcFs, boolean recursive) throws IOException {
+  private void delete(Path src, FileSystem srcFs, boolean recursive, 
+                      boolean skipTrash) throws IOException {
     if (srcFs.isDirectory(src) && !recursive) {
       throw new IOException("Cannot remove directory \"" + src +
                             "\", use -rmr instead");
     }
-    Trash trashTmp = new Trash(srcFs, getConf());
-    if (trashTmp.moveToTrash(src)) {
-      System.out.println("Moved to trash: " + src);
-      return;
+    
+    if(!skipTrash) {
+      Trash trashTmp = new Trash(srcFs, getConf());
+      if (trashTmp.moveToTrash(src)) {
+        System.out.println("Moved to trash: " + src);
+        return;
+      }
     }
+    
     if (srcFs.delete(src, true)) {
       System.out.println("Deleted " + src);
     } else {
@@ -1312,8 +1319,8 @@ public class FsShell extends Configured implements Tool {
       "The full syntax is: \n\n" +
       "hadoop fs [-fs <local | file system URI>] [-conf <configuration file>]\n\t" +
       "[-D <property=value>] [-ls <path>] [-lsr <path>] [-df [<path>]] [-du <path>]\n\t" + 
-      "[-dus <path>] [-mv <src> <dst>] [-cp <src> <dst>] [-rm <src>]\n\t" + 
-      "[-rmr <src>] [-put <localsrc> ... <dst>] [-copyFromLocal <localsrc> ... <dst>]\n\t" +
+      "[-dus <path>] [-mv <src> <dst>] [-cp <src> <dst>] [-rm [-skipTrash] <src>]\n\t" + 
+      "[-rmr [-skipTrash] <src>] [-put <localsrc> ... <dst>] [-copyFromLocal <localsrc> ... <dst>]\n\t" +
       "[-moveFromLocal <localsrc> ... <dst>] [" + 
       GET_SHORT_USAGE + "\n\t" +
       "[-getmerge <src> <localdst> [addnl]] [-cat <src>]\n\t" +
@@ -1381,11 +1388,15 @@ public class FsShell extends Configured implements Tool {
       "\t\tdestination.  When copying multiple files, the destination\n" +
       "\t\tmust be a directory. \n";
 
-    String rm = "-rm <src>: \tDelete all files that match the specified file pattern.\n" +
-      "\t\tEquivlent to the Unix command \"rm <src>\"\n";
+    String rm = "-rm [-skipTrash] <src>: \tDelete all files that match the specified file pattern.\n" +
+      "\t\tEquivalent to the Unix command \"rm <src>\"\n" +
+      "\t\t-skipTrash option bypasses trash, if enabled, and immediately\n" +
+      "deletes <src>";
 
-    String rmr = "-rmr <src>: \tRemove all directories which match the specified file \n" +
-      "\t\tpattern. Equivlent to the Unix command \"rm -rf <src>\"\n";
+    String rmr = "-rmr [-skipTrash] <src>: \tRemove all directories which match the specified file \n" +
+      "\t\tpattern. Equivalent to the Unix command \"rm -rf <src>\"\n" +
+      "\t\t-skipTrash option bypasses trash, if enabled, and immediately\n" +
+      "deletes <src>";
 
     String put = "-put <localsrc> ... <dst>: \tCopy files " + 
     "from the local file system \n\t\tinto fs. \n";
@@ -1573,8 +1584,6 @@ public class FsShell extends Configured implements Tool {
       System.out.println(Count.DESCRIPTION);
       System.out.println(help);
     }        
-
-                           
   }
 
   /**
@@ -1584,6 +1593,15 @@ public class FsShell extends Configured implements Tool {
   private int doall(String cmd, String argv[], int startindex) {
     int exitCode = 0;
     int i = startindex;
+    boolean rmSkipTrash = false;
+    
+    // Check for -skipTrash option in rm/rmr
+    if(("-rm".equals(cmd) || "-rmr".equals(cmd)) 
+        && "-skipTrash".equals(argv[i])) {
+      rmSkipTrash = true;
+      i++;
+    }
+    
     //
     // for each source file, issue the command
     //
@@ -1597,9 +1615,9 @@ public class FsShell extends Configured implements Tool {
         } else if ("-mkdir".equals(cmd)) {
           mkdir(argv[i]);
         } else if ("-rm".equals(cmd)) {
-          delete(argv[i], false);
+          delete(argv[i], false, rmSkipTrash);
         } else if ("-rmr".equals(cmd)) {
-          delete(argv[i], true);
+          delete(argv[i], true, rmSkipTrash);
         } else if ("-df".equals(cmd)) {
           df(argv[i]);
         } else if (Count.matches(cmd)) {
@@ -1708,8 +1726,8 @@ public class FsShell extends Configured implements Tool {
       System.err.println("           [" + Count.USAGE + "]");
       System.err.println("           [-mv <src> <dst>]");
       System.err.println("           [-cp <src> <dst>]");
-      System.err.println("           [-rm <path>]");
-      System.err.println("           [-rmr <path>]");
+      System.err.println("           [-rm [-skipTrash] <path>]");
+      System.err.println("           [-rmr [-skipTrash] <path>]");
       System.err.println("           [-expunge]");
       System.err.println("           [-put <localsrc> ... <dst>]");
       System.err.println("           [-copyFromLocal <localsrc> ... <dst>]");

src/test/core/org/apache/hadoop/fs/TestTrash.java

@@ -140,7 +140,7 @@ public class TestTrash extends TestCase {
 
     // Verify that we can recreate the file
     writeFile(fs, myFile);
-
+    
     // Verify that we succeed in removing the whole directory
     // along with the file inside it.
     {
@@ -256,6 +256,63 @@ public class TestTrash extends TestCase {
       assertTrue(val == -1);
       assertTrue(fs.exists(trashRoot));
     }
+    
+    // Verify skip trash option really works
+    
+    // recreate directory and file
+    mkdir(fs, myPath);
+    writeFile(fs, myFile);
+    
+    // Verify that skip trash option really skips the trash for files (rm)
+    {
+      String[] args = new String[3];
+      args[0] = "-rm";
+      args[1] = "-skipTrash";
+      args[2] = myFile.toString();
+      int val = -1;
+      try {
+        // Clear out trash
+        assertEquals(0, shell.run(new String [] { "-expunge" } ));
+        
+        val = shell.run(args);
+        
+      }catch (Exception e) {
+        System.err.println("Exception raised from Trash.run " +
+            e.getLocalizedMessage());
+      }
+      assertFalse(fs.exists(trashRoot)); // No new Current should be created
+      assertFalse(fs.exists(myFile));
+      assertTrue(val == 0);
+    }
+    
+    // recreate directory and file
+    mkdir(fs, myPath);
+    writeFile(fs, myFile);
+    
+    // Verify that skip trash option really skips the trash for rmr
+    {
+      String[] args = new String[3];
+      args[0] = "-rmr";
+      args[1] = "-skipTrash";
+      args[2] = myPath.toString();
+
+      int val = -1;
+      try {
+        // Clear out trash
+        assertEquals(0, shell.run(new String [] { "-expunge" } ));
+        
+        val = shell.run(args);
+        
+      }catch (Exception e) {
+        System.err.println("Exception raised from Trash.run " +
+            e.getLocalizedMessage());
+      }
+
+      assertFalse(fs.exists(trashRoot)); // No new Current should be created
+      assertFalse(fs.exists(myPath));
+      assertFalse(fs.exists(myFile));
+      assertTrue(val == 0);
+    }
   }
 
   public static void trashNonDefaultFS(Configuration conf) throws IOException {