HADOOP-3400. A new API FileSystem.deleteOnExit() that facilitates

handling of temporary files in HDFS. (dhruba)



git-svn-id: https://svn.apache.org/repos/asf/hadoop/core/trunk@659005 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -94,6 +94,9 @@ Trunk (unreleased changes)
     HADOOP-3336. Direct a subset of annotated FSNamesystem calls for audit
     logging. (cdouglas)
 
+    HADOOP-3400. A new API FileSystem.deleteOnExit() that facilitates
+    handling of temporary files in HDFS. (dhruba)
+
   IMPROVEMENTS
    
     HADOOP-2928. Remove deprecated FileSystem.getContentLength().

src/java/org/apache/hadoop/dfs/DistributedFileSystem.java

@@ -194,6 +194,7 @@ public class DistributedFileSystem extends FileSystem {
   /** {@inheritDoc} */
   public void close() throws IOException {
     try {
+      super.processDeleteOnExit();
       dfs.close();
     } finally {
       super.close();

src/java/org/apache/hadoop/fs/FileSystem.java

@@ -69,6 +69,12 @@ public abstract class FileSystem extends Configured implements Closeable {
    * The statistics for this file system.
    */
   protected final Statistics statistics;
+
+  /**
+   * A cache of files that should be deleted when filsystem is closed
+   * or the JVM is exited.
+   */
+  private Set<Path> deleteOnExit = new TreeSet<Path>();
   
   /**
    * Parse the cmd-line args, starting at i.  Remove consumed args
@@ -591,6 +597,41 @@ public abstract class FileSystem extends Configured implements Closeable {
    * @throws IOException
    */
   public abstract boolean delete(Path f, boolean recursive) throws IOException;
+
+  /** Mark a path to be deleted when FileSystem is closed or JVM exits.
+   *  The path has to exist in the file system.
+   *
+   * @param f the path to delete.
+   * @return  true if deleteOnExit is successful, otherwise false.
+   * @throws IOException
+   */
+  public boolean deleteOnExit(Path f) throws IOException {
+    if (!exists(f)) {
+      return false;
+    }
+    synchronized (deleteOnExit) {
+      deleteOnExit.add(f);
+    }
+    return true;
+  }
+
+  // Delete all files that were marked as delete-on-exit. This recursively
+  // deletes all files in the specified paths.
+  //
+  protected void processDeleteOnExit() {
+    synchronized (deleteOnExit) {
+      for (Iterator<Path> iter = deleteOnExit.iterator(); iter.hasNext();) {
+        Path path = iter.next();
+        try {
+          delete(path, true);
+        }
+        catch (IOException e) {
+          LOG.info("Ignoring failure to deleteOnExit for path " + path);
+        }
+        iter.remove();
+      }
+    }
+  }
   
   /** Check if exists.
    * @param f source file
@@ -1172,6 +1213,8 @@ public abstract class FileSystem extends Configured implements Closeable {
    * release any held locks.
    */
   public void close() throws IOException {
+    // delete all files that were marked as delete-on-exit.
+    processDeleteOnExit();
     CACHE.remove(new Cache.Key(this), this);
   }
 

src/test/org/apache/hadoop/dfs/TestFileCreation.java

@@ -202,6 +202,71 @@ public class TestFileCreation extends TestCase {
     }
   }
 
+  /**
+   * Test deleteOnExit
+   */
+  public void testDeleteOnExit() throws IOException {
+    Configuration conf = new Configuration();
+    if (simulatedStorage) {
+      conf.setBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, true);
+    }
+    MiniDFSCluster cluster = new MiniDFSCluster(conf, 1, true, null);
+    FileSystem fs = cluster.getFileSystem();
+    FileSystem localfs = FileSystem.getLocal(conf);
+
+    try {
+
+      // Creates files in HDFS and local file system.
+      //
+      Path file1 = new Path("filestatus.dat");
+      Path file2 = new Path("filestatus2.dat");
+      Path file3 = new Path("filestatus3.dat");
+      FSDataOutputStream stm1 = createFile(fs, file1, 1);
+      FSDataOutputStream stm2 = createFile(fs, file2, 1);
+      FSDataOutputStream stm3 = createFile(localfs, file3, 1);
+      System.out.println("DeleteOnExit: Created files.");
+
+      // write to files and close. Purposely, do not close file2.
+      writeFile(stm1);
+      writeFile(stm3);
+      stm1.close();
+      stm3.close();
+
+      // set delete on exit flag on files.
+      fs.deleteOnExit(file1);
+      fs.deleteOnExit(file2);
+      localfs.deleteOnExit(file3);
+
+      // close the file system. This should make the above files
+      // disappear.
+      fs.close();
+      localfs.close();
+      fs = null;
+      localfs = null;
+
+      // reopen file system and verify that file does not exist.
+      fs = cluster.getFileSystem();
+      localfs = FileSystem.getLocal(conf);
+
+      assertTrue(file1 + " still exists inspite of deletOnExit set.",
+                 !fs.exists(file1));
+      assertTrue(file2 + " still exists inspite of deletOnExit set.",
+                 !fs.exists(file2));
+      assertTrue(file3 + " still exists inspite of deletOnExit set.",
+                 !localfs.exists(file3));
+      System.out.println("DeleteOnExit successful.");
+
+    } finally {
+      if (fs != null) {
+        fs.close();
+      }
+      if (localfs != null) {
+        localfs.close();
+      }
+      cluster.shutdown();
+    }
+  }
+
   /**
    * Test that file data does not become corrupted even in the face of errors.
    */