MAPREDUCE-130. Delete the jobconf copy from the log directory of the JobTracker when the job is retired. Contributed by Amar Kamat.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.20@788662 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -147,6 +147,9 @@ Release 0.20.1 - Unreleased
     MAPREDUCE-2. Fixes a bug in KeyFieldBasedPartitioner in handling empty
     keys. (Amar Kamat via sharad)
 
+    MAPREDUCE-130. Delete the jobconf copy from the log directory of the 
+    JobTracker when the job is retired. (Amar Kamat via sharad)
+
 Release 0.20.0 - 2009-04-15
 
   INCOMPATIBLE CHANGES

src/mapred/org/apache/hadoop/mapred/JobHistory.java

@@ -832,6 +832,19 @@ public class JobHistory {
       }
     }
 
+    /**
+     * Deletes job data from the local disk.
+     * For now just deletes the localized copy of job conf
+     */
+    static void cleanupJob(JobID id) {
+      String localJobFilePath =  JobInfo.getLocalJobFilePath(id);
+      File f = new File (localJobFilePath);
+      LOG.info("Deleting localized job conf at " + f);
+      if (!f.delete()) {
+        LOG.debug("Failed to delete file " + f);
+      }
+    }
+
     /**
      * Log job submitted event to history. Creates a new file in history 
      * for the job. if history file creation fails, it disables history 

src/mapred/org/apache/hadoop/mapred/JobTracker.java

@@ -426,6 +426,9 @@ public class JobTracker implements MRConstants, InterTrackerProtocol,
                     LOG.info("Retired job with id: '" + 
                              job.getProfile().getJobID() + "' of user '" +
                              jobUser + "'");
+
+                    // clean up job files from the local disk
+                    JobHistory.JobInfo.cleanupJob(job.getProfile().getJobID());
                   }
                 }
               }

src/test/org/apache/hadoop/mapred/TestJobHistory.java

@@ -778,10 +778,14 @@ public class TestJobHistory extends TestCase {
   public void testJobHistoryFile() throws IOException {
     MiniMRCluster mr = null;
     try {
-      mr = new MiniMRCluster(2, "file:///", 3);
+      JobConf conf = new JobConf();
+      // keep for less time
+      conf.setLong("mapred.jobtracker.retirejob.check", 1000);
+      conf.setLong("mapred.jobtracker.retirejob.interval", 1000);
+      mr = new MiniMRCluster(2, "file:///", 3, null, null, conf);
 
       // run the TCs
-      JobConf conf = mr.createJobConf();
+      conf = mr.createJobConf();
 
       FileSystem fs = FileSystem.get(conf);
       // clean up
@@ -802,6 +806,15 @@ public class TestJobHistory extends TestCase {
       validateJobHistoryFileFormat(job.getID(), conf, "SUCCESS", false);
       validateJobHistoryFileContent(mr, job, conf);
 
+      // get the job conf filename
+      String name = JobHistory.JobInfo.getLocalJobFilePath(job.getID());
+      File file = new File(name);
+
+      // check if the file get deleted
+      while (file.exists()) {
+        LOG.info("Waiting for " + file + " to be deleted");
+        UtilsForTests.waitFor(100);
+      }
     } finally {
       if (mr != null) {
         cleanupLocalFiles(mr);