HADOOP-1921. Save the configuration of completed/failed jobs and make them available via the web-ui.

git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk@579212 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -190,6 +190,9 @@ Trunk (unreleased changes)
 
   IMPROVEMENTS
 
+    HADOOP-1921. Save the configuration of completed/failed jobs and make them
+    available via the web-ui. (Amar Kamat via devaraj)
+
     HADOOP-1266. Remove dependency of package org.apache.hadoop.net on 
     org.apache.hadoop.dfs.  (Hairong Kuang via dhruba)
 

src/java/org/apache/hadoop/mapred/JobHistory.java

@@ -315,24 +315,35 @@ public class JobHistory {
      */
     public Map<String, Task> getAllTasks() { return allTasks; }
     
+    /**
+     * Get the path of the locally stored job file
+     * @param jobId id of the job
+     * @return the path of the job file on the local file system 
+     */
+    public static String getLocalJobFilePath(String jobId){
+      return LOG_DIR + File.separator + jobId + "_conf.xml";
+    }
+    
     /**
      * Log job submitted event to history. Creates a new file in history 
      * for the job. if history file creation fails, it disables history 
      * for all other events. 
-     * @param jobId job id assigned by job tracker. 
-     * @param jobName job name as given by user in job conf
-     * @param user user name
+     * @param jobId job id assigned by job tracker.
+     * @param jobConf job conf of the job
+     * @param jobConfPath path to job conf xml file in HDFS.
      * @param submitTime time when job tracker received the job
-     * @param jobConf path to job conf xml file in HDFS. 
      */
-    public static void logSubmitted(String jobId, String jobName, String user, 
-                                    long submitTime, String jobConf){
-      
+    public static void logSubmitted(String jobId, JobConf jobConf, 
+                                    String jobConfPath, long submitTime) {
+      String jobName = jobConf.getJobName();
+      String user = jobConf.getUser(); 
       if (!disableHistory){
         synchronized(MASTER_INDEX_LOG_FILE){
           JobHistory.log(masterIndex, RecordTypes.Job, 
                          new Enum[]{Keys.JOBID, Keys.JOBNAME, Keys.USER, Keys.SUBMIT_TIME, Keys.JOBCONF }, 
-                         new String[]{jobId, jobName, user, String.valueOf(submitTime), jobConf });
+                         new String[]{jobId, jobName, user, 
+                                      String.valueOf(submitTime), jobConfPath}
+                        );
         }
         // setup the history log file for this job
         String logFileName =  JOBTRACKER_START_TIME + "_" + jobId; 
@@ -344,13 +355,28 @@ public class JobHistory {
           // add to writer as well 
           JobHistory.log(writer, RecordTypes.Job, 
                          new Enum[]{Keys.JOBID, Keys.JOBNAME, Keys.USER, Keys.SUBMIT_TIME, Keys.JOBCONF }, 
-                         new String[]{jobId, jobName, user, String.valueOf(submitTime) , jobConf}); 
+                         new String[]{jobId, jobName, user, 
+                                      String.valueOf(submitTime) , jobConfPath}
+                        ); 
              
         }catch(IOException e){
           LOG.error("Failed creating job history log file, disabling history", e);
           disableHistory = true; 
         }
       }
+      /* Storing the job conf on the local file system */
+      String localJobFilePath =  JobInfo.getLocalJobFilePath(jobId); 
+      File localJobFile = new File(localJobFilePath);
+      try {
+        FileOutputStream jobOut = new FileOutputStream(localJobFile);
+        jobConf.write(jobOut);
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Job conf for " + jobId + " stored at " 
+                    + localJobFile.getAbsolutePath());
+        }
+      } catch (IOException ioe) {
+        LOG.error("Failed to store job conf on the local filesystem ", ioe);
+      }
     }
     /**
      * Logs launch time of job. 

src/java/org/apache/hadoop/mapred/JobInProgress.java

@@ -159,9 +159,8 @@ class JobInProgress {
     this.mapFailuresPercent = conf.getMaxMapTaskFailuresPercent();
     this.reduceFailuresPercent = conf.getMaxReduceTaskFailuresPercent();
         
-    JobHistory.JobInfo.logSubmitted(jobid, conf.getJobName(), conf.getUser(), 
-                                    System.currentTimeMillis(), 
-                                    jobFile.toString()); 
+    JobHistory.JobInfo.logSubmitted(jobid, conf, jobFile.toString(), 
+                                    System.currentTimeMillis()); 
         
     MetricsContext metricsContext = MetricsUtil.getContext("mapred");
     this.jobMetrics = MetricsUtil.createRecord(metricsContext, "job");

src/java/org/apache/hadoop/mapred/JobTracker.java

@@ -1832,6 +1832,14 @@ public class JobTracker implements MRConstants, InterTrackerProtocol, JobSubmiss
     }
   }
 
+  /**
+   * Get the localized job file path on the job trackers local file system
+   * @param jobId id of the job
+   * @return the path of the job conf file on the local file system
+   */
+  public static String getLocalJobFilePath(String jobId){
+    return JobHistory.JobInfo.getLocalJobFilePath(jobId);
+  }
   ////////////////////////////////////////////////////////////
   // main()
   ////////////////////////////////////////////////////////////

src/webapps/job/jobconf.jsp

@@ -26,28 +26,12 @@
 <h2>Job Configuration: JobId - <%= jobId %></h2><br>
 
 <%
-  JobInProgress job = (JobInProgress)tracker.getJob(jobId);
-  if (job == null) {
-    out.print("<h4>Job '" + jobId + "' not found!</h4><br>\n");
-    return;
-  }
-  
-  JobStatus status = job.getStatus();
-  int runState = status.getRunState();
-  if (runState != JobStatus.RUNNING) {
-    out.print("<h4>Job '" + jobId + "' not running!</h4><br>\n");
-    return;
-  }
-  
+  String jobFilePath = tracker.getLocalJobFilePath(jobId);
   try {
-    JobConf jobConf = job.getJobConf();
-    ByteArrayOutputStream jobConfXML = new ByteArrayOutputStream();
-    jobConf.write(jobConfXML);
+    JobConf jobConf = new JobConf(jobFilePath);
     XMLUtils.transform(
         jobConf.getConfResourceAsInputStream("webapps/static/jobconf.xsl"),
-	    new ByteArrayInputStream(jobConfXML.toByteArray()), 
-	    out
-	  );
+        new FileInputStream(jobFilePath), out);
   } catch (Exception e) {
     out.println("Failed to retreive job configuration for job '" + jobId + "!");
     out.println(e);

src/webapps/job/jobdetails.jsp

@@ -146,12 +146,8 @@
     int flakyTaskTrackers = job.getNoOfBlackListedTrackers();
     out.print("<b>User:</b> " + profile.getUser() + "<br>\n");
     out.print("<b>Job Name:</b> " + profile.getJobName() + "<br>\n");
-    if (runState == JobStatus.RUNNING) {
-      out.print("<b>Job File:</b> <a href=\"jobconf.jsp?jobid=" + jobId + "\">" + 
-          profile.getJobFile() + "</a><br>\n");
-    } else {
-      out.print("<b>Job File:</b> " + profile.getJobFile() + "<br>\n");
-    }
+    out.print("<b>Job File:</b> <a href=\"jobconf.jsp?jobid=" + jobId + "\">" 
+              + profile.getJobFile() + "</a><br>\n");
     if (runState == JobStatus.RUNNING) {
       out.print("<b>Status:</b> Running<br>\n");
       out.print("<b>Started at:</b> " + new Date(job.getStartTime()) + "<br>\n");

src/webapps/job/jobdetailshistory.jsp

@@ -24,7 +24,8 @@
 
 <b>User: </b> <%=job.get(Keys.USER) %><br/> 
 <b>JobName: </b> <%=job.get(Keys.JOBNAME) %><br/> 
-<b>JobConf: </b> <%=job.get(Keys.JOBCONF) %><br/> 
+<b>JobConf: </b> <a href="jobconf.jsp?jobid=<%=jobid %>"> 
+                 <%=job.get(Keys.JOBCONF) %></a><br/> 
 <b>Submitted At: </b> <%=StringUtils.getFormattedTimeWithDiff(dateFormat, job.getLong(Keys.SUBMIT_TIME), 0 )  %><br/> 
 <b>Launched At: </b> <%=StringUtils.getFormattedTimeWithDiff(dateFormat, job.getLong(Keys.LAUNCH_TIME), job.getLong(Keys.SUBMIT_TIME)) %><br/>
 <b>Finished At: </b>  <%=StringUtils.getFormattedTimeWithDiff(dateFormat, job.getLong(Keys.FINISH_TIME), job.getLong(Keys.LAUNCH_TIME)) %><br/>