MAPREDUCE-6640. mapred job -history command should be able to take Job ID (rkanter)

hadoop-mapreduce-project/CHANGES.txt

@@ -319,6 +319,9 @@ Release 2.9.0 - UNRELEASED
     MAPREDUCE-6627. Add machine-readable output to mapred job -history
     command (rkanter)
 
+    MAPREDUCE-6640. mapred job -history command should be able to take
+    Job ID (rkanter)
+
   OPTIMIZATIONS
 
   BUG FIXES

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java

@@ -327,7 +327,8 @@ public class TypeConverter {
         jobreport.getMapProgress(), jobreport.getReduceProgress(),
         jobreport.getCleanupProgress(), fromYarn(jobreport.getJobState()),
         jobPriority, jobreport.getUser(), jobreport.getJobName(),
-        jobreport.getJobFile(), trackingUrl, jobreport.isUber());
+        jobreport.getJobFile(), trackingUrl, jobreport.isUber(),
+        jobreport.getHistoryFile());
     jobStatus.setStartTime(jobreport.getStartTime());
     jobStatus.setFinishTime(jobreport.getFinishTime());
     jobStatus.setFailureInfo(jobreport.getDiagnostics());

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/JobReport.java

@@ -40,6 +40,7 @@ public interface JobReport {
   public abstract List<AMInfo> getAMInfos();
   public abstract boolean isUber();
   public abstract Priority getJobPriority();
+  public abstract String getHistoryFile();
 
   public abstract void setJobId(JobId jobId);
   public abstract void setJobState(JobState jobState);
@@ -58,4 +59,5 @@ public interface JobReport {
   public abstract void setAMInfos(List<AMInfo> amInfos);
   public abstract void setIsUber(boolean isUber);
   public abstract void setJobPriority(Priority priority);
+  public abstract void setHistoryFile(String historyFile);
 }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/impl/pb/JobReportPBImpl.java

@@ -380,4 +380,16 @@ public class JobReportPBImpl extends ProtoBase<JobReportProto> implements
     }
     this.jobPriority = priority;
   }
+
+  @Override
+  public synchronized String getHistoryFile() {
+    JobReportProtoOrBuilder p = viaProto ? proto : builder;
+    return p.getHistoryFile();
+  }
+
+  @Override
+  public synchronized void setHistoryFile(String historyFile) {
+    maybeInitBuilder();
+    builder.setHistoryFile(historyFile);
+  }
 }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/mr_protos.proto

@@ -147,6 +147,7 @@ message JobReportProto {
   optional int64 submit_time = 15;
   optional bool is_uber = 16 [default = false];
   optional hadoop.yarn.PriorityProto jobPriority = 17;
+  optional string historyFile = 18;
 }
 
 message AMInfoProto {

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobStatus.java

@@ -201,12 +201,12 @@ public class JobStatus extends org.apache.hadoop.mapreduce.JobStatus {
    * @param jp Priority of the job.
    * @param user userid of the person who submitted the job.
    * @param jobName user-specified job name.
-   * @param jobFile job configuration file. 
+   * @param jobFile job configuration file.
    * @param trackingUrl link to the web-ui for details of the job.
    */
    public JobStatus(JobID jobid, float setupProgress, float mapProgress,
-                    float reduceProgress, float cleanupProgress, 
-                    int runState, JobPriority jp, String user, String jobName, 
+                    float reduceProgress, float cleanupProgress,
+                    int runState, JobPriority jp, String user, String jobName,
                     String jobFile, String trackingUrl) {
      this(jobid, setupProgress, mapProgress, reduceProgress, cleanupProgress,
          runState, jp, user, jobName, "default", jobFile, trackingUrl);
@@ -223,17 +223,43 @@ public class JobStatus extends org.apache.hadoop.mapreduce.JobStatus {
     * @param jp Priority of the job.
     * @param user userid of the person who submitted the job.
     * @param jobName user-specified job name.
-    * @param jobFile job configuration file. 
+    * @param jobFile job configuration file.
     * @param trackingUrl link to the web-ui for details of the job.
     * @param isUber Whether job running in uber mode
     */
     public JobStatus(JobID jobid, float setupProgress, float mapProgress,
-                     float reduceProgress, float cleanupProgress, 
-                     int runState, JobPriority jp, String user, String jobName, 
+                     float reduceProgress, float cleanupProgress,
+                     int runState, JobPriority jp, String user, String jobName,
                      String jobFile, String trackingUrl, boolean isUber) {
       this(jobid, setupProgress, mapProgress, reduceProgress, cleanupProgress,
           runState, jp, user, jobName, "default", jobFile, trackingUrl, isUber);
-    }   
+  }
+
+   /**
+    * Create a job status object for a given jobid.
+    * @param jobid The jobid of the job
+    * @param setupProgress The progress made on the setup
+    * @param mapProgress The progress made on the maps
+    * @param reduceProgress The progress made on the reduces
+    * @param cleanupProgress The progress made on the cleanup
+    * @param runState The current state of the job
+    * @param jp Priority of the job.
+    * @param user userid of the person who submitted the job.
+    * @param jobName user-specified job name.
+    * @param jobFile job configuration file.
+    * @param trackingUrl link to the web-ui for details of the job.
+    * @param isUber Whether job running in uber mode
+    * @param historyFile history file
+    */
+  public JobStatus(JobID jobid, float setupProgress, float mapProgress,
+                   float reduceProgress, float cleanupProgress,
+                   int runState, JobPriority jp, String user, String jobName,
+                   String jobFile, String trackingUrl, boolean isUber,
+                   String historyFile) {
+      this(jobid, setupProgress, mapProgress, reduceProgress, cleanupProgress,
+          runState, jp, user, jobName, "default", jobFile, trackingUrl, isUber,
+          historyFile);
+  }
    
    /**
     * Create a job status object for a given jobid.
@@ -281,11 +307,39 @@ public class JobStatus extends org.apache.hadoop.mapreduce.JobStatus {
        int runState, JobPriority jp, 
        String user, String jobName, String queue, 
        String jobFile, String trackingUrl, boolean isUber) {
-     super(jobid, setupProgress, mapProgress, reduceProgress, cleanupProgress,
-         getEnum(runState), org.apache.hadoop.mapreduce.JobPriority.valueOf(jp.name()),
-         user, jobName, queue, jobFile, trackingUrl, isUber);
+     this(jobid, setupProgress, mapProgress, reduceProgress, cleanupProgress,
+         runState, jp, user, jobName, queue, jobFile, trackingUrl, isUber, "");
    }
 
+  /**
+   * Create a job status object for a given jobid.
+   * @param jobid The jobid of the job
+   * @param setupProgress The progress made on the setup
+   * @param mapProgress The progress made on the maps
+   * @param reduceProgress The progress made on the reduces
+   * @param cleanupProgress The progress made on the cleanup
+   * @param runState The current state of the job
+   * @param jp Priority of the job.
+   * @param user userid of the person who submitted the job.
+   * @param jobName user-specified job name.
+   * @param queue job queue name.
+   * @param jobFile job configuration file.
+   * @param trackingUrl link to the web-ui for details of the job.
+   * @param isUber Whether job running in uber mode
+   * @param historyFile history file
+   */
+  public JobStatus(JobID jobid, float setupProgress, float mapProgress,
+                   float reduceProgress, float cleanupProgress,
+                   int runState, JobPriority jp,
+                   String user, String jobName, String queue,
+                   String jobFile, String trackingUrl, boolean isUber,
+                   String historyFile) {
+    super(jobid, setupProgress, mapProgress, reduceProgress, cleanupProgress,
+        getEnum(runState),
+        org.apache.hadoop.mapreduce.JobPriority.valueOf(jp.name()),
+        user, jobName, queue, jobFile, trackingUrl, isUber, historyFile);
+  }
+
   public static JobStatus downgrade(org.apache.hadoop.mapreduce.JobStatus stat){
     JobStatus old = new JobStatus(JobID.downgrade(stat.getJobID()),
       stat.getSetupProgress(), stat.getMapProgress(), stat.getReduceProgress(),

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobStatus.java

@@ -152,43 +152,71 @@ public class JobStatus implements Writable, Cloneable {
     }
 
    /**
-    * Create a job status object for a given jobid.
-    * @param jobid The jobid of the job
-    * @param setupProgress The progress made on the setup
-    * @param mapProgress The progress made on the maps
-    * @param reduceProgress The progress made on the reduces
-    * @param cleanupProgress The progress made on the cleanup
-    * @param runState The current state of the job
-    * @param jp Priority of the job.
-    * @param user userid of the person who submitted the job.
-    * @param jobName user-specified job name.
-    * @param queue queue name
-    * @param jobFile job configuration file.
-    * @param trackingUrl link to the web-ui for details of the job.
-    * @param isUber Whether job running in uber mode
-    */
-    public JobStatus(JobID jobid, float setupProgress, float mapProgress,
-                     float reduceProgress, float cleanupProgress,
-                     State runState, JobPriority jp,
-                     String user, String jobName, String queue,
-                     String jobFile, String trackingUrl, boolean isUber) {
-      this.jobid = jobid;
-      this.setupProgress = setupProgress;
-      this.mapProgress = mapProgress;
-      this.reduceProgress = reduceProgress;
-      this.cleanupProgress = cleanupProgress;
-      this.runState = runState;
-      this.user = user;
-      this.queue = queue;
-      if (jp == null) {
-        throw new IllegalArgumentException("Job Priority cannot be null.");
-      }
-      priority = jp;
-      this.jobName = jobName;
-      this.jobFile = jobFile;
-      this.trackingUrl = trackingUrl;
-      this.isUber = isUber;
+   * Create a job status object for a given jobid.
+   * @param jobid The jobid of the job
+   * @param setupProgress The progress made on the setup
+   * @param mapProgress The progress made on the maps
+   * @param reduceProgress The progress made on the reduces
+   * @param cleanupProgress The progress made on the cleanup
+   * @param runState The current state of the job
+   * @param jp Priority of the job.
+   * @param user userid of the person who submitted the job.
+   * @param jobName user-specified job name.
+   * @param queue queue name
+   * @param jobFile job configuration file.
+   * @param trackingUrl link to the web-ui for details of the job.
+   * @param isUber Whether job running in uber mode
+   */
+  public JobStatus(JobID jobid, float setupProgress, float mapProgress,
+                    float reduceProgress, float cleanupProgress,
+                    State runState, JobPriority jp,
+                    String user, String jobName, String queue,
+                    String jobFile, String trackingUrl, boolean isUber) {
+     this(jobid, setupProgress, mapProgress, reduceProgress, cleanupProgress,
+         runState, jp, user, jobName, queue, jobFile, trackingUrl, isUber, "");
+  }
+
+ /**
+  * Create a job status object for a given jobid.
+  * @param jobid The jobid of the job
+  * @param setupProgress The progress made on the setup
+  * @param mapProgress The progress made on the maps
+  * @param reduceProgress The progress made on the reduces
+  * @param cleanupProgress The progress made on the cleanup
+  * @param runState The current state of the job
+  * @param jp Priority of the job.
+  * @param user userid of the person who submitted the job.
+  * @param jobName user-specified job name.
+  * @param queue queue name
+  * @param jobFile job configuration file.
+  * @param trackingUrl link to the web-ui for details of the job.
+  * @param isUber Whether job running in uber mode
+  * @param historyFile history file
+  */
+  public JobStatus(JobID jobid, float setupProgress, float mapProgress,
+                   float reduceProgress, float cleanupProgress,
+                   State runState, JobPriority jp,
+                   String user, String jobName, String queue,
+                   String jobFile, String trackingUrl, boolean isUber,
+                   String historyFile) {
+    this.jobid = jobid;
+    this.setupProgress = setupProgress;
+    this.mapProgress = mapProgress;
+    this.reduceProgress = reduceProgress;
+    this.cleanupProgress = cleanupProgress;
+    this.runState = runState;
+    this.user = user;
+    this.queue = queue;
+    if (jp == null) {
+      throw new IllegalArgumentException("Job Priority cannot be null.");
     }
+    priority = jp;
+    this.jobName = jobName;
+    this.jobFile = jobFile;
+    this.trackingUrl = trackingUrl;
+    this.isUber = isUber;
+    this.historyFile = historyFile;
+  }
 
 
   /**

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java

@@ -95,7 +95,7 @@ public class CLI extends Configured implements Tool {
     String submitJobFile = null;
     String jobid = null;
     String taskid = null;
-    String historyFile = null;
+    String historyFileOrJobId = null;
     String historyOutFile = null;
     String historyOutFormat = HistoryViewer.HUMAN_FORMAT;
     String counterGroupName = null;
@@ -188,15 +188,15 @@ public class CLI extends Configured implements Tool {
       // Some arguments are optional while others are not, and some require
       // second arguments.  Due to this, the indexing can vary depending on
       // what's specified and what's left out, as summarized in the below table:
-      // [all] <jobHistoryFile> [-outfile <file>] [-format <human|json>]
-      //   1          2             3       4         5         6
-      //   1          2             3       4
-      //   1          2                               3         4
-      //   1          2
-      //              1             2       3         4         5
-      //              1             2       3
-      //              1                               2         3
-      //              1
+      // [all] <jobHistoryFile|jobId> [-outfile <file>] [-format <human|json>]
+      //   1                  2            3       4         5         6
+      //   1                  2            3       4
+      //   1                  2                              3         4
+      //   1                  2
+      //                      1            2       3         4         5
+      //                      1            2       3
+      //                      1                              2         3
+      //                      1
 
       // "all" is optional, but comes first if specified
       int index = 1;
@@ -208,8 +208,8 @@ public class CLI extends Configured implements Tool {
           return exitCode;
         }
       }
-      // Get the job history file argument
-      historyFile = argv[index++];
+      // Get the job history file or job id argument
+      historyFileOrJobId = argv[index++];
       // "-outfile" is optional, but if specified requires a second argument
       if (argv.length > index + 1 && "-outfile".equals(argv[index])) {
         index++;
@@ -379,9 +379,28 @@ public class CLI extends Configured implements Tool {
           exitCode = 0;
         } 
       } else if (viewHistory) {
-        viewHistory(historyFile, viewAllHistory, historyOutFile,
-            historyOutFormat);
-        exitCode = 0;
+        // If it ends with .jhist, assume it's a jhist file; otherwise, assume
+        // it's a Job ID
+        if (historyFileOrJobId.endsWith(".jhist")) {
+          viewHistory(historyFileOrJobId, viewAllHistory, historyOutFile,
+              historyOutFormat);
+          exitCode = 0;
+        } else {
+          Job job = getJob(JobID.forName(historyFileOrJobId));
+          if (job == null) {
+            System.out.println("Could not find job " + jobid);
+          } else {
+            String historyUrl = job.getHistoryUrl();
+            if (historyUrl == null || historyUrl.isEmpty()) {
+              System.out.println("History file for job " + historyFileOrJobId +
+                  " is currently unavailable.");
+            } else {
+              viewHistory(historyUrl, viewAllHistory, historyOutFile,
+                  historyOutFormat);
+              exitCode = 0;
+            }
+          }
+        }
       } else if (listEvents) {
         listEvents(getJob(JobID.forName(jobid)), fromEvent, nEvents);
         exitCode = 0;
@@ -493,8 +512,8 @@ public class CLI extends Configured implements Tool {
       System.err.println(prefix + "[" + cmd + 
         " <job-id> <from-event-#> <#-of-events>]. Event #s start from 1.");
     } else if ("-history".equals(cmd)) {
-      System.err.println(prefix + "[" + cmd +
-          " [all] <jobHistoryFile> [-outfile <file>] [-format <human|json>]]");
+      System.err.println(prefix + "[" + cmd + " [all] <jobHistoryFile|jobId> " +
+          "[-outfile <file>] [-format <human|json>]]");
     } else if ("-list".equals(cmd)) {
       System.err.println(prefix + "[" + cmd + " [all]]");
     } else if ("-kill-task".equals(cmd) || "-fail-task".equals(cmd)) {
@@ -527,8 +546,8 @@ public class CLI extends Configured implements Tool {
           "Valid values for priorities are: " + jobPriorityValues +
           ". In addition to this, integers also can be used." + "%n");
       System.err.printf("\t[-events <job-id> <from-event-#> <#-of-events>]%n");
-      System.err.printf("\t[-history [all] <jobHistoryFile> [-outfile <file>]" +
-          " [-format <human|json>]]%n");
+      System.err.printf("\t[-history [all] <jobHistoryFile|jobId> " +
+          "[-outfile <file>] [-format <human|json>]]%n");
       System.err.printf("\t[-list [all]]%n");
       System.err.printf("\t[-list-active-trackers]%n");
       System.err.printf("\t[-list-blacklisted-trackers]%n");

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapredCommands.md

@@ -81,7 +81,7 @@ Copy file or directories recursively. More information can be found at
 
 Command to interact with Map Reduce Jobs.
 
-Usage: `mapred job | [GENERIC_OPTIONS] | [-submit <job-file>] | [-status <job-id>] | [-counter <job-id> <group-name> <counter-name>] | [-kill <job-id>] | [-events <job-id> <from-event-#> <#-of-events>] | [-history [all] <jobHistoryFile> [-outfile <file>] [-format <human|json>]] | [-list [all]] | [-kill-task <task-id>] | [-fail-task <task-id>] | [-set-priority <job-id> <priority>] | [-list-active-trackers] | [-list-blacklisted-trackers] | [-list-attempt-ids <job-id> <task-type> <task-state>] [-logs <job-id> <task-attempt-id>]`
+Usage: `mapred job | [GENERIC_OPTIONS] | [-submit <job-file>] | [-status <job-id>] | [-counter <job-id> <group-name> <counter-name>] | [-kill <job-id>] | [-events <job-id> <from-event-#> <#-of-events>] | [-history [all] <jobHistoryFile|jobId> [-outfile <file>] [-format <human|json>]] | [-list [all]] | [-kill-task <task-id>] | [-fail-task <task-id>] | [-set-priority <job-id> <priority>] | [-list-active-trackers] | [-list-blacklisted-trackers] | [-list-attempt-ids <job-id> <task-type> <task-state>] [-logs <job-id> <task-attempt-id>]`
 
 | COMMAND\_OPTION | Description |
 |:---- |:---- |
@@ -90,7 +90,7 @@ Usage: `mapred job | [GENERIC_OPTIONS] | [-submit <job-file>] | [-status <job-id
 | -counter *job-id* *group-name* *counter-name* | Prints the counter value. |
 | -kill *job-id* | Kills the job. |
 | -events *job-id* *from-event-\#* *\#-of-events* | Prints the events' details received by jobtracker for the given range. |
-| -history [all] *jobHistoryFile* [-outfile *file*] [-format *human|json*] | Prints job details, failed and killed task details. More details about the job such as successful tasks, task attempts made for each task, task counters, etc can be viewed by specifying the [all] option. An optional file output path (instead of stdout) can be specified. The format defaults to human-readable but can also be changed to JSON with the [-format] option. |
+| -history [all] *jobHistoryFile|jobId* [-outfile *file*] [-format *human|json*] | Prints job details, failed and killed task details. More details about the job such as successful tasks, task attempts made for each task, task counters, etc can be viewed by specifying the [all] option. An optional file output path (instead of stdout) can be specified. The format defaults to human-readable but can also be changed to JSON with the [-format] option. |
 | -list [all] | Displays jobs which are yet to complete. `-list all` displays all jobs. |
 | -kill-task *task-id* | Kills the task. Killed tasks are NOT counted against failed attempts. |
 | -fail-task *task-id* | Fails the task. Failed tasks are counted against failed attempts. |

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java

@@ -166,6 +166,7 @@ public class CompletedJob implements org.apache.hadoop.mapreduce.v2.app.job.Job
     report.setTrackingUrl(historyUrl);
     report.setAMInfos(getAMInfos());
     report.setIsUber(isUber());
+    report.setHistoryFile(info.getHistoryFile().toString());
   }
 
   @Override

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java

@@ -454,10 +454,10 @@ public class HistoryFileManager extends AbstractService {
     }
 
     /**
-     * Return the history file.  This should only be used for testing.
+     * Return the history file.
      * @return the history file.
      */
-    synchronized Path getHistoryFile() {
+    public synchronized Path getHistoryFile() {
       return historyFile;
     }
     

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEntities.java

@@ -90,6 +90,7 @@ public class TestJobHistoryEntities {
   public void testCompletedJob() throws Exception {
     HistoryFileInfo info = mock(HistoryFileInfo.class);
     when(info.getConfFile()).thenReturn(fullConfPath);
+    when(info.getHistoryFile()).thenReturn(fullHistoryPath);
     //Re-initialize to verify the delayed load.
     completedJob =
       new CompletedJob(conf, jobId, fullHistoryPath, loadTasks, "user",
@@ -109,12 +110,14 @@ public class TestJobHistoryEntities {
     JobReport jobReport = completedJob.getReport();
     assertEquals("user", jobReport.getUser());
     assertEquals(JobState.SUCCEEDED, jobReport.getJobState());
+    assertEquals(fullHistoryPath.toString(), jobReport.getHistoryFile());
   }
   
   @Test (timeout=100000)
   public void testCopmletedJobReportWithZeroTasks() throws Exception {
     HistoryFileInfo info = mock(HistoryFileInfo.class);
     when(info.getConfFile()).thenReturn(fullConfPath);
+    when(info.getHistoryFile()).thenReturn(fullHistoryPathZeroReduces);
     completedJob =
       new CompletedJob(conf, jobId, fullHistoryPathZeroReduces, loadTasks, "user",
           info, jobAclsManager);
@@ -124,6 +127,8 @@ public class TestJobHistoryEntities {
     assertEquals(0, completedJob.getCompletedReduces());
     // Verify that the reduce progress is 1.0 (not NaN)
     assertEquals(1.0, jobReport.getReduceProgress(), 0.001);
+    assertEquals(fullHistoryPathZeroReduces.toString(),
+        jobReport.getHistoryFile());
   }
 
   @Test (timeout=10000)

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/TestJobInfo.java

@@ -68,6 +68,7 @@ public class TestJobInfo {
 
     HistoryFileInfo info = mock(HistoryFileInfo.class);
     when(info.getConfFile()).thenReturn(fullConfPath);
+    when(info.getHistoryFile()).thenReturn(fulleHistoryPath);
 
     JobId jobId = MRBuilderUtils.newJobId(1329348432655l, 1, 1);
     CompletedJob completedJob =

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java

@@ -29,7 +29,10 @@ import java.io.OutputStream;
 import java.io.PipedInputStream;
 import java.io.PipedOutputStream;
 import java.io.PrintStream;
+import java.util.Arrays;
 
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.RemoteIterator;
 import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
 import org.junit.Assert;
@@ -83,6 +86,7 @@ public class TestMRJobClient extends ClusterMapReduceTestCase {
 
   public static int runTool(Configuration conf, Tool tool, String[] args,
       OutputStream out) throws Exception {
+    LOG.info("args = " + Arrays.toString(args));
     PrintStream oldOut = System.out;
     PrintStream newOut = new PrintStream(out, true);
     try {
@@ -140,7 +144,7 @@ public class TestMRJobClient extends ClusterMapReduceTestCase {
     // test list of events
     testJobEvents(jobId, conf);
     // test job history
-    testJobHistory(conf);
+    testJobHistory(jobId, conf);
     // test tracker list
     testListTrackers(conf);
     // attempts list
@@ -354,111 +358,127 @@ public class TestMRJobClient extends ClusterMapReduceTestCase {
   /**
    * print job history from file 
    */
-  private void testJobHistory(Configuration conf) throws Exception {
+  private void testJobHistory(String jobId, Configuration conf)
+      throws Exception {
     CLI jc = createJobClient();
     ByteArrayOutputStream out = new ByteArrayOutputStream();
-    File f = new File("src/test/resources/job_1329348432655_0001-10.jhist");
-    FileSystem localFs = FileSystem.getLocal(conf);
-    String historyFileUri = new Path(f.getAbsolutePath())
-        .makeQualified(localFs.getUri(), localFs.getWorkingDirectory()).toUri()
-        .toString();
 
-    // Try a bunch of different valid combinations of the command to test
-    // argument parsing
-    int exitCode = runTool(conf, jc, new String[] {
-        "-history",
-        "all",
-        historyFileUri,
+    // Find jhist file
+    String historyFileUri = null;
+    RemoteIterator<LocatedFileStatus> it =
+        getFileSystem().listFiles(new Path("/"), true);
+    while (it.hasNext() && historyFileUri == null) {
+      LocatedFileStatus file = it.next();
+      if (file.getPath().getName().endsWith(".jhist")) {
+        historyFileUri = file.getPath().toUri().toString();
+      }
+    }
+    assertNotNull("Could not find jhist file", historyFileUri);
+
+    for (String historyFileOrJobId : new String[]{historyFileUri, jobId}) {
+      // Try a bunch of different valid combinations of the command
+      int exitCode = runTool(conf, jc, new String[]{
+          "-history",
+          "all",
+          historyFileOrJobId,
       }, out);
-    assertEquals("Exit code", 0, exitCode);
-    checkHistoryHumanOutput(out);
-    File outFile = File.createTempFile("myout", ".txt");
-    exitCode = runTool(conf, jc, new String[] {
-        "-history",
-        "all",
-        historyFileUri,
-        "-outfile",
-        outFile.getAbsolutePath()
+      assertEquals("Exit code", 0, exitCode);
+      checkHistoryHumanOutput(jobId, out);
+      File outFile = File.createTempFile("myout", ".txt");
+      exitCode = runTool(conf, jc, new String[]{
+          "-history",
+          "all",
+          historyFileOrJobId,
+          "-outfile",
+          outFile.getAbsolutePath()
       }, out);
-    assertEquals("Exit code", 0, exitCode);
-    checkHistoryHumanFileOutput(out, outFile);
-    outFile = File.createTempFile("myout", ".txt");
-    exitCode = runTool(conf, jc, new String[] {
-        "-history",
-        "all",
-        historyFileUri,
-        "-outfile",
-        outFile.getAbsolutePath(),
-        "-format",
-        "human"
+      assertEquals("Exit code", 0, exitCode);
+      checkHistoryHumanFileOutput(jobId, out, outFile);
+      outFile = File.createTempFile("myout", ".txt");
+      exitCode = runTool(conf, jc, new String[]{
+          "-history",
+          "all",
+          historyFileOrJobId,
+          "-outfile",
+          outFile.getAbsolutePath(),
+          "-format",
+          "human"
       }, out);
-    assertEquals("Exit code", 0, exitCode);
-    checkHistoryHumanFileOutput(out, outFile);
-
-    exitCode = runTool(conf, jc, new String[] {
-        "-history",
-        historyFileUri,
-        "-format",
-        "human"
+      assertEquals("Exit code", 0, exitCode);
+      checkHistoryHumanFileOutput(jobId, out, outFile);
+      exitCode = runTool(conf, jc, new String[]{
+          "-history",
+          historyFileOrJobId,
+          "-format",
+          "human"
       }, out);
-    assertEquals("Exit code", 0, exitCode);
-    checkHistoryHumanOutput(out);
-
-    exitCode = runTool(conf, jc, new String[] {
-        "-history",
-        "all",
-        historyFileUri,
-        "-format",
-        "json"
+      assertEquals("Exit code", 0, exitCode);
+      checkHistoryHumanOutput(jobId, out);
+      exitCode = runTool(conf, jc, new String[]{
+          "-history",
+          "all",
+          historyFileOrJobId,
+          "-format",
+          "json"
       }, out);
-    assertEquals("Exit code", 0, exitCode);
-    checkHistoryJSONOutput(out);
-    outFile = File.createTempFile("myout", ".txt");
-    exitCode = runTool(conf, jc, new String[] {
-        "-history",
-        "all",
-        historyFileUri,
-        "-outfile",
-        outFile.getAbsolutePath(),
-        "-format",
-        "json"
+      assertEquals("Exit code", 0, exitCode);
+      checkHistoryJSONOutput(jobId, out);
+      outFile = File.createTempFile("myout", ".txt");
+      exitCode = runTool(conf, jc, new String[]{
+          "-history",
+          "all",
+          historyFileOrJobId,
+          "-outfile",
+          outFile.getAbsolutePath(),
+          "-format",
+          "json"
       }, out);
-    assertEquals("Exit code", 0, exitCode);
-    checkHistoryJSONFileOutput(out, outFile);
-    exitCode = runTool(conf, jc, new String[] {
-        "-history",
-        historyFileUri,
-        "-format",
-        "json"
+      assertEquals("Exit code", 0, exitCode);
+      checkHistoryJSONFileOutput(jobId, out, outFile);
+      exitCode = runTool(conf, jc, new String[]{
+          "-history",
+          historyFileOrJobId,
+          "-format",
+          "json"
       }, out);
-    assertEquals("Exit code", 0, exitCode);
-    checkHistoryJSONOutput(out);
-
-    // Check some bad arguments
-    exitCode = runTool(conf, jc, new String[] {
-        "-history",
-        historyFileUri,
-        "foo"
-    }, out);
-    assertEquals("Exit code", -1, exitCode);
-    exitCode = runTool(conf, jc, new String[] {
-        "-history",
-        historyFileUri,
-        "-format"
+      assertEquals("Exit code", 0, exitCode);
+      checkHistoryJSONOutput(jobId, out);
+
+      // Check some bad arguments
+      exitCode = runTool(conf, jc, new String[]{
+          "-history",
+          historyFileOrJobId,
+          "foo"
       }, out);
-    assertEquals("Exit code", -1, exitCode);
-    runTool(conf, jc, new String[] {
-        "-history",
-        historyFileUri,
-        "-outfile",
-    }, out);
-    assertEquals("Exit code", -1, exitCode);
+      assertEquals("Exit code", -1, exitCode);
+      exitCode = runTool(conf, jc, new String[]{
+          "-history",
+          historyFileOrJobId,
+          "-format"
+      }, out);
+      assertEquals("Exit code", -1, exitCode);
+      exitCode = runTool(conf, jc, new String[]{
+          "-history",
+          historyFileOrJobId,
+          "-outfile",
+      }, out);
+      assertEquals("Exit code", -1, exitCode);
+      try {
+        runTool(conf, jc, new String[]{
+            "-history",
+            historyFileOrJobId,
+            "-format",
+            "foo"
+        }, out);
+        fail();
+      } catch (IllegalArgumentException e) {
+        // Expected
+      }
+    }
     try {
       runTool(conf, jc, new String[]{
           "-history",
-          historyFileUri,
-          "-format",
-          "foo"
+          "not_a_valid_history_file_or_job_id",
       }, out);
       fail();
     } catch (IllegalArgumentException e) {
@@ -466,45 +486,47 @@ public class TestMRJobClient extends ClusterMapReduceTestCase {
     }
   }
 
-  private void checkHistoryHumanOutput(ByteArrayOutputStream out)
+  private void checkHistoryHumanOutput(String jobId, ByteArrayOutputStream out)
       throws IOException, JSONException {
     BufferedReader br = new BufferedReader(new InputStreamReader(
         new ByteArrayInputStream(out.toByteArray())));
     br.readLine();
     String line = br.readLine();
     br.close();
-    assertEquals("Hadoop job: job_1329348432655_0001", line);
+    assertEquals("Hadoop job: " + jobId, line);
     out.reset();
   }
 
-  private void checkHistoryJSONOutput(ByteArrayOutputStream out)
+  private void checkHistoryJSONOutput(String jobId, ByteArrayOutputStream out)
       throws IOException, JSONException {
     BufferedReader br = new BufferedReader(new InputStreamReader(
         new ByteArrayInputStream(out.toByteArray())));
     String line = org.apache.commons.io.IOUtils.toString(br);
     br.close();
     JSONObject json = new JSONObject(line);
-    assertEquals("job_1329348432655_0001", json.getString("hadoopJob"));
+    assertEquals(jobId, json.getString("hadoopJob"));
     out.reset();
   }
 
-  private void checkHistoryHumanFileOutput(ByteArrayOutputStream out,
-      File outFile) throws IOException, JSONException {
+  private void checkHistoryHumanFileOutput(String jobId,
+      ByteArrayOutputStream out, File outFile)
+      throws IOException, JSONException {
     BufferedReader br = new BufferedReader(new FileReader(outFile));
     br.readLine();
     String line = br.readLine();
     br.close();
-    assertEquals("Hadoop job: job_1329348432655_0001", line);
+    assertEquals("Hadoop job: " + jobId, line);
     assertEquals(0, out.size());
   }
 
-  private void checkHistoryJSONFileOutput(ByteArrayOutputStream out,
-      File outFile) throws IOException, JSONException {
+  private void checkHistoryJSONFileOutput(String jobId,
+      ByteArrayOutputStream out, File outFile)
+      throws IOException, JSONException {
     BufferedReader br = new BufferedReader(new FileReader(outFile));
     String line = org.apache.commons.io.IOUtils.toString(br);
     br.close();
     JSONObject json = new JSONObject(line);
-    assertEquals("job_1329348432655_0001", json.getString("hadoopJob"));
+    assertEquals(jobId, json.getString("hadoopJob"));
     assertEquals(0, out.size());
   }