Merge -r 700020:700021 from trunk onto 0.19 branch. Fixes HADOOP-4189.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/core/branches/branch-0.19@700024 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -755,6 +755,9 @@ Release 0.19.0 - Unreleased
     memory management in TaskTracker are disabled on Windows.
     (Vinod K V via rangadi)
 
+    HADOOP-4189. Fixes the history blocksize & intertracker protocol version
+    issues introduced as part of HADOOP-3245. (Amar Kamat via ddas)
+
 Release 0.18.2 - Unreleased
 
   BUG FIXES

conf/hadoop-default.xml

@@ -762,19 +762,11 @@ creations/deletions), or "all".</description>
 
 <property>
   <name>mapred.jobtracker.job.history.block.size</name>
-  <value>0</value>
+  <value>3145728></value>
   <description>The block size of the job history file. Since the job recovery
                uses job history, its important to dump job history to disk as 
-               soon as possible.
-  </description>
-</property>
-
-<property>
-  <name>mapred.jobtracker.job.history.buffer.size</name>
-  <value>4096</value>
-  <description>The buffer size for the job history file. Since the job 
-               recovery uses job history, its important to frequently flush the 
-               job history to disk. This will minimize the loss in recovery.
+               soon as possible. Note that this is an expert level parameter.
+               The default value is set to 3 MB.
   </description>
 </property>
 

src/mapred/org/apache/hadoop/mapred/InterTrackerProtocol.java

@@ -48,6 +48,8 @@ interface InterTrackerProtocol extends VersionedProtocol {
    * Version 16: adds ResourceStatus to TaskTrackerStatus for HADOOP-3759
    * Version 17: Changed format of Task and TaskStatus for HADOOP-3150
    * Version 18: Changed status message due to changes in TaskStatus
+   *             Changed heartbeat to piggyback JobTracker restart information
+                 so that the TaskTracker can synchronize itself.
    */
   public static final long versionID = 18L;
   

src/mapred/org/apache/hadoop/mapred/JobHistory.java

@@ -85,7 +85,6 @@ public class JobHistory {
   private static boolean disableHistory = false; 
   private static final String SECONDARY_FILE_SUFFIX = ".recover";
   private static long jobHistoryBlockSize = 0;
-  private static int jobHistoryBufferSize = 0;
   private static String jobtrackerHostname;
   /**
    * Record types are identifiers for each line of log in history files. 
@@ -148,14 +147,10 @@ public class JobHistory {
       }
       conf.set("hadoop.job.history.location", LOG_DIR);
       disableHistory = false;
-      // set the job history block size
+      // set the job history block size (default is 3MB)
       jobHistoryBlockSize = 
         conf.getLong("mapred.jobtracker.job.history.block.size", 
-                     fs.getDefaultBlockSize());
-      // set the job history buffer size
-      jobHistoryBufferSize = 
-        Integer.parseInt(conf.get("mapred.jobtracker.job.history.buffer.size", 
-                                  "4096"));
+                     3 * 1024 * 1024);
     } catch(IOException e) {
         LOG.error("Failed to initialize JobHistory log file", e); 
         disableHistory = true;
@@ -729,8 +724,10 @@ public class JobHistory {
             
             logFile = recoverJobHistoryFile(jobConf, logFile);
             
+            int defaultBufferSize = 
+              fs.getConf().getInt("io.file.buffer.size", 4096);
             out = fs.create(logFile, FsPermission.getDefault(), true, 
-                            jobHistoryBufferSize, 
+                            defaultBufferSize, 
                             fs.getDefaultReplication(), 
                             jobHistoryBlockSize, null);
             writer = new PrintWriter(out);

src/webapps/job/loadhistory.jsp

@@ -24,9 +24,16 @@
     // then remove the attribute
     // if the job has not yet finished, remove the attribute sothat it 
     // gets refreshed.
+    boolean isJobComplete = false;
+    if (null != job) {
+      String jobStatus = job.get(Keys.JOB_STATUS);
+      isJobComplete = Values.SUCCESS.name() == jobStatus
+                      || Values.FAILED.name() == jobStatus
+                      || Values.KILLED.name() == jobStatus;
+    }
     if (null != job && 
        (!jobId.equals(job.get(Keys.JOBID)) 
-         || job.get(Keys.JOB_STATUS) == "")) {
+         || !isJobComplete)) {
       // remove jobInfo from session, keep only one job in session at a time
       request.getSession().removeAttribute("job"); 
       job = null ;