HADOOP-243. Fix rounding in the display of task and job progress so that things are not shown as 100% complete until they are in fact finished. Contributed by Owen.

git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk@443452 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -1,6 +1,13 @@
 Hadoop Change Log
 
 
+Trunk (unreleased changes)
+
+1. HADOOP-243.  Fix rounding in the display of task and job progress
+   so that things are not shown to be 100% complete until they are in
+   fact finished.  (omalley via cutting) 
+
+
 Release 0.6.1 - 2006-08-13
 
  1. HADOOP-520.  Fix a bug in libhdfs, where write failures were not

src/java/org/apache/hadoop/mapred/JobClient.java

@@ -342,9 +342,10 @@ public class JobClient extends ToolBase implements MRConstants  {
               break;
             }
             running = jc.getJob(jobId);
-            String report = (" map " + Math.round(running.mapProgress()*100)+
-                             "%  reduce " + 
-                             Math.round(running.reduceProgress()*100) + "%");
+            String report = 
+              (" map " + StringUtils.formatPercent(running.mapProgress())+
+               " reduce " + 
+               StringUtils.formatPercent(running.reduceProgress()));
             if (!report.equals(lastReport)) {
               LOG.info(report);
               lastReport = report;
@@ -402,7 +403,6 @@ public class JobClient extends ToolBase implements MRConstants  {
         init();
         
         // Process args
-        String jobTrackerSpec = null;
         String submitJobFile = null;
         String jobid = null;
         boolean getStatus = false;

src/java/org/apache/hadoop/mapred/JobInProgress.java

@@ -293,13 +293,16 @@ class JobInProgress {
           if (maps.length == 0) {
             this.status.setMapProgress(1.0f);
           } else {
-            this.status.mapProgress += (progressDelta / maps.length);
+            this.status.setMapProgress((float) (this.status.mapProgress() +
+                                                progressDelta / maps.length));
           }
         } else {
           if (reduces.length == 0) {
             this.status.setReduceProgress(1.0f);
           } else {
-            this.status.reduceProgress += (progressDelta / reduces.length);
+            this.status.setReduceProgress
+                 ((float) (this.status.reduceProgress() +
+                           (progressDelta / reduces.length)));
           }
         }
     }   
@@ -477,6 +480,9 @@ class JobInProgress {
             }
         }
         if (allDone) {
+            if (tip.isMapTask()) {
+              this.status.setMapProgress(1.0f);              
+            }
             for (int i = 0; i < reduces.length; i++) {
                 if (! reduces[i].isComplete()) {
                     allDone = false;
@@ -489,7 +495,8 @@ class JobInProgress {
         // If all tasks are complete, then the job is done!
         //
         if (status.getRunState() == JobStatus.RUNNING && allDone) {
-            this.status.runState = JobStatus.SUCCEEDED;
+            this.status.setRunState(JobStatus.SUCCEEDED);
+            this.status.setReduceProgress(1.0f);
             this.finishTime = System.currentTimeMillis();
             garbageCollect();
             LOG.info("Job " + this.status.getJobId() + 

src/java/org/apache/hadoop/mapred/JobStatus.java

@@ -41,12 +41,12 @@ public class JobStatus implements Writable {
     public static final int FAILED = 3;
     public static final int PREP = 4;
 
-    String jobid;
-    float mapProgress;
-    float reduceProgress;
-    int runState;
-    long startTime;
-    String user;
+    private String jobid;
+    private float mapProgress;
+    private float reduceProgress;
+    private int runState;
+    private long startTime;
+    private String user;
     /**
      */
     public JobStatus() {
@@ -81,7 +81,10 @@ public class JobStatus implements Writable {
      * Sets the map progress of this job
      * @param p The value of map progress to set to
      */
-    void setMapProgress(float p) { this.mapProgress = p; }
+    void setMapProgress(float p) { 
+      this.mapProgress = (float) Math.min(1.0, Math.max(0.0, p)); 
+    
+    }
     
     /**
      * @return Percentage of progress in reduce 
@@ -92,13 +95,22 @@ public class JobStatus implements Writable {
      * Sets the reduce progress of this Job
      * @param p The value of reduce progress to set to
      */
-    void setReduceProgress(float p) { this.reduceProgress = p; }
+    void setReduceProgress(float p) { 
+      this.reduceProgress = (float) Math.min(1.0, Math.max(0.0, p)); 
+    }
     
     /**
      * @return running state of the job
      */
     public int getRunState() { return runState; }
     
+    /**
+     * Change the current run state of the job.
+     */
+    public void setRunState(int state) {
+      this.runState = state;
+    }
+    
     /** 
      * Set the start time of the job
      * @param startTime The startTime of the job

src/java/org/apache/hadoop/mapred/LocalJobRunner.java

@@ -49,7 +49,7 @@ class LocalJobRunner implements JobSubmissionProtocol {
     private JobConf job;
     private Random random = new Random();
 
-    private JobStatus status = new JobStatus();
+    private JobStatus status;
     private ArrayList mapIds = new ArrayList();
     private MapOutputFile mapoutputFile;
     private JobProfile profile;
@@ -73,8 +73,7 @@ class LocalJobRunner implements JobSubmissionProtocol {
       this.job = new JobConf(localFile);
       profile = new JobProfile(job.getUser(), id, file, 
                                "http://localhost:8080/", job.getJobName());
-      this.status.jobid = id;
-      this.status.runState = JobStatus.RUNNING;
+      status = new JobStatus(id, 0.0f, 0.0f, JobStatus.RUNNING);
 
       jobs.put(id, this);
 
@@ -134,10 +133,10 @@ class LocalJobRunner implements JobSubmissionProtocol {
         }
         this.mapoutputFile.removeAll(reduceId);
         
-        this.status.runState = JobStatus.SUCCEEDED;
+        this.status.setRunState(JobStatus.SUCCEEDED);
 
       } catch (Throwable t) {
-        this.status.runState = JobStatus.FAILED;
+        this.status.setRunState(JobStatus.FAILED);
         LOG.warn(id, t);
 
       } finally {
@@ -163,9 +162,9 @@ class LocalJobRunner implements JobSubmissionProtocol {
       float taskIndex = mapIds.indexOf(taskId);
       if (taskIndex >= 0) {                       // mapping
         float numTasks = mapIds.size();
-        status.mapProgress = (taskIndex/numTasks)+(progress/numTasks);
+        status.setMapProgress(taskIndex/numTasks + progress/numTasks);
       } else {
-        status.reduceProgress = progress;
+        status.setReduceProgress(progress);
       }
     }
 
@@ -180,9 +179,9 @@ class LocalJobRunner implements JobSubmissionProtocol {
     public void done(String taskId) throws IOException {
       int taskIndex = mapIds.indexOf(taskId);
       if (taskIndex >= 0) {                       // mapping
-        status.mapProgress = 1.0f;
+        status.setMapProgress(1.0f);
       } else {
-        status.reduceProgress = 1.0f;
+        status.setReduceProgress(1.0f);
       }
     }
 

src/java/org/apache/hadoop/util/StringUtils.java

@@ -52,7 +52,7 @@ public class StringUtils {
     return fullHostname;
   }
 
-  private static DecimalFormat numFormat = new DecimalFormat("0.0");
+  private static DecimalFormat oneDecimal = new DecimalFormat("0.0");
   
   /**
    * Given an integer, return a string that is in an approximate, but human 
@@ -77,7 +77,15 @@ public class StringUtils {
       result = number / (1024.0 * 1024 * 1024);
       suffix = "g";
     }
-    return numFormat.format(result) + suffix;
+    return oneDecimal.format(result) + suffix;
+  }
+  
+  private static DecimalFormat percentFormat = new DecimalFormat("0.00%");
+  
+  public static String formatPercent(double done) {
+    final int scale = 10000;
+    double rounded = Math.floor(done * scale);
+    return percentFormat.format(rounded / scale);
   }
   
   /**

src/java/org/apache/hadoop/util/ToolBase.java

@@ -80,7 +80,7 @@ import org.apache.hadoop.fs.Path;
  *
  */
 public abstract class ToolBase implements Tool {
-    public static final Log LOG = LogFactory.getLog(
+    private static final Log LOG = LogFactory.getLog(
             "org.apache.hadoop.util.ToolBase");
     public Configuration conf;
 

src/webapps/job/jobdetails.jsp

@@ -36,7 +36,7 @@
     }
     out.print("<tr><th><a href=\"/jobtasks.jsp?jobid=" + jobId + 
               "&type="+ kind + "&pagenum=1\">" + kind + "</a></th><td>" + 
-              percentFormat.format(100.0 * completePercent) +
+              StringUtils.formatPercent(completePercent) +
               "</td><td>" + totalTasks + "</td><td>" + 
               (totalTasks - runningTasks - finishedTasks) + "</td><td>" +
               runningTasks + "</td><td>" +

src/webapps/job/jobtasks.jsp

@@ -63,7 +63,8 @@
           out.print("<tr><td><a href=\"taskdetails.jsp?jobid=" + jobid + 
                     "&taskid=" + report.getTaskId() + "\">"  + 
                     report.getTaskId() + "</a></td>");
-         out.print("<td>" + report.getProgress() + "</td>");
+         out.print("<td>" + StringUtils.formatPercent(report.getProgress()) + 
+                   "</td>");
          out.print("<td>"  + report.getState() + "</td>");
          String[] diagnostics = report.getDiagnostics();
          for (int j = 0; j < diagnostics.length ; j++) {

src/webapps/job/jobtracker.jsp

@@ -44,11 +44,11 @@
                   "<td>" + profile.getUser() + "</td>" 
                     + "<td>" + ("".equals(name) ? "&nbsp;" : name) + "</td>" + 
                     "<td>" + 
-                    percentFormat.format(100.0 * status.mapProgress()) + 
-                    "%</td><td>" + 
+                    StringUtils.formatPercent(status.mapProgress()) + 
+                    "</td><td>" + 
                     desiredMaps + "</td><td>" + completedMaps + "</td><td>" + 
-                    percentFormat.format(100.0 * status.reduceProgress()) + 
-                    "%</td><td>" + 
+                    StringUtils.formatPercent(status.reduceProgress()) + 
+                    "</td><td>" + 
                     desiredReduces + "</td><td> " + completedReduces + 
                     "</td></tr>\n");
         }

src/webapps/job/machines.jsp

@@ -12,7 +12,6 @@
   JobTracker tracker = JobTracker.getTracker();
   String trackerLabel = 
            StringUtils.simpleHostname(tracker.getJobTrackerMachine());
-  private static DecimalFormat percentFormat = new DecimalFormat("##0.00");
 
   public void generateTaskTrackerTable(JspWriter out) throws IOException {
     Collection c = tracker.taskTrackers();

src/webapps/job/taskdetails.jsp

@@ -6,6 +6,7 @@
   import="java.lang.String"
   import="java.util.*"
   import="org.apache.hadoop.mapred.*"
+  import="org.apache.hadoop.util.*"
 %>
 <%
   String jobid = request.getParameter("jobid");
@@ -65,7 +66,8 @@
       out.print("<td>");
       writeString(out, status.getRunState()); 
       out.print("</td>");
-      out.print("<td>"+status.getProgress()+"</td>");
+      out.print("<td>"+ StringUtils.formatPercent(status.getProgress()) + 
+                "</td>");
       out.print("<td><pre>" + status.getDiagnosticInfo() + "</pre></td>");
       out.print("</tr>\n");
     }