HADOOP-560. Add a 'killed' task state. Contributed by Owen.

git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk@450959 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -84,6 +84,12 @@ Trunk (unreleased changes)
     returned after seeking to a random location.
     (Milind Bhandarkar via cutting)
 
+22. HADOOP-560.  Add a "killed" task state.  This can be used to
+    distinguish kills from other failures.  Task state has also been
+    converted to use an enum type instead of an int, uncovering a bug
+    elsewhere.  The web interface is also updated to display killed
+    tasks.  (omalley via cutting)
+
 
 Release 0.6.2 (unreleased)
 

src/java/org/apache/hadoop/dfs/NameNode.java

@@ -436,6 +436,13 @@ public class NameNode implements ClientProtocol, DatanodeProtocol, FSConstants {
       return namesystem.isInSafeMode();
     }
 
+    /**
+     * Is the cluster currently in safe mode?
+     */
+    boolean isInSafeMode() {
+      return namesystem.isInSafeMode();
+    }
+    
     ////////////////////////////////////////////////////////////////
     // DatanodeProtocol
     ////////////////////////////////////////////////////////////////

src/java/org/apache/hadoop/mapred/InterTrackerProtocol.java

@@ -25,7 +25,8 @@ import org.apache.hadoop.ipc.VersionedProtocol;
  * The JobTracker is the Server, which implements this protocol.
  */ 
 interface InterTrackerProtocol extends VersionedProtocol {
-  public static final long versionID = 1L;
+  // version 2 introduced to replace TaskStatus.State with an enum
+  public static final long versionID = 2L;
   
   public final static int TRACKERS_OK = 0;
   public final static int UNKNOWN_TASKTRACKER = 1;

src/java/org/apache/hadoop/mapred/JobInProgress.java

@@ -274,9 +274,11 @@ class JobInProgress {
         boolean wasComplete = tip.isComplete();
         boolean change = tip.updateStatus(status);
         if (change) {
-          if (status.getRunState() == TaskStatus.SUCCEEDED) {
+          TaskStatus.State state = status.getRunState();
+          if (state == TaskStatus.State.SUCCEEDED) {
             completedTask(tip, status, metrics);
-          } else if (status.getRunState() == TaskStatus.FAILED) {
+          } else if (state == TaskStatus.State.FAILED ||
+                     state == TaskStatus.State.KILLED) {
             // Tell the job to fail the relevant task
             failedTask(tip, status.getTaskId(), status, status.getTaskTracker(),
                        wasRunning, wasComplete);
@@ -494,7 +496,7 @@ class JobInProgress {
         //
         // If all tasks are complete, then the job is done!
         //
-        if (status.getRunState() == JobStatus.RUNNING && allDone) {
+        if (this.status.getRunState() == JobStatus.RUNNING && allDone) {
             this.status.setRunState(JobStatus.SUCCEEDED);
             this.status.setReduceProgress(1.0f);
             this.finishTime = System.currentTimeMillis();
@@ -597,7 +599,7 @@ class JobInProgress {
        TaskStatus status = new TaskStatus(taskid,
                                           tip.isMapTask(),
                                           0.0f,
-                                          TaskStatus.FAILED,
+                                          TaskStatus.State.FAILED,
                                           reason,
                                           reason,
                                           trackerName, phase);
@@ -659,7 +661,7 @@ class JobInProgress {
        if (tip.isComplete()) {
          TaskStatus[] statuses = tip.getTaskStatuses();
          for(int i=0; i < statuses.length; i++) {
-           if (statuses[i].getRunState() == TaskStatus.SUCCEEDED) {
+           if (statuses[i].getRunState() == TaskStatus.State.SUCCEEDED) {
              return statuses[i];
            }
          }

src/java/org/apache/hadoop/mapred/TaskInProgress.java

@@ -69,6 +69,7 @@ class TaskInProgress {
     private long execFinishTime = 0 ;
     private int completes = 0;
     private boolean failed = false;
+    private boolean killed = false;
     private TreeSet usableTaskIds = new TreeSet();
     private TreeSet recentTasks = new TreeSet();
     private JobConf conf;
@@ -188,7 +189,8 @@ class TaskInProgress {
         if (status == null) {
             return false;
         }
-        return ((completes > 0) && (status.getRunState() == TaskStatus.SUCCEEDED));
+        return ((completes > 0) && 
+                (status.getRunState() == TaskStatus.State.SUCCEEDED));
     }
     /**
      */
@@ -281,16 +283,17 @@ class TaskInProgress {
           diagHistory.add(diagInfo);
         }
         if (oldStatus != null) {
-          int oldState = oldStatus.getRunState();
-          int newState = status.getRunState();
+          TaskStatus.State oldState = oldStatus.getRunState();
+          TaskStatus.State newState = status.getRunState();
           
           // The task is not allowed to move from completed back to running.
           // We have seen out of order status messagesmoving tasks from complete
           // to running. This is a spot fix, but it should be addressed more
           // globally.
-          if (newState == TaskStatus.RUNNING &&
-              (oldState == TaskStatus.FAILED || 
-               oldState == TaskStatus.SUCCEEDED)) {
+          if (newState == TaskStatus.State.RUNNING &&
+              (oldState == TaskStatus.State.FAILED || 
+               oldState == TaskStatus.State.KILLED || 
+               oldState == TaskStatus.State.SUCCEEDED)) {
             return false;
           }
           
@@ -315,7 +318,7 @@ class TaskInProgress {
         LOG.info("Task '" + taskid + "' has been lost.");
         TaskStatus status = (TaskStatus) taskStatuses.get(taskid);
         if (status != null) {
-            status.setRunState(TaskStatus.FAILED);
+            status.setRunState(TaskStatus.State.FAILED);
             // tasktracker went down and failed time was not reported. 
             if( 0 == status.getFinishTime() ){
               status.setFinishTime(System.currentTimeMillis());
@@ -341,7 +344,7 @@ class TaskInProgress {
     public void completed(String taskid) {
         LOG.info("Task '" + taskid + "' has completed.");
         TaskStatus status = (TaskStatus) taskStatuses.get(taskid);
-        status.setRunState(TaskStatus.SUCCEEDED);
+        status.setRunState(TaskStatus.State.SUCCEEDED);
         recentTasks.remove(taskid);
 
         //
@@ -369,9 +372,18 @@ class TaskInProgress {
             return;
         }
         this.failed = true;
+        killed = true;
         recomputeProgress();
     }
 
+    /**
+     * Was the task killed?
+     * @return true if the task killed
+     */
+    public boolean wasKilled() {
+      return killed;
+    }
+    
     /**
      * This method is called whenever there's a status change
      * for one of the TIP's sub-tasks.  It recomputes the overall 
@@ -391,11 +403,11 @@ class TaskInProgress {
             for (Iterator it = taskStatuses.keySet().iterator(); it.hasNext(); ) {
                 String taskid = (String) it.next();
                 TaskStatus status = (TaskStatus) taskStatuses.get(taskid);
-                if (status.getRunState() == TaskStatus.SUCCEEDED) {
+                if (status.getRunState() == TaskStatus.State.SUCCEEDED) {
                     bestProgress = 1;
                     bestState = status.getStateString();
                     break;
-                } else if (status.getRunState() == TaskStatus.RUNNING) {
+                } else if (status.getRunState() == TaskStatus.State.RUNNING) {
                   if (status.getProgress() >= bestProgress) {
                     bestProgress = status.getProgress();
                     bestState = status.getStateString();

src/java/org/apache/hadoop/mapred/TaskStatus.java

@@ -19,7 +19,7 @@ import org.apache.hadoop.io.*;
 
 import java.io.*;
 // enumeration for reporting current phase of a task. 
-enum Phase{STARTING, MAP, SHUFFLE, SORT, REDUCE};
+enum Phase{STARTING, MAP, SHUFFLE, SORT, REDUCE}
 
 /**************************************************
  * Describes the current status of a task.  This is
@@ -28,15 +28,12 @@ enum Phase{STARTING, MAP, SHUFFLE, SORT, REDUCE};
  * @author Mike Cafarella
  **************************************************/
 class TaskStatus implements Writable {
-    public static final int RUNNING = 0;
-    public static final int SUCCEEDED = 1;
-    public static final int FAILED = 2;
-    public static final int UNASSIGNED = 3;
+    public static enum State {RUNNING, SUCCEEDED, FAILED, UNASSIGNED, KILLED};
     
     private String taskid;
     private boolean isMap;
     private float progress;
-    private int runState;
+    private State runState;
     private String diagnosticInfo;
     private String stateString;
     private String taskTracker;
@@ -53,7 +50,7 @@ class TaskStatus implements Writable {
     public TaskStatus() {}
 
     public TaskStatus(String taskid, boolean isMap, float progress,
-                      int runState, String diagnosticInfo,
+                      State runState, String diagnosticInfo,
                       String stateString, String taskTracker,
                       Phase phase) {
         this.taskid = taskid;
@@ -70,10 +67,10 @@ class TaskStatus implements Writable {
     public boolean getIsMap() { return isMap; }
     public float getProgress() { return progress; }
     public void setProgress(float progress) { this.progress = progress; } 
-    public int getRunState() { return runState; }
+    public State getRunState() { return runState; }
     public String getTaskTracker() {return taskTracker;}
     public void setTaskTracker(String tracker) { this.taskTracker = tracker;}
-    public void setRunState(int runState) { this.runState = runState; }
+    public void setRunState(State runState) { this.runState = runState; }
     public String getDiagnosticInfo() { return diagnosticInfo; }
     public void setDiagnosticInfo(String info) { this.diagnosticInfo = info; }
     public String getStateString() { return stateString; }
@@ -183,7 +180,7 @@ class TaskStatus implements Writable {
         UTF8.writeString(out, taskid);
         out.writeBoolean(isMap);
         out.writeFloat(progress);
-        out.writeInt(runState);
+        WritableUtils.writeEnum(out, runState);
         UTF8.writeString(out, diagnosticInfo);
         UTF8.writeString(out, stateString);
         WritableUtils.writeEnum(out, phase);
@@ -199,7 +196,7 @@ class TaskStatus implements Writable {
         this.taskid = UTF8.readString(in);
         this.isMap = in.readBoolean();
         this.progress = in.readFloat();
-        this.runState = in.readInt();
+        this.runState = WritableUtils.readEnum(in, State.class);
         this.diagnosticInfo = UTF8.readString(in);
         this.stateString = UTF8.readString(in);
         this.phase = WritableUtils.readEnum(in, Phase.class); 

src/java/org/apache/hadoop/mapred/TaskTracker.java

@@ -284,7 +284,7 @@ public class TaskTracker
         tip.setJobConf(jobConf);
         tip.launchTask();
       } catch (Throwable ie) {
-        tip.runstate = TaskStatus.FAILED;
+        tip.runstate = TaskStatus.State.FAILED;
         try {
           tip.cleanup();
         } catch (Throwable ie2) {
@@ -460,7 +460,7 @@ public class TaskTracker
               for (Iterator it = taskReports.iterator();
                    it.hasNext(); ) {
                   TaskStatus taskStatus = (TaskStatus) it.next();
-                  if (taskStatus.getRunState() != TaskStatus.RUNNING) {
+                  if (taskStatus.getRunState() != TaskStatus.State.RUNNING) {
                       if (taskStatus.getIsMap()) {
                           mapTotal--;
                       } else {
@@ -509,7 +509,7 @@ public class TaskTracker
                     TaskInProgress tip = (TaskInProgress) it.next();
                     long timeSinceLastReport = System.currentTimeMillis() - 
                                                tip.getLastProgressReport();
-                    if ((tip.getRunState() == TaskStatus.RUNNING) &&
+                    if ((tip.getRunState() == TaskStatus.State.RUNNING) &&
                         (timeSinceLastReport > this.taskTimeout) &&
                         !tip.wasKilled) {
                         String msg = "Task failed to report status for " +
@@ -577,7 +577,7 @@ public class TaskTracker
 
                 for (Iterator it = runningTasks.values().iterator(); it.hasNext(); ) {
                   TaskInProgress tip = (TaskInProgress) it.next();
-                  if ((tip.getRunState() == TaskStatus.RUNNING) &&
+                  if ((tip.getRunState() == TaskStatus.State.RUNNING) &&
                       !tip.wasKilled) {
                         	
                     if (killMe == null) {
@@ -770,7 +770,7 @@ public class TaskTracker
     class TaskInProgress {
         Task task;
         float progress;
-        int runstate;
+        TaskStatus.State runstate;
         long lastProgressReport;
         StringBuffer diagnosticInfo = new StringBuffer();
         TaskRunner runner;
@@ -788,7 +788,7 @@ public class TaskTracker
         public TaskInProgress(Task task, JobConf conf) {
             this.task = task;
             this.progress = 0.0f;
-            this.runstate = TaskStatus.UNASSIGNED;
+            this.runstate = TaskStatus.State.UNASSIGNED;
             this.lastProgressReport = System.currentTimeMillis();
             this.defaultJobConf = conf;
             localJobConf = null;
@@ -862,7 +862,7 @@ public class TaskTracker
          */
         public synchronized void launchTask() throws IOException {
             localizeTask(task);
-            this.runstate = TaskStatus.RUNNING;
+            this.runstate = TaskStatus.State.RUNNING;
             this.runner = task.createRunner(TaskTracker.this);
             this.runner.start();
             this.taskStatus.setStartTime(System.currentTimeMillis());
@@ -874,7 +874,7 @@ public class TaskTracker
         public synchronized void reportProgress(float p, String state, Phase newPhase) {
             LOG.info(task.getTaskId()+" "+p+"% "+state);
             this.progress = p;
-            this.runstate = TaskStatus.RUNNING;
+            this.runstate = TaskStatus.State.RUNNING;
             this.lastProgressReport = System.currentTimeMillis();
             Phase oldPhase = taskStatus.getPhase() ;
             if( oldPhase != newPhase ){
@@ -896,7 +896,7 @@ public class TaskTracker
 
         /**
          */
-        public int getRunState() {
+        public TaskStatus.State getRunState() {
             return runstate;
         }
 
@@ -941,16 +941,19 @@ public class TaskTracker
             boolean needCleanup = false;
             synchronized (this) {
               if (done) {
-                  runstate = TaskStatus.SUCCEEDED;
+                  runstate = TaskStatus.State.SUCCEEDED;
               } else {
                   if (!wasKilled) {
                     failures += 1;
+                    runstate = TaskStatus.State.FAILED;
+                  } else {
+                    runstate = TaskStatus.State.KILLED;
                   }
-                  runstate = TaskStatus.FAILED;
                   progress = 0.0f;
               }
               this.taskStatus.setFinishTime(System.currentTimeMillis());
-              needCleanup = runstate == TaskStatus.FAILED;
+              needCleanup = (runstate == TaskStatus.State.FAILED) |
+                            (runstate == TaskStatus.State.KILLED);
             }
 
             //
@@ -973,7 +976,7 @@ public class TaskTracker
          */
         public synchronized void jobHasFinished() throws IOException {
         	 
-            if (getRunState() == TaskStatus.RUNNING) {
+            if (getRunState() == TaskStatus.State.RUNNING) {
                 killAndCleanup(false);
             } else {
                 cleanup();
@@ -991,7 +994,7 @@ public class TaskTracker
          */
         public synchronized void killAndCleanup(boolean wasFailure
                                                 ) throws IOException {
-            if (runstate == TaskStatus.RUNNING) {
+            if (runstate == TaskStatus.State.RUNNING) {
                 wasKilled = true;
                 if (wasFailure) {
                   failures += 1;
@@ -1005,9 +1008,9 @@ public class TaskTracker
          */
         public synchronized void mapOutputLost(String failure
                                                ) throws IOException {
-            if (runstate == TaskStatus.SUCCEEDED) {
+            if (runstate == TaskStatus.State.SUCCEEDED) {
               LOG.info("Reporting output lost:"+task.getTaskId());
-              runstate = TaskStatus.FAILED;       // change status to failure
+              runstate = TaskStatus.State.FAILED;    // change status to failure
               progress = 0.0f;
               reportDiagnosticInfo("Map output lost, rescheduling: " + 
                                    failure);
@@ -1029,7 +1032,7 @@ public class TaskTracker
             synchronized (TaskTracker.this) {
                tasks.remove(taskId);
                if (alwaysKeepTaskFiles ||
-                   (runstate == TaskStatus.FAILED && 
+                   (runstate == TaskStatus.State.FAILED && 
                        keepFailedTaskFiles)) {
                  return;
                }

src/java/org/apache/hadoop/mapred/TaskTrackerStatus.java

@@ -104,32 +104,34 @@ class TaskTrackerStatus implements Writable {
      * Return the current MapTask count
      */
     public int countMapTasks() {
-        int mapCount = 0;
-        for (Iterator it = taskReports.iterator(); it.hasNext(); ) {
-            TaskStatus ts = (TaskStatus) it.next();
-	    if (ts.getIsMap() &&
-		((ts.getRunState() == TaskStatus.RUNNING)
-                 || (ts.getRunState() == TaskStatus.UNASSIGNED))) {
-		mapCount++;
-	    }
-	}
-        return mapCount;
+      int mapCount = 0;
+      for (Iterator it = taskReports.iterator(); it.hasNext(); ) {
+        TaskStatus ts = (TaskStatus) it.next();
+        TaskStatus.State state = ts.getRunState();
+        if (ts.getIsMap() &&
+            ((state == TaskStatus.State.RUNNING) ||
+                (state == TaskStatus.State.UNASSIGNED))) {
+          mapCount++;
+        }
+      }
+      return mapCount;
     }
 
     /**
      * Return the current ReduceTask count
      */
     public int countReduceTasks() {
-	int reduceCount = 0;
-        for (Iterator it = taskReports.iterator(); it.hasNext(); ) {
-            TaskStatus ts = (TaskStatus) it.next();
-            if ((!ts.getIsMap()) &&
-		((ts.getRunState() == TaskStatus.RUNNING)
-		 || (ts.getRunState() == TaskStatus.UNASSIGNED))) {
-                reduceCount++;
-            }
+      int reduceCount = 0;
+      for (Iterator it = taskReports.iterator(); it.hasNext(); ) {
+        TaskStatus ts = (TaskStatus) it.next();
+        TaskStatus.State state = ts.getRunState();
+        if ((!ts.getIsMap()) &&
+            ((state == TaskStatus.State.RUNNING) ||  
+                (state == TaskStatus.State.UNASSIGNED))) {
+          reduceCount++;
         }
-        return reduceCount;
+      }
+      return reduceCount;
     }
 
     /**

src/test/org/apache/hadoop/dfs/MiniDFSCluster.java

@@ -30,7 +30,7 @@ public class MiniDFSCluster {
       }
       try {
         long[] sizes = node.getStats();
-        return sizes[0] != 0;
+        return !node.isInSafeMode() && sizes[0] != 0;
       } catch (IOException ie) {
         return false;
       }

src/test/org/apache/hadoop/mapred/TestMiniMRDFSCaching.java

@@ -38,17 +38,16 @@ public class TestMiniMRDFSCaching extends TestCase {
     String namenode = null;
     FileSystem fileSys = null;
     try {
-      Configuration conf = new Configuration();
+      JobConf conf = new JobConf();
       dfs = new MiniDFSCluster(65314, conf, true);
       fileSys = dfs.getFileSystem();
       namenode = fileSys.getName();
       mr = new MiniMRCluster(50050, 50060, 2, namenode, true);
-      JobConf jconf = new JobConf();
       // run the wordcount example with caching
       boolean ret = MRCaching.launchMRCache("localhost:50050",
                                             "/testing/wc/input",
                                             "/testing/wc/output", namenode,
-                                            jconf,
+                                            conf,
                                             "The quick brown fox\nhas many silly\n"
                                                 + "red fox sox\n");
       assertTrue("Archives not matching", ret);

src/webapps/job/jobdetails.jsp

@@ -23,6 +23,7 @@
     int totalTasks = tasks.length;
     int runningTasks = 0;
     int finishedTasks = 0;
+    int killedTasks = 0;
     int failures = 0;
     for(int i=0; i < totalTasks; ++i) {
       TaskInProgress task = tasks[i];
@@ -30,6 +31,8 @@
         finishedTasks += 1;
       } else if (task.isRunning()) {
         runningTasks += 1;
+      } else if (task.wasKilled()) {
+        killedTasks += 1;
       }
       failures += task.numTaskFailures();
     }
@@ -37,9 +40,11 @@
               "&type="+ kind + "&pagenum=1\">" + kind + "</a></th><td>" + 
               StringUtils.formatPercent(completePercent, 2) +
               "</td><td>" + totalTasks + "</td><td>" + 
-              (totalTasks - runningTasks - finishedTasks) + "</td><td>" +
+              (totalTasks - runningTasks - finishedTasks - killedTasks) + 
+              "</td><td>" +
               runningTasks + "</td><td>" +
-              finishedTasks + 
+              finishedTasks + "</td><td>" +
+              killedTasks +
               "</td><td><a href=\"/jobfailures.jsp?jobid=" + jobId +
               "&kind=" + kind + "\">" +
               failures + "</a></td></tr>\n");
@@ -74,6 +79,7 @@
     out.print("<table border=2 cellpadding=\"5\" cellspacing=\"2\">");
     out.print("<tr><th>Kind</th><th>% Complete</th><th>Num Tasks</th>" +
               "<th>Pending</th><th>Running</th><th>Complete</th>" +
+              "<th>Killed</th>" +
               "<th><a href=\"/jobfailures.jsp?jobid=" + jobId + 
               "\">Failures</a></th></tr>\n");
     printTaskSummary(out, jobId, "map", status.mapProgress(), 

src/webapps/job/jobfailures.jsp

@@ -19,7 +19,7 @@
     TaskStatus[] statuses = tip.getTaskStatuses();
     String tipId = tip.getTIPId();
     for(int i=0; i < statuses.length; ++i) {
-      if (statuses[i].getRunState() == TaskStatus.FAILED) {
+      if (statuses[i].getRunState() == TaskStatus.State.FAILED) {
         String taskTrackerName = statuses[i].getTaskTracker();
         TaskTrackerStatus taskTracker = tracker.getTaskTracker(taskTrackerName);
         out.print("<tr><td>" + statuses[i].getTaskId() +

src/webapps/job/taskdetails.jsp

@@ -19,28 +19,6 @@
   TaskStatus[] ts = (job != null) ? tracker.getTaskStatuses(jobid, tipid): null;
 %>
 
-<%! 
-  public void writeString(JspWriter out, int state) throws IOException{
-    String sstate;
-    if (state == TaskStatus.RUNNING){
-      sstate = "RUNNING";
-    }
-    else if (state == TaskStatus.SUCCEEDED){
-      sstate = "SUCCEDED";
-    }
-    else if (state == TaskStatus.FAILED){
-      sstate = "FAILED";
-    }
-    else if (state == TaskStatus.UNASSIGNED){
-      sstate = "UNASSIGNED";
-    }
-    else{
-      sstate = "ERROR IN STATUS";
-   }
-    out.print(sstate);
-  }
-%>
-
 <html>
 <title>Hadoop Task Details</title>
 <body>
@@ -80,9 +58,7 @@
                   taskTracker.getHttpPort() + "\">" +  taskTracker.getHost() + 
                   "</a></td>");
       }
-      out.print("<td>");
-      writeString(out, status.getRunState()); 
-      out.print("</td>");
+      out.print("<td>" + status.getRunState() + "</td>");
       out.print("<td>"+ StringUtils.formatPercent(status.getProgress(),2) + 
                 "</td>");
       out.print("<td>" + StringUtils.getFormattedTimeWithDiff(dateFormat,  

src/webapps/task/tasktracker.jsp

@@ -8,24 +8,6 @@
   import="org.apache.hadoop.mapred.*"
   import="org.apache.hadoop.util.*"
 %>
-<%!
-  private static DecimalFormat percentFormat = new DecimalFormat("##0.00");
-  
-  private String stringifyState(int state) {
-    if (state == TaskStatus.RUNNING){
-      return "RUNNING";
-    } else if (state == TaskStatus.SUCCEEDED){
-      return "SUCCEDED";
-    } else if (state == TaskStatus.FAILED){
-      return "FAILED";
-    } else if (state == TaskStatus.UNASSIGNED){
-      return "UNASSIGNED";
-    }
-    return "unknown status";
-  }
-  
- %>
-
 <%
   TaskTracker tracker = (TaskTracker) application.getAttribute("task.tracker");
   String trackerName = tracker.getName();
@@ -50,9 +32,9 @@
      while (itr.hasNext()) {
        TaskStatus status = (TaskStatus) itr.next();
        out.print("<tr><td>" + status.getTaskId());
-       out.print("</td><td>" + stringifyState(status.getRunState())); 
+       out.print("</td><td>" + status.getRunState()); 
        out.print("</td><td>" + 
-                 percentFormat.format(100.0 * status.getProgress()));
+                 StringUtils.formatPercent(status.getProgress(), 2));
        out.print("</td><td><pre>" + status.getDiagnosticInfo() + "</pre></td>");
        out.print("</tr>\n");
      }