HADOOP-298. Improved progress reports for CopyFiles, the distributed file copier. Contributed by Owen.

git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk@413984 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -7,6 +7,9 @@ Trunk (unreleased changes)
     selector, and a thread per connection is no longer required.  This
     should permit larger clusters.  (Devaraj Das via cutting)
 
+ 2. HADOOP-298.  Improved progress reports for CopyFiles utility, the
+    distributed file copier.  (omalley via cutting)
+
 
 Release 0.3.2 - 2006-06-09
 

src/java/org/apache/hadoop/util/CopyFiles.java

@@ -19,9 +19,9 @@ package org.apache.hadoop.util;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.text.DecimalFormat;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.Date;
 import java.util.Enumeration;
 import java.util.Iterator;
 import java.util.StringTokenizer;
@@ -75,11 +75,13 @@ public class CopyFiles extends MapReduceBase implements Reducer {
     private static final long reportInterval = 1L << 25;
     private long bytesSinceLastReport = 0L;
     private long totalBytesCopied = 0L;
+    private static DecimalFormat percentFormat = new DecimalFormat("0.00");
     
     private void copy(String src, Reporter reporter) throws IOException {
       // open source file
       Path srcFile = new Path(srcPath, src);
       FSDataInputStream in = srcFileSys.open(srcFile);
+      long totalBytes = srcFileSys.getLength(srcFile);
       
       // create directories to hold destination file and create destFile
       Path destFile = new Path(destPath, src);
@@ -96,7 +98,12 @@ public class CopyFiles extends MapReduceBase implements Reducer {
         if (bytesSinceLastReport > reportInterval) {
             totalBytesCopied += bytesSinceLastReport;
             bytesSinceLastReport = 0L;
-            reporter.setStatus("Total bytes copied: "+totalBytesCopied);
+            reporter.setStatus("Copy "+ src + ": " + 
+                               percentFormat.format(100.0 * totalBytesCopied / 
+                                                    totalBytes) +
+                               "% and " +
+                               StringUtils.humanReadableInt(totalBytesCopied) +
+                               " bytes");
         }
       }
       
@@ -105,7 +112,8 @@ public class CopyFiles extends MapReduceBase implements Reducer {
       // report at least once for each file
       totalBytesCopied += bytesSinceLastReport;
       bytesSinceLastReport = 0L;
-      reporter.setStatus("Total bytes copied: "+totalBytesCopied);
+      reporter.setStatus("Finished. Bytes copied: " + 
+                         StringUtils.humanReadableInt(totalBytesCopied));
     }
     
     /** Mapper configuration.

src/java/org/apache/hadoop/util/StringUtils.java

@@ -18,6 +18,7 @@ package org.apache.hadoop.util;
 
 import java.io.PrintWriter;
 import java.io.StringWriter;
+import java.text.DecimalFormat;
 
 /**
  * General string utils
@@ -51,4 +52,31 @@ public class StringUtils {
     return fullHostname;
   }
 
+  private static DecimalFormat numFormat = new DecimalFormat("0.0");
+  
+  /**
+   * Given an integer, return a string that is in an approximate, but human 
+   * readable format. 
+   * It uses the bases 'k', 'm', and 'g' for 1024, 1024**2, and 1024**3.
+   * @param number the number to format
+   * @return a human readable form of the integer
+   */
+  public static String humanReadableInt(long number) {
+    long absNumber = Math.abs(number);
+    double result = number;
+    String suffix = "";
+    if (absNumber < 1024) {
+      // nothing
+    } else if (absNumber < 1024 * 1024) {
+      result = number / 1024.0;
+      suffix = "k";
+    } else if (absNumber < 1024 * 1024 * 1024) {
+      result = number / (1024.0 * 1024);
+      suffix = "m";
+    } else {
+      result = number / (1024.0 * 1024 * 1024);
+      suffix = "g";
+    }
+    return numFormat.format(result) + suffix;
+  }
 }