HADOOP-4861. Add disk usage with human-readable size (-duh). Contributed by Todd Lipcon.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/core/trunk@779102 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -128,6 +128,9 @@ Trunk (unreleased changes)
     HADOOP-5815. Sqoop: A database import tool for Hadoop.
     (Aaron Kimball via tomwhite)
 
+    HADOOP-4861. Add disk usage with human-readable size (-duh).
+    (Todd Lipcon via tomwhite)
+
   IMPROVEMENTS
 
     HADOOP-4565. Added CombineFileInputFormat to use data locality information

build.xml

@@ -709,6 +709,7 @@
     <copy file="${test.src.dir}/hdfs/org/apache/hadoop/cli/clitest_data/data30bytes" todir="${test.cache.data}"/>
     <copy file="${test.src.dir}/hdfs/org/apache/hadoop/cli/clitest_data/data60bytes" todir="${test.cache.data}"/>
     <copy file="${test.src.dir}/hdfs/org/apache/hadoop/cli/clitest_data/data120bytes" todir="${test.cache.data}"/>
+    <copy file="${test.src.dir}/hdfs/org/apache/hadoop/cli/clitest_data/data1k" todir="${test.cache.data}"/>
     <copy file="${test.src.dir}/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/fsimageV18" todir="${test.cache.data}"/>
     <copy file="${test.src.dir}/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/fsimageV19" todir="${test.cache.data}"/>
   </target>

src/core/org/apache/hadoop/fs/FsShell.java

@@ -61,6 +61,7 @@ public class FsShell extends Configured implements Tool {
   static final String COPYTOLOCAL_SHORT_USAGE = GET_SHORT_USAGE.replace(
       "-get", "-copyToLocal");
   static final String TAIL_USAGE="-tail [-f] <file>";
+  static final String DU_USAGE="-du [-s] [-h] <paths...>";
 
   /**
    */
@@ -670,58 +671,98 @@ public class FsShell extends Configured implements Tool {
 
   /**
    * Show the size of all files that match the file pattern <i>src</i>
-   * @param src a file pattern specifying source files
+   * @param cmd
+   * @param pos ignore anything before this pos in cmd
    * @throws IOException  
    * @see org.apache.hadoop.fs.FileSystem#globStatus(Path)
    */
-  void du(String src) throws IOException {
-    Path srcPath = new Path(src);
-    FileSystem srcFs = srcPath.getFileSystem(getConf());
-    Path[] pathItems = FileUtil.stat2Paths(srcFs.globStatus(srcPath), 
-                                           srcPath);
-    FileStatus items[] = srcFs.listStatus(pathItems);
-    if ((items == null) || ((items.length == 0) && 
-        (!srcFs.exists(srcPath)))){
-      throw new FileNotFoundException("Cannot access " + src
-            + ": No such file or directory.");
-    } else {
-      System.out.println("Found " + items.length + " items");
-      int maxLength = 10;
-      
-      long length[] = new long[items.length];
-      for (int i = 0; i < items.length; i++) {
-        length[i] = items[i].isDir() ?
-          srcFs.getContentSummary(items[i].getPath()).getLength() :
-          items[i].getLen();
-        int len = String.valueOf(length[i]).length();
-        if (len > maxLength) maxLength = len;
+  void du(String[] cmd, int pos) throws IOException {
+    CommandFormat c = new CommandFormat(
+      "du", 0, Integer.MAX_VALUE, "h", "s");
+    List<String> params;
+    try {
+      params = c.parse(cmd, pos);
+    } catch (IllegalArgumentException iae) {
+      System.err.println("Usage: java FsShell " + DU_USAGE);
+      throw iae;
+    }
+    boolean humanReadable = c.getOpt("h");
+    boolean summary = c.getOpt("s");
+
+    // Default to cwd
+    if (params.isEmpty()) {
+      params.add(".");
+    }
+
+    List<UsagePair> usages = new ArrayList<UsagePair>();
+
+    for (String src : params) {
+      Path srcPath = new Path(src);
+      FileSystem srcFs = srcPath.getFileSystem(getConf());
+      FileStatus globStatus[] = srcFs.globStatus(srcPath);
+      FileStatus statusToPrint[];
+
+      if (summary) {
+        statusToPrint = globStatus;
+      } else {
+        Path statPaths[] = FileUtil.stat2Paths(globStatus, srcPath);
+        statusToPrint = srcFs.listStatus(statPaths);
       }
-      for(int i = 0; i < items.length; i++) {
-        System.out.printf("%-"+ (maxLength + BORDER) +"d", length[i]);
-        System.out.println(items[i].getPath());
+      if ((statusToPrint == null) || ((statusToPrint.length == 0) &&
+                                      (!srcFs.exists(srcPath)))){
+        throw new FileNotFoundException("Cannot access " + src
+                                        + ": No such file or directory.");
+      }
+
+      if (!summary) {
+        System.out.println("Found " + statusToPrint.length + " items");
+      }
+
+      for (FileStatus stat : statusToPrint) {
+        long length;
+        if (summary || stat.isDir()) {
+          length = srcFs.getContentSummary(stat.getPath()).getLength();
+        } else {
+          length = stat.getLen();
+        }
+
+        usages.add(new UsagePair(String.valueOf(stat.getPath()), length));
       }
     }
+    printUsageSummary(usages, humanReadable);
   }
     
   /**
    * Show the summary disk usage of each dir/file 
    * that matches the file pattern <i>src</i>
-   * @param src a file pattern specifying source files
+   * @param cmd
+   * @param pos ignore anything before this pos in cmd
    * @throws IOException  
    * @see org.apache.hadoop.fs.FileSystem#globStatus(Path)
    */
-  void dus(String src) throws IOException {
-    Path srcPath = new Path(src);
-    FileSystem srcFs = srcPath.getFileSystem(getConf());
-    FileStatus status[] = srcFs.globStatus(new Path(src));
-    if (status==null || status.length==0) {
-      throw new FileNotFoundException("Cannot access " + src + 
-          ": No such file or directory.");
+  void dus(String[] cmd, int pos) throws IOException {
+    String newcmd[] = new String[cmd.length + 1];
+    System.arraycopy(cmd, 0, newcmd, 0, cmd.length);
+    newcmd[cmd.length] = "-s";
+    du(newcmd, pos);
+  }
+
+  private void printUsageSummary(List<UsagePair> usages,
+                                 boolean humanReadable) {
+    int maxColumnWidth = 0;
+    for (UsagePair usage : usages) {
+      String toPrint = humanReadable ?
+        StringUtils.humanReadableInt(usage.bytes) : String.valueOf(usage.bytes);
+      if (toPrint.length() > maxColumnWidth) {
+        maxColumnWidth = toPrint.length();
+      }
     }
-    for(int i=0; i<status.length; i++) {
-      long totalSize = srcFs.getContentSummary(status[i].getPath()).getLength();
-      String pathStr = status[i].getPath().toString();
-      System.out.println(("".equals(pathStr)?".":pathStr) + "\t" + totalSize);
+
+    for (UsagePair usage : usages) {
+      String toPrint = humanReadable ?
+        StringUtils.humanReadableInt(usage.bytes) : String.valueOf(usage.bytes);
+      System.out.printf("%-"+ (maxColumnWidth + BORDER) +"s", toPrint);
+      System.out.println(usage.path);
     }
   }
 
@@ -1558,10 +1599,6 @@ public class FsShell extends Configured implements Tool {
           delete(argv[i], true);
         } else if ("-df".equals(cmd)) {
           df(argv[i]);
-        } else if ("-du".equals(cmd)) {
-          du(argv[i]);
-        } else if ("-dus".equals(cmd)) {
-          dus(argv[i]);
         } else if (Count.matches(cmd)) {
           new Count(argv, i, getConf()).runAll();
         } else if ("-ls".equals(cmd)) {
@@ -1809,17 +1846,9 @@ public class FsShell extends Configured implements Tool {
           df(null);
         }
       } else if ("-du".equals(cmd)) {
-        if (i < argv.length) {
-          exitCode = doall(cmd, argv, i);
-        } else {
-          du(".");
-        }
+        du(argv, i);
       } else if ("-dus".equals(cmd)) {
-        if (i < argv.length) {
-          exitCode = doall(cmd, argv, i);
-        } else {
-          dus(".");
-        }         
+        dus(argv, i);
       } else if (Count.matches(cmd)) {
         exitCode = new Count(argv, i, getConf()).runAll();
       } else if ("-mkdir".equals(cmd)) {
@@ -1922,4 +1951,18 @@ public class FsShell extends Configured implements Tool {
           throw new IOException("Multiple IOExceptions: " + exceptions);
     }
   }
+
+
+  /**
+   * Utility class for a line of du output
+   */
+  private static class UsagePair {
+    public String path;
+    public long bytes;
+
+    public UsagePair(String path, long bytes) {
+      this.path = path;
+      this.bytes = bytes;
+    }
+  }
 }

src/core/org/apache/hadoop/util/StringUtils.java

@@ -88,7 +88,8 @@ public class StringUtils {
     double result = number;
     String suffix = "";
     if (absNumber < 1024) {
-      // nothing
+      // since no division has occurred, don't format with a decimal point
+      return String.valueOf(number);
     } else if (absNumber < 1024 * 1024) {
       result = number / 1024.0;
       suffix = "k";

src/test/hdfs/org/apache/hadoop/cli/clitest_data/data1k

@@ -0,0 +1,71 @@
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234
+12345678901234

src/test/hdfs/org/apache/hadoop/cli/testHDFSConf.xml

@@ -1132,6 +1132,33 @@
         </comparator>
       </comparators>
     </test>
+
+    <test> <!-- TESTED -->
+      <description>duh: Test for hdfs:// path - directory</description>
+      <test-commands>
+        <command>-fs NAMENODE -mkdir hdfs:///dir0</command>
+        <command>-fs NAMENODE -put CLITEST_DATA/data15bytes hdfs:///dir0/data15bytes</command>
+        <command>-fs NAMENODE -put CLITEST_DATA/data1k hdfs:///dir0/data1k</command>
+        <command>-fs NAMENODE -du -h hdfs:///dir0</command>
+      </test-commands>
+      <cleanup-commands>
+        <command>-fs NAMENODE -rmr hdfs:///dir0</command>
+      </cleanup-commands>
+      <comparators>
+        <comparator>
+          <type>TokenComparator</type>
+          <expected-output>Found 2 items</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^15( |\t)*hdfs://localhost[.a-z]*:[0-9]*/dir0/data15bytes</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^1.0k( |\t)*hdfs://localhost[.a-z]*:[0-9]*/dir0/data1k</expected-output>
+        </comparator>
+      </comparators>
+    </test>
     
     <test> <!-- TESTED -->
       <description>du: Test for hdfs:// path - directory using globbing</description>
@@ -1314,7 +1341,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir0( |\t)*450</expected-output>
+          <expected-output>^450\s+hdfs://localhost[.a-z]*:[0-9]*/dir0</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -1346,7 +1373,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/user/[a-z]*/dir0( |\t)*450</expected-output>
+          <expected-output>^450\s+hdfs://localhost[.a-z]*:[0-9]*/user/[a-z]*/dir0</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -1384,7 +1411,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir0( |\t)*450</expected-output>
+          <expected-output>^450\s+hdfs://localhost[.a-z]*:[0-9]*/dir0</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -1417,7 +1444,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir0( |\t)*450</expected-output>
+          <expected-output>^450\s+hdfs://localhost[.a-z]*:[0-9]*/dir0</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -1455,7 +1482,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir0( |\t)*450</expected-output>
+          <expected-output>^450\s+hdfs://localhost[.a-z]*:[0-9]*/dir0</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -1487,7 +1514,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir0( |\t)*450</expected-output>
+          <expected-output>^450\s+hdfs://localhost[.a-z]*:[0-9]*/dir0</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -1525,7 +1552,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir0( |\t)*450</expected-output>
+          <expected-output>^450\s+hdfs://localhost[.a-z]*:[0-9]*/dir0</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -5485,7 +5512,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir0(|\t)*0</expected-output>
+          <expected-output>^0\s+hdfs://localhost[.a-z]*:[0-9]*/dir0</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -5502,7 +5529,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/user/[a-z]*/dir0(|\t)*0</expected-output>
+          <expected-output>^0\s+hdfs://localhost[.a-z]*:[0-9]*/user/[a-z]*/dir0</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -5522,19 +5549,19 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir0(|\t)*0</expected-output>
+          <expected-output>^0\s+hdfs://localhost[.a-z]*:[0-9]*/dir0</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir1(|\t)*0</expected-output>
+          <expected-output>^0\s+hdfs://localhost[.a-z]*:[0-9]*/dir1</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir2(|\t)*0</expected-output>
+          <expected-output>^0\s+hdfs://localhost[.a-z]*:[0-9]*/dir2</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir3(|\t)*0</expected-output>
+          <expected-output>^0\s+hdfs://localhost[.a-z]*:[0-9]*/dir3</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -5554,19 +5581,19 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/user/[a-z]*/dir0(|\t)*0</expected-output>
+          <expected-output>^0\s+hdfs://localhost[.a-z]*:[0-9]*/user/[a-z]*/dir0</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/user/[a-z]*/dir1(|\t)*0</expected-output>
+          <expected-output>^0\s+hdfs://localhost[.a-z]*:[0-9]*/user/[a-z]*/dir1</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/user/[a-z]*/dir2(|\t)*0</expected-output>
+          <expected-output>^0\s+hdfs://localhost[.a-z]*:[0-9]*/user/[a-z]*/dir2</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/user/[a-z]*/dir3(|\t)*0</expected-output>
+          <expected-output>^0\s+hdfs://localhost[.a-z]*:[0-9]*/user/[a-z]*/dir3</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -5617,7 +5644,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir0(|\t)*0</expected-output>
+          <expected-output>^0\s+hdfs://localhost[.a-z]*:[0-9]*/dir0(|\t)*</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -5634,19 +5661,19 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir0(|\t)*0</expected-output>
+          <expected-output>^0\s+hdfs://localhost[.a-z]*:[0-9]*/dir0</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir1(|\t)*0</expected-output>
+          <expected-output>^0\s+hdfs://localhost[.a-z]*:[0-9]*/dir1</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir2(|\t)*0</expected-output>
+          <expected-output>^0\s+hdfs://localhost[.a-z]*:[0-9]*/dir2</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir3(|\t)*0</expected-output>
+          <expected-output>^0\s+hdfs://localhost[.a-z]*:[0-9]*/dir3</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -5697,7 +5724,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir0(|\t)*0</expected-output>
+          <expected-output>^0\s+hdfs://localhost[.a-z]*:[0-9]*/dir0</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -5714,19 +5741,19 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir0(|\t)*0</expected-output>
+          <expected-output>^0\s+hdfs://localhost[.a-z]*:[0-9]*/dir0</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir1(|\t)*0</expected-output>
+          <expected-output>^0\s+hdfs://localhost[.a-z]*:[0-9]*/dir1</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir2(|\t)*0</expected-output>
+          <expected-output>^0\s+hdfs://localhost[.a-z]*:[0-9]*/dir2</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^hdfs://localhost[.a-z]*:[0-9]*/dir3(|\t)*0</expected-output>
+          <expected-output>^0\s+hdfs://localhost[.a-z]*:[0-9]*/dir3</expected-output>
         </comparator>
       </comparators>
     </test>