HADOOP-7713. dfs -count -q should label output column (Jonathan Allen via aw)

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -154,6 +154,9 @@ Trunk (Unreleased)
     HADOOP-10976. moving the source code of hadoop-tools docs to the
     directory under hadoop-tools (Masatake Iwasaki via aw)
 
+    HADOOP-7713. dfs -count -q should label output column (Jonathan Allen
+    via aw)
+
   BUG FIXES
 
     HADOOP-11473. test-patch says "-1 overall" even when all checks are +1

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java

@@ -97,28 +97,35 @@ public class ContentSummary implements Writable{
     this.spaceConsumed = in.readLong();
     this.spaceQuota = in.readLong();
   }
-  
-  /** 
+
+  /**
    * Output format:
    * <----12----> <----12----> <-------18------->
-   *    DIR_COUNT   FILE_COUNT       CONTENT_SIZE FILE_NAME    
+   *    DIR_COUNT   FILE_COUNT       CONTENT_SIZE
    */
-  private static final String STRING_FORMAT = "%12s %12s %18s ";
-  /** 
+  private static final String SUMMARY_FORMAT = "%12s %12s %18s ";
+  /**
    * Output format:
-   * <----12----> <----15----> <----15----> <----15----> <----12----> <----12----> <-------18------->
-   *    QUOTA   REMAINING_QUATA SPACE_QUOTA SPACE_QUOTA_REM DIR_COUNT   FILE_COUNT   CONTENT_SIZE     FILE_NAME    
+   * <----12----> <------15-----> <------15-----> <------15----->
+   *        QUOTA       REM_QUOTA     SPACE_QUOTA REM_SPACE_QUOTA
+   * <----12----> <----12----> <-------18------->
+   *    DIR_COUNT   FILE_COUNT       CONTENT_SIZE
    */
-  private static final String QUOTA_STRING_FORMAT = "%12s %15s ";
-  private static final String SPACE_QUOTA_STRING_FORMAT = "%15s %15s ";
-  
+  private static final String QUOTA_SUMMARY_FORMAT = "%12s %15s ";
+  private static final String SPACE_QUOTA_SUMMARY_FORMAT = "%15s %15s ";
+
+  private static final String[] HEADER_FIELDS = new String[] { "DIR_COUNT",
+      "FILE_COUNT", "CONTENT_SIZE"};
+  private static final String[] QUOTA_HEADER_FIELDS = new String[] { "QUOTA",
+      "REM_QUOTA", "SPACE_QUOTA", "REM_SPACE_QUOTA" };
+
   /** The header string */
   private static final String HEADER = String.format(
-      STRING_FORMAT.replace('d', 's'), "directories", "files", "bytes");
+      SUMMARY_FORMAT, (Object[]) HEADER_FIELDS);
 
   private static final String QUOTA_HEADER = String.format(
-      QUOTA_STRING_FORMAT + SPACE_QUOTA_STRING_FORMAT, 
-      "name quota", "rem name quota", "space quota", "rem space quota") +
+      QUOTA_SUMMARY_FORMAT + SPACE_QUOTA_SUMMARY_FORMAT,
+      (Object[]) QUOTA_HEADER_FIELDS) +
       HEADER;
   
   /** Return the header of the output.
@@ -131,7 +138,25 @@ public class ContentSummary implements Writable{
   public static String getHeader(boolean qOption) {
     return qOption ? QUOTA_HEADER : HEADER;
   }
-  
+
+  /**
+   * Returns the names of the fields from the summary header.
+   * 
+   * @return names of fields as displayed in the header
+   */
+  public static String[] getHeaderFields() {
+    return HEADER_FIELDS;
+  }
+
+  /**
+   * Returns the names of the fields used in the quota summary.
+   * 
+   * @return names of quota fields as displayed in the header
+   */
+  public static String[] getQuotaHeaderFields() {
+    return QUOTA_HEADER_FIELDS;
+  }
+
   @Override
   public String toString() {
     return toString(true);
@@ -175,11 +200,11 @@ public class ContentSummary implements Writable{
         spaceQuotaRem = formatSize(spaceQuota - spaceConsumed, hOption);
       }
       
-      prefix = String.format(QUOTA_STRING_FORMAT + SPACE_QUOTA_STRING_FORMAT, 
+      prefix = String.format(QUOTA_SUMMARY_FORMAT + SPACE_QUOTA_SUMMARY_FORMAT,
                              quotaStr, quotaRem, spaceQuotaStr, spaceQuotaRem);
     }
     
-    return prefix + String.format(STRING_FORMAT,
+    return prefix + String.format(SUMMARY_FORMAT,
      formatSize(directoryCount, hOption),
      formatSize(fileCount, hOption),
      formatSize(length, hOption));

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java

@@ -21,6 +21,7 @@ import java.io.IOException;
 import java.util.Arrays;
 import java.util.LinkedList;
 
+import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
@@ -44,18 +45,26 @@ public class Count extends FsCommand {
 
   private static final String OPTION_QUOTA = "q";
   private static final String OPTION_HUMAN = "h";
+  private static final String OPTION_HEADER = "v";
 
   public static final String NAME = "count";
   public static final String USAGE =
-      "[-" + OPTION_QUOTA + "] [-" + OPTION_HUMAN + "] <path> ...";
-  public static final String DESCRIPTION = 
+      "[-" + OPTION_QUOTA + "] [-" + OPTION_HUMAN + "] [-" + OPTION_HEADER
+          + "] <path> ...";
+  public static final String DESCRIPTION =
       "Count the number of directories, files and bytes under the paths\n" +
-      "that match the specified file pattern.  The output columns are:\n" +
-      "DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME or\n" +
-      "QUOTA REMAINING_QUOTA SPACE_QUOTA REMAINING_SPACE_QUOTA \n" +
-      "      DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME\n" +
-      "The -h option shows file sizes in human readable format.";
-  
+          "that match the specified file pattern.  The output columns are:\n" +
+          StringUtils.join(ContentSummary.getHeaderFields(), ' ') +
+          " PATHNAME\n" +
+          "or, with the -" + OPTION_QUOTA + " option:\n" +
+          StringUtils.join(ContentSummary.getQuotaHeaderFields(), ' ') + "\n" +
+          "      " +
+          StringUtils.join(ContentSummary.getHeaderFields(), ' ') +
+          " PATHNAME\n" +
+          "The -" + OPTION_HUMAN +
+          " option shows file sizes in human readable format.\n" +
+          "The -" + OPTION_HEADER + " option displays a header line.";
+
   private boolean showQuotas;
   private boolean humanReadable;
 
@@ -65,7 +74,7 @@ public class Count extends FsCommand {
   /** Constructor
    * @deprecated invoke via {@link FsShell}
    * @param cmd the count command
-   * @param pos the starting index of the arguments 
+   * @param pos the starting index of the arguments
    * @param conf configuration
    */
   @Deprecated
@@ -77,13 +86,16 @@ public class Count extends FsCommand {
   @Override
   protected void processOptions(LinkedList<String> args) {
     CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE,
-      OPTION_QUOTA, OPTION_HUMAN);
+        OPTION_QUOTA, OPTION_HUMAN, OPTION_HEADER);
     cf.parse(args);
     if (args.isEmpty()) { // default path is the current working directory
       args.add(".");
     }
     showQuotas = cf.getOpt(OPTION_QUOTA);
     humanReadable = cf.getOpt(OPTION_HUMAN);
+    if (cf.getOpt(OPTION_HEADER)) {
+      out.println(ContentSummary.getHeader(showQuotas) + "PATHNAME");
+    }
   }
 
   @Override

hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm

@@ -154,17 +154,19 @@ bin/hadoop fs <args>
 
 * count
 
-   Usage: <<<hadoop fs -count [-q] [-h] <paths> >>>
+   Usage: <<<hadoop fs -count [-q] [-h] [-v] <paths> >>>
 
    Count the number of directories, files and bytes under the paths that match
    the specified file pattern.  The output columns with -count are: DIR_COUNT,
-   FILE_COUNT, CONTENT_SIZE FILE_NAME
+   FILE_COUNT, CONTENT_SIZE PATHNAME
 
    The output columns with -count -q are: QUOTA, REMAINING_QUATA, SPACE_QUOTA,
-   REMAINING_SPACE_QUOTA, DIR_COUNT, FILE_COUNT, CONTENT_SIZE, FILE_NAME
+   REMAINING_SPACE_QUOTA, DIR_COUNT, FILE_COUNT, CONTENT_SIZE, PATHNAME
 
    The -h option shows sizes in human readable format.
 
+   The -v option displays a header line.
+
    Example:
 
      * <<<hadoop fs -count hdfs://nn1.example.com/file1 hdfs://nn2.example.com/file2>>>
@@ -173,6 +175,8 @@ bin/hadoop fs <args>
 
      * <<<hadoop fs -count -q -h hdfs://nn1.example.com/file1>>>
 
+     * <<<hdfs dfs -count -q -h -v hdfs://nn1.example.com/file1>>>
+
    Exit Code:
 
    Returns 0 on success and -1 on error.
@@ -743,4 +747,4 @@ permissions userid groupid modification_date modification_time dirname
 
    Usage: <<<hadoop fs -usage command>>>
 
-   Return the help for an individual command.
+   Return the help for an individual command.

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestContentSummary.java

@@ -137,15 +137,15 @@ public class TestContentSummary {
   // check the header with quotas
   @Test
   public void testGetHeaderWithQuota() {
-    String header = "  name quota  rem name quota     space quota "
-        + "rem space quota  directories        files              bytes ";
+    String header = "       QUOTA       REM_QUOTA     SPACE_QUOTA "
+        + "REM_SPACE_QUOTA    DIR_COUNT   FILE_COUNT       CONTENT_SIZE ";
     assertEquals(header, ContentSummary.getHeader(true));
   }
 
   // check the header without quotas
   @Test
   public void testGetHeaderNoQuota() {
-    String header = " directories        files              bytes ";
+    String header = "   DIR_COUNT   FILE_COUNT       CONTENT_SIZE ";
     assertEquals(header, ContentSummary.getHeader(false));
   }
 

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java

@@ -31,6 +31,7 @@ import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FilterFileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.shell.CommandFormat.NotEnoughArgumentsException;
 import org.junit.Test;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -47,7 +48,6 @@ public class TestCount {
   private static Configuration conf;
   private static FileSystem mockFs;
   private static FileStatus fileStat;
-  private static ContentSummary mockCs;
 
   @BeforeClass
   public static void setup() {
@@ -55,7 +55,6 @@ public class TestCount {
     conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
     mockFs = mock(FileSystem.class);
     fileStat = mock(FileStatus.class);
-    mockCs = mock(ContentSummary.class);
     when(fileStat.isFile()).thenReturn(true);
   }
 
@@ -87,6 +86,85 @@ public class TestCount {
     assertTrue(count.isHumanReadable());
   }
 
+  // check no options is handled correctly
+  @Test
+  public void processOptionsNoOptions() {
+    LinkedList<String> options = new LinkedList<String>();
+    options.add("dummy");
+    Count count = new Count();
+    count.processOptions(options);
+    assertFalse(count.isShowQuotas());
+  }
+
+  // check -q is handled correctly
+  @Test
+  public void processOptionsShowQuotas() {
+    LinkedList<String> options = new LinkedList<String>();
+    options.add("-q");
+    options.add("dummy");
+    Count count = new Count();
+    count.processOptions(options);
+    assertTrue(count.isShowQuotas());
+  }
+
+  // check missing arguments is handled correctly
+  @Test
+  public void processOptionsMissingArgs() {
+    LinkedList<String> options = new LinkedList<String>();
+    Count count = new Count();
+    try {
+      count.processOptions(options);
+      fail("Count.processOptions - NotEnoughArgumentsException not thrown");
+    } catch (NotEnoughArgumentsException e) {
+    }
+    assertFalse(count.isShowQuotas());
+  }
+
+  // check the correct header is produced with no quotas (-v)
+  @Test
+  public void processOptionsHeaderNoQuotas() {
+    LinkedList<String> options = new LinkedList<String>();
+    options.add("-v");
+    options.add("dummy");
+
+    PrintStream out = mock(PrintStream.class);
+
+    Count count = new Count();
+    count.out = out;
+
+    count.processOptions(options);
+
+    String noQuotasHeader =
+    // <----12----> <----12----> <-------18------->
+      "   DIR_COUNT   FILE_COUNT       CONTENT_SIZE PATHNAME";
+    verify(out).println(noQuotasHeader);
+    verifyNoMoreInteractions(out);
+  }
+
+  // check the correct header is produced with quotas (-q -v)
+  @Test
+  public void processOptionsHeaderWithQuotas() {
+    LinkedList<String> options = new LinkedList<String>();
+    options.add("-q");
+    options.add("-v");
+    options.add("dummy");
+
+    PrintStream out = mock(PrintStream.class);
+
+    Count count = new Count();
+    count.out = out;
+
+    count.processOptions(options);
+
+    String withQuotasHeader =
+    // <----12----> <-----15------> <-----15------> <-----15------>
+      "       QUOTA       REM_QUOTA     SPACE_QUOTA REM_SPACE_QUOTA " +
+    // <----12----> <----12----> <-------18------->
+      "   DIR_COUNT   FILE_COUNT       CONTENT_SIZE PATHNAME";
+    verify(out).println(withQuotasHeader);
+    verifyNoMoreInteractions(out);
+  }
+
   // check quotas are reported correctly
   @Test
   public void processPathShowQuotas() throws Exception {
@@ -211,29 +289,48 @@ public class TestCount {
   public void getUsage() {
     Count count = new Count();
     String actual = count.getUsage();
-    String expected = "-count [-q] [-h] <path> ...";
+    String expected = "-count [-q] [-h] [-v] <path> ...";
     assertEquals("Count.getUsage", expected, actual);
   }
 
+  // check the correct description is returned
+  @Test
+  public void getDescription() {
+    Count count = new Count();
+    String actual = count.getDescription();
+    String expected =
+        "Count the number of directories, files and bytes under the paths\n"
+        + "that match the specified file pattern.  The output columns are:\n"
+        + "DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME\n"
+        + "or, with the -q option:\n"
+        + "QUOTA REM_QUOTA SPACE_QUOTA REM_SPACE_QUOTA\n"
+        + "      DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME\n"
+        + "The -h option shows file sizes in human readable format.\n"
+        + "The -v option displays a header line.";
+
+    assertEquals("Count.getDescription", expected, actual);
+  }
+
 
   // mock content system
   static class MockContentSummary extends ContentSummary {
-    
-    public MockContentSummary() {}
+
+    public MockContentSummary() {
+    }
 
     @Override
     public String toString(boolean qOption, boolean hOption) {
       if (qOption) {
         if (hOption) {
-          return(HUMAN + WITH_QUOTAS);
+          return (HUMAN + WITH_QUOTAS);
         } else {
-          return(BYTES + WITH_QUOTAS);
+          return (BYTES + WITH_QUOTAS);
         }
       } else {
         if (hOption) {
-          return(HUMAN + NO_QUOTAS);
+          return (HUMAN + NO_QUOTAS);
         } else {
-          return(BYTES + NO_QUOTAS);
+          return (BYTES + NO_QUOTAS);
         }
       }
     }

hadoop-common-project/hadoop-common/src/test/resources/testConf.xml

@@ -238,7 +238,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-count \[-q\] \[-h\] &lt;path&gt; \.\.\. :( )*</expected-output>
+          <expected-output>^-count \[-q\] \[-h\] \[-v\] &lt;path&gt; \.\.\. :( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
@@ -250,20 +250,28 @@
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME or( )*</expected-output>
+          <expected-output>^( |\t)*DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*QUOTA REMAINING_QUOTA SPACE_QUOTA REMAINING_SPACE_QUOTA( )*</expected-output>
+          <expected-output>^( |\t)*or, with the -q option:( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME( )*</expected-output>
+          <expected-output>^( |\t)*QUOTA REM_QUOTA SPACE_QUOTA REM_SPACE_QUOTA( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
           <expected-output>^( |\t)*The -h option shows file sizes in human readable format.( )*</expected-output>
         </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*The -v option displays a header line.( )*</expected-output>
+        </comparator>
       </comparators>
     </test>
 

hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml

@@ -8699,6 +8699,48 @@
       </comparators>
     </test>
 
+    <test> <!-- TESTED -->
+      <description>count: file using absolute path showing header record</description>
+      <test-commands>
+        <command>-fs NAMENODE -touchz /file1</command>
+        <command>-fs NAMENODE -count -v /file1</command>
+      </test-commands>
+      <cleanup-commands>
+        <command>-fs NAMENODE -rm /file1</command>
+      </cleanup-commands>
+      <comparators>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>( |\t)*DIR_COUNT   FILE_COUNT       CONTENT_SIZE PATHNAME</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>( |\t)*0( |\t)*1( |\t)*0 /file1</expected-output>
+        </comparator>
+      </comparators>
+    </test>
+
+    <test> <!-- TESTED -->
+      <description>count: file using absolute path with -q option and showing header record</description>
+      <test-commands>
+        <command>-fs NAMENODE -touchz /file1</command>
+        <command>-fs NAMENODE -count -q -v /file1</command>
+      </test-commands>
+      <cleanup-commands>
+        <command>-fs NAMENODE -rm /file1</command>
+      </cleanup-commands>
+      <comparators>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>( |\t)*QUOTA       REM_QUOTA     SPACE_QUOTA REM_SPACE_QUOTA    DIR_COUNT   FILE_COUNT       CONTENT_SIZE PATHNAME</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>( |\t)*none( |\t)*inf( |\t)*none( |\t)*inf( |\t)*0( |\t)*1( |\t)*0 /file1</expected-output>
+        </comparator>
+      </comparators>
+    </test>
+
     <!-- Tests for chmod -->
     <test> <!-- TESTED -->
       <description>chmod: change permission(octal mode) of file in absolute path</description>