svn merge -c 1095761 from trunk for HADOOP-7233.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/yahoo-merge@1126778 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -32,6 +32,9 @@ Trunk (unreleased changes)
     HADOOP-7230. Move "fs -help" shell command tests from HDFS to COMMOM; see
     also HDFS-1844.  (Daryn Sharp via szetszwo)
 
+    HADOOP-7233. Refactor ls to conform to new FsCommand class.  (Daryn Sharp
+    via szetszwo)
+
   OPTIMIZATIONS
 
   BUG FIXES

src/java/org/apache/hadoop/fs/FsShell.java

@@ -609,87 +609,6 @@ public class FsShell extends Configured implements Tool {
     }
   }
     
-    
-  /**
-   * Get a listing of all files in that match the file pattern <i>srcf</i>.
-   * @param srcf a file pattern specifying source files
-   * @param recursive if need to list files in subdirs
-   * @throws IOException  
-   * @see org.apache.hadoop.fs.FileSystem#globStatus(Path)
-   */
-  private int ls(String srcf, boolean recursive) throws IOException {
-    Path srcPath = new Path(srcf);
-    FileSystem srcFs = srcPath.getFileSystem(this.getConf());
-    FileStatus[] srcs = srcFs.globStatus(srcPath);
-    if (srcs==null || srcs.length==0) {
-      throw new FileNotFoundException("Cannot access " + srcf + 
-          ": No such file or directory.");
-    }
- 
-    boolean printHeader = (srcs.length == 1) ? true: false;
-    int numOfErrors = 0;
-    for(int i=0; i<srcs.length; i++) {
-      numOfErrors += ls(srcs[i], srcFs, recursive, printHeader);
-    }
-    return numOfErrors == 0 ? 0 : -1;
-  }
-
-  /* list all files under the directory <i>src</i>
-   * ideally we should provide "-l" option, that lists like "ls -l".
-   */
-  private int ls(FileStatus src, FileSystem srcFs, boolean recursive,
-      boolean printHeader) throws IOException {
-    final String cmd = recursive? "lsr": "ls";
-    final FileStatus[] items = shellListStatus(cmd, srcFs, src);
-    if (items == null) {
-      return 1;
-    } else {
-      int numOfErrors = 0;
-      if (!recursive && printHeader) {
-        if (items.length != 0) {
-          System.out.println("Found " + items.length + " items");
-        }
-      }
-      
-      int maxReplication = 3, maxLen = 10, maxOwner = 0,maxGroup = 0;
-
-      for(int i = 0; i < items.length; i++) {
-        FileStatus stat = items[i];
-        int replication = String.valueOf(stat.getReplication()).length();
-        int len = String.valueOf(stat.getLen()).length();
-        int owner = String.valueOf(stat.getOwner()).length();
-        int group = String.valueOf(stat.getGroup()).length();
-        
-        if (replication > maxReplication) maxReplication = replication;
-        if (len > maxLen) maxLen = len;
-        if (owner > maxOwner)  maxOwner = owner;
-        if (group > maxGroup)  maxGroup = group;
-      }
-      
-      for (int i = 0; i < items.length; i++) {
-        FileStatus stat = items[i];
-        Path cur = stat.getPath();
-        String mdate = dateForm.format(new Date(stat.getModificationTime()));
-        
-        System.out.print((stat.isDirectory() ? "d" : "-") + 
-          stat.getPermission() + " ");
-        System.out.printf("%"+ maxReplication + 
-          "s ", (stat.isFile() ? stat.getReplication() : "-"));
-        if (maxOwner > 0)
-          System.out.printf("%-"+ maxOwner + "s ", stat.getOwner());
-        if (maxGroup > 0)
-          System.out.printf("%-"+ maxGroup + "s ", stat.getGroup());
-        System.out.printf("%"+ maxLen + "d ", stat.getLen());
-        System.out.print(mdate + " ");
-        System.out.println(cur.toUri().getPath());
-        if (recursive && stat.isDirectory()) {
-          numOfErrors += ls(stat,srcFs, recursive, printHeader);
-        }
-      }
-      return numOfErrors;
-    }
-  }
-
    /**
    * Show the size of a partition in the filesystem that contains
    * the specified <i>path</i>.
@@ -1387,7 +1306,7 @@ public class FsShell extends Configured implements Tool {
     String summary = "hadoop fs is the command to execute fs commands. " +
       "The full syntax is: \n\n" +
       "hadoop fs [-fs <local | file system URI>] [-conf <configuration file>]\n\t" +
-      "[-D <property=value>] [-ls <path>] [-lsr <path>] [-df [<path>]] [-du [-s] [-h] <path>]\n\t" +
+      "[-D <property=value>] [-df [<path>]] [-du [-s] [-h] <path>]\n\t" +
       "[-dus <path>] [-mv <src> <dst>] [-cp <src> <dst>] [-rm [-skipTrash] <src>]\n\t" + 
       "[-rmr [-skipTrash] <src>] [-put <localsrc> ... <dst>] [-copyFromLocal <localsrc> ... <dst>]\n\t" +
       "[-moveFromLocal <localsrc> ... <dst>] [" + 
@@ -1416,21 +1335,6 @@ public class FsShell extends Configured implements Tool {
       "\t\tappear first on the command line.  Exactly one additional\n" +
       "\t\targument must be specified. \n";
 
-        
-    String ls = "-ls <path>: \tList the contents that match the specified file pattern. If\n" + 
-      "\t\tpath is not specified, the contents of /user/<currentUser>\n" +
-      "\t\twill be listed. Directory entries are of the form \n" +
-      "\t\t\tdirName (full path) <dir> \n" +
-      "\t\tand file entries are of the form \n" + 
-      "\t\t\tfileName(full path) <r n> size \n" +
-      "\t\twhere n is the number of replicas specified for the file \n" + 
-      "\t\tand size is the size of the file, in bytes.\n";
-
-    String lsr = "-lsr <path>: \tRecursively list the contents that match the specified\n" +
-      "\t\tfile pattern.  Behaves very similarly to hadoop fs -ls,\n" + 
-      "\t\texcept that the data is shown for all the entries in the\n" +
-      "\t\tsubtree.\n";
-
     String df = "-df [<path>]: \tShows the capacity, free and used space of the filesystem.\n"+
       "\t\tIf the filesystem has multiple partitions, and no path to a particular partition\n"+
       "\t\tis specified, then the status of the root partitions will be shown.\n";
@@ -1560,17 +1464,13 @@ public class FsShell extends Configured implements Tool {
 
     Command instance = commandFactory.getInstance("-" + cmd);
     if (instance != null) {
-      System.out.println(instance.getDescription());
+      printHelp(instance);
     } else if ("fs".equals(cmd)) {
       System.out.println(fs);
     } else if ("conf".equals(cmd)) {
       System.out.println(conf);
     } else if ("D".equals(cmd)) {
       System.out.println(D);
-    } else if ("ls".equals(cmd)) {
-      System.out.println(ls);
-    } else if ("lsr".equals(cmd)) {
-      System.out.println(lsr);
     } else if ("df".equals(cmd)) {
       System.out.println(df);
     } else if ("du".equals(cmd)) {
@@ -1631,13 +1531,11 @@ public class FsShell extends Configured implements Tool {
       System.out.println(summary);
       for (String thisCmdName : commandFactory.getNames()) {
         instance = commandFactory.getInstance(thisCmdName);
-        System.out.println(instance.getUsage());
+        System.out.println("\t[" + instance.getUsage() + "]");
       }
       System.out.println("\t[-help [cmd]]\n");
       
       System.out.println(fs);
-      System.out.println(ls);
-      System.out.println(lsr);
       System.out.println(df);
       System.out.println(du);
       System.out.println(dus);
@@ -1665,14 +1563,29 @@ public class FsShell extends Configured implements Tool {
       System.out.println(chgrp);
 
       for (String thisCmdName : commandFactory.getNames()) {
-        instance = commandFactory.getInstance(thisCmdName);
-        System.out.println(instance.getDescription());
+        printHelp(commandFactory.getInstance(thisCmdName));
       }
 
       System.out.println(help);
     }        
   }
 
+  // TODO: will eventually auto-wrap the text, but this matches the expected
+  // output for the hdfs tests...
+  private void printHelp(Command instance) {
+    boolean firstLine = true;
+    for (String line : instance.getDescription().split("\n")) {
+      String prefix;
+      if (firstLine) {
+        prefix = instance.getUsage() + ":\t";
+        firstLine = false;
+      } else {
+        prefix = "\t\t";
+      }
+      System.out.println(prefix + line);
+    }    
+  }
+  
   /**
    * Apply operation specified by 'cmd' on all parameters
    * starting from argv[startindex].
@@ -1707,10 +1620,6 @@ public class FsShell extends Configured implements Tool {
           delete(argv[i], true, rmSkipTrash);
         } else if ("-df".equals(cmd)) {
           df(argv[i]);
-        } else if ("-ls".equals(cmd)) {
-          exitCode = ls(argv[i], false);
-        } else if ("-lsr".equals(cmd)) {
-          exitCode = ls(argv[i], true);
         } else if ("-touchz".equals(cmd)) {
           touchz(argv[i]);
         } else if ("-text".equals(cmd)) {
@@ -1766,8 +1675,7 @@ public class FsShell extends Configured implements Tool {
     } else if ("-D".equals(cmd)) {
       System.err.println("Usage: java FsShell" + 
                          " [-D <[property=value>]");
-    } else if ("-ls".equals(cmd) || "-lsr".equals(cmd) ||
-               "-du".equals(cmd) || "-dus".equals(cmd) ||
+    } else if ("-du".equals(cmd) || "-dus".equals(cmd) ||
                "-touchz".equals(cmd) || "-mkdir".equals(cmd) ||
                "-text".equals(cmd)) {
       System.err.println("Usage: java FsShell" + 
@@ -1807,8 +1715,6 @@ public class FsShell extends Configured implements Tool {
       System.err.println("Usage: java FsShell [" + TAIL_USAGE + "]");
     } else {
       System.err.println("Usage: java FsShell");
-      System.err.println("           [-ls <path>]");
-      System.err.println("           [-lsr <path>]");
       System.err.println("           [-df [<path>]]");
       System.err.println("           [-du [-s] [-h] <path>]");
       System.err.println("           [-dus <path>]");
@@ -1948,18 +1854,6 @@ public class FsShell extends Configured implements Tool {
                  "-chown".equals(cmd) ||
                  "-chgrp".equals(cmd)) {
         exitCode = FsShellPermissions.changePermissions(fs, cmd, argv, i, this);
-      } else if ("-ls".equals(cmd)) {
-        if (i < argv.length) {
-          exitCode = doall(cmd, argv, i);
-        } else {
-          exitCode = ls(Path.CUR_DIR, false);
-        } 
-      } else if ("-lsr".equals(cmd)) {
-        if (i < argv.length) {
-          exitCode = doall(cmd, argv, i);
-        } else {
-          exitCode = ls(Path.CUR_DIR, true);
-        } 
       } else if ("-mv".equals(cmd)) {
         exitCode = rename(argv, getConf());
       } else if ("-cp".equals(cmd)) {

src/java/org/apache/hadoop/fs/shell/Command.java

@@ -79,6 +79,14 @@ abstract public class Command extends Configured {
     name = cmdName;
   }
   
+  protected void setRecursive(boolean flag) {
+    recursive = flag;
+  }
+  
+  protected boolean isRecursive() {
+    return recursive;
+  }
+
   /** 
    * Execute the command on the input path
    * 
@@ -138,7 +146,8 @@ abstract public class Command extends Configured {
       displayError(e);
     }
     
-    return (numErrors == 0) ? exitCode : 1;
+    // TODO: -1 should be reserved for syntax error, 1 should be failure
+    return (numErrors == 0) ? exitCode : -1;
   }
 
   /**
@@ -244,7 +253,18 @@ abstract public class Command extends Configured {
   protected void processNonexistentPathArgument(PathData item)
   throws IOException {
     // TODO: this should be more posix-like: ex. "No such file or directory"
-    throw new FileNotFoundException("Can not find listing for " + item);
+    throw new FileNotFoundException(getFnfText(item.path));
+  }
+
+  /**
+   *  TODO: A crutch until the text is standardized across commands...
+   *  Eventually an exception that takes the path as an argument will
+   *  replace custom text
+   *  @param path the thing that doesn't exist
+   *  @returns String in printf format
+   */
+  protected String getFnfText(Path path) {
+    throw new RuntimeException(path + ": No such file or directory");
   }
 
   /**
@@ -309,7 +329,7 @@ abstract public class Command extends Configured {
     
     if (stats != null && stats.length == 0) { // glob failed to match
       // TODO: this should be more posix-like: ex. "No such file or directory"
-      throw new FileNotFoundException("Can not find listing for " + pattern);
+      throw new FileNotFoundException(getFnfText(path));
     }
     
     List<PathData> items = new LinkedList<PathData>();
@@ -371,7 +391,9 @@ abstract public class Command extends Configured {
    * @return "name options"
    */
   public String getUsage() {
-    return getCommandField("USAGE");
+    String cmd = "-" + getCommandName();
+    String usage = getCommandField("USAGE");
+    return usage.isEmpty() ? cmd : cmd + " " + usage; 
   }
 
   /**

src/java/org/apache/hadoop/fs/shell/Count.java

@@ -26,6 +26,7 @@ import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.ContentSummary;
 import org.apache.hadoop.fs.FsShell;
+import org.apache.hadoop.fs.Path;
 
 /**
  * Count the number of directories, files, bytes, quota, and remaining quota.
@@ -43,13 +44,13 @@ public class Count extends FsCommand {
   }
 
   public static final String NAME = "count";
-  public static final String USAGE = "-" + NAME + " [-q] <path> ...";
-  public static final String DESCRIPTION = CommandUtils.formatDescription(USAGE, 
-      "Count the number of directories, files and bytes under the paths",
-      "that match the specified file pattern.  The output columns are:",
-      "DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME or",
-      "QUOTA REMAINING_QUATA SPACE_QUOTA REMAINING_SPACE_QUOTA ",
-      "      DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME");
+  public static final String USAGE = "[-q] <path> ...";
+  public static final String DESCRIPTION = 
+      "Count the number of directories, files and bytes under the paths\n" +
+      "that match the specified file pattern.  The output columns are:\n" +
+      "DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME or\n" +
+      "QUOTA REMAINING_QUATA SPACE_QUOTA REMAINING_SPACE_QUOTA \n" +
+      "      DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME";
   
   private boolean showQuotas;
 
@@ -86,4 +87,10 @@ public class Count extends FsCommand {
     ContentSummary summary = src.fs.getContentSummary(src.path);
     out.println(summary.toString(showQuotas) + src.path);
   }
+
+  // TODO: remove when the error is commonized...
+  @Override
+  protected String getFnfText(Path path) {
+    return "Can not find listing for " + path;
+  }
 }

src/java/org/apache/hadoop/fs/shell/FsCommand.java

@@ -42,7 +42,8 @@ abstract public class FsCommand extends Command {
    * @param factory where to register the class
    */
   public static void registerCommands(CommandFactory factory) {
-    Count.registerCommands(factory);
+    factory.registerCommands(Count.class);
+    factory.registerCommands(Ls.class);
   }
 
   protected FsCommand() {}

src/java/org/apache/hadoop/fs/shell/Ls.java

@@ -0,0 +1,155 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.shell;
+
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.LinkedList;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * Get a listing of all files in that match the file patterns.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+
+class Ls extends FsCommand {
+  public static void registerCommands(CommandFactory factory) {
+    factory.addClass(Ls.class, "-ls");
+    factory.addClass(Lsr.class, "-lsr");
+  }
+  
+  public static final String NAME = "ls";
+  public static final String USAGE = "[<path> ...]";
+  public static final String DESCRIPTION =
+    "List the contents that match the specified file pattern. If\n" + 
+    "path is not specified, the contents of /user/<currentUser>\n" +
+    "will be listed. Directory entries are of the form \n" +
+    "\tdirName (full path) <dir> \n" +
+    "and file entries are of the form \n" + 
+    "\tfileName(full path) <r n> size \n" +
+    "where n is the number of replicas specified for the file \n" + 
+    "and size is the size of the file, in bytes.";
+
+  protected static final SimpleDateFormat dateFormat = 
+    new SimpleDateFormat("yyyy-MM-dd HH:mm");
+
+  protected int maxRepl = 3, maxLen = 10, maxOwner = 0, maxGroup = 0;
+  protected String lineFormat;
+
+  @Override
+  protected void processOptions(LinkedList<String> args)
+  throws IOException {
+    CommandFormat cf = new CommandFormat(null, 0, Integer.MAX_VALUE, "R");
+    cf.parse(args);
+    setRecursive(cf.getOpt("R"));
+    if (args.isEmpty()) args.add(Path.CUR_DIR);
+  }
+
+  @Override
+  protected void processPaths(PathData parent, PathData ... items)
+  throws IOException {
+    // implicitly recurse once for cmdline directories
+    if (parent == null && items[0].stat.isDirectory()) {
+      recursePath(items[0]);
+      return;
+    }
+
+    if (!isRecursive() && items.length != 0) {
+      out.println("Found " + items.length + " items");
+    }
+    adjustColumnWidths(items);
+    super.processPaths(parent, items);
+  }
+
+  @Override
+  protected void processPath(PathData item) throws IOException {
+    FileStatus stat = item.stat;
+    String line = String.format(lineFormat,
+        (stat.isDirectory() ? "d" : "-"),
+        stat.getPermission(),
+        (stat.isFile() ? stat.getReplication() : "-"),
+        stat.getOwner(),
+        stat.getGroup(),
+        stat.getLen(),
+        dateFormat.format(new Date(stat.getModificationTime())),
+        item.path.toUri().getPath()
+    );
+    out.println(line);
+  }
+
+  /**
+   * Compute column widths and rebuild the format string
+   * @param items to find the max field width for each column
+   */
+  private void adjustColumnWidths(PathData items[]) {
+    for (PathData item : items) {
+      FileStatus stat = item.stat;
+      maxRepl  = maxLength(maxRepl, stat.getReplication());
+      maxLen   = maxLength(maxLen, stat.getLen());
+      maxOwner = maxLength(maxOwner, stat.getOwner());
+      maxGroup = maxLength(maxGroup, stat.getGroup());
+    }
+
+    StringBuilder fmt = new StringBuilder();
+    fmt.append("%s%s "); // permission string
+    fmt.append("%"  + maxRepl  + "s ");
+    fmt.append("%-" + maxOwner + "s ");
+    fmt.append("%-" + maxGroup + "s ");
+    fmt.append("%"  + maxLen   + "d ");
+    fmt.append("%s %s"); // mod time & path
+    lineFormat = fmt.toString();
+  }
+
+  private int maxLength(int n, Object value) {
+    return Math.max(n, (value != null) ? String.valueOf(value).length() : 0);
+  }
+
+  // TODO: remove when the error is commonized...
+  @Override
+  protected String getFnfText(Path path) {
+    return "Cannot access " + path.toUri() + ": No such file or directory.";
+  }
+
+  /**
+   * Get a recursive listing of all files in that match the file patterns.
+   * Same as "-ls -R"
+   */
+  public static class Lsr extends Ls {
+    public static final String NAME = "lsr";
+    public static final String USAGE = Ls.USAGE;
+    public static final String DESCRIPTION =
+      "Recursively list the contents that match the specified\n" +
+      "file pattern.  Behaves very similarly to hadoop fs -ls,\n" + 
+      "except that the data is shown for all the entries in the\n" +
+      "subtree.";
+
+    @Override
+    protected void processOptions(LinkedList<String> args)
+    throws IOException {
+      args.addFirst("-R");
+      super.processOptions(args);
+    }
+  }
+}

src/test/core/org/apache/hadoop/cli/testConf.xml

@@ -54,7 +54,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-ls &lt;path&gt;:( |\t)*List the contents that match the specified file pattern. If( )*</expected-output>
+          <expected-output>^-ls \[&lt;path&gt; \.\.\.\]:( |\t)*List the contents that match the specified file pattern. If( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
@@ -101,7 +101,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-lsr &lt;path&gt;:( |\t)*Recursively list the contents that match the specified( )*</expected-output>
+          <expected-output>^-lsr \[&lt;path&gt; \.\.\.\]:( |\t)*Recursively list the contents that match the specified( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
@@ -222,7 +222,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-count \[-q\] &lt;path&gt; \.\.\.: Count the number of directories, files and bytes under the paths( )*</expected-output>
+          <expected-output>^-count \[-q\] &lt;path&gt; \.\.\.:( |\t)*Count the number of directories, files and bytes under the paths( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>