svn merge -c 1096988 from trunk for HADOOP-7235.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/yahoo-merge@1126782 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -35,6 +35,9 @@ Trunk (unreleased changes)
     HADOOP-7233. Refactor ls to conform to new FsCommand class.  (Daryn Sharp
     via szetszwo)
 
+    HADOOP-7235. Refactor the tail command to conform to new FsCommand class.
+    (Daryn Sharp via szetszwo)
+
   OPTIMIZATIONS
 
   BUG FIXES

src/java/org/apache/hadoop/fs/FsShell.java

@@ -72,7 +72,6 @@ public class FsShell extends Configured implements Tool {
   static final String GET_SHORT_USAGE = "-get [-ignoreCrc] [-crc] <src> <localdst>";
   static final String COPYTOLOCAL_SHORT_USAGE = GET_SHORT_USAGE.replace(
       "-get", "-copyToLocal");
-  static final String TAIL_USAGE="-tail [-f] <file>";
   static final String DU_USAGE="-du [-s] [-h] <paths...>";
 
   /**
@@ -1128,57 +1127,6 @@ public class FsShell extends Configured implements Tool {
     return getTrash().getCurrentTrashDir();
   }
 
-  /**
-   * Parse the incoming command string
-   * @param cmd
-   * @param pos ignore anything before this pos in cmd
-   * @throws IOException 
-   */
-  private void tail(String[] cmd, int pos) throws IOException {
-    CommandFormat c = new CommandFormat("tail", 1, 1, "f");
-    String src = null;
-    Path path = null;
-
-    try {
-      List<String> parameters = c.parse(cmd, pos);
-      src = parameters.get(0);
-    } catch(IllegalArgumentException iae) {
-      System.err.println("Usage: java FsShell " + TAIL_USAGE);
-      throw iae;
-    }
-    boolean foption = c.getOpt("f") ? true: false;
-    path = new Path(src);
-    FileSystem srcFs = path.getFileSystem(getConf());
-    FileStatus fileStatus = srcFs.getFileStatus(path);
-    if (fileStatus.isDirectory()) {
-      throw new IOException("Source must be a file.");
-    }
-
-    long fileSize = fileStatus.getLen();
-    long offset = (fileSize > 1024) ? fileSize - 1024: 0;
-
-    while (true) {
-      FSDataInputStream in = srcFs.open(path);
-      try {
-        in.seek(offset);
-        IOUtils.copyBytes(in, System.out, 1024);
-        offset = in.getPos();
-      } finally {
-        in.close();
-      }
-      if (!foption) {
-        break;
-      }
-      fileSize = srcFs.getFileStatus(path).getLen();
-      offset = (fileSize > offset) ? offset: fileSize;
-      try {
-        Thread.sleep(5000);
-      } catch (InterruptedException e) {
-        break;
-      }
-    }
-  }
-
   /**
    * This class runs a command on a given FileStatus. This can be used for
    * running various commands like chmod, chown etc.
@@ -1315,7 +1263,7 @@ public class FsShell extends Configured implements Tool {
       "[" + COPYTOLOCAL_SHORT_USAGE + "] [-moveToLocal <src> <localdst>]\n\t" +
       "[-mkdir <path>] [-report] [" + SETREP_SHORT_USAGE + "]\n\t" +
       "[-touchz <path>] [-test -[ezd] <path>] [-stat [format] <path>]\n\t" +
-      "[-tail [-f] <path>] [-text <path>]\n\t" +
+      "[-text <path>]\n\t" +
       "[" + FsShellPermissions.CHMOD_USAGE + "]\n\t" +
       "[" + FsShellPermissions.CHOWN_USAGE + "]\n\t" +
       "[" + FsShellPermissions.CHGRP_USAGE + "]";
@@ -1421,10 +1369,6 @@ public class FsShell extends Configured implements Tool {
       "\t\tin the specified format. Format accepts filesize in blocks (%b), filename (%n),\n" +
       "\t\tblock size (%o), replication (%r), modification date (%y, %Y)\n";
 
-    String tail = TAIL_USAGE
-      + ":  Show the last 1KB of the file. \n"
-      + "\t\tThe -f option shows appended data as the file grows. \n";
-
     String chmod = FsShellPermissions.CHMOD_USAGE + "\n" +
       "\t\tChanges permissions of a file.\n" +
       "\t\tThis works similar to shell's chmod with a few exceptions.\n\n" +
@@ -1517,8 +1461,6 @@ public class FsShell extends Configured implements Tool {
       System.out.println(text);
     } else if ("stat".equals(cmd)) {
       System.out.println(stat);
-    } else if ("tail".equals(cmd)) {
-      System.out.println(tail);
     } else if ("chmod".equals(cmd)) {
       System.out.println(chmod);
     } else if ("chown".equals(cmd)) {
@@ -1553,7 +1495,6 @@ public class FsShell extends Configured implements Tool {
       System.out.println(moveToLocal);
       System.out.println(mkdir);
       System.out.println(setrep);
-      System.out.println(tail);
       System.out.println(touchz);
       System.out.println(test);
       System.out.println(text);
@@ -1711,8 +1652,6 @@ public class FsShell extends Configured implements Tool {
     } else if ("-stat".equals(cmd)) {
       System.err.println("Usage: java FsShell" +
                          " [-stat [format] <path>]");
-    } else if ("-tail".equals(cmd)) {
-      System.err.println("Usage: java FsShell [" + TAIL_USAGE + "]");
     } else {
       System.err.println("Usage: java FsShell");
       System.err.println("           [-df [<path>]]");
@@ -1737,7 +1676,6 @@ public class FsShell extends Configured implements Tool {
       System.err.println("           [-touchz <path>]");
       System.err.println("           [-test -[ezd] <path>]");
       System.err.println("           [-stat [format] <path>]");
-      System.err.println("           [" + TAIL_USAGE + "]");
       System.err.println("           [" + FsShellPermissions.CHMOD_USAGE + "]");      
       System.err.println("           [" + FsShellPermissions.CHOWN_USAGE + "]");
       System.err.println("           [" + FsShellPermissions.CHGRP_USAGE + "]");
@@ -1892,8 +1830,6 @@ public class FsShell extends Configured implements Tool {
         } else {
           printHelp("");
         }
-      } else if ("-tail".equals(cmd)) {
-        tail(argv, i);           
       } else {
         exitCode = -1;
         System.err.println(cmd.substring(1) + ": Unknown command");

src/java/org/apache/hadoop/fs/shell/Command.java

@@ -259,12 +259,13 @@ abstract public class Command extends Configured {
   /**
    *  TODO: A crutch until the text is standardized across commands...
    *  Eventually an exception that takes the path as an argument will
-   *  replace custom text
+   *  replace custom text, until then, commands can supply custom text
+   *  for backwards compatibility
    *  @param path the thing that doesn't exist
    *  @returns String in printf format
    */
   protected String getFnfText(Path path) {
-    throw new RuntimeException(path + ": No such file or directory");
+    return path + ": No such file or directory";
   }
 
   /**

src/java/org/apache/hadoop/fs/shell/FsCommand.java

@@ -44,6 +44,7 @@ abstract public class FsCommand extends Command {
   public static void registerCommands(CommandFactory factory) {
     factory.registerCommands(Count.class);
     factory.registerCommands(Ls.class);
+    factory.registerCommands(Tail.class);
   }
 
   protected FsCommand() {}

src/java/org/apache/hadoop/fs/shell/Tail.java

@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.shell;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+
+/**
+ * Get a listing of all files in that match the file patterns.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+
+class Tail extends FsCommand {
+  public static void registerCommands(CommandFactory factory) {
+    factory.addClass(Tail.class, "-tail");
+  }
+  
+  public static final String NAME = "tail";
+  public static final String USAGE = "[-f] <file>";
+  public static final String DESCRIPTION =
+    "Show the last 1KB of the file.\n" +
+    "\t\tThe -f option shows appended data as the file grows.\n";
+
+  private long startingOffset = -1024;
+  private boolean follow = false;
+  private long followDelay = 5000; // milliseconds
+  
+  @Override
+  protected void processOptions(LinkedList<String> args) throws IOException {
+    CommandFormat cf = new CommandFormat(null, 1, 1, "f");
+    cf.parse(args);
+    follow = cf.getOpt("f");
+  }
+
+  // TODO: HADOOP-7234 will add glob support; for now, be backwards compat
+  @Override
+  protected List<PathData> expandArgument(String arg) throws IOException {
+    Path path = new Path(arg);
+    FileSystem fs = path.getFileSystem(getConf());
+    
+    List<PathData> items = new LinkedList<PathData>();
+    items.add(new PathData(fs, path));
+    return items;
+  }
+      
+  @Override
+  protected void processPath(PathData item) throws IOException {
+    if (item.stat.isDirectory()) {
+      throw new IOException("Source must be a file.");
+    }
+
+    long offset = dumpFromOffset(item, startingOffset);
+    while (follow) {
+      try {
+        Thread.sleep(followDelay);
+      } catch (InterruptedException e) {
+        break;
+      }
+      offset = dumpFromOffset(item, offset);
+    }
+  }
+
+  private long dumpFromOffset(PathData item, long offset) throws IOException {
+    long fileSize = item.refreshStatus().getLen();
+    if (offset > fileSize) return fileSize;
+    // treat a negative offset as relative to end of the file, floor of 0
+    if (offset < 0) {
+      offset = Math.max(fileSize + offset, 0);
+    }
+    
+    FSDataInputStream in = item.fs.open(item.path);
+    try {
+      in.seek(offset);
+      // use conf so the system configured io block size is used
+      IOUtils.copyBytes(in, System.out, getConf(), false);
+      offset = in.getPos();
+    } finally {
+      in.close();
+    }
+    return offset;
+  }
+  
+  @Override
+  protected String getFnfText(Path path) {
+    return "File does not exist: " + path;
+  }
+}

src/test/core/org/apache/hadoop/cli/testConf.xml

@@ -606,7 +606,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-tail \[-f\] &lt;file&gt;:  Show the last 1KB of the file.( )*</expected-output>
+          <expected-output>^-tail \[-f\] &lt;file&gt;:( |\t)+Show the last 1KB of the file.( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>