HADOOP-1436. Improved command line APIs. Contributed by Enis Soztutar.

git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk@567824 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -61,6 +61,10 @@ Trunk (unreleased changes)
     HADOOP-1231.  Add generics to Mapper and Reducer interfaces.
     (tomwhite via cutting)
 
+    HADOOP-1436.  Improved command-line APIs, so that all tools need
+    not subclass ToolBase, and generic parameter parser is public.
+    (Enis Soztutar via cutting)
+
 
 Release 0.14.0 - 2007-08-17
 

src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamedMerge.java

@@ -22,7 +22,6 @@ import java.io.DataOutputStream;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.LineNumberInputStream;
 import java.io.OutputStream;
 import java.net.ServerSocket;
 import java.net.Socket;
@@ -32,10 +31,11 @@ import java.util.Arrays;
 import junit.framework.TestCase;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FsShell;
 import org.apache.hadoop.dfs.MiniDFSCluster;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsShell;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.ToolRunner;
 
 /**
  * This JUnit test is not pure-Java and is not run as 
@@ -105,7 +105,7 @@ public class TestStreamedMerge extends TestCase {
   void lsr() {
     try {
       System.out.println("lsr /");
-      new FsShell().doMain(conf_, new String[]{ "-lsr", "/" });
+      ToolRunner.run(conf_, new FsShell(), new String[]{ "-lsr", "/" });
     } catch (Exception e) {
       e.printStackTrace();
     }
@@ -156,6 +156,7 @@ public class TestStreamedMerge extends TestCase {
         Socket client;
         InputStream in;
       
+        @Override
         InputStream connectInputStream() throws IOException {
           listen = new ServerSocket(SOC_PORT);
           client = listen.accept();
@@ -163,6 +164,7 @@ public class TestStreamedMerge extends TestCase {
           return in;
         }
       
+        @Override
         void close() throws IOException
         {
           listen.close();
@@ -184,6 +186,7 @@ public class TestStreamedMerge extends TestCase {
     
     abstract void close() throws IOException;
     
+    @Override
     public void run() {
       try {
         in_ = connectInputStream();

src/java/org/apache/hadoop/conf/Configured.java

@@ -23,6 +23,11 @@ public class Configured implements Configurable {
 
   private Configuration conf;
 
+  /** Construct a Configured. */
+  public Configured() {
+    this(null);
+  }
+  
   /** Construct a Configured. */
   public Configured(Configuration conf) {
     setConf(conf);

src/java/org/apache/hadoop/dfs/DFSAdmin.java

@@ -19,12 +19,13 @@ package org.apache.hadoop.dfs;
 
 import java.io.IOException;
 
-import org.apache.hadoop.fs.FsShell;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ipc.RemoteException;
-import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.dfs.DistributedFileSystem.DiskStatus;
 import org.apache.hadoop.dfs.FSConstants.UpgradeAction;
+import org.apache.hadoop.fs.FsShell;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.util.ToolRunner;
 
 /**
  * This class provides some DFS administrative access.
@@ -35,9 +36,16 @@ public class DFSAdmin extends FsShell {
    * Construct a DFSAdmin object.
    */
   public DFSAdmin() {
-    super();
+    this(null);
   }
 
+  /**
+   * Construct a DFSAdmin object.
+   */
+  public DFSAdmin(Configuration conf) {
+    super(conf);
+  }
+  
   /**
    * Gives a report on how the FileSystem is doing.
    * @exception IOException if the filesystem does not exist.
@@ -325,6 +333,7 @@ public class DFSAdmin extends FsShell {
    * @exception Exception if the filesystem does not exist.
    * @return 0 on success, non zero on error.
    */
+  @Override
   public int run(String[] argv) throws Exception {
 
     if (argv.length < 1) {
@@ -442,7 +451,7 @@ public class DFSAdmin extends FsShell {
    * @exception Exception if the filesystem does not exist.
    */
   public static void main(String[] argv) throws Exception {
-    int res = new DFSAdmin().doMain(new Configuration(), argv);
+    int res = ToolRunner.run(new DFSAdmin(), argv);
     System.exit(res);
   }
 }

src/java/org/apache/hadoop/dfs/DFSck.java

@@ -25,10 +25,12 @@ import java.net.URL;
 import java.net.URLConnection;
 import java.net.URLEncoder;
 
-import org.apache.commons.logging.*;
-
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.ToolBase;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
 
 /**
  * This class provides rudimentary checking of DFS volumes for errors and
@@ -53,7 +55,7 @@ import org.apache.hadoop.util.ToolBase;
  *  factors of each file.
  *  
  */
-public class DFSck extends ToolBase {
+public class DFSck extends Configured implements Tool {
   private static final Log LOG = LogFactory.getLog(DFSck.class.getName());
 
   DFSck() {}
@@ -64,13 +66,13 @@ public class DFSck extends ToolBase {
    * @throws Exception
    */
   public DFSck(Configuration conf) throws Exception {
-    setConf(conf);
+    super(conf);
   }
   
   private String getInfoServer() throws IOException {
     InetSocketAddress addr = 
-      DataNode.createSocketAddr(conf.get("fs.default.name"));
-    int infoPort = conf.getInt("dfs.info.port", 50070);
+      DataNode.createSocketAddr(getConf().get("fs.default.name"));
+    int infoPort = getConf().getInt("dfs.info.port", 50070);
     return addr.getHostName() + ":" + infoPort;
   }
   
@@ -121,7 +123,7 @@ public class DFSck extends ToolBase {
   }
 
   public static void main(String[] args) throws Exception {
-    int res = new DFSck().doMain(new Configuration(), args);
+    int res = ToolRunner.run(new DFSck(new Configuration()), args);
     System.exit(res);
   }
 }

src/java/org/apache/hadoop/fs/FsShell.java

@@ -18,16 +18,19 @@
 package org.apache.hadoop.fs;
 
 import java.io.*;
-import java.util.*;
 import java.text.DecimalFormat;
 import java.text.SimpleDateFormat;
+import java.util.*;
 
-import org.apache.hadoop.conf.*;
-import org.apache.hadoop.ipc.*;
-import org.apache.hadoop.util.ToolBase;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
 
 /** Provide command line access to a FileSystem. */
-public class FsShell extends ToolBase {
+public class FsShell extends Configured implements Tool {
 
   protected FileSystem fs;
   private Trash trash;
@@ -45,17 +48,22 @@ public class FsShell extends ToolBase {
   /**
    */
   public FsShell() {
+    this(null);
+  }
+
+  public FsShell(Configuration conf) {
+    super(conf);
     fs = null;
     trash = null;
   }
-
+  
   protected void init() throws IOException {
-    conf.setQuietMode(true);
+    getConf().setQuietMode(true);
     if (this.fs == null) {
-      this.fs = FileSystem.get(conf);
+      this.fs = FileSystem.get(getConf());
     }
     if (this.trash == null) {
-      this.trash = new Trash(conf);
+      this.trash = new Trash(getConf());
     }
   }
 
@@ -64,7 +72,7 @@ public class FsShell extends ToolBase {
    */
   private void copyBytes(InputStream in, OutputStream out) throws IOException {
     PrintStream ps = out instanceof PrintStream ? (PrintStream)out : null;
-    byte buf[] = new byte[conf.getInt("io.file.buffer.size", 4096)];
+    byte buf[] = new byte[getConf().getInt("io.file.buffer.size", 4096)];
     int bytesRead = in.read(buf);
     while (bytesRead >= 0) {
       out.write(buf, 0, bytesRead);
@@ -268,10 +276,10 @@ public class FsShell extends ToolBase {
     for(int i=0; i<srcs.length; i++) {
       if (endline) {
         FileUtil.copyMerge(fs, srcs[i], 
-                           FileSystem.getLocal(conf), dst, false, conf, "\n");
+                           FileSystem.getLocal(getConf()), dst, false, getConf(), "\n");
       } else {
         FileUtil.copyMerge(fs, srcs[i], 
-                           FileSystem.getLocal(conf), dst, false, conf, null);
+                           FileSystem.getLocal(getConf()), dst, false, getConf(), null);
       }
     }
   }      
@@ -301,6 +309,7 @@ public class FsShell extends ToolBase {
     //  zzz
 
     new DelayedExceptionThrowing() {
+      @Override
       void process(Path p) throws IOException {
         printToStdout(p);
       }
@@ -851,6 +860,7 @@ public class FsShell extends ToolBase {
     //  rm: cannot remove `y.txt': No such file or directory
 
     new DelayedExceptionThrowing() {
+      @Override
       void process(Path p) throws IOException {
         delete(p, recursive);
       }
@@ -1321,49 +1331,49 @@ public class FsShell extends ToolBase {
         else
           copyMergeToLocal(argv[i++], new Path(argv[i++]));
       } else if ("-cat".equals(cmd)) {
-        exitCode = doall(cmd, argv, conf, i);
+        exitCode = doall(cmd, argv, getConf(), i);
       } else if ("-moveToLocal".equals(cmd)) {
         moveToLocal(argv[i++], new Path(argv[i++]));
       } else if ("-setrep".equals(cmd)) {
         setReplication(argv, i);           
       } else if ("-ls".equals(cmd)) {
         if (i < argv.length) {
-          exitCode = doall(cmd, argv, conf, i);
+          exitCode = doall(cmd, argv, getConf(), i);
         } else {
           ls(Path.CUR_DIR, false);
         } 
       } else if ("-lsr".equals(cmd)) {
         if (i < argv.length) {
-          exitCode = doall(cmd, argv, conf, i);
+          exitCode = doall(cmd, argv, getConf(), i);
         } else {
           ls(Path.CUR_DIR, true);
         } 
       } else if ("-mv".equals(cmd)) {
-        exitCode = rename(argv, conf);
+        exitCode = rename(argv, getConf());
       } else if ("-cp".equals(cmd)) {
-        exitCode = copy(argv, conf);
+        exitCode = copy(argv, getConf());
       } else if ("-rm".equals(cmd)) {
-        exitCode = doall(cmd, argv, conf, i);
+        exitCode = doall(cmd, argv, getConf(), i);
       } else if ("-rmr".equals(cmd)) {
-        exitCode = doall(cmd, argv, conf, i);
+        exitCode = doall(cmd, argv, getConf(), i);
       } else if ("-expunge".equals(cmd)) {
         expunge();
       } else if ("-du".equals(cmd)) {
         if (i < argv.length) {
-          exitCode = doall(cmd, argv, conf, i);
+          exitCode = doall(cmd, argv, getConf(), i);
         } else {
           du("");
         }
       } else if ("-dus".equals(cmd)) {
         if (i < argv.length) {
-          exitCode = doall(cmd, argv, conf, i);
+          exitCode = doall(cmd, argv, getConf(), i);
         } else {
           dus("");
         }         
       } else if ("-mkdir".equals(cmd)) {
-        exitCode = doall(cmd, argv, conf, i);
+        exitCode = doall(cmd, argv, getConf(), i);
       } else if ("-touchz".equals(cmd)) {
-        exitCode = doall(cmd, argv, conf, i);
+        exitCode = doall(cmd, argv, getConf(), i);
       } else if ("-test".equals(cmd)) {
         exitCode = test(argv, i);
       } else if ("-stat".equals(cmd)) {
@@ -1426,7 +1436,7 @@ public class FsShell extends ToolBase {
     FsShell shell = new FsShell();
     int res;
     try {
-      res = shell.doMain(new Configuration(), argv);
+      res = ToolRunner.run(shell, argv);
     } finally {
       shell.close();
     }

src/java/org/apache/hadoop/fs/s3/MigrationTool.java

@@ -26,9 +26,10 @@ import java.net.URLEncoder;
 import java.util.Set;
 import java.util.TreeSet;
 
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.util.ToolBase;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
 import org.jets3t.service.S3Service;
 import org.jets3t.service.S3ServiceException;
 import org.jets3t.service.impl.rest.httpclient.RestS3Service;
@@ -46,13 +47,13 @@ import org.jets3t.service.security.AWSCredentials;
  * - no datafiles are touched.
  * </p>
  */
-public class MigrationTool extends ToolBase {
+public class MigrationTool extends Configured implements Tool {
   
   private S3Service s3Service;
   private S3Bucket bucket;
   
   public static void main(String[] args) throws Exception {
-    int res = new MigrationTool().doMain(new Configuration(), args);
+    int res = ToolRunner.run(new MigrationTool(), args);
     System.exit(res);
   }
   
@@ -66,10 +67,10 @@ public class MigrationTool extends ToolBase {
     
     URI uri = URI.create(args[0]);
     
-    initialize(uri, conf);
+    initialize(uri);
     
     FileSystemStore newStore = new Jets3tFileSystemStore();
-    newStore.initialize(uri, conf);
+    newStore.initialize(uri, getConf());
     
     if (get("%2F") != null) { 
       System.err.println("Current version number is [unversioned].");
@@ -105,9 +106,9 @@ public class MigrationTool extends ToolBase {
     
   }
   
-  public void initialize(URI uri, Configuration conf) throws IOException {
+  public void initialize(URI uri) throws IOException {
+    
     
-    this.conf = conf;
     
     try {
       String accessKey = null;
@@ -123,10 +124,10 @@ public class MigrationTool extends ToolBase {
         }
       }
       if (accessKey == null) {
-        accessKey = conf.get("fs.s3.awsAccessKeyId");
+        accessKey = getConf().get("fs.s3.awsAccessKeyId");
       }
       if (secretAccessKey == null) {
-        secretAccessKey = conf.get("fs.s3.awsSecretAccessKey");
+        secretAccessKey = getConf().get("fs.s3.awsSecretAccessKey");
       }
       if (accessKey == null && secretAccessKey == null) {
         throw new IllegalArgumentException("AWS " +

src/java/org/apache/hadoop/mapred/JobClient.java

@@ -17,18 +17,46 @@
  */
 package org.apache.hadoop.mapred;
 
-import org.apache.commons.logging.*;
-
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.io.*;
-import org.apache.hadoop.io.retry.*;
-import org.apache.hadoop.ipc.*;
-import org.apache.hadoop.conf.*;
-import org.apache.hadoop.util.*;
-import org.apache.hadoop.filecache.*;
-import java.io.*;
-import java.net.*;
-import java.util.*;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.net.InetSocketAddress;
+import java.net.SocketTimeoutException;
+import java.net.URI;
+import java.net.URL;
+import java.net.URLConnection;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.filecache.DistributedCache;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.io.retry.RetryPolicies;
+import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.hadoop.io.retry.RetryProxy;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
 
 /*******************************************************
  * JobClient interacts with the JobTracker network interface.
@@ -37,7 +65,7 @@ import java.util.*;
  * with the networked job system.
  *
  *******************************************************/
-public class JobClient extends ToolBase implements MRConstants  {
+public class JobClient extends Configured implements MRConstants, Tool  {
   private static final Log LOG = LogFactory.getLog("org.apache.hadoop.mapred.JobClient");
   public static enum TaskStatusFilter { NONE, FAILED, SUCCEEDED, ALL }
   private TaskStatusFilter taskOutputFilter = TaskStatusFilter.FAILED; 
@@ -171,6 +199,7 @@ public class JobClient extends ToolBase implements MRConstants  {
     /**
      * Dump stats to screen
      */
+    @Override
     public String toString() {
       try {
         ensureFreshStatus();
@@ -261,7 +290,7 @@ public class JobClient extends ToolBase implements MRConstants  {
   public synchronized FileSystem getFs() throws IOException {
     if (this.fs == null) {
       String fsName = jobSubmitClient.getFilesystemName();
-      this.fs = FileSystem.getNamed(fsName, this.conf);
+      this.fs = FileSystem.getNamed(fsName, getConf());
     }
     return fs;
   }
@@ -824,7 +853,7 @@ public class JobClient extends ToolBase implements MRConstants  {
   /**
    */
   public static void main(String argv[]) throws Exception {
-    int res = new JobClient().doMain(new Configuration(), argv);
+    int res = ToolRunner.run(new JobClient(), argv);
     System.exit(res);
   }
 }

src/java/org/apache/hadoop/util/CopyFiles.java

@@ -60,7 +60,7 @@ import org.apache.hadoop.mapred.SequenceFileInputFormat;
  * A Map-reduce program to recursively copy directories between
  * different file-systems.
  */
-public class CopyFiles extends ToolBase {
+public class CopyFiles implements Tool {
   private static final String HDFS = "hdfs";
   private static final String S3 = "s3";
   
@@ -74,6 +74,8 @@ public class CopyFiles extends ToolBase {
   private static final String readFailuresAttribute = 
     "distcp.ignore.read.failures";
   
+  private JobConf conf;
+  
   public void setConf(Configuration conf) {
     if (conf instanceof JobConf) {
       this.conf = (JobConf) conf;
@@ -82,6 +84,17 @@ public class CopyFiles extends ToolBase {
     }
   }
   
+  public Configuration getConf() {
+    return conf;
+  }
+  
+  public CopyFiles() {
+  }
+  
+  public CopyFiles(Configuration conf) {
+    setConf(conf);
+  }
+  
   /**
    * Base-class for all mappers for distcp
    */
@@ -250,6 +263,7 @@ public class CopyFiles extends ToolBase {
      * @param logPath : The log Path.
      * @param ignoreReadFailures : Ignore read failures?
      */
+    @Override
     public void setup(Configuration conf, JobConf jobConf, 
                       String[] srcPaths, String destPath, 
                       Path logPath, boolean ignoreReadFailures) 
@@ -354,6 +368,7 @@ public class CopyFiles extends ToolBase {
       
     }
     
+    @Override
     public void cleanup(Configuration conf, JobConf jobConf, 
                         String srcPath, String destPath) 
       throws IOException
@@ -372,6 +387,7 @@ public class CopyFiles extends ToolBase {
      * top-level paths on source and destination directories.
      * Gets the named file systems, to be used later in map.
      */
+    @Override
     public void configure(JobConf job) 
     {
       String srcfs = job.get("copy.src.fs", "local");
@@ -421,6 +437,7 @@ public class CopyFiles extends ToolBase {
       }
     }
     
+    @Override
     public void close() {
       // nothing
     }
@@ -445,6 +462,7 @@ public class CopyFiles extends ToolBase {
      * @param logPath : The log Path.
      * @param ignoreReadFailures : Ignore read failures?
      */
+    @Override
     public void setup(Configuration conf, JobConf jobConf, 
                       String[] srcPaths, String destPath, 
                       Path logPath, boolean ignoreReadFailures) 
@@ -491,6 +509,7 @@ public class CopyFiles extends ToolBase {
       }
     }	
     
+    @Override
     public void cleanup(Configuration conf, JobConf jobConf, 
                         String srcPath, String destPath) 
       throws IOException
@@ -504,6 +523,7 @@ public class CopyFiles extends ToolBase {
       }
     }
     
+    @Override
     public void configure(JobConf job)
     {
       //Save jobConf
@@ -861,9 +881,10 @@ public class CopyFiles extends ToolBase {
   }
   
   public static void main(String[] args) throws Exception {
-    int res = new CopyFiles().doMain(
-                                     new JobConf(new Configuration(), CopyFiles.class), 
-                                     args);
+    JobConf job = new JobConf(new Configuration(), CopyFiles.class);
+    CopyFiles distcp = new CopyFiles();
+    distcp.setConf(job);
+    int res = ToolRunner.run(distcp, args);
     System.exit(res);
   }
 }

src/java/org/apache/hadoop/util/GenericOptionsParser.java

@@ -0,0 +1,213 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+
+/*************************************************************
+ * This class can be used as a utility to parse command line
+ * arguments generic to the Hadoop framework. This class 
+ * recognizes several command line arguments, which allow a user 
+ * to specify a namenode, a job tracker etc. Generic options 
+ * supported are 
+ * <p>-conf <configuration file>     specify an application configuration file
+ * <p>-D <property=value>            use value for given property
+ * <p>-fs <local|namenode:port>      specify a namenode
+ * <p>-jt <local|jobtracker:port>    specify a job tracker
+ * <br>
+ * <p>The general command line syntax is
+ * <p>bin/hadoop command [genericOptions] [commandOptions]
+ * <br>
+ * Generic command line arguments <strong>might</strong> modify 
+ * <code>Configuration </code> objects, given to constructors  
+ * <br><br>
+ * The functionality is implemented using Commons CLI.
+ * <br>
+ * <p>Examples using generic options are
+ * <p>bin/hadoop dfs -fs darwin:8020 -ls /data
+ * <p><blockquote><pre>
+ *     list /data directory in dfs with namenode darwin:8020
+ * </pre></blockquote>
+ * <p>bin/hadoop dfs -D fs.default.name=darwin:8020 -ls /data
+ * <p><blockquote><pre>
+ *     list /data directory in dfs with namenode darwin:8020
+ * </pre></blockquote>
+ * <p>bin/hadoop dfs -conf hadoop-site.xml -ls /data
+ * <p><blockquote><pre>
+ *     list /data directory in dfs with conf specified in hadoop-site.xml
+ * </pre></blockquote>
+ * <p>bin/hadoop job -D mapred.job.tracker=darwin:50020 -submit job.xml
+ * <p><blockquote><pre>
+ *     submit a job to job tracker darwin:50020
+ * </pre></blockquote>
+ * <p>bin/hadoop job -jt darwin:50020 -submit job.xml
+ * <p><blockquote><pre>
+ *     submit a job to job tracker darwin:50020
+ * </pre></blockquote>
+ * <p>bin/hadoop job -jt local -submit job.xml
+ * <p><blockquote><pre>
+ *     submit a job to local runner
+ * </pre></blockquote>
+ *
+ * @see Tool
+ * @see ToolRunner
+ */
+public class GenericOptionsParser {
+
+  private static final Log LOG = LogFactory.getLog(GenericOptionsParser.class);
+
+  private CommandLine commandLine;
+
+  /** Instantinates a GenericOptionsParser to parse only
+   * the generic Hadoop  arguments. The array of string arguments 
+   * other than the generic arguments can be obtained by 
+   * {@link #getRemainingArgs()}
+   * @param conf the configuration to modify
+   * @param args User-specified arguments
+   */
+  public GenericOptionsParser(Configuration conf, String[] args) {
+    this(conf, new Options(), args); 
+  }
+
+  /** 
+   * Instantinates a GenericOptionsParser to parse given options 
+   * as well as generic Hadoop options. The resulting <code>
+   * CommandLine</code> object can be obtained by {@link #getCommandLine()}
+   * @param conf the configuration to modify  
+   * @param options options built by the caller 
+   * @param args User-specified arguments
+   */
+  public GenericOptionsParser(Configuration conf, Options options, String[] args) {
+    parseGeneralOptions(options, conf, args);
+  }
+
+  /**
+   * Returns an array of Strings containing only command-specific 
+   * arguments.
+   * @return String array of remaining arguments not parsed
+   */
+  public String[] getRemainingArgs() {
+    return commandLine.getArgs();
+  }
+
+  /**
+   * Returns the commons-cli <code>CommandLine</code> object 
+   * to process the parsed arguments. Note : if the object is 
+   * created with <code>GenericCommandLineParser(Configuration, String[])</code>, 
+   * then returned object will only contain parsed generic 
+   * options.
+   * @return CommandLine object representing list of arguments 
+   * parsed against Options descriptor.
+   */
+  public CommandLine getCommandLine() {
+    return commandLine;
+  }
+
+  /**
+   * Specify properties of each generic option
+   */
+  @SuppressWarnings("static-access")
+  private Options buildGeneralOptions(Options opts) {
+    Option fs = OptionBuilder.withArgName("local|namenode:port")
+    .hasArg()
+    .withDescription("specify a namenode")
+    .create("fs");
+    Option jt = OptionBuilder.withArgName("local|jobtracker:port")
+    .hasArg()
+    .withDescription("specify a job tracker")
+    .create("jt");
+    Option oconf = OptionBuilder.withArgName("configuration file")
+    .hasArg()
+    .withDescription("specify an application configuration file")
+    .create("conf");
+    Option property = OptionBuilder.withArgName("property=value")
+    .hasArgs()
+    .withArgPattern("=", 1)
+    .withDescription("use value for given property")
+    .create('D');
+
+    opts.addOption(fs);
+    opts.addOption(jt);
+    opts.addOption(oconf);
+    opts.addOption(property);
+    
+    return opts;
+  }
+
+  /**
+   * Modify configuration according user-specified generic options
+   * @param conf Configuration to be modified
+   * @param line User-specified generic options
+   */
+  private void processGeneralOptions(Configuration conf,
+      CommandLine line) {
+    if (line.hasOption("fs")) {
+      conf.set("fs.default.name", line.getOptionValue("fs"));
+    }
+
+    if (line.hasOption("jt")) {
+      conf.set("mapred.job.tracker", line.getOptionValue("jt"));
+    }
+    if (line.hasOption("conf")) {
+      conf.addFinalResource(new Path(line.getOptionValue("conf")));
+    }
+    if (line.hasOption('D')) {
+      String[] property = line.getOptionValues('D');
+      for(int i=0; i<property.length-1; i=i+2) {
+        if (property[i]!=null)
+          conf.set(property[i], property[i+1]);
+      }
+    }
+  }
+
+  /**
+   * Parse the user-specified options, get the generic options, and modify
+   * configuration accordingly
+   * @param conf Configuration to be modified
+   * @param args User-specified arguments
+   * @return Command-specific arguments
+   */
+  private String[] parseGeneralOptions(Options opts, Configuration conf, 
+      String[] args) {
+    opts = buildGeneralOptions(opts);
+    CommandLineParser parser = new GnuParser();
+    try {
+      commandLine = parser.parse(opts, args, true);
+      processGeneralOptions(conf, commandLine);
+      return commandLine.getArgs();
+    } catch(ParseException e) {
+      LOG.warn("options parsing failed: "+e.getMessage());
+
+      HelpFormatter formatter = new HelpFormatter();
+      formatter.printHelp("general options are: ", opts);
+    }
+    return args;
+  }
+
+}

src/java/org/apache/hadoop/util/Tool.java

@@ -26,7 +26,7 @@ import org.apache.hadoop.conf.Configurable;
  */
 public interface Tool extends Configurable {
   /**
-   * execute the command with the given arguments
+   * Execute the command with the given arguments.
    * @param args command specific arguments
    * @return exit code
    * @throws Exception

src/java/org/apache/hadoop/util/ToolBase.java

@@ -18,70 +18,19 @@
 
 package org.apache.hadoop.util;
 
-import java.io.IOException;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.GnuParser;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
 
 /*************************************************************
- * This is a base class to support generic commonad options.
- * Generic command options allow a user to specify a namenode,
- * a job tracker etc. Generic options supported are 
- * <p>-conf <configuration file>     specify an application configuration file
- * <p>-D <property=value>            use value for given property
- * <p>-fs <local|namenode:port>      specify a namenode
- * <p>-jt <local|jobtracker:port>    specify a job tracker
- * <br>
- * <p>The general command line syntax is
- * <p>bin/hadoop command [genericOptions] [commandOptions]
- * 
- * <p>For every tool that inherits from ToolBase, generic options are 
- * handled by ToolBase while command options are passed to the tool.
- * Generic options handling is implemented using Common CLI.
- * 
- * <p>Tools that inherit from ToolBase in Hadoop are
- * DFSShell, DFSck, JobClient, and CopyFiles.
- * <br>
- * <p>Examples using generic options are
- * <p>bin/hadoop dfs -fs darwin:8020 -ls /data
- * <p><blockquote><pre>
- *     list /data directory in dfs with namenode darwin:8020
- * </pre></blockquote>
- * <p>bin/hadoop dfs -D fs.default.name=darwin:8020 -ls /data
- * <p><blockquote><pre>
- *     list /data directory in dfs with namenode darwin:8020
- * </pre></blockquote>
- * <p>bin/hadoop dfs -conf hadoop-site.xml -ls /data
- * <p><blockquote><pre>
- *     list /data directory in dfs with conf specified in hadoop-site.xml
- * </pre></blockquote>
- * <p>bin/hadoop job -D mapred.job.tracker=darwin:50020 -submit job.xml
- * <p><blockquote><pre>
- *     submit a job to job tracker darwin:50020
- * </pre></blockquote>
- * <p>bin/hadoop job -jt darwin:50020 -submit job.xml
- * <p><blockquote><pre>
- *     submit a job to job tracker darwin:50020
- * </pre></blockquote>
- * <p>bin/hadoop job -jt local -submit job.xml
- * <p><blockquote><pre>
- *     submit a job to local runner
- * </pre></blockquote>
- *        
+ * @deprecated This class is depracated. Classes 
+ * extending ToolBase should rather implement {@link Tool} 
+ * interface, and use {@link ToolRunner} for execution 
+ * functionality. Alternativelly, {@link GenericOptionsParser} 
+ * can be used to parse generic arguments related to hadoop 
+ * framework. 
  */
+@Deprecated
 public abstract class ToolBase implements Tool {
-  private static final Log LOG = LogFactory.getLog(
-                                                   "org.apache.hadoop.util.ToolBase");
+  
   public Configuration conf;
 
   public void setConf(Configuration conf) {
@@ -91,87 +40,7 @@ public abstract class ToolBase implements Tool {
   public Configuration getConf() {
     return conf;
   }
-    
-  /*
-   * Specify properties of each generic option
-   */
-  static private Options buildGeneralOptions() {
-    Option fs = OptionBuilder.withArgName("local|namenode:port")
-      .hasArg()
-      .withDescription("specify a namenode")
-      .create("fs");
-    Option jt = OptionBuilder.withArgName("local|jobtracker:port")
-      .hasArg()
-      .withDescription("specify a job tracker")
-      .create("jt");
-    Option oconf = OptionBuilder.withArgName("configuration file")
-      .hasArg()
-      .withDescription("specify an application configuration file")
-      .create("conf");
-    Option property = OptionBuilder.withArgName("property=value")
-      .hasArgs()
-      .withArgPattern("=", 1)
-      .withDescription("use value for given property")
-      .create('D');
-    Options opts = new Options();
-    opts.addOption(fs);
-    opts.addOption(jt);
-    opts.addOption(oconf);
-    opts.addOption(property);
-        
-    return opts;
-  }
-    
-  /*
-   * Modify configuration according user-specified generic options
-   * @param conf Configuration to be modified
-   * @param line User-specified generic options
-   */
-  static private void processGeneralOptions(Configuration conf,
-                                            CommandLine line) {
-    if (line.hasOption("fs")) {
-      conf.set("fs.default.name", line.getOptionValue("fs"));
-    }
-        
-    if (line.hasOption("jt")) {
-      conf.set("mapred.job.tracker", line.getOptionValue("jt"));
-    }
-    if (line.hasOption("conf")) {
-      conf.addFinalResource(new Path(line.getOptionValue("conf")));
-    }
-    if (line.hasOption('D')) {
-      String[] property = line.getOptionValues('D');
-      for(int i=0; i<property.length-1; i=i+2) {
-        if (property[i]!=null)
-          conf.set(property[i], property[i+1]);
-      }
-    }           
-  }
- 
-  /**
-   * Parse the user-specified options, get the generic options, and modify
-   * configuration accordingly
-   * @param conf Configuration to be modified
-   * @param args User-specified arguments
-   * @return Commoand-specific arguments
-   */
-  static private String[] parseGeneralOptions(Configuration conf, 
-                                              String[] args) {
-    Options opts = buildGeneralOptions();
-    CommandLineParser parser = new GnuParser();
-    try {
-      CommandLine line = parser.parse(opts, args, true);
-      processGeneralOptions(conf, line);
-      return line.getArgs();
-    } catch(ParseException e) {
-      LOG.warn("options parsing failed: "+e.getMessage());
-
-      HelpFormatter formatter = new HelpFormatter();
-      formatter.printHelp("general options are: ", opts);
-    }
-    return args;
-  }
-
+     
   /**
    * Work as a main program: execute a command and handle exception if any
    * @param conf Application default configuration
@@ -182,9 +51,7 @@ public abstract class ToolBase implements Tool {
    * positive non-zero values can be used to return application-specific codes.
    */
   public final int doMain(Configuration conf, String[] args) throws Exception {
-    String [] commandOptions = parseGeneralOptions(conf, args);
-    setConf(conf);
-    return this.run(commandOptions);
+    return ToolRunner.run(this, args);
   }
 
 }

src/java/org/apache/hadoop/util/ToolRunner.java

@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * ToolRunner can be used to run classes implementing {@link Tool}
+ * interface. Static method {@link #run(Tool, String[])} is used.
+ * {@link GenericOptionsParser} is used to parse the hadoop generic 
+ * arguments to modify the <code>Configuration</code>.
+ */
+public class ToolRunner {
+ 
+  /**
+   * Runs the given Tool by {@link Tool#run(String[])}, with the 
+   * given arguments. Uses the given configuration, or builds one if null.
+   * Sets the possibly modified version of the conf by Tool#setConf()  
+   * 
+   * @param conf Configuration object to use
+   * @param tool The Tool to run
+   * @param args the arguments to the tool(including generic arguments
+   * , see {@link GenericOptionsParser})
+   * @return exit code of the {@link Tool#run(String[])} method
+   */
+  public static int run(Configuration conf, Tool tool, String[] args) 
+    throws Exception{
+    if(conf == null) {
+      conf = new Configuration();
+    }
+    GenericOptionsParser parser = new GenericOptionsParser(conf, args);
+    //set the configuration back, so that Tool can configure itself
+    tool.setConf(conf);
+    
+    //get the args w/o generic hadoop args
+    String[] toolArgs = parser.getRemainingArgs();
+    return tool.run(toolArgs);
+  }
+  
+  /**
+   * Runs the tool with the tool's Configuration
+   * Equivalent to <code>run(tool.getConf(), tool, args)</code>.
+   * @param tool The Tool to run
+   * @param args the arguments to the tool(including generic arguments
+   * , see {@link GenericOptionsParser})
+   * @return exit code of the {@link Tool#run(String[])} method
+   */
+  public static int run(Tool tool, String[] args) 
+    throws Exception{
+    return run(tool.getConf(), tool, args);
+  }
+  
+}

src/test/org/apache/hadoop/dfs/MiniDFSCluster.java

@@ -17,14 +17,17 @@
  */
 package org.apache.hadoop.dfs;
 
-import java.io.*;
-import java.net.*;
+import java.io.File;
+import java.io.IOException;
+import java.net.InetSocketAddress;
 import java.util.ArrayList;
 import java.util.Collection;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.dfs.FSConstants.StartupOption;
-import org.apache.hadoop.fs.*;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.util.ToolRunner;
 
 /**
  * This class creates a single-process DFS cluster for junit testing.
@@ -215,7 +218,7 @@ public class MiniDFSCluster {
       throw new IllegalStateException("Attempting to finalize "
                                       + "Namenode but it is not running");
     }
-    new DFSAdmin().doMain(conf, new String[] {"-finalizeUpgrade"});
+    ToolRunner.run(new DFSAdmin(conf), new String[] {"-finalizeUpgrade"});
   }
   
   /**

src/test/org/apache/hadoop/dfs/TestDFSShellGenericOptions.java

@@ -25,11 +25,10 @@ import java.io.PrintWriter;
 import junit.framework.TestCase;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FsShell;
-import org.apache.hadoop.dfs.DataNode;
-import org.apache.hadoop.dfs.MiniDFSCluster;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsShell;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.ToolRunner;
 
 public class TestDFSShellGenericOptions extends TestCase {
 
@@ -99,7 +98,7 @@ public class TestDFSShellGenericOptions extends TestCase {
     FsShell shell=new FsShell();
     FileSystem fs=null;
     try {
-      shell.doMain(new Configuration(), args);
+      ToolRunner.run(shell, args);
       fs = new DistributedFileSystem(
                                      DataNode.createSocketAddr(namenode), 
                                      shell.getConf());

src/test/org/apache/hadoop/dfs/TestFsck.java

@@ -20,9 +20,12 @@ package org.apache.hadoop.dfs;
 
 import java.io.ByteArrayOutputStream;
 import java.io.PrintStream;
-import junit.framework.*;
+
+import junit.framework.TestCase;
+
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.*;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.util.ToolRunner;
 
 /**
  * A JUnit test for doing fsck
@@ -35,9 +38,11 @@ public class TestFsck extends TestCase {
 
   
   
+  @Override
   protected void setUp() throws Exception {
   }
 
+  @Override
   protected void tearDown() throws Exception {
   }
   
@@ -54,7 +59,7 @@ public class TestFsck extends TestCase {
       ByteArrayOutputStream bStream = new ByteArrayOutputStream();
       PrintStream newOut = new PrintStream(bStream, true);
       System.setOut(newOut);
-      assertEquals(0, new DFSck().doMain(conf, new String[] {"/"}));
+      assertEquals(0, ToolRunner.run(new DFSck(conf), new String[] {"/"}));
       System.setOut(oldOut);
       String outStr = bStream.toString();
       assertTrue(-1 != outStr.indexOf("HEALTHY"));
@@ -67,7 +72,7 @@ public class TestFsck extends TestCase {
       bStream = new ByteArrayOutputStream();
       newOut = new PrintStream(bStream, true);
       System.setOut(newOut);
-      assertEquals(0, new DFSck().doMain(conf, new String[] {"/"}));
+      assertEquals(0, ToolRunner.run(new DFSck(conf), new String[] {"/"}));
       System.setOut(oldOut);
       outStr = bStream.toString();
       // expect the result is corrupt
@@ -95,7 +100,7 @@ public class TestFsck extends TestCase {
       ByteArrayOutputStream bStream = new ByteArrayOutputStream();
       PrintStream newOut = new PrintStream(bStream, true);
       System.setOut(newOut);
-      assertEquals(0, new DFSck().doMain(conf, new String[] {"/non-existent"}));
+      assertEquals(0, ToolRunner.run(new DFSck(conf), new String[] {"/non-existent"}));
       System.setOut(oldOut);
       String outStr = bStream.toString();
       assertEquals(-1, outStr.indexOf("HEALTHY"));

src/test/org/apache/hadoop/fs/TestCopyFiles.java

@@ -19,12 +19,15 @@
 package org.apache.hadoop.fs;
 
 import java.io.IOException;
-import java.util.Random;
 import java.net.URI;
-import junit.framework.*;
+import java.util.Random;
+
+import junit.framework.TestCase;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.dfs.MiniDFSCluster;
 import org.apache.hadoop.util.CopyFiles;
+import org.apache.hadoop.util.ToolRunner;
 
 
 /**
@@ -83,9 +86,11 @@ public class TestCopyFiles extends TestCase {
 
   
   
+  @Override
   protected void setUp() throws Exception {
   }
 
+  @Override
   protected void tearDown() throws Exception {
   }
   
@@ -163,7 +168,7 @@ public class TestCopyFiles extends TestCase {
   /** copy files from local file system to local file system */
   public void testCopyFromLocalToLocal() throws Exception {
     MyFile[] files = createFiles("local", TEST_ROOT_DIR+"/srcdat");
-    new CopyFiles().doMain(new Configuration(),
+    ToolRunner.run(new CopyFiles(new Configuration()),
                            new String[] {"file://"+TEST_ROOT_DIR+"/srcdat",
                                          "file://"+TEST_ROOT_DIR+"/destdat"});
     assertTrue("Source and destination directories do not match.",
@@ -182,7 +187,7 @@ public class TestCopyFiles extends TestCase {
       namenode = conf.get("fs.default.name", "local");
       if (!"local".equals(namenode)) {
         MyFile[] files = createFiles(namenode, "/srcdat");
-        new CopyFiles().doMain(conf, new String[] {"hdfs://"+namenode+"/srcdat",
+        ToolRunner.run(new CopyFiles(conf), new String[] {"hdfs://"+namenode+"/srcdat",
                                                    "hdfs://"+namenode+"/destdat",
                                                    "-log",
                                                    "hdfs://"+namenode+"/logs"});
@@ -210,7 +215,7 @@ public class TestCopyFiles extends TestCase {
       namenode = conf.get("fs.default.name", "local");
       if (!"local".equals(namenode)) {
         MyFile[] files = createFiles("local", TEST_ROOT_DIR+"/srcdat");
-        new CopyFiles().doMain(conf, new String[] {"file://"+TEST_ROOT_DIR+"/srcdat",
+        ToolRunner.run(new CopyFiles(conf), new String[] {"file://"+TEST_ROOT_DIR+"/srcdat",
                                                    "hdfs://"+namenode+"/destdat",
                                                    "-log",
                                                    "hdfs://"+namenode+"/logs"});
@@ -238,7 +243,7 @@ public class TestCopyFiles extends TestCase {
       namenode = conf.get("fs.default.name", "local");
       if (!"local".equals(namenode)) {
         MyFile[] files = createFiles(namenode, "/srcdat");
-        new CopyFiles().doMain(conf, new String[] {"hdfs://"+namenode+"/srcdat",
+        ToolRunner.run(new CopyFiles(conf), new String[] {"hdfs://"+namenode+"/srcdat",
                                                    "file://"+TEST_ROOT_DIR+"/destdat",
                                                    "-log",
                                                    "/logs"});

src/test/org/apache/hadoop/fs/TestFileSystem.java

@@ -18,17 +18,31 @@
 
 package org.apache.hadoop.fs;
 
-import java.io.*;
-import java.util.*;
-import junit.framework.TestCase;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Arrays;
+import java.util.Random;
 
-import org.apache.commons.logging.*;
+import junit.framework.TestCase;
 
-import org.apache.hadoop.mapred.*;
-import org.apache.hadoop.mapred.lib.*;
-import org.apache.hadoop.io.*;
+import org.apache.commons.logging.Log;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.UTF8;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
 import org.apache.hadoop.io.SequenceFile.CompressionType;
-import org.apache.hadoop.conf.*;
+import org.apache.hadoop.mapred.InputFormatBase;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.SequenceFileInputFormat;
+import org.apache.hadoop.mapred.lib.LongSumReducer;
 
 public class TestFileSystem extends TestCase {
   private static final Log LOG = InputFormatBase.LOG;