HADOOP-3962. Shell command fs -count should support paths with different file systems. (Tsz Wo (Nicholas), SZE via mahadev)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/core/trunk@686840 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -329,6 +329,9 @@ Trunk (unreleased changes)
     HADOOP-3933. DataNode sometimes sends up to io.byte.per.checksum bytes 
     more than required to client. (Ning Li via rangadi)
 
+    HADOOP-3962. Shell command "fs -count" should support paths with different
+    file systems. (Tsz Wo (Nicholas), SZE via mahadev)
+
 Release 0.18.0 - 2008-08-19
 
   INCOMPATIBLE CHANGES

src/core/org/apache/hadoop/fs/FsShell.java

@@ -1522,7 +1522,7 @@ public class FsShell extends Configured implements Tool {
         } else if ("-dus".equals(cmd)) {
           dus(argv[i]);
         } else if (Count.matches(cmd)) {
-          new Count(argv, i, fs).runAll();
+          new Count(argv, i, getConf()).runAll();
         } else if ("-ls".equals(cmd)) {
           ls(argv[i], false);
         } else if ("-lsr".equals(cmd)) {
@@ -1772,7 +1772,7 @@ public class FsShell extends Configured implements Tool {
           dus(".");
         }         
       } else if (Count.matches(cmd)) {
-        exitCode = new Count(argv, i, fs).runAll();
+        exitCode = new Count(argv, i, getConf()).runAll();
       } else if ("-mkdir".equals(cmd)) {
         exitCode = doall(cmd, argv, i);
       } else if ("-touchz".equals(cmd)) {

src/core/org/apache/hadoop/fs/shell/Command.java

@@ -19,6 +19,8 @@ package org.apache.hadoop.fs.shell;
 
 import java.io.*;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -27,13 +29,12 @@ import org.apache.hadoop.ipc.RemoteException;
 /**
  * An abstract class for the execution of a file system command
  */
-abstract public class Command {
-  final protected FileSystem fs;
+abstract public class Command extends Configured {
   protected String[] args;
   
   /** Constructor */
-  protected Command(FileSystem fs) {
-    this.fs = fs;
+  protected Command(Configuration conf) {
+    super(conf);
   }
   
   /** Return the command's name excluding the leading character - */
@@ -57,6 +58,7 @@ abstract public class Command {
     for (String src : args) {
       try {
         Path srcPath = new Path(src);
+        FileSystem fs = srcPath.getFileSystem(getConf());
         FileStatus[] statuses = fs.globStatus(srcPath);
         if (statuses == null) {
           System.err.println("Can not find listing for " + src);

src/core/org/apache/hadoop/fs/shell/Count.java

@@ -20,6 +20,7 @@ package org.apache.hadoop.fs.shell;
 import java.io.*;
 import java.util.List;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 
@@ -41,10 +42,9 @@ public class Count extends Command {
    * 
    * @param cmd the count command
    * @param pos the starting index of the arguments 
-   * @param fs the file system handler
    */
-  public Count(String[] cmd, int pos, FileSystem fs) {
-    super(fs);
+  public Count(String[] cmd, int pos, Configuration conf) {
+    super(conf);
     CommandFormat c = new CommandFormat(NAME, 1, Integer.MAX_VALUE, "q");
     List<String> parameters = c.parse(cmd, pos);
     this.args = parameters.toArray(new String[parameters.size()]);
@@ -70,6 +70,7 @@ public class Count extends Command {
 
   @Override
   protected void run(Path path) throws IOException {
+    FileSystem fs = path.getFileSystem(getConf());
     System.out.println(fs.getContentSummary(path).toString(qOption) + path);
   }
 }

src/hdfs/org/apache/hadoop/hdfs/tools/DFSAdmin.java

@@ -47,13 +47,15 @@ public class DFSAdmin extends FsShell {
    * An abstract class for the execution of a file system command
    */
   abstract private static class DFSAdminCommand extends Command {
+    final DistributedFileSystem dfs;
     /** Constructor */
     public DFSAdminCommand(FileSystem fs) {
-      super(fs);
+      super(fs.getConf());
       if (!(fs instanceof DistributedFileSystem)) {
         throw new IllegalArgumentException("FileSystem " + fs.getUri() + 
             " is not a distributed file system");
       }
+      this.dfs = (DistributedFileSystem)fs;
     }
   }
   
@@ -92,7 +94,7 @@ public class DFSAdmin extends FsShell {
 
     @Override
     public void run(Path path) throws IOException {
-      ((DistributedFileSystem)fs).clearQuota(path);
+      dfs.clearQuota(path);
     }
   }
   
@@ -139,7 +141,7 @@ public class DFSAdmin extends FsShell {
 
     @Override
     public void run(Path path) throws IOException {
-      ((DistributedFileSystem)fs).setQuota(path, quota);
+      dfs.setQuota(path, quota);
     }
   }
   

src/test/org/apache/hadoop/hdfs/TestDFSShell.java

@@ -673,10 +673,20 @@ public class TestDFSShell extends TestCase {
       String root = createTree(dfs, "count");
 
       // Verify the counts
-      runCount(root, 2, 4, dfs);
-      runCount(root + "2", 2, 1, dfs);
-      runCount(root + "2/f1", 0, 1, dfs);
-      runCount(root + "2/sub", 1, 0, dfs);
+      runCount(root, 2, 4, conf);
+      runCount(root + "2", 2, 1, conf);
+      runCount(root + "2/f1", 0, 1, conf);
+      runCount(root + "2/sub", 1, 0, conf);
+
+      final FileSystem localfs = FileSystem.getLocal(conf);
+      Path localpath = new Path(TEST_ROOT_DIR, "testcount");
+      localpath = localpath.makeQualified(localfs);
+      localfs.mkdirs(localpath);
+      
+      final String localstr = localpath.toString();
+      System.out.println("localstr=" + localstr);
+      runCount(localstr, 1, 0, conf);
+      assertEquals(0, new Count(new String[]{root, localstr}, 0, conf).runAll());
     } finally {
       try {
         dfs.close();
@@ -685,17 +695,17 @@ public class TestDFSShell extends TestCase {
       cluster.shutdown();
     }
   }
-  private void runCount(String path, long dirs, long files, FileSystem fs
+  private void runCount(String path, long dirs, long files, Configuration conf
     ) throws IOException {
     ByteArrayOutputStream bytes = new ByteArrayOutputStream(); 
     PrintStream out = new PrintStream(bytes);
     PrintStream oldOut = System.out;
     System.setOut(out);
     Scanner in = null;
+    String results = null;
     try {
-      new Count(new String[]{path}, 0, fs).runAll();
-      String results = bytes.toString();
-      System.out.println(results);
+      new Count(new String[]{path}, 0, conf).runAll();
+      results = bytes.toString();
       in = new Scanner(results);
       assertEquals(dirs, in.nextLong());
       assertEquals(files, in.nextLong());
@@ -703,6 +713,7 @@ public class TestDFSShell extends TestCase {
       if (in!=null) in.close();
       IOUtils.closeStream(out);
       System.setOut(oldOut);
+      System.out.println("results:\n" + results);
     }
   }