HADOOP-369. Add -getmerge option to dfs command that appends all files in a directory into a single local file. Contributed by Johan Oskarson.

git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk@427719 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -120,10 +120,14 @@ Trunk (unreleased changes)
 34. HADOOP-345.  Permit iteration over Configuration key/value pairs.
     (Michel Tourn via cutting)
 
-35. HADOOP-409.  Streaming contrib module: make Hadoop configuration
+35. HADOOP-409.  Streaming contrib module: make configuration
     properties available to commands as environment variables.
     (Michel Tourn via cutting)
 
+36. HADOOP-369.  Add -getmerge option to dfs command that appends all
+    files in a directory into a single local file.
+    (Johan Oskarson via cutting)
+
 
 Release 0.4.0 - 2006-06-28
 

src/java/org/apache/hadoop/dfs/DFSShell.java

@@ -59,6 +59,34 @@ public class DFSShell extends ToolBase {
     void copyToLocal(String srcf, Path dst) throws IOException {
         fs.copyToLocalFile(new Path(srcf), dst);
     }
+    
+    /**
+     * Get all the files in the directory and output them to
+     * only one file on local fs 
+     * srcf is kept.
+     */
+    void copyMergeToLocal(String srcf, Path dst) throws IOException {
+        copyMergeToLocal(srcf, dst, false);
+    }    
+    
+
+    /**
+     * Get all the files in the directory and output them to
+     * only one file on local fs 
+     * srcf is kept.
+     * 
+     * Also adds a string between the files (useful for adding \n
+     * to a text file)
+     */
+    void copyMergeToLocal(String srcf, Path dst, boolean endline) throws IOException {
+        if(endline) {
+            FileUtil.copyMerge(fs, new Path(srcf), 
+                    FileSystem.getNamed("local", conf), dst, false, conf, "\n");
+        } else {
+            FileUtil.copyMerge(fs, new Path(srcf), 
+                    FileSystem.getNamed("local", conf), dst, false, conf, null);
+        }
+    }      
 
     /**
      * Obtain the indicated DFS file and copy to the local name.
@@ -300,8 +328,8 @@ public class DFSShell extends ToolBase {
                     " [-conf <configuration file>] [-D <[property=value>]"+
                     " [-ls <path>] [-lsr <path>] [-du <path>] [-mv <src> <dst>] [-cp <src> <dst>] [-rm <src>]" +
                     " [-put <localsrc> <dst>] [-copyFromLocal <localsrc> <dst>] [-moveFromLocal <localsrc> <dst>]" + 
-                    " [-get <src> <localdst>] [-cat <src>] [-copyToLocal <src> <localdst>] [-moveToLocal <src> <localdst>]" +
-                    " [-mkdir <path>] [-report] [-setrep [-R] <rep> <path/file>]");
+                    " [-get <src> <localdst>] [-getmerge <src> <localdst> [addnl]] [-cat <src>] [-copyToLocal <src> <localdst>]" +
+                    " [-moveToLocal <src> <localdst>] [-mkdir <path>] [-report] [-setrep [-R] <rep> <path/file>]");
             return -1;
         }
 
@@ -318,6 +346,12 @@ public class DFSShell extends ToolBase {
                 moveFromLocal(new Path(argv[i++]), argv[i++]);
             } else if ("-get".equals(cmd) || "-copyToLocal".equals(cmd)) {
                 copyToLocal(argv[i++], new Path(argv[i++]));
+            } else if ("-getmerge".equals(cmd)) {
+                if(argv.length>i+2)
+                    copyMergeToLocal(argv[i++], new Path(argv[i++]), Boolean.parseBoolean(argv[i++]));
+                else
+                    copyMergeToLocal(argv[i++], new Path(argv[i++]));
+                
             } else if ("-cat".equals(cmd)) {
                 cat(argv[i++]);
             } else if ("-moveToLocal".equals(cmd)) {

src/java/org/apache/hadoop/fs/FileUtil.java

@@ -82,6 +82,45 @@ public class FileUtil {
       return true;
     }
   }
+  
+  /** Copy all files in a directory to one output file (merge). */
+  public static boolean copyMerge(FileSystem srcFS, Path srcDir, 
+                             FileSystem dstFS, Path dstFile, 
+                             boolean deleteSource,
+                             Configuration conf, String addString) throws IOException {
+      dstFile = checkDest(srcDir.getName(), dstFS, dstFile);
+
+    if (!srcFS.isDirectory(srcDir))
+      return false;
+   
+    OutputStream out = dstFS.create(dstFile);
+    
+    try {
+      Path contents[] = srcFS.listPaths(srcDir);
+      for (int i = 0; i < contents.length; i++) {
+        if (srcFS.isFile(contents[i])) {
+          InputStream in = srcFS.open(contents[i]);
+          try {
+            copyContent(in, out, conf, false);
+            if(addString!=null)
+              out.write(addString.getBytes("UTF-8"));
+                
+          } finally {
+            in.close();
+          } 
+        }
+      }
+    } finally {
+      out.close();
+    }
+    
+
+    if (deleteSource) {
+      return srcFS.delete(srcDir);
+    } else {
+      return true;
+    }
+  }  
 
   /** Copy local files to a FileSystem. */
   public static boolean copy(File src,
@@ -142,7 +181,13 @@ public class FileUtil {
   }
 
   private static void copyContent(InputStream in, OutputStream out,
-                                  Configuration conf) throws IOException {
+          Configuration conf) throws IOException {
+    copyContent(in, out, conf, true);
+  }
+
+  
+  private static void copyContent(InputStream in, OutputStream out,
+                                  Configuration conf, boolean close) throws IOException {
     byte buf[] = new byte[conf.getInt("io.file.buffer.size", 4096)];
     try {
       int bytesRead = in.read(buf);
@@ -151,7 +196,8 @@ public class FileUtil {
         bytesRead = in.read(buf);
       }
     } finally {
-      out.close();
+      if(close)
+        out.close();
     }
   }