HADOOP-559. Add file name globbing (pattern matching) support to the FileSystem API and use it in DFSShell ('bin/hadoop dfs') commands. Contributed by Hairong.

git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk@450208 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -76,6 +76,10 @@ Trunk (unreleased changes)
 19. HADOOP-487.  Throw a more informative exception for unknown RPC
     hosts.  (Sameer Paranjpye via cutting)
 
+20. HADOOP-559.  Add file name globbing (pattern matching) support to
+    the FileSystem API, and use it in DFSShell ('bin/hadoop dfs')
+    commands.  (Hairong Kuang via cutting)
+
 
 Release 0.6.2 (unreleased)
 

src/java/org/apache/hadoop/dfs/DFSShell.java

@@ -53,17 +53,35 @@ public class DFSShell extends ToolBase {
     }
 
     /**
-     * Obtain the indicated DFS file and copy to the local name.
-     * srcf is kept.
+     * Obtain the indicated DFS files that match the file pattern <i>srcf</i>
+     * and copy them to the local name. srcf is kept.
+     * When copying mutiple files, the destination must be a directory. 
+     * Otherwise, IOException is thrown.
+     * @param srcf: a file pattern specifying source files
+     * @param dstf: a destination local file/directory 
+     * @exception: IOException  
+     * @see org.apache.hadoop.fs.FileSystem.globPaths 
      */
-    void copyToLocal(String srcf, Path dst) throws IOException {
-        fs.copyToLocalFile(new Path(srcf), dst);
+    void copyToLocal(String srcf, String dstf) throws IOException {
+      Path [] srcs = fs.globPaths( new Path(srcf) );
+      if( srcs.length > 1 && !new File( dstf ).isDirectory()) {
+        throw new IOException( "When copy multiple files, " 
+            + "destination should be a directory." );
+      }
+      Path dst = new Path( dstf );
+      for( int i=0; i<srcs.length; i++ ) {
+        fs.copyToLocalFile( srcs[i], dst );
+      }
     }
     
     /**
-     * Get all the files in the directory and output them to
-     * only one file on local fs 
+     * Get all the files in the directories that match the source file 
+     * pattern and merge and sort them to only one file on local fs 
      * srcf is kept.
+     * @param srcf: a file pattern specifying source files
+     * @param dstf: a destination local file/directory 
+     * @exception: IOException  
+     * @see org.apache.hadoop.fs.FileSystem.globPaths 
      */
     void copyMergeToLocal(String srcf, Path dst) throws IOException {
         copyMergeToLocal(srcf, dst, false);
@@ -71,21 +89,29 @@ public class DFSShell extends ToolBase {
     
 
     /**
-     * Get all the files in the directory and output them to
-     * only one file on local fs 
+     * Get all the files in the directories that match the source file pattern
+     * and merge and sort them to only one file on local fs 
      * srcf is kept.
      * 
      * Also adds a string between the files (useful for adding \n
      * to a text file)
+     * @param srcf: a file pattern specifying source files
+     * @param dstf: a destination local file/directory
+     * @param endline: if an end of line character is added to a text file 
+     * @exception: IOException  
+     * @see org.apache.hadoop.fs.FileSystem.globPaths 
      */
     void copyMergeToLocal(String srcf, Path dst, boolean endline) throws IOException {
+      Path [] srcs = fs.globPaths( new Path( srcf ) );
+      for( int i=0; i<srcs.length; i++ ) {
         if(endline) {
-            FileUtil.copyMerge(fs, new Path(srcf), 
+            FileUtil.copyMerge(fs, srcs[i], 
                     FileSystem.getNamed("local", conf), dst, false, conf, "\n");
         } else {
-            FileUtil.copyMerge(fs, new Path(srcf), 
+            FileUtil.copyMerge(fs, srcs[i], 
                     FileSystem.getNamed("local", conf), dst, false, conf, null);
         }
+      }
     }      
 
     /**
@@ -96,8 +122,24 @@ public class DFSShell extends ToolBase {
         System.err.println("Option '-moveToLocal' is not implemented yet.");
     }
 
+    /**
+     * Fetch all DFS files that match the file pattern <i>srcf</i> and display
+     * their content on stdout. 
+     * @param srcf: a file pattern specifying source files
+     * @exception: IOException
+     * @see org.apache.hadoop.fs.FileSystem.globPaths 
+     */
     void cat(String srcf) throws IOException {
-      FSDataInputStream in = fs.open(new Path(srcf));
+      Path [] srcs = fs.globPaths( new Path( srcf ) );
+      for( int i=0; i<srcs.length; i++ ) {
+        cat(srcs[i]);
+      }
+    }
+    
+
+    /* print the content of src to screen */
+    private void cat(Path src) throws IOException {
+      FSDataInputStream in = fs.open(src);
       try {
         BufferedReader din = new BufferedReader(new InputStreamReader(in));
         String line;
@@ -138,15 +180,29 @@ public class DFSShell extends ToolBase {
         System.exit(-1);
       }
       
-      setReplication(rep, new Path(cmd[pos]), recursive);
+      setReplication(rep, cmd[pos], recursive);
     }
     
     /**
-     * Set the replication for the path argument
+     * Set the replication for files that match file pattern <i>srcf</i>
      * if it's a directory and recursive is true,
      * set replication for all the subdirs and those files too
+     * @param newRep: new replication factor
+     * @param srcf: a file pattern specifying source files
+     * @param recursive: if need to set replication factor for files in subdirs
+     * @exception: IOException  
+     * @see org.apache.hadoop.fs.FileSystem.globPaths 
      */
-    public void setReplication(short newRep, Path src, boolean recursive) throws IOException {
+    public void setReplication(short newRep, String srcf, boolean recursive)
+        throws IOException {
+      Path[] srcs = fs.globPaths( new Path(srcf) );
+      for( int i=0; i<srcs.length; i++ ) {
+        setReplication( newRep, srcs[i], recursive );
+      }
+    }
+    
+    private void setReplication(short newRep, Path src, boolean recursive)
+      throws IOException {
   	
     	if(!fs.isDirectory(src)) {
     		setFileReplication(src, newRep);
@@ -172,8 +228,8 @@ public class DFSShell extends ToolBase {
     /**
      * Actually set the replication for this file
      * If it fails either throw IOException or print an error msg
-     * @param file
-     * @param newRep
+     * @param file: a dfs file/directory
+     * @param newRep: new replication factor
      * @throws IOException
      */
     private void setFileReplication(Path file, short newRep) throws IOException {
@@ -187,14 +243,27 @@ public class DFSShell extends ToolBase {
     
     
     /**
-     * Get a listing of all files in DFS at the indicated name
+     * Get a listing of all files in DFS that match the file pattern <i>srcf</i>
+     * @param srcf: a file pattern specifying source files
+     * @param recursive: if need to list files in subdirs
+     * @exception: IOException  
+     * @see org.apache.hadoop.fs.FileSystem.globPaths 
      */
-    public void ls(String src, boolean recursive) throws IOException {
-        Path items[] = fs.listPaths(new Path(src));
+    public void ls(String srcf, boolean recursive) throws IOException {
+      Path[] srcs = fs.globPaths( new Path(srcf) );
+      boolean printHeader = (srcs.length == 1) ? true: false;
+      for(int i=0; i<srcs.length; i++) {
+        ls(srcs[i], recursive, printHeader);
+      }
+    }
+
+    /* list all files in dfs under the directory <i>src</i>*/
+    private void ls(Path src, boolean recursive, boolean printHeader ) throws IOException {
+        Path items[] = fs.listPaths(src);
         if (items == null) {
             System.out.println("Could not get listing for " + src);
         } else {
-            if(!recursive) {
+            if(!recursive && printHeader ) {
             	System.out.println("Found " + items.length + " items");
             }
             for (int i = 0; i < items.length; i++) {
@@ -205,16 +274,20 @@ public class DFSShell extends ToolBase {
                                         ("<r " + fs.getReplication(cur) 
                                             + ">\t" + fs.getLength(cur))));
                 if(recursive && fs.isDirectory(cur)) {
-                  ls(cur.toString(), recursive);
+                  ls(cur, recursive, printHeader);
                 }
             }
         }
     }
 
     /**
+     * Show the size of all files in DFS that match the file pattern <i>srcf</i>
+     * @param srcf: a file pattern specifying source files
+     * @exception: IOException  
+     * @see org.apache.hadoop.fs.FileSystem.globPaths 
      */
     public void du(String src) throws IOException {
-        Path items[] = fs.listPaths(new Path(src));
+        Path items[] = fs.listPaths( fs.globPaths( new Path(src) ) );
         if (items == null) {
             System.out.println("Could not get listing for " + src);
         } else {
@@ -235,38 +308,79 @@ public class DFSShell extends ToolBase {
     }
     
     /**
-     * Rename an DFS file
+     * Move DFS files that match the file pattern <i>srcf</i>
+     * to a destination dfs file.
+     * When moving mutiple files, the destination must be a directory. 
+     * Otherwise, IOException is thrown.
+     * @param srcf: a file pattern specifying source files
+     * @param dstf: a destination local file/directory 
+     * @exception: IOException  
+     * @see org.apache.hadoop.fs.FileSystem.globPaths 
      */
     public void rename(String srcf, String dstf) throws IOException {
-        if (fs.rename(new Path(srcf), new Path(dstf))) {
-            System.out.println("Renamed " + srcf + " to " + dstf);
+      Path [] srcs = fs.globPaths( new Path(srcf) );
+      Path dst = new Path(dstf);
+      if( srcs.length > 1 && !fs.isDirectory(dst)) {
+        throw new IOException( "When moving multiple files, " 
+            + "destination should be a directory." );
+      }
+      for( int i=0; i<srcs.length; i++ ) {
+        if (fs.rename(srcs[i], dst)) {
+            System.out.println("Renamed " + srcs[i] + " to " + dstf);
         } else {
-            System.out.println("Rename failed");
+            System.out.println("Rename failed " + srcs[i]);
         }
+      }
     }
 
     /**
-     * Copy an DFS file
+     * Copy DFS files that match the file pattern <i>srcf</i>
+     * to a destination dfs file.
+     * When copying mutiple files, the destination must be a directory. 
+     * Otherwise, IOException is thrown.
+     * @param srcf: a file pattern specifying source files
+     * @param dstf: a destination local file/directory 
+     * @exception: IOException  
+     * @see org.apache.hadoop.fs.FileSystem.globPaths 
      */
     public void copy(String srcf, String dstf, Configuration conf) throws IOException {
-      FileUtil.copy(fs, new Path(srcf), fs, new Path(dstf), false, conf);
+      Path [] srcs = fs.globPaths( new Path(srcf) );
+      Path dst = new Path(dstf);
+      if( srcs.length > 1 && !fs.isDirectory(dst)) {
+        throw new IOException( "When copying multiple files, " 
+            + "destination should be a directory." );
+      }
+      for( int i=0; i<srcs.length; i++ ) {
+        FileUtil.copy(fs, srcs[i], fs, dst, false, conf);
+      }
     }
 
     /**
-     * Delete an DFS file
+     * Delete all files in DFS that match the file pattern <i>srcf</i>
+     * @param srcf: a file pattern specifying source files
+     * @param recursive: if need to delete subdirs
+     * @exception: IOException  
+     * @see org.apache.hadoop.fs.FileSystem.globPaths 
      */
     public void delete(String srcf, boolean recursive) throws IOException {
-      Path srcp = new Path(srcf);
-      if (fs.isDirectory(srcp) && !recursive) {
-        System.out.println("Cannot remove directory \"" + srcf +
+      Path [] srcs = fs.globPaths( new Path(srcf) );
+      for( int i=0; i<srcs.length; i++ ) {
+        delete(srcs[i], recursive);
+      }
+    }
+    
+    /* delete an DFS file */
+    private void delete(Path src, boolean recursive ) throws IOException {
+      if (fs.isDirectory(src) && !recursive) {
+        System.out.println("Cannot remove directory \"" + src +
                            "\", use -rmr instead");
         return;
       }
 
-      if (fs.delete(srcp)) {
-        System.out.println("Deleted " + srcf);
+      if (fs.delete(src)) {
+        System.out.println("Deleted " + src);
       } else {
-        System.out.println("Delete failed");
+        System.out.println("Delete failed " + src);
       }
     }
 
@@ -403,7 +517,7 @@ public class DFSShell extends ToolBase {
             } else if ("-moveFromLocal".equals(cmd)) {
                 moveFromLocal(new Path(argv[i++]), argv[i++]);
             } else if ("-get".equals(cmd) || "-copyToLocal".equals(cmd)) {
-                copyToLocal(argv[i++], new Path(argv[i++]));
+                copyToLocal(argv[i++], argv[i++]);
             } else if ("-getmerge".equals(cmd)) {
                 if(argv.length>i+2)
                     copyMergeToLocal(argv[i++], new Path(argv[i++]), Boolean.parseBoolean(argv[i++]));

src/java/org/apache/hadoop/fs/FileSystem.java

@@ -18,6 +18,7 @@ package org.apache.hadoop.fs;
 import java.io.*;
 import java.net.*;
 import java.util.*;
+import java.util.regex.Pattern;
 
 import org.apache.commons.logging.*;
 
@@ -481,15 +482,17 @@ public abstract class FileSystem extends Configured {
       return result;
     }
 
+    final private static PathFilter DEFAULT_FILTER = new PathFilter() {
+      public boolean accept(Path file) {
+        return !isChecksumFile(file);
+      }     
+    };
+  
     /** List files in a directory. */
     public Path[] listPaths(Path f) throws IOException {
-      return listPaths(f, new PathFilter() {
-          public boolean accept(Path file) {
-            return !isChecksumFile(file);
-          }
-        });
+      return listPaths(f, DEFAULT_FILTER);
     }
-
+    
     /** List files in a directory. */
     public abstract Path[] listPathsRaw(Path f) throws IOException;
 
@@ -509,21 +512,253 @@ public abstract class FileSystem extends Configured {
       }
       return result;
     }
-
-    /** Filter files in a directory. */
-    public Path[] listPaths(Path f, PathFilter filter) throws IOException {
-        Vector results = new Vector();
-        Path listing[] = listPathsRaw(f);
-        if (listing != null) {
-          for (int i = 0; i < listing.length; i++) {
-            if (filter.accept(listing[i])) {
-              results.add(listing[i]);
-            }
+    
+    /** Filter raw files in a directory. */
+    private void listPaths(ArrayList<Path> results, Path f, PathFilter filter)
+      throws IOException {
+      Path listing[] = listPathsRaw(f);
+      if (listing != null) {
+        for (int i = 0; i < listing.length; i++) {
+          if (filter.accept(listing[i])) {
+            results.add(listing[i]);
           }
         }
+      }      
+    }
+    
+    /** Filter raw files in a directory. */
+    public Path[] listPaths(Path f, PathFilter filter) throws IOException {
+        ArrayList<Path> results = new ArrayList<Path>();
+        listPaths(results, f, filter);
         return (Path[]) results.toArray(new Path[results.size()]);
     }
 
+    /** 
+     * Filter raw files in a list directories using the default checksum filter. 
+     * @param files: a list of paths
+     * @return a list of files under the source paths
+     * @exception IOException
+     */
+    public Path[] listPaths(Path[] files ) throws IOException {
+      return listPaths( files, DEFAULT_FILTER );
+    }
+    
+    /** 
+     * Filter raw files in a list directories using user-supplied path filter. 
+     * @param files: a list of paths
+     * @return a list of files under the source paths
+     * @exception IOException
+     */
+    public Path[] listPaths(Path[] files, PathFilter filter)
+    throws IOException {
+      ArrayList<Path> results = new ArrayList<Path>();
+      for(int i=0; i<files.length; i++) {
+        listPaths(results, files[i], filter);
+      }
+      return (Path[]) results.toArray(new Path[results.size()]);
+    }
+    
+    /**
+     * <p>Return all the files that match filePattern and are not checksum
+     * files. Results are sorted by their names.
+     * 
+     * <p>
+     * A filename pattern is composed of <i>regular</i> characters and
+     * <i>special pattern matching</i> characters, which are:
+     *
+     * <dl>
+     *  <dd>
+     *   <dl>
+     *    <p>
+     *    <dt> <tt> ? </tt>
+     *    <dd> Matches any single character.
+     *
+     *    <p>
+     *    <dt> <tt> * </tt>
+     *    <dd> Matches zero or more characters.
+     *
+     *    <p>
+     *    <dt> <tt> [<i>abc</i>] </tt>
+     *    <dd> Matches a single character from character set
+     *     <tt>{<i>a,b,c</i>}</tt>.
+     *
+     *    <p>
+     *    <dt> <tt> [<i>a</i>-<i>b</i>] </tt>
+     *    <dd> Matches a single character from the character range
+     *     <tt>{<i>a...b</i>}</tt>.  Note that character <tt><i>a</i></tt> must be
+     *     lexicographically less than or equal to character <tt><i>b</i></tt>.
+     *
+     *    <p>
+     *    <dt> <tt> [^<i>a</i>] </tt>
+     *    <dd> Matches a single character that is not from character set or range
+     *     <tt>{<i>a</i>}</tt>.  Note that the <tt>^</tt> character must occur
+     *     immediately to the right of the opening bracket.
+     *
+     *    <p>
+     *    <dt> <tt> \<i>c</i> </tt>
+     *    <dd> Removes (escapes) any special meaning of character <i>c</i>.
+     *
+     *   </dl>
+     *  </dd>
+     * </dl>
+     *
+     * @param filePattern: a regular expression specifying file pattern
+
+     * @return an array of paths that match the file pattern
+     * @throws IOException
+     */
+    public Path[] globPaths(Path filePattern) throws IOException {
+      return globPaths(filePattern, DEFAULT_FILTER);
+    }
+    
+    /** glob all the file names that matches filePattern
+     * and is accepted by filter
+     * @param 
+     */
+    public Path[] globPaths(Path filePattern, PathFilter filter) 
+        throws IOException {
+      Path [] parents = new Path[1];
+      int level = 0;
+      
+      String filename = filePattern.toString();
+      if("".equals(filename) || Path.SEPARATOR.equals(filename)) {
+        parents[0] = filePattern;
+        return parents;
+      }
+      
+      String [] components = filename.split(Path.SEPARATOR);
+      if(filePattern.isAbsolute()) {
+        parents[0] = new Path(Path.SEPARATOR);
+        level = 1;
+      } else {
+        parents[0] = new Path( "" );
+      }
+      
+      Path[] results = globPathsLevel(parents, components, level, filter);
+      Arrays.sort(results);
+      return results;
+    }
+    
+    private Path[] globPathsLevel(Path[] parents,
+        String [] filePattern, int level, PathFilter filter) throws IOException {
+      if (level == filePattern.length)
+        return parents;
+      GlobFilter fp = new GlobFilter(filePattern[level], filter);
+      if( fp.hasPattern()) {
+        parents = listPaths(parents, fp);
+      } else {
+        for(int i=0; i<parents.length; i++) {
+          parents[i] = new Path(parents[i], filePattern[level]);
+        }
+      }
+      return globPathsLevel(parents, filePattern, level+1, filter);      
+    }
+ 
+    private static class GlobFilter implements PathFilter {
+      private PathFilter userFilter = DEFAULT_FILTER;
+      private Pattern regex;
+      private boolean hasPattern = false;
+      
+      /** Default pattern character: Escape any special meaning. */
+      private static final char  PAT_ESCAPE =  '\\';
+      /** Default pattern character: Any single character. */
+      private static final char  PAT_ANY = '.';
+      /** Default pattern character: Character set close. */
+      private static final char  PAT_SET_CLOSE = ']';
+      
+      GlobFilter() {
+      }
+      
+      GlobFilter(String filePattern) throws IOException {
+        setRegex(filePattern);
+      }
+      
+      GlobFilter(String filePattern, PathFilter filter) throws IOException {
+        userFilter = filter;
+        setRegex(filePattern);
+      }
+      
+      void setRegex(String filePattern) throws IOException {
+        int len;
+        int setOpen;
+        boolean setRange;
+        StringBuffer fileRegex = new StringBuffer();
+
+        // Validate the pattern
+        len = filePattern.length();
+        if (len == 0)
+            return;
+
+        setOpen =  0;
+        setRange = false;
+
+        for (int i = 0;  i < len;  i++)
+        {
+            char  pCh;
+
+            // Examine a single pattern character
+            pCh = filePattern.charAt(i);            
+            if( pCh == PAT_ESCAPE ) {
+              fileRegex.append( pCh );
+              i++;
+              if (i >= len)
+                  error( "An escaped character does not present",
+                      filePattern, i);
+              pCh = filePattern.charAt(i);
+            } else if( pCh == '.' ) {
+              fileRegex.append( PAT_ESCAPE );
+            } else if( pCh == '*' ) {
+                fileRegex.append( PAT_ANY );
+                hasPattern = true;
+            } else if( pCh == '?' ) {
+                pCh = PAT_ANY ;
+                hasPattern = true;
+            } else if( pCh == '[' && setOpen == 0 ) {
+                setOpen++;
+                hasPattern = true;
+            } else if( pCh == '^' && setOpen > 0) {
+            } else if (pCh == '-'  &&  setOpen > 0) {
+                // Character set range
+                setRange = true;
+            } else if (pCh == PAT_SET_CLOSE  &&  setRange) {
+                // Incomplete character set range
+                error("Incomplete character set range", filePattern, i);
+            } else if (pCh == PAT_SET_CLOSE  &&  setOpen > 0) {
+                // End of a character set
+                if (setOpen < 2)
+                    error("Unexpected end of set", filePattern, i);
+                setOpen = 0;
+            } else if (setOpen > 0) {
+                // Normal character, or the end of a character set range
+                setOpen++;
+                setRange = false;
+            }
+            fileRegex.append( pCh );
+        }
+
+        // Check for a well-formed pattern
+        if (setOpen > 0  ||  setRange)
+        {
+            // Incomplete character set or character range
+            error("Expecting set closure character or end of range", filePattern, len);
+        }
+        regex = Pattern.compile(fileRegex.toString());
+      }
+      
+      boolean hasPattern() {
+        return hasPattern;
+      }
+      
+      public boolean accept(Path path) {
+        return regex.matcher(path.getName()).matches() && userFilter.accept(path);
+      }
+      
+      private void error(String s, String pattern, int pos) throws IOException {
+        throw new IOException("Illegal file pattern: "
+                                 +s+" for glob "+pattern + " at " + pos);
+      }
+    }
+    
     /**
      * Set the current working directory for the given file system.
      * All relative paths will be resolved relative to it.

src/java/org/apache/hadoop/fs/LocalFileSystem.java

@@ -218,15 +218,26 @@ public class LocalFileSystem extends FileSystem {
     }
 
     public Path[] listPathsRaw(Path f) throws IOException {
-        String[] names = pathToFile(f).list();
-        if (names == null) {
+        File localf = pathToFile(f);
+        Path[] results;
+
+        if(!localf.exists())
           return null;
+        else if(localf.isFile()) {
+          results = new Path[1];
+          results[0] = f;
+          return results;
+        } else { //directory
+          String[] names = localf.list();
+          if (names == null) {
+            return null;
+          }
+          results = new Path[names.length];
+          for (int i = 0; i < names.length; i++) {
+            results[i] = new Path(f, names[i]);
+          }
+          return results;
         }
-        Path[] results = new Path[names.length];
-        for (int i = 0; i < names.length; i++) {
-          results[i] = new Path(f, names[i]);
-        }
-        return results;
     }
     
     /**

src/test/org/apache/hadoop/fs/TestGlobPaths.java

@@ -0,0 +1,230 @@
+package org.apache.hadoop.fs;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.dfs.MiniDFSCluster;
+
+import junit.framework.TestCase;
+
+public class TestGlobPaths extends TestCase {
+  
+  static private MiniDFSCluster dfsCluster;
+  static private FileSystem fs;
+  static final private int NUM_OF_PATHS = 4;
+  static final String USER_DIR = "/user/"+System.getProperty("user.name");
+  private Path[] path = new Path[NUM_OF_PATHS];
+  
+  protected void setUp() throws Exception {
+    try {
+      Configuration conf = new Configuration();
+      dfsCluster = new MiniDFSCluster(8889, conf, true);
+      fs = FileSystem.get(conf);
+    } catch (IOException e) {
+      e.printStackTrace();
+    }
+  }
+  
+  protected void tearDown() throws Exception {
+    dfsCluster.shutdown();
+  }
+  
+  public void testGlob() {
+    try {
+      pTestLiteral();
+      pTestAny();
+      pTestClosure();
+      pTestSet();
+      pTestRange();
+      pTestSetExcl();
+      pTestCombination();
+      pTestRelativePath();
+    } catch( IOException e) {
+      e.printStackTrace();
+    } 
+  }
+  
+  private void pTestLiteral() throws IOException {
+    try {
+      String [] files = new String[2];
+      files[0] = USER_DIR+"/a2c";
+      files[1] = USER_DIR+"/ab\\[c.d";
+      Path[] matchedPath = prepareTesting( USER_DIR+"/ab\\[c.d", files );
+      assertEquals( matchedPath.length, 1 );
+      assertEquals( matchedPath[0], path[1] );
+    } finally {
+      cleanupDFS();
+    }
+  }
+  
+  private void pTestAny() throws IOException {
+    try {
+      String [] files = new String[4];
+      files[0] = USER_DIR+"/abc";
+      files[1] = USER_DIR+"/a2c";
+      files[2] = USER_DIR+"/a.c";
+      files[3] = USER_DIR+"/abcd";
+      Path[] matchedPath = prepareTesting(USER_DIR+"/a?c", files);
+      assertEquals( matchedPath.length, 3 );
+      assertEquals( matchedPath[0], path[2] );
+      assertEquals( matchedPath[1], path[1] );
+      assertEquals( matchedPath[2], path[0] );
+    } finally {
+      cleanupDFS();
+    }
+  }
+  
+  private void pTestClosure() throws IOException {
+    pTestClosure1();
+    pTestClosure2();
+    pTestClosure3();
+  }
+  
+  private void pTestClosure1() throws IOException {
+    try {
+      String [] files = new String[4];
+      files[0] = USER_DIR+"/a";
+      files[1] = USER_DIR+"/abc";
+      files[2] = USER_DIR+"/abc.p";
+      files[3] = USER_DIR+"/bacd";
+      Path[] matchedPath = prepareTesting(USER_DIR+"/a*", files);
+      assertEquals( matchedPath.length, 3 );
+      assertEquals( matchedPath[0], path[0] );
+      assertEquals( matchedPath[1], path[1] );
+      assertEquals( matchedPath[2], path[2] );
+    } finally {
+      cleanupDFS();
+    }
+  }
+  
+  private void pTestClosure2() throws IOException {
+    try {
+      String [] files = new String[4];
+      files[0] = USER_DIR+"/a.";
+      files[1] = USER_DIR+"/a.txt";
+      files[2] = USER_DIR+"/a.old.java";
+      files[3] = USER_DIR+"/.java";
+      Path[] matchedPath = prepareTesting(USER_DIR+"/a.*", files);
+      assertEquals( matchedPath.length, 3 );
+      assertEquals( matchedPath[0], path[0] );
+      assertEquals( matchedPath[1], path[2] );
+      assertEquals( matchedPath[2], path[1] );
+    } finally {
+      cleanupDFS();
+    }
+  }
+  
+  private void pTestClosure3() throws IOException {
+    try {    
+      String [] files = new String[4];
+      files[0] = USER_DIR+"/a.txt.x";
+      files[1] = USER_DIR+"/ax";
+      files[2] = USER_DIR+"/ab37x";
+      files[3] = USER_DIR+"/bacd";
+      Path[] matchedPath = prepareTesting(USER_DIR+"/a*x", files);
+      assertEquals( matchedPath.length, 3 );
+      assertEquals( matchedPath[0], path[0] );
+      assertEquals( matchedPath[1], path[2] );
+      assertEquals( matchedPath[2], path[1] );
+    } finally {
+      cleanupDFS();
+    } 
+  }
+  
+  private void pTestSet() throws IOException {
+    try {    
+      String [] files = new String[4];
+      files[0] = USER_DIR+"/a.c";
+      files[1] = USER_DIR+"/a.cpp";
+      files[2] = USER_DIR+"/a.hlp";
+      files[3] = USER_DIR+"/a.hxy";
+      Path[] matchedPath = prepareTesting(USER_DIR+"/a.[ch]??", files);
+      assertEquals( matchedPath.length, 3 );
+      assertEquals( matchedPath[0], path[1] );
+      assertEquals( matchedPath[1], path[2] );
+      assertEquals( matchedPath[2], path[3] );
+    } finally {
+      cleanupDFS();
+    }
+  }
+  
+  private void pTestRange() throws IOException {
+    try {    
+      String [] files = new String[4];
+      files[0] = USER_DIR+"/a.d";
+      files[1] = USER_DIR+"/a.e";
+      files[2] = USER_DIR+"/a.f";
+      files[3] = USER_DIR+"/a.h";
+      Path[] matchedPath = prepareTesting(USER_DIR+"/a.[d-fm]", files);
+      assertEquals( matchedPath.length, 3 );
+      assertEquals( matchedPath[0], path[0] );
+      assertEquals( matchedPath[1], path[1] );
+      assertEquals( matchedPath[2], path[2] );
+    } finally {
+      cleanupDFS();
+    }
+  }
+  
+  private void pTestSetExcl() throws IOException {
+    try {    
+      String [] files = new String[4];
+      files[0] = USER_DIR+"/a.d";
+      files[1] = USER_DIR+"/a.e";
+      files[2] = USER_DIR+"/a.0";
+      files[3] = USER_DIR+"/a.h";
+      Path[] matchedPath = prepareTesting(USER_DIR+"/a.[^a-cg-z0-9]", files);
+      assertEquals( matchedPath.length, 2 );
+      assertEquals( matchedPath[0], path[0] );
+      assertEquals( matchedPath[1], path[1] );
+    } finally {
+      cleanupDFS();
+    }
+  }
+
+  private void pTestCombination() throws IOException {
+    try {    
+      String [] files = new String[4];
+      files[0] = "/user/aa/a.c";
+      files[1] = "/user/bb/a.cpp";
+      files[2] = "/user1/cc/b.hlp";
+      files[3] = "/user/dd/a.hxy";
+      Path[] matchedPath = prepareTesting("/use?/*/a.[ch]??", files);
+      assertEquals( matchedPath.length, 2 );
+      assertEquals( matchedPath[0], path[1] );
+      assertEquals( matchedPath[1], path[3] );
+    } finally {
+      cleanupDFS();
+    }
+  }
+  
+  private void pTestRelativePath() throws IOException {
+    try {
+      String [] files = new String[4];
+      files[0] = "a";
+      files[1] = "abc";
+      files[2] = "abc.p";
+      files[3] = "bacd";
+      Path[] matchedPath = prepareTesting("a*", files);
+      assertEquals( matchedPath.length, 3 );
+      assertEquals( matchedPath[0], new Path(USER_DIR, path[0]) );
+      assertEquals( matchedPath[1], new Path(USER_DIR, path[1]) );
+      assertEquals( matchedPath[2], new Path(USER_DIR, path[2]) );
+    } finally {
+      cleanupDFS();
+    }
+  }
+  
+  private Path[] prepareTesting( String pattern, String[] files)
+  throws IOException {
+    for(int i=0; i<Math.min(NUM_OF_PATHS, files.length); i++) {
+      path[i] = new Path( files[i] );
+      fs.mkdirs( path[i] );
+    }
+    return fs.globPaths( new Path(pattern) );
+  }
+  
+  private void cleanupDFS( ) throws IOException {
+    fs.delete( new Path("/user"));
+  }
+  
+}