HADOOP-2151. FileSystem.globPaths validates the list of Paths that

it returns.  (Lohit Vijayarenu via dhruba)



git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk@594270 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -88,6 +88,9 @@ Trunk (unreleased changes)
     Now invalid options are caught, logged and jobs are failed early. (Lohit
     Vijayarenu via acmurthy)
 
+    HADOOP-2151. FileSyste.globPaths validates the list of Paths that
+    it returns.  (Lohit Vijayarenu via dhruba)
+
 Release 0.15.1 -
 
   BUG FIXES

src/java/org/apache/hadoop/fs/FileSystem.java

@@ -616,24 +616,34 @@ public abstract class FileSystem extends Configured {
       parents[0] = new Path(Path.CUR_DIR);
     }
       
-    Path[] results = globPathsLevel(parents, components, level, filter);
+    Path[] results = globPathsLevel(parents, components, level, filter, false);
     Arrays.sort(results);
     return results;
   }
     
-  private Path[] globPathsLevel(Path[] parents,
-                                String [] filePattern, int level, PathFilter filter) throws IOException {
+  private Path[] globPathsLevel(Path[] parents, String [] filePattern, 
+            int level, PathFilter filter, boolean hasGlob) throws IOException {
     if (level == filePattern.length)
       return parents;
     GlobFilter fp = new GlobFilter(filePattern[level], filter);
     if (fp.hasPattern()) {
       parents = listPaths(parents, fp);
+       hasGlob = true;
     } else {
+      int goodPathCount = 0;
       for(int i=0; i<parents.length; i++) {
-        parents[i] = new Path(parents[i], filePattern[level]);
+        Path tmpPath = new Path(parents[i], filePattern[level]);
+        if (!hasGlob || exists(tmpPath)) {
+          parents[goodPathCount++] = tmpPath;
+        }
+      }
+      if (goodPathCount != parents.length) {
+        Path [] goodParents = new Path[goodPathCount];
+        System.arraycopy(parents, 0, goodParents, 0, goodPathCount);
+        parents = goodParents;
       }
     }
-    return globPathsLevel(parents, filePattern, level+1, filter);      
+    return globPathsLevel(parents, filePattern, level+1, filter, hasGlob);
   }
  
   private static class GlobFilter implements PathFilter {

src/test/org/apache/hadoop/fs/TestGlobPaths.java

@@ -102,6 +102,8 @@ public class TestGlobPaths extends TestCase {
     pTestClosure1();
     pTestClosure2();
     pTestClosure3();
+    pTestClosure4();
+    pTestClosure5();
   }
   
   private void pTestClosure1() throws IOException {
@@ -145,7 +147,33 @@ public class TestGlobPaths extends TestCase {
       cleanupDFS();
     } 
   }
+
+  private void pTestClosure4() throws IOException {
+    try {
+      String [] files = new String[] {USER_DIR+"/dir1/file1", 
+                                      USER_DIR+"/dir2/file2", 
+                                       USER_DIR+"/dir3/file1"};
+      Path[] matchedPath = prepareTesting(USER_DIR+"/*/file1", files);
+      assertEquals(matchedPath.length, 2);
+      assertEquals(matchedPath[0], path[0]);
+      assertEquals(matchedPath[1], path[2]);
+    } finally {
+      cleanupDFS();
+    }
+  }
   
+  private void pTestClosure5() throws IOException {
+    try {
+      String [] files = new String[] {USER_DIR+"/dir1/file1", 
+                                      USER_DIR+"/file1"};
+      Path[] matchedPath = prepareTesting(USER_DIR+"/*/file1", files);
+      assertEquals(matchedPath.length, 1);
+      assertEquals(matchedPath[0], path[0]);
+    } finally {
+      cleanupDFS();
+    }
+  }
+
   private void pTestSet() throws IOException {
     try {    
       String [] files = new String[] {USER_DIR+"/a.c", USER_DIR+"/a.cpp",