HADOOP-3816. Faster directory listing in KFS. (Sriram Rao via omalley)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/core/trunk@685405 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -204,6 +204,8 @@ Trunk (unreleased changes)
     HADOOP-3864. Prevent the JobTracker from locking up when a job is being
     initialized. (acmurthy via omalley)
 
+    HADOOP-3816. Faster directory listing in KFS. (Sriram Rao via omalley)
+
   BUG FIXES
 
     HADOOP-3563.  Refactor the distributed upgrade code so that it is 

src/core/org/apache/hadoop/fs/kfs/IFSImpl.java

@@ -31,12 +31,15 @@ import java.io.*;
 
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
 
 interface IFSImpl {
     public boolean exists(String path) throws IOException;
     public boolean isDirectory(String path) throws IOException;
     public boolean isFile(String path) throws IOException;
     public String[] readdir(String path) throws IOException;
+    public FileStatus[] readdirplus(Path path) throws IOException;
 
     public int mkdirs(String path) throws IOException;
     public int rename(String source, String dest) throws IOException;

src/core/org/apache/hadoop/fs/kfs/KFSImpl.java

@@ -24,8 +24,11 @@ import java.io.*;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
 
 import org.kosmix.kosmosfs.access.KfsAccess;
+import org.kosmix.kosmosfs.access.KfsFileAttr;
 
 class KFSImpl implements IFSImpl {
     private KfsAccess kfsAccess = null;
@@ -59,6 +62,40 @@ class KFSImpl implements IFSImpl {
         return kfsAccess.kfs_readdir(path);
     }
 
+    public FileStatus[] readdirplus(Path path) throws IOException {
+        String srep = path.toUri().getPath();
+        KfsFileAttr[] fattr = kfsAccess.kfs_readdirplus(srep);
+        if (fattr == null)
+            return null;
+        int numEntries = 0;
+        for (int i = 0; i < fattr.length; i++) {
+            if ((fattr[i].filename.compareTo(".") == 0) || (fattr[i].filename.compareTo("..") == 0))
+                continue;
+            numEntries++;
+        }
+        FileStatus[] fstatus = new FileStatus[numEntries];
+        int j = 0;
+        for (int i = 0; i < fattr.length; i++) {
+            if ((fattr[i].filename.compareTo(".") == 0) || (fattr[i].filename.compareTo("..") == 0))
+                continue;
+            Path fn = new Path(path, fattr[i].filename);
+
+            if (fattr[i].isDirectory)
+                fstatus[j] = new FileStatus(0, true, 1, 0, fattr[i].modificationTime, fn);
+            else
+                fstatus[j] = new FileStatus(fattr[i].filesize, fattr[i].isDirectory,
+                                            fattr[i].replication,
+                                            (long)
+                                            (1 << 26),
+                                            fattr[i].modificationTime,
+                                            fn);
+
+            j++;
+        }
+        return fstatus;
+    }
+
+
     public int mkdirs(String path) throws IOException {
         return kfsAccess.kfs_mkdirs(path);
     }

src/core/org/apache/hadoop/fs/kfs/KosmosFileSystem.java

@@ -128,36 +128,13 @@ public class KosmosFileSystem extends FileSystem {
     }
 
     public FileStatus[] listStatus(Path path) throws IOException {
-	Path absolute = makeAbsolute(path);
+        Path absolute = makeAbsolute(path);
         String srep = absolute.toUri().getPath();
 
-	if (kfsImpl.isFile(srep))
-	    return new FileStatus[] { getFileStatus(path) } ;
-
-	String[] entries = kfsImpl.readdir(srep);
-
-        if (entries == null)
-            return null;
+        if (kfsImpl.isFile(srep))
+                return new FileStatus[] { getFileStatus(path) } ;
 
-        // kfsreaddir() returns "." and ".."; strip them before
-        // passing back to hadoop fs.
-	int numEntries = 0;
-	for (int i = 0; i < entries.length; i++) {
-	    if ((entries[i].compareTo(".") == 0) || (entries[i].compareTo("..") == 0))
-		continue;
-	    numEntries++;
-	}
-
-	FileStatus[] pathEntries = new FileStatus[numEntries];
-	int j = 0;
-	for (int i = 0; i < entries.length; i++) {
-	    if ((entries[i].compareTo(".") == 0) || (entries[i].compareTo("..") == 0))
-		continue;
-
-	    pathEntries[j] = getFileStatus(new Path(path, entries[i]));
-	    j++;
-	}
-	return pathEntries;
+        return kfsImpl.readdirplus(absolute);
     }
 
     public FileStatus getFileStatus(Path path) throws IOException {

src/test/org/apache/hadoop/fs/kfs/KFSEmulationImpl.java

@@ -65,6 +65,10 @@ public class KFSEmulationImpl implements IFSImpl {
         return entries;
     }
 
+    public FileStatus[] readdirplus(Path path) throws IOException {
+        return localFS.listStatus(path);
+    }
+
     public int mkdirs(String path) throws IOException {
         if (localFS.mkdirs(new Path(path)))
             return 0;