commit 7322dbc945d42b1276f8bbb657c1021014d63e58

Author: Rajesh Balamohan <rbala@yahoo-inc.com>
Date:   Fri Jan 21 09:54:14 2011 +0530

    . Reducing new Path(), RawFileStatus() creation overhead in LocalDirAllocator


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/yahoo-merge@1079162 13f79535-47bb-0310-9956-ffa450edef68

src/java/org/apache/hadoop/fs/LocalDirAllocator.java

@@ -239,7 +239,7 @@ public class LocalDirAllocator {
     private FileSystem localFS;
     private DF[] dirDF;
     private String contextCfgItemName;
-    private String[] localDirs;
+    private Path[] localDirsPath;
     private String savedLocalDirs = "";
 
     public AllocatorPerContext(String contextCfgItemName) {
@@ -253,7 +253,7 @@ public class LocalDirAllocator {
                                           ) throws IOException {
       String newLocalDirs = conf.get(contextCfgItemName);
       if (!newLocalDirs.equals(savedLocalDirs)) {
-        localDirs = conf.getTrimmedStrings(contextCfgItemName);
+        String[] localDirs = conf.getTrimmedStrings(contextCfgItemName);
         localFS = FileSystem.getLocal(conf);
         int numDirs = localDirs.length;
         ArrayList<String> dirs = new ArrayList<String>(numDirs);
@@ -279,7 +279,10 @@ public class LocalDirAllocator {
                 ie.getMessage() + "\n" + StringUtils.stringifyException(ie));
           } //ignore
         }
-        localDirs = dirs.toArray(new String[dirs.size()]);
+        localDirsPath = new Path[dirs.size()];
+        for(int i=0;i<localDirsPath.length;i++) {
+        	localDirsPath[i] = new Path(dirs.get(i));
+        }
         dirDF = dfList.toArray(new DF[dirs.size()]);
         savedLocalDirs = newLocalDirs;
         
@@ -288,10 +291,10 @@ public class LocalDirAllocator {
       }
     }
 
-    private Path createPath(String path, 
+    private Path createPath(Path path, 
     		boolean checkWrite) throws IOException {
-      Path file = new Path(new Path(localDirs[dirNumLastAccessed]),
-                                    path);
+      Path file = new Path(localDirsPath[dirNumLastAccessed], path);
+    	
       if (checkWrite) {
     	//check whether we are able to create a directory here. If the disk
     	//happens to be RDONLY we will fail
@@ -325,7 +328,7 @@ public class LocalDirAllocator {
     	                      Configuration conf, boolean checkWrite
     	                      ) throws IOException {
       confChanged(conf);
-      int numDirs = localDirs.length;
+      int numDirs = localDirsPath.length;
       int numDirsSearched = 0;
       //remove the leading slash from the path (to make sure that the uri
       //resolution results in a valid path on the dir being checked)
@@ -333,6 +336,7 @@ public class LocalDirAllocator {
         pathStr = pathStr.substring(1);
       }
       Path returnPath = null;
+      Path path = new Path(pathStr);
       
       if(size == SIZE_UNKNOWN) {  //do roulette selection: pick dir with probability 
                     //proportional to available size
@@ -355,7 +359,7 @@ public class LocalDirAllocator {
             dir++;
           }
           dirNumLastAccessed = dir;
-          returnPath = createPath(pathStr, checkWrite);
+          returnPath = createPath(path, checkWrite);
           if (returnPath == null) {
             totalAvailable -= availableOnDisk[dir];
             availableOnDisk[dir] = 0; // skip this disk
@@ -366,7 +370,7 @@ public class LocalDirAllocator {
         while (numDirsSearched < numDirs && returnPath == null) {
           long capacity = dirDF[dirNumLastAccessed].getAvailable();
           if (capacity > size) {
-            returnPath = createPath(pathStr, checkWrite);
+          	returnPath = createPath(path, checkWrite);
           }
           dirNumLastAccessed++;
           dirNumLastAccessed = dirNumLastAccessed % numDirs; 
@@ -409,15 +413,16 @@ public class LocalDirAllocator {
     public synchronized Path getLocalPathToRead(String pathStr, 
         Configuration conf) throws IOException {
       confChanged(conf);
-      int numDirs = localDirs.length;
+      int numDirs = localDirsPath.length;
       int numDirsSearched = 0;
       //remove the leading slash from the path (to make sure that the uri
       //resolution results in a valid path on the dir being checked)
       if (pathStr.startsWith("/")) {
         pathStr = pathStr.substring(1);
       }
+      Path childPath = new Path(pathStr);
       while (numDirsSearched < numDirs) {
-        Path file = new Path(localDirs[numDirsSearched], pathStr);
+        Path file = new Path(localDirsPath[numDirsSearched], childPath);
         if (localFS.exists(file)) {
           return file;
         }
@@ -434,10 +439,10 @@ public class LocalDirAllocator {
       private final FileSystem fs;
       private final String pathStr;
       private int i = 0;
-      private final String[] rootDirs;
+      private final Path[] rootDirs;
       private Path next = null;
 
-      private PathIterator(FileSystem fs, String pathStr, String[] rootDirs
+      private PathIterator(FileSystem fs, String pathStr, Path[] rootDirs
                            ) throws IOException {
         this.fs = fs;
         this.pathStr = pathStr;
@@ -496,7 +501,7 @@ public class LocalDirAllocator {
       if (pathStr.startsWith("/")) {
         pathStr = pathStr.substring(1);
       }
-      return new PathIterator(localFS, pathStr, localDirs);
+      return new PathIterator(localFS, pathStr, localDirsPath);
     }
 
     /** We search through all the configured dirs for the file's existence
@@ -504,15 +509,16 @@ public class LocalDirAllocator {
      */
     public synchronized boolean ifExists(String pathStr,Configuration conf) {
       try {
-        int numDirs = localDirs.length;
+        int numDirs = localDirsPath.length;
         int numDirsSearched = 0;
         //remove the leading slash from the path (to make sure that the uri
         //resolution results in a valid path on the dir being checked)
         if (pathStr.startsWith("/")) {
           pathStr = pathStr.substring(1);
         }
+        Path childPath = new Path(pathStr);
         while (numDirsSearched < numDirs) {
-          Path file = new Path(localDirs[numDirsSearched], pathStr);
+          Path file = new Path(localDirsPath[numDirsSearched], childPath);
           if (localFS.exists(file)) {
             return true;
           }

src/java/org/apache/hadoop/fs/LocalFileSystem.java

@@ -48,6 +48,18 @@ public class LocalFileSystem extends ChecksumFileSystem {
     super(rawLocalFileSystem);
     rfs = rawLocalFileSystem;
   }
+  
+  /**
+   * Get file status.
+   */
+  public boolean exists(Path f) throws IOException {
+    File path = pathToFile(f);
+    if (path.exists()) {
+  	  return true;
+    } else {
+      return false;
+    }
+  }
     
   /** Convert a path to a File. */
   public File pathToFile(Path path) {

src/java/org/apache/hadoop/fs/Path.java

@@ -157,8 +157,12 @@ public class Path implements Comparable {
 
   private String normalizePath(String path) {
     // remove double slashes & backslashes
-    path = path.replace("//", "/");
-    path = path.replace("\\", "/");
+    if (path.indexOf("//") != -1) {
+      path = path.replace("//", "/");
+    }
+    if (path.indexOf("\\") != -1) {	
+      path = path.replace("\\", "/");
+    }
     
     // trim trailing slash from non-root path (ignoring windows drive)
     int minLength = hasWindowsDrive(path, true) ? 4 : 1;