Merge -r 678510:678511 from trunk to branch 0.18 to fix HADOOP-3762.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/core/branches/branch-0.18@678527 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -541,12 +541,14 @@ Release 0.18.0 - Unreleased
     HADOOP-3454. Fix Text::find to search only valid byte ranges. (Chad Whipkey
     via cdouglas)
 
-    HADOOP-3417. Removes the static configuration variable, commandLineConfig from 
-    JobClient. Moves the cli parsing from JobShell to GenericOptionsParser. 
-    Thus removes the class org.apache.hadoop.mapred.JobShell.
-    (Amareshwari Sriramadasu via ddas) 
+    HADOOP-3417. Removes the static configuration variable,
+    commandLineConfig from JobClient. Moves the cli parsing from
+    JobShell to GenericOptionsParser.  Thus removes the class
+    org.apache.hadoop.mapred.JobShell.  (Amareshwari Sriramadasu via
+    ddas)
 
-    HADOOP-2132. Only RUNNING/PREP jobs can be killed. (Jothi Padmanabhan via ddas)
+    HADOOP-2132. Only RUNNING/PREP jobs can be killed. (Jothi Padmanabhan 
+    via ddas)
 
     HADOOP-3472 MapFile.Reader getClosest() function returns incorrect results
     when before is true (Todd Lipcon via Stack)
@@ -579,8 +581,9 @@ Release 0.18.0 - Unreleased
     HADOOP-3135. Get the system directory from the JobTracker instead of from
     the conf. (Subramaniam Krishnan via ddas)
 
-    HADOOP-3503. Fix a race condition when client and namenode start simultaneous
-    recovery of the same block.  (dhruba & Tsz Wo (Nicholas), SZE)
+    HADOOP-3503. Fix a race condition when client and namenode start
+    simultaneous recovery of the same block.  (dhruba & Tsz Wo
+    (Nicholas), SZE)
 
     HADOOP-3440. Fixes DistributedCache to not create symlinks for paths which
     don't have fragments even when createSymLink is true. 
@@ -590,7 +593,8 @@ Release 0.18.0 - Unreleased
 
     HADOOP-3489. Fix NPE in SafeModeMonitor. (Lohit Vijayarenu via shv)
 
-    HADOOP-3509. Fix NPE in FSNamesystem.close. (Tsz Wo (Nicholas), SZE via shv)
+    HADOOP-3509. Fix NPE in FSNamesystem.close. (Tsz Wo (Nicholas), SZE via 
+    shv)
 
     HADOOP-3491. Name-node shutdown causes InterruptedException in 
     ResolutionMonitor. (Lohit Vijayarenu via shv)
@@ -765,6 +769,9 @@ Release 0.18.0 - Unreleased
     HADOOP-3774. Fix typos in shell output. (Tsz Wo (Nicholas), SZE via
     cdouglas)
 
+    HADOOP-3762. Fixed FileSystem cache to work with the default port. (cutting
+    via omalley)
+
     HADOOP-3798. Fix tests compilation. (Mukund Madhugiri via omalley)
 
 Release 0.17.2 - Unreleased

src/core/org/apache/hadoop/fs/FileSystem.java

@@ -60,6 +60,10 @@ public abstract class FileSystem extends Configured implements Closeable {
 
   /** FileSystem cache */
   private static final Cache CACHE = new Cache();
+
+  /** The key this instance is stored under in the cache. */
+  private Cache.Key key;
+
   /** Recording statistics per a FileSystem class */
   private static final Map<Class<? extends FileSystem>, Statistics> 
     statisticsTable =
@@ -312,9 +316,9 @@ public abstract class FileSystem extends Configured implements Closeable {
             thisAuthority.equalsIgnoreCase(defaultUri.getAuthority()))
           return;
       }
-      throw new IllegalArgumentException("Wrong FS: "+path+
-                                         ", expected: "+this.getUri());
     }
+    throw new IllegalArgumentException("Wrong FS: "+path+
+                                       ", expected: "+this.getUri());
   }
 
   /**
@@ -1237,7 +1241,7 @@ public abstract class FileSystem extends Configured implements Closeable {
   public void close() throws IOException {
     // delete all files that were marked as delete-on-exit.
     processDeleteOnExit();
-    CACHE.remove(new Cache.Key(this), this);
+    CACHE.remove(this.key, this);
   }
 
   /** Return the total size of all files in the filesystem.*/
@@ -1341,13 +1345,15 @@ public abstract class FileSystem extends Configured implements Closeable {
     private final Map<Key, FileSystem> map = new HashMap<Key, FileSystem>();
 
     synchronized FileSystem get(URI uri, Configuration conf) throws IOException{
-      FileSystem fs = map.get(new Key(uri, conf));
+      Key key = new Key(uri, conf);
+      FileSystem fs = map.get(key);
       if (fs == null) {
         fs = createFileSystem(uri, conf);
         if (map.isEmpty() && !clientFinalizer.isAlive()) {
           Runtime.getRuntime().addShutdownHook(clientFinalizer);
         }
-        map.put(new Key(fs), fs);
+        fs.key = key;
+        map.put(key, fs);
       }
       return fs;
     }
@@ -1395,10 +1401,6 @@ public abstract class FileSystem extends Configured implements Closeable {
       final String authority;
       final String username;
 
-      Key(FileSystem fs) throws IOException {
-        this(fs.getUri(), fs.getConf());
-      }
-
       Key(URI uri, Configuration conf) throws IOException {
         scheme = uri.getScheme();
         authority = uri.getAuthority();

src/hdfs/org/apache/hadoop/dfs/DistributedFileSystem.java

@@ -69,6 +69,21 @@ public class DistributedFileSystem extends FileSystem {
     this.workingDir = getHomeDirectory();
   }
 
+  /** Permit paths which explicitly specify the default port. */
+  protected void checkPath(Path path) {
+    URI thisUri = this.getUri();
+    URI thatUri = path.toUri();
+    String thatAuthority = thatUri.getAuthority();
+    if (thatUri.getScheme() != null
+        && thatUri.getScheme().equalsIgnoreCase(thisUri.getScheme())
+        && thatUri.getPort() == NameNode.DEFAULT_PORT
+        && thisUri.getPort() == -1
+        && thatAuthority.substring(0,thatAuthority.indexOf(":"))
+        .equalsIgnoreCase(thisUri.getAuthority()))
+      return;
+    super.checkPath(path);
+  }
+
   public Path getWorkingDirectory() {
     return workingDir;
   }

src/test/org/apache/hadoop/fs/TestFileSystem.java

@@ -32,6 +32,7 @@ import org.apache.commons.logging.Log;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.dfs.MiniDFSCluster;
+import org.apache.hadoop.dfs.NameNode;
 import org.apache.hadoop.fs.shell.CommandFormat;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.SequenceFile;
@@ -506,22 +507,51 @@ public class TestFileSystem extends TestCase {
     }
     
     {
-      MiniDFSCluster cluster = null;
       try {
-        cluster = new MiniDFSCluster(new Configuration(), 2, true, null);
-        URI uri = cluster.getFileSystem().getUri();
+        runTestCache(NameNode.DEFAULT_PORT);
+      } catch(java.net.BindException be) {
+        LOG.warn("Cannot test NameNode.DEFAULT_PORT (="
+            + NameNode.DEFAULT_PORT + ")", be);
+      }
+
+      runTestCache(0);
+    }
+  }
+  
+  static void runTestCache(int port) throws Exception {
+    Configuration conf = new Configuration();
+    MiniDFSCluster cluster = null;
+    try {
+      cluster = new MiniDFSCluster(port, conf, 2, true, true, null, null);
+      URI uri = cluster.getFileSystem().getUri();
+      LOG.info("uri=" + uri);
+
+      {
         FileSystem fs = FileSystem.get(uri, new Configuration());
         checkPath(cluster, fs);
         for(int i = 0; i < 100; i++) {
           assertTrue(fs == FileSystem.get(uri, new Configuration()));
         }
-      } finally {
-        cluster.shutdown(); 
       }
+      
+      if (port == NameNode.DEFAULT_PORT) {
+        //test explicit default port
+        URI uri2 = new URI(uri.getScheme(), uri.getUserInfo(),
+            uri.getHost(), NameNode.DEFAULT_PORT, uri.getPath(),
+            uri.getQuery(), uri.getFragment());  
+        LOG.info("uri2=" + uri2);
+        FileSystem fs = FileSystem.get(uri2, conf);
+        checkPath(cluster, fs);
+        for(int i = 0; i < 100; i++) {
+          assertTrue(fs == FileSystem.get(uri2, new Configuration()));
+        }
+      }
+    } finally {
+      if (cluster != null) cluster.shutdown(); 
     }
   }
   
-  private void checkPath(MiniDFSCluster cluster, FileSystem fileSys) throws IOException {
+  static void checkPath(MiniDFSCluster cluster, FileSystem fileSys) throws IOException {
     InetSocketAddress add = cluster.getNameNode().getNameNodeAddress();
     // Test upper/lower case
     fileSys.checkPath(new Path("hdfs://" + add.getHostName().toUpperCase() + ":" + add.getPort()));