HADOOP-1967. Use authority from default filesystem when none specified.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/core/trunk@632403 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -41,6 +41,12 @@ Trunk (unreleased changes)
     presence of snapshots. Needed for supporting appends to HDFS
     files. (dhruba) 
 
+    HADOOP-1967.  When a Path specifies the same scheme as the default
+    FileSystem but no authority, the default FileSystem's authority is
+    used.  Also add warnings for old-format FileSystem names, accessor
+    methods for fs.default.name, and check for null authority in HDFS.
+    (cutting)
+
   OPTIMIZATIONS
 
   BUG FIXES

src/java/org/apache/hadoop/dfs/DistributedFileSystem.java

@@ -63,6 +63,9 @@ public class DistributedFileSystem extends FileSystem {
     setConf(conf);
     String host = uri.getHost();
     int port = uri.getPort();
+    if (host == null || port == -1) {
+      throw new IOException("Incomplete HDFS URI, no host/port: "+ uri);
+    }
     this.dfs = new DFSClient(new InetSocketAddress(host, port), conf);
     this.uri = URI.create("hdfs://"+host+":"+port);
     this.workingDir = getHomeDirectory();

src/java/org/apache/hadoop/fs/FileSystem.java

@@ -51,6 +51,8 @@ import org.apache.hadoop.ipc.RemoteException;
  * implementation is {@link DistributedFileSystem}.
  *****************************************************************/
 public abstract class FileSystem extends Configured {
+  private static final String FS_DEFAULT_NAME_KEY = "fs.default.name";
+
   public static final Log LOG = LogFactory.getLog("org.apache.hadoop.fs.FileSystem");
 
   // cache indexed by URI scheme and authority
@@ -91,7 +93,31 @@ public abstract class FileSystem extends Configured {
 
   /** Returns the configured filesystem implementation.*/
   public static FileSystem get(Configuration conf) throws IOException {
-    return getNamed(conf.get("fs.default.name", "file:///"), conf);
+    return get(getDefaultUri(conf), conf);
+  }
+  
+  /** Get the default filesystem URI from a configuration.
+   * @param conf the configuration to access
+   * @return the uri of the default filesystem
+   */
+  public static URI getDefaultUri(Configuration conf) {
+    return URI.create(fixName(conf.get(FS_DEFAULT_NAME_KEY, "file:///")));
+  }
+
+  /** Set the default filesystem URI in a configuration.
+   * @param conf the configuration to alter
+   * @param uri the new default filesystem uri
+   */
+  public static void setDefaultUri(Configuration conf, URI uri) {
+    conf.set(FS_DEFAULT_NAME_KEY, uri.toString());
+  }
+
+  /** Set the default filesystem URI in a configuration.
+   * @param conf the configuration to alter
+   * @param uri the new default filesystem uri
+   */
+  public static void setDefaultUri(Configuration conf, String uri) {
+    setDefaultUri(conf, URI.create(fixName(uri)));
   }
 
   /** Called after a new FileSystem instance is constructed.
@@ -111,15 +137,24 @@ public abstract class FileSystem extends Configured {
   /** @deprecated call #get(URI,Configuration) instead. */
   public static FileSystem getNamed(String name, Configuration conf)
     throws IOException {
+    return get(URI.create(fixName(name)), conf);
+  }
 
+  /** Update old-format filesystem names, for back-compatibility.  This should
+   * eventually be replaced with a checkName() method that throws an exception
+   * for old-format names. */ 
+  private static String fixName(String name) {
     // convert old-format name to new-format name
     if (name.equals("local")) {         // "local" is now "file:///".
+      LOG.warn("\"local\" is a deprecated filesystem name."
+               +" Use \"file:///\" instead.");
       name = "file:///";
     } else if (name.indexOf('/')==-1) {   // unqualified is "hdfs://"
+      LOG.warn("\""+name+"\" is a deprecated filesystem name."
+               +" Use \"hdfs://"+name+"/\" instead.");
       name = "hdfs://"+name;
     }
-
-    return get(URI.create(name), conf);
+    return name;
   }
 
   /**
@@ -147,6 +182,14 @@ public abstract class FileSystem extends Configured {
       return get(conf);
     }
 
+    if (authority == null) {                       // no authority
+      URI defaultUri = getDefaultUri(conf);
+      if (scheme.equals(defaultUri.getScheme())    // if scheme matches default
+          && defaultUri.getAuthority() != null) {  // & default has authority
+        return get(defaultUri, conf);              // return default
+      }
+    }
+
     Map<String,FileSystem> authorityToFs = CACHE.get(scheme);
     if (authorityToFs == null) {
       if (CACHE.isEmpty()) {
@@ -260,11 +303,22 @@ public abstract class FileSystem extends Configured {
     URI uri = path.toUri();
     if (uri.getScheme() == null)                // fs is relative 
       return;
+    String thisScheme = this.getUri().getScheme();
+    String thatScheme = uri.getScheme();
     String thisAuthority = this.getUri().getAuthority();
     String thatAuthority = uri.getAuthority();
-    if (!(this.getUri().getScheme().equals(uri.getScheme()) &&
-           (thisAuthority == thatAuthority || 
-             (thisAuthority != null && thisAuthority.equals(thatAuthority))))){
+    if (thisScheme.equals(thatScheme)) {          // schemes match
+      if (thisAuthority == thatAuthority ||       // & authorities match
+          (thisAuthority != null && thisAuthority.equals(thatAuthority)))
+        return;
+
+      if (thatAuthority == null &&                // path's authority is null
+          thisAuthority != null) {                // fs has an authority
+        URI defaultUri = getDefaultUri(getConf()); // & is the default fs
+        if (thisScheme.equals(defaultUri.getScheme()) &&
+            thisAuthority.equals(defaultUri.getAuthority()))
+          return;
+      }
       throw new IllegalArgumentException("Wrong FS: "+path+
                                          ", expected: "+this.getUri());
     }

src/test/org/apache/hadoop/dfs/TestDFSShell.java

@@ -382,6 +382,19 @@ public class TestDFSShell extends TestCase {
       confirmOwner(null, "herbivores", dstFs, parent, path);
       runCmd(shell, "-chown", "-R", ":reptiles", dstFs.getUri().toString() + "/");
       confirmOwner(null, "reptiles", dstFs, root, parent, path);
+      //check if default hdfs:/// works 
+      argv[0] = "-cat";
+      argv[1] = "hdfs:///furi";
+      ret = ToolRunner.run(shell, argv);
+      assertTrue(" default works for cat", (ret == 0));
+      argv[0] = "-ls";
+      argv[1] = "hdfs:///";
+      ret = ToolRunner.run(shell, argv);
+      assertTrue("default works for ls ", (ret == 0));
+      argv[0] = "-rmr";
+      argv[1] = "hdfs:///furi";
+      ret = ToolRunner.run(shell, argv);
+      assertTrue("default works for rm/rmr", (ret ==0));
     } finally {
       System.setProperty("test.build.data", bak);
       if (null != srcCluster) {