svn merge -c 1190109 from trunk for HADOOP-7360.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1190111 13f79535-47bb-0310-9956-ffa450edef68

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -420,6 +420,9 @@ Release 0.23.0 - Unreleased
 
     MAPREDUCE-2764. Fix renewal of dfs delegation tokens. (Owen via jitendra)
 
+    HADOOP-7360. Preserve relative paths that do not contain globs in FsShell.
+    (Daryn Sharp and Kihwal Lee via szetszwo)
+
   OPTIMIZATIONS
   
     HADOOP-7333. Performance improvement in PureJavaCrc32. (Eric Caspole

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java

@@ -27,10 +27,8 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.ChecksumFileSystem;
 import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.LocalFileSystem;
-import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.shell.PathExceptions.PathExistsException;
 import org.apache.hadoop.fs.shell.PathExceptions.PathIOException;
 import org.apache.hadoop.fs.shell.PathExceptions.PathOperationException;
@@ -131,12 +129,10 @@ class CopyCommands {
     private static final String COPYTOLOCAL_PREFIX = "_copyToLocal_";
     private boolean copyCrc;
     private boolean verifyChecksum;
-    private LocalFileSystem localFs;
 
     @Override
     protected void processOptions(LinkedList<String> args)
     throws IOException {
-      localFs = FileSystem.getLocal(getConf());
       CommandFormat cf = new CommandFormat(
           1, Integer.MAX_VALUE, "crc", "ignoreCrc");
       cf.parse(args);
@@ -157,16 +153,15 @@ class CopyCommands {
         copyCrc = false;
       }      
 
-      File targetFile = localFs.pathToFile(target.path);
       if (src.stat.isFile()) {
         // copy the file and maybe its crc
-        copyFileToLocal(src, target.path);
+        copyFileToLocal(src, target);
         if (copyCrc) {
-          copyCrcToLocal(src, target.path);
+          copyFileToLocal(src.getChecksumFile(), target.getChecksumFile());
         }
       } else if (src.stat.isDirectory()) {
         // create the remote directory structure locally
-        if (!targetFile.mkdirs()) {
+        if (!target.toFile().mkdirs()) {
           throw new PathIOException(target.toString());
         }
       } else {
@@ -174,9 +169,9 @@ class CopyCommands {
       }
     }
 
-    private void copyFileToLocal(PathData src, Path target)
+    private void copyFileToLocal(PathData src, PathData target)
     throws IOException {
-      File targetFile = localFs.pathToFile(target);
+      File targetFile = target.toFile();
       File tmpFile = FileUtil.createLocalTempFile(
           targetFile, COPYTOLOCAL_PREFIX, true);
       // too bad we can't tell exactly why it failed...
@@ -195,14 +190,6 @@ class CopyCommands {
         throw e;
       }
     }
-
-    private void copyCrcToLocal(PathData src, Path target)
-    throws IOException {
-      ChecksumFileSystem srcFs = (ChecksumFileSystem)src.fs;
-      Path srcPath = srcFs.getChecksumFile(src.path);
-      src = new PathData(srcFs.getRawFileSystem(), srcPath);
-      copyFileToLocal(src, localFs.getChecksumFile(target));    
-    }
   }
 
   /**

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java

@@ -81,6 +81,6 @@ public class Count extends FsCommand {
   @Override
   protected void processPath(PathData src) throws IOException {
     ContentSummary summary = src.fs.getContentSummary(src.path);
-    out.println(summary.toString(showQuotas) + src.path);
+    out.println(summary.toString(showQuotas) + src);
   }
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java

@@ -113,7 +113,7 @@ class Ls extends FsCommand {
         stat.getGroup(),
         formatSize(stat.getLen()),
         dateFormat.format(new Date(stat.getModificationTime())),
-        item.path.toUri().getPath()
+        item
     );
     out.println(line);
   }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java

@@ -21,27 +21,34 @@ package org.apache.hadoop.fs.shell;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.ChecksumFileSystem;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.shell.PathExceptions.PathIOException;
+import org.apache.hadoop.fs.shell.PathExceptions.PathIsDirectoryException;
 import org.apache.hadoop.fs.shell.PathExceptions.PathIsNotDirectoryException;
+import org.apache.hadoop.fs.shell.PathExceptions.PathNotFoundException;
 
 /**
  * Encapsulates a Path (path), its FileStatus (stat), and its FileSystem (fs).
  * The stat field will be null if the path does not exist.
  */
 @InterfaceAudience.Private
-@InterfaceStability.Evolving
+@InterfaceStability.Unstable
 
 public class PathData {
-  protected String string = null;
+  protected final URI uri;
+  public final FileSystem fs;
   public final Path path;
   public FileStatus stat;
-  public final FileSystem fs;
   public boolean exists;
 
   /**
@@ -53,10 +60,7 @@ public class PathData {
    * @throws IOException if anything goes wrong...
    */
   public PathData(String pathString, Configuration conf) throws IOException {
-    this.string = pathString;
-    this.path = new Path(pathString);
-    this.fs = path.getFileSystem(conf);
-    setStat(getStat(fs, path));
+    this(FileSystem.get(URI.create(pathString), conf), pathString);
   }
   
   /**
@@ -68,87 +72,116 @@ public class PathData {
    * @throws IOException if anything goes wrong...
    */
   public PathData(File localPath, Configuration conf) throws IOException {
-    this.string = localPath.toString();
-    this.path = new Path(this.string);
-    this.fs = FileSystem.getLocal(conf);
-    setStat(getStat(fs, path));
+    this(FileSystem.getLocal(conf), localPath.toString());
   }
 
   /**
-   * Creates an object to wrap the given parameters as fields. 
-   * @param fs the FileSystem
-   * @param path a Path
-   * @param stat the FileStatus (may be null if the path doesn't exist)
-   */
-  public PathData(FileSystem fs, Path path, FileStatus stat) {
-    this.string = path.toString();
-    this.path = path;
-    this.fs = fs;
-    setStat(stat);
-  }
-
-  /**
-   * Convenience ctor that looks up the file status for a path.  If the path
+   * Looks up the file status for a path.  If the path
    * doesn't exist, then the status will be null
    * @param fs the FileSystem for the path
-   * @param path the pathname to lookup 
+   * @param pathString a string for a path 
    * @throws IOException if anything goes wrong
    */
-  public PathData(FileSystem fs, Path path) throws IOException {
-    this(fs, path, getStat(fs, path));
+  private PathData(FileSystem fs, String pathString) throws IOException {
+    this(fs, pathString, lookupStat(fs, pathString, true));
   }
 
   /**
    * Creates an object to wrap the given parameters as fields.  The string
    * used to create the path will be recorded since the Path object does not
-   * return exactly the same string used to initialize it.  If the FileStatus
-   * is not null, then its Path will be used to initialized the path, else
-   * the string of the path will be used.
+   * return exactly the same string used to initialize it.
    * @param fs the FileSystem
    * @param pathString a String of the path
    * @param stat the FileStatus (may be null if the path doesn't exist)
    */
-  public PathData(FileSystem fs, String pathString, FileStatus stat) {
-    this.string = pathString;
-    this.path = (stat != null) ? stat.getPath() : new Path(pathString);
+  private PathData(FileSystem fs, String pathString, FileStatus stat)
+  throws IOException {
     this.fs = fs;
+    this.uri = stringToUri(pathString);
+    this.path = fs.makeQualified(new Path(uri));
     setStat(stat);
   }
 
   // need a static method for the ctor above
-  private static FileStatus getStat(FileSystem fs, Path path)
-  throws IOException {  
+  /**
+   * Get the FileStatus info
+   * @param ignoreFNF if true, stat will be null if the path doesn't exist
+   * @return FileStatus for the given path
+   * @throws IOException if anything goes wrong
+   */
+  private static
+  FileStatus lookupStat(FileSystem fs, String pathString, boolean ignoreFNF)
+  throws IOException {
     FileStatus status = null;
     try {
-      status = fs.getFileStatus(path);
-    } catch (FileNotFoundException e) {} // ignore FNF
+      status = fs.getFileStatus(new Path(pathString));
+    } catch (FileNotFoundException e) {
+      if (!ignoreFNF) throw new PathNotFoundException(pathString);
+    }
+    // TODO: should consider wrapping other exceptions into Path*Exceptions
     return status;
   }
   
-  private void setStat(FileStatus theStat) {
-    stat = theStat;
+  private void setStat(FileStatus stat) {
+    this.stat = stat;
     exists = (stat != null);
   }
 
-  /**
-   * Convenience ctor that extracts the path from the given file status
-   * @param fs the FileSystem for the FileStatus
-   * @param stat the FileStatus 
-   */
-  public PathData(FileSystem fs, FileStatus stat) {
-    this(fs, stat.getPath(), stat);
-  }
-  
   /**
    * Updates the paths's file status
    * @return the updated FileStatus
    * @throws IOException if anything goes wrong...
    */
   public FileStatus refreshStatus() throws IOException {
-    setStat(fs.getFileStatus(path));
-    return stat;
+    FileStatus status = null;
+    try {
+      status = lookupStat(fs, toString(), false);
+    } finally {
+      // always set the status.  the caller must get the correct result
+      // if it catches the exception and later interrogates the status
+      setStat(status);
+    }
+    return status;
+  }
+
+  protected enum FileTypeRequirement {
+    SHOULD_NOT_BE_DIRECTORY, SHOULD_BE_DIRECTORY
+  };
+
+  /**
+   * Ensure that the file exists and if it is or is not a directory
+   * @param typeRequirement Set it to the desired requirement.
+   * @throws PathIOException if file doesn't exist or the type does not match
+   * what was specified in typeRequirement.
+   */
+  private void checkIfExists(FileTypeRequirement typeRequirement) 
+  throws PathIOException {
+    if (!exists) {
+      throw new PathNotFoundException(toString());      
+    }
+
+    if ((typeRequirement == FileTypeRequirement.SHOULD_BE_DIRECTORY)
+       && !stat.isDirectory()) {
+      throw new PathIsNotDirectoryException(toString());
+    } else if ((typeRequirement == FileTypeRequirement.SHOULD_NOT_BE_DIRECTORY)
+              && stat.isDirectory()) {
+      throw new PathIsDirectoryException(toString());
+    }
   }
   
+  /**
+   * Return the corresponding crc data for a file.  Avoids exposing the fs
+   * contortions to the caller.  
+   * @return PathData of the crc file
+   * @throws IOException is anything goes wrong
+   */
+  public PathData getChecksumFile() throws IOException {
+    checkIfExists(FileTypeRequirement.SHOULD_NOT_BE_DIRECTORY);
+    ChecksumFileSystem srcFs = (ChecksumFileSystem)fs;
+    Path srcPath = srcFs.getChecksumFile(path);
+    return new PathData(srcFs.getRawFileSystem(), srcPath.toString());
+  }
+
   /**
    * Returns a list of PathData objects of the items contained in the given
    * directory.
@@ -156,18 +189,13 @@ public class PathData {
    * @throws IOException if anything else goes wrong...
    */
   public PathData[] getDirectoryContents() throws IOException {
-    if (!stat.isDirectory()) {
-      throw new PathIsNotDirectoryException(string);
-    }
-
+    checkIfExists(FileTypeRequirement.SHOULD_BE_DIRECTORY);
     FileStatus[] stats = fs.listStatus(path);
     PathData[] items = new PathData[stats.length];
     for (int i=0; i < stats.length; i++) {
       // preserve relative paths
-      String basename = stats[i].getPath().getName();
-      String parent = string;
-      if (!parent.endsWith(Path.SEPARATOR)) parent += Path.SEPARATOR;
-      items[i] = new PathData(fs, parent + basename, stats[i]);
+      String child = getStringForChildPath(stats[i].getPath());
+      items[i] = new PathData(fs, child, stats[i]);
     }
     return items;
   }
@@ -179,12 +207,30 @@ public class PathData {
    * @throws IOException if this object does not exist or is not a directory
    */
   public PathData getPathDataForChild(PathData child) throws IOException {
-    if (!stat.isDirectory()) {
-      throw new PathIsNotDirectoryException(string);
-    }
-    return new PathData(fs, new Path(path, child.path.getName()));
+    checkIfExists(FileTypeRequirement.SHOULD_BE_DIRECTORY);
+    return new PathData(fs, getStringForChildPath(child.path));
   }
 
+  /**
+   * Given a child of this directory, use the directory's path and the child's
+   * basename to construct the string to the child.  This preserves relative
+   * paths since Path will fully qualify.
+   * @param child a path contained within this directory
+   * @return String of the path relative to this directory
+   */
+  private String getStringForChildPath(Path childPath) {
+    String basename = childPath.getName();
+    if (Path.CUR_DIR.equals(toString())) {
+      return basename;
+    }
+    // check getPath() so scheme slashes aren't considered part of the path
+    String separator = uri.getPath().endsWith(Path.SEPARATOR)
+        ? "" : Path.SEPARATOR;
+    return uri + separator + basename;
+  }
+  
+  protected enum PathType { HAS_SCHEME, SCHEMELESS_ABSOLUTE, RELATIVE };
+  
   /**
    * Expand the given path as a glob pattern.  Non-existent paths do not
    * throw an exception because creation commands like touch and mkdir need
@@ -207,35 +253,184 @@ public class PathData {
     if (stats == null) {
       // not a glob & file not found, so add the path with a null stat
       items = new PathData[]{ new PathData(fs, pattern, null) };
-    } else if (
-        // this is very ugly, but needed to avoid breaking hdfs tests...
-        // if a path has no authority, then the FileStatus from globStatus
-        // will add the "-fs" authority into the path, so we need to sub
-        // it back out to satisfy the tests
-        stats.length == 1
-        &&
-        stats[0].getPath().equals(fs.makeQualified(globPath)))
-    {
-      // if the fq path is identical to the pattern passed, use the pattern
-      // to initialize the string value
-      items = new PathData[]{ new PathData(fs, pattern, stats[0]) };
     } else {
+      // figure out what type of glob path was given, will convert globbed
+      // paths to match the type to preserve relativity
+      PathType globType;
+      URI globUri = globPath.toUri();
+      if (globUri.getScheme() != null) {
+        globType = PathType.HAS_SCHEME;
+      } else if (new File(globUri.getPath()).isAbsolute()) {
+        globType = PathType.SCHEMELESS_ABSOLUTE;
+      } else {
+        globType = PathType.RELATIVE;
+      }
+
       // convert stats to PathData
       items = new PathData[stats.length];
       int i=0;
       for (FileStatus stat : stats) {
-        items[i++] = new PathData(fs, stat);
+        URI matchUri = stat.getPath().toUri();
+        String globMatch = null;
+        switch (globType) {
+          case HAS_SCHEME: // use as-is, but remove authority if necessary
+            if (globUri.getAuthority() == null) {
+              matchUri = removeAuthority(matchUri);
+            }
+            globMatch = matchUri.toString();
+            break;
+          case SCHEMELESS_ABSOLUTE: // take just the uri's path
+            globMatch = matchUri.getPath();
+            break;
+          case RELATIVE: // make it relative to the current working dir
+            URI cwdUri = fs.getWorkingDirectory().toUri();
+            globMatch = relativize(cwdUri, matchUri, stat.isDirectory());
+            break;
+        }
+        items[i++] = new PathData(fs, globMatch, stat);
       }
     }
     return items;
   }
 
+  private static URI removeAuthority(URI uri) {
+    try {
+      uri = new URI(
+          uri.getScheme(), "",
+          uri.getPath(), uri.getQuery(), uri.getFragment()
+      );
+    } catch (URISyntaxException e) {
+      throw new IllegalArgumentException(e.getLocalizedMessage());
+    }
+    return uri;
+  }
+  
+  private static String relativize(URI cwdUri, URI srcUri, boolean isDir) {
+    String uriPath = srcUri.getPath();
+    String cwdPath = cwdUri.getPath();
+    if (cwdPath.equals(uriPath)) {
+      return Path.CUR_DIR;
+    }
+
+    // find common ancestor
+    int lastSep = findLongestDirPrefix(cwdPath, uriPath, isDir);
+    
+    StringBuilder relPath = new StringBuilder();    
+    // take the remaining path fragment after the ancestor
+    if (lastSep < uriPath.length()) {
+      relPath.append(uriPath.substring(lastSep+1));
+    }
+
+    // if cwd has a path fragment after the ancestor, convert them to ".."
+    if (lastSep < cwdPath.length()) {
+      while (lastSep != -1) {
+        if (relPath.length() != 0) relPath.insert(0, Path.SEPARATOR);
+        relPath.insert(0, "..");
+        lastSep = cwdPath.indexOf(Path.SEPARATOR, lastSep+1);
+      }
+    }
+    return relPath.toString();
+  }
+
+  private static int findLongestDirPrefix(String cwd, String path, boolean isDir) {
+    // add the path separator to dirs to simplify finding the longest match
+    if (!cwd.endsWith(Path.SEPARATOR)) {
+      cwd += Path.SEPARATOR;
+    }
+    if (isDir && !path.endsWith(Path.SEPARATOR)) {
+      path += Path.SEPARATOR;
+    }
+
+    // find longest directory prefix 
+    int len = Math.min(cwd.length(), path.length());
+    int lastSep = -1;
+    for (int i=0; i < len; i++) {
+      if (cwd.charAt(i) != path.charAt(i)) break;
+      if (cwd.charAt(i) == Path.SEPARATOR_CHAR) lastSep = i;
+    }
+    return lastSep;
+  }
+  
   /**
    * Returns the printable version of the path that is either the path
    * as given on the commandline, or the full path
    * @return String of the path
    */
   public String toString() {
-    return (string != null) ? string : path.toString();
+    String scheme = uri.getScheme();
+    // No interpretation of symbols. Just decode % escaped chars.
+    String decodedRemainder = uri.getSchemeSpecificPart();
+
+    if (scheme == null) {
+      return decodedRemainder;
+    } else {
+      StringBuilder buffer = new StringBuilder();
+      buffer.append(scheme);
+      buffer.append(":");
+      buffer.append(decodedRemainder);
+      return buffer.toString();
+    }
+  }
+  
+  /**
+   * Get the path to a local file
+   * @return File representing the local path
+   * @throws IllegalArgumentException if this.fs is not the LocalFileSystem
+   */
+  public File toFile() {
+    if (!(fs instanceof LocalFileSystem)) {
+       throw new IllegalArgumentException("Not a local path: " + path);
+    }
+    return ((LocalFileSystem)fs).pathToFile(path);
+  }
+
+  /** Construct a URI from a String with unescaped special characters
+   *  that have non-standard sematics. e.g. /, ?, #. A custom parsing
+   *  is needed to prevent misbihaviors.
+   *  @param pathString The input path in string form
+   *  @return URI
+   */
+  private static URI stringToUri(String pathString) {
+    // We can't use 'new URI(String)' directly. Since it doesn't do quoting
+    // internally, the internal parser may fail or break the string at wrong
+    // places. Use of multi-argument ctors will quote those chars for us,
+    // but we need to do our own parsing and assembly.
+    
+    // parse uri components
+    String scheme = null;
+    String authority = null;
+
+    int start = 0;
+
+    // parse uri scheme, if any
+    int colon = pathString.indexOf(':');
+    int slash = pathString.indexOf('/');
+    if (colon > 0 && (slash == colon +1)) {
+      // has a non zero-length scheme
+      scheme = pathString.substring(0, colon);
+      start = colon + 1;
+    }
+
+    // parse uri authority, if any
+    if (pathString.startsWith("//", start) &&
+        (pathString.length()-start > 2)) {
+      start += 2;
+      int nextSlash = pathString.indexOf('/', start);
+      int authEnd = nextSlash > 0 ? nextSlash : pathString.length();
+      authority = pathString.substring(start, authEnd);
+      start = authEnd;
+    }
+
+    // uri path is the rest of the string. ? or # are not interpreated,
+    // but any occurrence of them will be quoted by the URI ctor.
+    String path = pathString.substring(start, pathString.length());
+
+    // Construct the URI
+    try {
+      return new URI(scheme, authority, path, null, null);
+    } catch (URISyntaxException e) {
+      throw new IllegalArgumentException(e);
+    }
   }
+
 }

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/CLITestHelper.java

@@ -180,7 +180,7 @@ public class CLITestHelper {
           LOG.info("         Comparision result:   [" + 
                    (resultBoolean ? "pass" : "fail") + "]");
           LOG.info("            Expected output:   [" + 
-                   cd.getExpectedOutput() + "]");
+                   expandCommand(cd.getExpectedOutput()) + "]");
           LOG.info("              Actual output:   [" + 
                    cd.getActualOutput() + "]");
         }
@@ -290,7 +290,7 @@ public class CLITestHelper {
           comparatorType);
         ComparatorBase comp = (ComparatorBase) comparatorClass.newInstance();
         compareOutput = comp.compare(cmdResult.getCommandOutput(), 
-          compdata.getExpectedOutput());
+          expandCommand(compdata.getExpectedOutput()));
       } catch (Exception e) {
         LOG.info("Error in instantiating the comparator" + e);
       }

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java

@@ -17,80 +17,180 @@
  */
 package org.apache.hadoop.fs.shell;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.util.Arrays;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.junit.BeforeClass;
 import org.junit.Test;
-import static org.junit.Assert.*;
 
 public class TestPathData {
   protected static Configuration conf;
   protected static FileSystem fs;
   protected static String dirString;
-  protected static Path dir;
+  protected static Path testDir;
   protected static PathData item;
-
+  
+  protected static String[] d1Paths =
+    new String[] { "d1/f1", "d1/f1.1", "d1/f2" };
+  protected static String[] d2Paths =
+    new String[] { "d2/f3" };
+        
   @BeforeClass
   public static void initialize() throws Exception {
     conf = new Configuration();
-    fs = FileSystem.getLocal(conf); 
+    fs = FileSystem.getLocal(conf);
+    testDir = new Path(
+        System.getProperty("test.build.data", "build/test/data") + "/testPD"
+    );
+    // don't want scheme on the path, just an absolute path
+    testDir = new Path(fs.makeQualified(testDir).toUri().getPath());
+    FileSystem.setDefaultUri(conf, fs.getUri());    
+    fs.setWorkingDirectory(testDir);
+    fs.mkdirs(new Path("d1"));
+    fs.createNewFile(new Path("d1", "f1"));
+    fs.createNewFile(new Path("d1", "f1.1"));
+    fs.createNewFile(new Path("d1", "f2"));
+    fs.mkdirs(new Path("d2"));
+    fs.create(new Path("d2","f3"));
   }
 
   @Test
-  public void testWithFsAndPath() throws Exception {
-    dirString = "/tmp";
-    dir = new Path(dirString);
-    item = new PathData(fs, dir);
+  public void testWithDirStringAndConf() throws Exception {
+    dirString = "d1";
+    item = new PathData(dirString, conf);
     checkPathData();
-  }
 
-  @Test
-  public void testWithStringAndConf() throws Exception {
-    dirString = "/tmp";
-    dir = new Path(dirString);
+    // properly implementing symlink support in various commands will require
+    // trailing slashes to be retained
+    dirString = "d1/";
     item = new PathData(dirString, conf);
     checkPathData();
   }
 
   @Test
   public void testUnqualifiedUriContents() throws Exception {
-    dirString = "/tmp";
+    dirString = "d1";
     item = new PathData(dirString, conf);
     PathData[] items = item.getDirectoryContents();
-    for (PathData item : items) {
-      assertTrue(item.toString().startsWith(dirString));
-    }
+    assertEquals(
+        sortedString("d1/f1", "d1/f1.1", "d1/f2"),
+        sortedString(items)
+    );
   }
 
   @Test
   public void testQualifiedUriContents() throws Exception {
-    dirString = "file:/tmp";
+    dirString = fs.makeQualified(new Path("d1")).toString();
     item = new PathData(dirString, conf);
     PathData[] items = item.getDirectoryContents();
-    for (PathData item : items) {
-      assertTrue(item.toString().startsWith(dirString));
-    }
+    assertEquals(
+        sortedString(dirString+"/f1", dirString+"/f1.1", dirString+"/f2"),
+        sortedString(items)
+    );
+  }
+
+  @Test
+  public void testCwdContents() throws Exception {
+    dirString = Path.CUR_DIR;
+    item = new PathData(dirString, conf);
+    PathData[] items = item.getDirectoryContents();
+    assertEquals(
+        sortedString("d1", "d2"),
+        sortedString(items)
+    );
+  }
+
+
+	@Test
+	public void testToFile() throws Exception {
+    item = new PathData(".", conf);
+    assertEquals(new File(testDir.toString()), item.toFile());
+	  item = new PathData("d1/f1", conf);
+	  assertEquals(new File(testDir+"/d1/f1"), item.toFile());
+    item = new PathData(testDir+"/d1/f1", conf);
+    assertEquals(new File(testDir+"/d1/f1"), item.toFile());
+	}
+	
+  @Test
+  public void testAbsoluteGlob() throws Exception {
+    PathData[] items = PathData.expandAsGlob(testDir+"/d1/f1*", conf);
+    assertEquals(
+        sortedString(testDir+"/d1/f1", testDir+"/d1/f1.1"),
+        sortedString(items)
+    );
+  }
+
+  @Test
+  public void testRelativeGlob() throws Exception {
+    PathData[] items = PathData.expandAsGlob("d1/f1*", conf);
+    assertEquals(
+        sortedString("d1/f1", "d1/f1.1"),
+        sortedString(items)
+    );
+  }
+
+  @Test
+  public void testRelativeGlobBack() throws Exception {
+    fs.setWorkingDirectory(new Path("d1"));
+    PathData[] items = PathData.expandAsGlob("../d2/*", conf);
+    assertEquals(
+        sortedString("../d2/f3"),
+        sortedString(items)
+    );
   }
 
   @Test
   public void testWithStringAndConfForBuggyPath() throws Exception {
     dirString = "file:///tmp";
-    dir = new Path(dirString);
+    testDir = new Path(dirString);
     item = new PathData(dirString, conf);
     // this may fail some day if Path is fixed to not crunch the uri
     // if the authority is null, however we need to test that the PathData
     // toString() returns the given string, while Path toString() does
     // the crunching
-    assertEquals("file:/tmp", dir.toString());
+    assertEquals("file:/tmp", testDir.toString());
     checkPathData();
   }
 
   public void checkPathData() throws Exception {
-    assertEquals(fs, item.fs);
-    assertEquals(dirString, item.toString());
-    assertEquals(dir, item.path);
-    assertTrue(item.stat != null);
-    assertTrue(item.stat.isDirectory());
+    assertEquals("checking fs", fs, item.fs);
+    assertEquals("checking string", dirString, item.toString());
+    assertEquals("checking path",
+        fs.makeQualified(new Path(item.toString())), item.path
+    );
+    assertTrue("checking exist", item.stat != null);
+    assertTrue("checking isDir", item.stat.isDirectory());
+  }
+  
+  /* junit does a lousy job of comparing arrays
+   * if the array lengths differ, it just says that w/o showing contents
+   * this sorts the paths, and builds a string of "i:<value>, ..." suitable
+   * for a string compare
+   */
+  private static String sortedString(Object ... list) {
+    String[] strings = new String[list.length];
+    for (int i=0; i < list.length; i++) {
+      strings[i] = String.valueOf(list[i]);
+    }
+    Arrays.sort(strings);
+    
+    StringBuilder result = new StringBuilder();
+    for (int i=0; i < strings.length; i++) {
+      if (result.length() > 0) {
+        result.append(", ");
+      }
+      result.append(i+":<"+strings[i]+">");
+    }
+    return result.toString();
+  }
+  
+  private static String sortedString(PathData ... items) {
+    return sortedString((Object[])items);
   }
 }