HDFS-4329. DFSShell issues with directories with spaces in name (Cristina L. Abad via jeagles)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1516928 13f79535-47bb-0310-9956-ffa450edef68

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java

@@ -247,7 +247,7 @@ public class PathData implements Comparable<PathData> {
     // check getPath() so scheme slashes aren't considered part of the path
     String separator = uri.getPath().endsWith(Path.SEPARATOR)
         ? "" : Path.SEPARATOR;
-    return uri + separator + basename;
+    return uriToString(uri) + separator + basename;
   }
   
   protected enum PathType { HAS_SCHEME, SCHEMELESS_ABSOLUTE, RELATIVE };
@@ -300,7 +300,7 @@ public class PathData implements Comparable<PathData> {
             if (globUri.getAuthority() == null) {
               matchUri = removeAuthority(matchUri);
             }
-            globMatch = matchUri.toString();
+            globMatch = uriToString(matchUri);
             break;
           case SCHEMELESS_ABSOLUTE: // take just the uri's path
             globMatch = matchUri.getPath();
@@ -381,19 +381,14 @@ public class PathData implements Comparable<PathData> {
    * @return String of the path
    */
   public String toString() {
+    return uriToString(uri);
+  }
+  
+  private static String uriToString(URI uri) {
     String scheme = uri.getScheme();
     // No interpretation of symbols. Just decode % escaped chars.
-    String decodedRemainder = uri.getSchemeSpecificPart();
-
-    if (scheme == null) {
-      return decodedRemainder;
-    } else {
-      StringBuilder buffer = new StringBuilder();
-      buffer.append(scheme);
-      buffer.append(":");
-      buffer.append(decodedRemainder);
-      return buffer.toString();
-    }
+    String ssp = uri.getSchemeSpecificPart();
+    return (scheme != null) ? scheme + ":" + ssp : ssp;
   }
   
   /**

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CommandExecutor.java

@@ -24,6 +24,9 @@ import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.PrintStream;
 import java.util.StringTokenizer;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.ArrayList;
 
 /**
  *
@@ -31,24 +34,32 @@ import java.util.StringTokenizer;
  */
 public abstract class CommandExecutor {  
   protected String[] getCommandAsArgs(final String cmd, final String masterKey,
-		                                       final String master) {
-    StringTokenizer tokenizer = new StringTokenizer(cmd, " ");
-    String[] args = new String[tokenizer.countTokens()];
+                                                       final String master) {
+    String regex = "\'([^\']*)\'|\"([^\"]*)\"|(\\S+)";
+    Matcher matcher = Pattern.compile(regex).matcher(cmd);
+
+    ArrayList<String> args = new ArrayList<String>();
+    String arg = null;
     
-    int i = 0;
-    while (tokenizer.hasMoreTokens()) {
-      args[i] = tokenizer.nextToken();
+    while (matcher.find()) {
+      if (matcher.group(1) != null) {
+        arg = matcher.group(1);
+      } else if (matcher.group(2) != null) {
+        arg = matcher.group(2);
+      } else {
+        arg = matcher.group(3);
+      }
 
-      args[i] = args[i].replaceAll(masterKey, master);
-      args[i] = args[i].replaceAll("CLITEST_DATA", 
+      arg = arg.replaceAll(masterKey, master);
+      arg = arg.replaceAll("CLITEST_DATA", 
         new File(CLITestHelper.TEST_CACHE_DATA_DIR).
         toURI().toString().replace(' ', '+'));
-      args[i] = args[i].replaceAll("USERNAME", System.getProperty("user.name"));
+      arg = arg.replaceAll("USERNAME", System.getProperty("user.name"));
 
-      i++;
+      args.add(arg);
     }
-    
-    return args;
+
+    return args.toArray(new String[0]);
   }
   
   public Result executeCommand(final String cmd) throws Exception {

hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt

@@ -16,6 +16,9 @@ Release 0.23.10 - UNRELEASED
 
     HDFS-4994. Audit log getContentSummary() calls. (Robert Parker via kihwal)
 
+    HDFS-4329. DFSShell issues with directories with spaces in name (Cristina
+    L. Abad via jeagles)
+
   OPTIMIZATIONS
 
   BUG FIXES

hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml

@@ -443,6 +443,153 @@
       </comparators>
     </test>
 
+    <test> <!-- TESTED -->
+      <description>ls: whitespaces in an absolute path to a file</description>
+      <test-commands>
+        <command>-fs NAMENODE -mkdir -p "/a path with/whitespaces in directories"</command>
+        <command>-fs NAMENODE -touchz "/a path with/whitespaces in directories/and file names"</command>
+        <command>-fs NAMENODE -ls "/a path with/whitespaces in directories"</command>
+      </test-commands>
+      <cleanup-commands>
+        <command>-fs NAMENODE -rm -r "/a path with"</command>
+      </cleanup-commands>
+      <comparators>
+        <comparator>
+          <type>TokenComparator</type>
+          <expected-output>Found 1 items</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^-rw-r--r--( )*1( )*[a-z]*( )*supergroup( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*/a path with/whitespaces in directories/and file names</expected-output>
+        </comparator>
+      </comparators>
+    </test>
+
+    <test> <!-- TESTED -->
+      <description>ls: whitespaces in a relative path to a file</description>
+      <test-commands>
+        <command>-fs NAMENODE -mkdir -p "a path with/whitespaces in directories"</command>
+        <command>-fs NAMENODE -touchz "a path with/whitespaces in directories/and file names"</command>
+        <command>-fs NAMENODE -ls "a path with/whitespaces in directories"</command>
+      </test-commands>
+      <cleanup-commands>
+        <command>-fs NAMENODE -rm -r "a path with"</command>
+      </cleanup-commands>
+      <comparators>
+        <comparator>
+          <type>TokenComparator</type>
+          <expected-output>Found 1 items</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^-rw-r--r--( )*1( )*[a-z]*( )*supergroup( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*a path with/whitespaces in directories/and file names</expected-output>
+        </comparator>
+      </comparators>
+    </test>
+
+    <test> <!-- TESTED -->
+      <description>ls: whitespaces in a scheme-qualified path to a file</description>
+      <test-commands>
+        <command>-fs NAMENODE -mkdir -p "NAMENODE/a path with/whitespaces in directories"</command>
+        <command>-fs NAMENODE -touchz "NAMENODE/a path with/whitespaces in directories/and file names"</command>
+        <command>-fs NAMENODE -ls "NAMENODE/a*/w*"</command>
+      </test-commands>
+      <cleanup-commands>
+        <command>-fs NAMENODE -rm -r "NAMENODE/a path with"</command>
+      </cleanup-commands>
+      <comparators>
+        <comparator>
+          <type>TokenComparator</type>
+          <expected-output>Found 1 items</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^-rw-r--r--( )*1( )*[a-z]*( )*supergroup( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*NAMENODE/a path with/whitespaces in directories/and file names</expected-output>
+        </comparator>
+      </comparators>
+    </test>
+
+    <test> <!-- TESTED -->
+      <description>ls: whitespaces in an absolute path to a file, using globbing</description>
+      <test-commands>
+        <command>-fs NAMENODE -mkdir -p "/a path with/whitespaces in directories"</command>
+        <command>-fs NAMENODE -touchz "/a path with/whitespaces in directories/and file names"</command>
+        <command>-fs NAMENODE -touchz "/a path with/whitespaces in directories/and file names 2"</command>
+        <command>-fs NAMENODE -ls "/a*/w*"</command>
+      </test-commands>
+      <cleanup-commands>
+        <command>-fs NAMENODE -rm -r "/a path with"</command>
+      </cleanup-commands>
+      <comparators>
+        <comparator>
+          <type>TokenComparator</type>
+          <expected-output>Found 2 items</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^-rw-r--r--( )*1( )*[a-z]*( )*supergroup( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*/a path with/whitespaces in directories/and file names</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^-rw-r--r--( )*1( )*[a-z]*( )*supergroup( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*/a path with/whitespaces in directories/and file names 2</expected-output>
+        </comparator>
+      </comparators>
+    </test>
+
+    <test> <!-- TESTED -->
+      <description>ls: whitespaces in a relative path to a file, using globbing</description>
+      <test-commands>
+        <command>-fs NAMENODE -mkdir -p "a path with/whitespaces in directories"</command>
+        <command>-fs NAMENODE -touchz "a path with/whitespaces in directories/and file names"</command>
+        <command>-fs NAMENODE -touchz "a path with/whitespaces in directories/and file names 2"</command>
+        <command>-fs NAMENODE -ls "a*/w*"</command>
+      </test-commands>
+      <cleanup-commands>
+        <command>-fs NAMENODE -rm -r "a path with"</command>
+      </cleanup-commands>
+      <comparators>
+        <comparator>
+          <type>TokenComparator</type>
+          <expected-output>Found 2 items</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^-rw-r--r--( )*1( )*[a-z]*( )*supergroup( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*a path with/whitespaces in directories/and file names</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^-rw-r--r--( )*1( )*[a-z]*( )*supergroup( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*a path with/whitespaces in directories/and file names 2</expected-output>
+        </comparator>
+      </comparators>
+    </test>
+
+    <test> <!-- TESTED -->
+      <description>ls: whitespaces in a scheme-qualified path to a file, using globbing</description>
+      <test-commands>
+        <command>-fs NAMENODE -mkdir -p "NAMENODE/a path with/whitespaces in directories"</command>
+        <command>-fs NAMENODE -touchz "NAMENODE/a path with/whitespaces in directories/and file names"</command>
+        <command>-fs NAMENODE -touchz "NAMENODE/a path with/whitespaces in directories/and file names 2"</command>
+        <command>-fs NAMENODE -ls "NAMENODE/a*/w*"</command>
+      </test-commands>
+      <cleanup-commands>
+        <command>-fs NAMENODE -rm -r "NAMENODE/a path with"</command>
+      </cleanup-commands>
+      <comparators>
+        <comparator>
+          <type>TokenComparator</type>
+          <expected-output>Found 2 items</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^-rw-r--r--( )*1( )*[a-z]*( )*supergroup( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*NAMENODE/a path with/whitespaces in directories/and file names</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^-rw-r--r--( )*1( )*[a-z]*( )*supergroup( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*NAMENODE/a path with/whitespaces in directories/and file names 2</expected-output>
+        </comparator>
+      </comparators>
+    </test>
+
     <!-- Tests for ls -R -->
     <test> <!-- TESTED -->
       <description>ls: files/directories using absolute path</description>
@@ -6503,23 +6650,23 @@
       <comparators>
         <comparator>
           <type>TokenComparator</type>
-          <expected-output>"data15bytes-15"</expected-output>
+          <expected-output>data15bytes-15</expected-output>
         </comparator>
         <comparator>
           <type>TokenComparator</type>
-          <expected-output>"data30bytes-30"</expected-output>
+          <expected-output>data30bytes-30</expected-output>
         </comparator>
         <comparator>
           <type>TokenComparator</type>
-          <expected-output>"data60bytes-60"</expected-output>
+          <expected-output>data60bytes-60</expected-output>
         </comparator>
         <comparator>
           <type>TokenComparator</type>
-          <expected-output>"data120bytes-120"</expected-output>
+          <expected-output>data120bytes-120</expected-output>
         </comparator>
         <comparator>
           <type>TokenComparator</type>
-          <expected-output>"datadir-0"</expected-output>
+          <expected-output>datadir-0</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -6542,23 +6689,23 @@
       <comparators>
         <comparator>
           <type>TokenComparator</type>
-          <expected-output>"data15bytes-15"</expected-output>
+          <expected-output>data15bytes-15</expected-output>
         </comparator>
         <comparator>
           <type>TokenComparator</type>
-          <expected-output>"data30bytes-30"</expected-output>
+          <expected-output>data30bytes-30</expected-output>
         </comparator>
         <comparator>
           <type>TokenComparator</type>
-          <expected-output>"data60bytes-60"</expected-output>
+          <expected-output>data60bytes-60</expected-output>
         </comparator>
         <comparator>
           <type>TokenComparator</type>
-          <expected-output>"data120bytes-120"</expected-output>
+          <expected-output>data120bytes-120</expected-output>
         </comparator>
         <comparator>
           <type>TokenComparator</type>
-          <expected-output>"datadir-0"</expected-output>
+          <expected-output>datadir-0</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -6644,23 +6791,23 @@
       <comparators>
         <comparator>
           <type>TokenComparator</type>
-          <expected-output>"data15bytes-15"</expected-output>
+          <expected-output>data15bytes-15</expected-output>
         </comparator>
         <comparator>
           <type>TokenComparator</type>
-          <expected-output>"data30bytes-30"</expected-output>
+          <expected-output>data30bytes-30</expected-output>
         </comparator>
         <comparator>
           <type>TokenComparator</type>
-          <expected-output>"data60bytes-60"</expected-output>
+          <expected-output>data60bytes-60</expected-output>
         </comparator>
         <comparator>
           <type>TokenComparator</type>
-          <expected-output>"data120bytes-120"</expected-output>
+          <expected-output>data120bytes-120</expected-output>
         </comparator>
         <comparator>
           <type>TokenComparator</type>
-          <expected-output>"datadir-0"</expected-output>
+          <expected-output>datadir-0</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -6731,23 +6878,23 @@
       <comparators>
         <comparator>
           <type>TokenComparator</type>
-          <expected-output>"data15bytes-15"</expected-output>
+          <expected-output>data15bytes-15</expected-output>
         </comparator>
         <comparator>
           <type>TokenComparator</type>
-          <expected-output>"data30bytes-30"</expected-output>
+          <expected-output>data30bytes-30</expected-output>
         </comparator>
         <comparator>
           <type>TokenComparator</type>
-          <expected-output>"data60bytes-60"</expected-output>
+          <expected-output>data60bytes-60</expected-output>
         </comparator>
         <comparator>
           <type>TokenComparator</type>
-          <expected-output>"data120bytes-120"</expected-output>
+          <expected-output>data120bytes-120</expected-output>
         </comparator>
         <comparator>
           <type>TokenComparator</type>
-          <expected-output>"datadir-0"</expected-output>
+          <expected-output>datadir-0</expected-output>
         </comparator>
       </comparators>
     </test>