YARN-2720. Windows: Wildcard classpath variables not expanded against resources contained in archives. Contributed by Craig Welch.

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java

@@ -1212,10 +1212,11 @@ public class FileUtil {
    * @param pwd Path to working directory to save jar
    * @param callerEnv Map<String, String> caller's environment variables to use
    *   for expansion
-   * @return String absolute path to new jar
+   * @return String[] with absolute path to new jar in position 0 and
+   *   unexpanded wild card entry path in position 1
    * @throws IOException if there is an I/O error while writing the jar file
    */
-  public static String createJarWithClassPath(String inputClassPath, Path pwd,
+  public static String[] createJarWithClassPath(String inputClassPath, Path pwd,
       Map<String, String> callerEnv) throws IOException {
     // Replace environment variables, case-insensitive on Windows
     @SuppressWarnings("unchecked")
@@ -1235,6 +1236,7 @@ public class FileUtil {
       LOG.debug("mkdirs false for " + workingDir + ", execution will continue");
     }
 
+    StringBuilder unexpandedWildcardClasspath = new StringBuilder();
     // Append all entries
     List<String> classPathEntryList = new ArrayList<String>(
       classPathEntries.length);
@@ -1243,16 +1245,22 @@ public class FileUtil {
         continue;
       }
       if (classPathEntry.endsWith("*")) {
+        boolean foundWildCardJar = false;
         // Append all jars that match the wildcard
         Path globPath = new Path(classPathEntry).suffix("{.jar,.JAR}");
         FileStatus[] wildcardJars = FileContext.getLocalFSFileContext().util()
           .globStatus(globPath);
         if (wildcardJars != null) {
           for (FileStatus wildcardJar: wildcardJars) {
+            foundWildCardJar = true;
             classPathEntryList.add(wildcardJar.getPath().toUri().toURL()
               .toExternalForm());
           }
         }
+        if (!foundWildCardJar) {
+          unexpandedWildcardClasspath.append(File.pathSeparator);
+          unexpandedWildcardClasspath.append(classPathEntry);
+        }
       } else {
         // Append just this entry
         File fileCpEntry = null;
@@ -1300,7 +1308,8 @@ public class FileUtil {
     } finally {
       IOUtils.cleanup(LOG, jos, bos, fos);
     }
-
-    return classPathJar.getCanonicalPath();
+    String[] jarCp = {classPathJar.getCanonicalPath(),
+                        unexpandedWildcardClasspath.toString()};
+    return jarCp;
   }
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Classpath.java

@@ -94,7 +94,7 @@ public final class Classpath {
       final String tmpJarPath;
       try {
         tmpJarPath = FileUtil.createJarWithClassPath(classPath, workingDir,
-          System.getenv());
+          System.getenv())[0];
       } catch (IOException e) {
         terminate(1, "I/O error creating jar: " + e.getMessage());
         return;

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java

@@ -1036,8 +1036,10 @@ public class TestFileUtil {
     List<String> classPaths = Arrays.asList("", "cp1.jar", "cp2.jar", wildcardPath,
       "cp3.jar", nonExistentSubdir);
     String inputClassPath = StringUtils.join(File.pathSeparator, classPaths);
-    String classPathJar = FileUtil.createJarWithClassPath(inputClassPath,
+    String[] jarCp = FileUtil.createJarWithClassPath(inputClassPath + File.pathSeparator + "unexpandedwildcard/*",
       new Path(tmp.getCanonicalPath()), System.getenv());
+    String classPathJar = jarCp[0];
+    assertNotEquals("Unexpanded wildcard was not placed in extra classpath", jarCp[1].indexOf("unexpanded"), -1);
 
     // verify classpath by reading manifest from jar file
     JarFile jarFile = null;

hadoop-yarn-project/CHANGES.txt

@@ -708,6 +708,9 @@ Release 2.6.0 - UNRELEASED
     YARN-2717. Avoided duplicate logging when container logs are not found. (Xuan
     Gong via zjshen)
 
+    YARN-2720. Windows: Wildcard classpath variables not expanded against
+    resources contained in archives. (Craig Welch via cnauroth)
+
 Release 2.5.1 - 2014-09-05
 
   INCOMPATIBLE CHANGES

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java

@@ -148,10 +148,12 @@ public class WindowsSecureContainerExecutor extends DefaultContainerExecutor {
      // Passing CLASSPATH explicitly is *way* too long for command line.
      String classPath = System.getProperty("java.class.path");
      Map<String, String> env = new HashMap<String, String>(System.getenv());
-     String classPathJar = FileUtil.createJarWithClassPath(classPath, appStorageDir, env);
+     String[] jarCp = FileUtil.createJarWithClassPath(classPath, appStorageDir, env);
+     String classPathJar = jarCp[0];
      localizeClasspathJar(new Path(classPathJar), user);
+     String replacementClassPath = classPathJar + jarCp[1];
      command.add("-classpath");
-     command.add(classPathJar);
+     command.add(replacementClassPath);
      
      String javaLibPath = System.getProperty("java.library.path");
      if (javaLibPath != null) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java

@@ -764,11 +764,13 @@ public class ContainerLaunch implements Callable<Integer> {
           System.getenv());
         mergedEnv.putAll(environment);
 
-        String classPathJar = FileUtil.createJarWithClassPath(
+        String[] jarCp = FileUtil.createJarWithClassPath(
           newClassPath.toString(), pwd, mergedEnv);
+        String classPathJar = jarCp[0];
         // In a secure cluster the classpath jar must be localized to grant access
         this.exec.localizeClasspathJar(new Path(classPathJar), container.getUser());
-        environment.put(Environment.CLASSPATH.name(), classPathJar);
+        String replacementClassPath = classPathJar + jarCp[1];
+        environment.put(Environment.CLASSPATH.name(), replacementClassPath);
       }
     }
     // put AuxiliaryService data to environment