MAPREDUCE-4983. Fixed various platform specific assumptions in various tests, so that they can pass on Windows too. Contributed by Chris Nauroth.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-trunk-win@1443317 13f79535-47bb-0310-9956-ffa450edef68

hadoop-mapreduce-project/CHANGES.branch-trunk-win.txt

@@ -13,3 +13,5 @@ branch-trunk-win changes - unreleased
 
   MAPREDUCE-4870. Fix TestMRJobsWithHistoryService. (Chris Nauroth via acmurthy)
 
+  MAPREDUCE-4983. Fixed various platform specific assumptions in various tests,
+  so that they can pass on Windows too. (Chris Nauroth via vinodkv)

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java

@@ -22,6 +22,7 @@ import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.net.URI;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -39,6 +40,8 @@ import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
 import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
 import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
 import org.apache.hadoop.mapreduce.v2.util.MRApps;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.LocalResourceType;
@@ -69,9 +72,9 @@ public class TestMRApps {
   }
   
   private static void delete(File dir) throws IOException {
-    Path p = new Path("file://"+dir.getAbsolutePath());
     Configuration conf = new Configuration();
-    FileSystem fs = p.getFileSystem(conf);
+    FileSystem fs = FileSystem.getLocal(conf);
+    Path p = fs.makeQualified(new Path(dir.getAbsolutePath()));
     fs.delete(p, true);
   }
 
@@ -165,18 +168,21 @@ public class TestMRApps {
     Job job = Job.getInstance();
     Map<String, String> environment = new HashMap<String, String>();
     MRApps.setClasspath(environment, job.getConfiguration());
-    assertTrue(environment.get("CLASSPATH").startsWith("$PWD:"));
+    assertTrue(environment.get("CLASSPATH").startsWith(
+      ApplicationConstants.Environment.PWD.$() + File.pathSeparator));
     String yarnAppClasspath = 
         job.getConfiguration().get(
             YarnConfiguration.YARN_APPLICATION_CLASSPATH);
     if (yarnAppClasspath != null) {
-      yarnAppClasspath = yarnAppClasspath.replaceAll(",\\s*", ":").trim();
+      yarnAppClasspath = yarnAppClasspath.replaceAll(",\\s*", File.pathSeparator)
+        .trim();
     }
     assertTrue(environment.get("CLASSPATH").contains(yarnAppClasspath));
     String mrAppClasspath = 
         job.getConfiguration().get(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH);
     if (mrAppClasspath != null) {
-      mrAppClasspath = mrAppClasspath.replaceAll(",\\s*", ":").trim();
+      mrAppClasspath = mrAppClasspath.replaceAll(",\\s*", File.pathSeparator)
+        .trim();
     }
     assertTrue(environment.get("CLASSPATH").contains(mrAppClasspath));
   }
@@ -188,16 +194,18 @@ public class TestMRApps {
     out.close();
     Job job = Job.getInstance();
     Configuration conf = job.getConfiguration();
-    conf.set(MRJobConfig.CLASSPATH_ARCHIVES, "file://" 
-        + testTGZ.getAbsolutePath());
-    conf.set(MRJobConfig.CACHE_ARCHIVES, "file://"
-        + testTGZ.getAbsolutePath() + "#testTGZ");
+    String testTGZQualifiedPath = FileSystem.getLocal(conf).makeQualified(new Path(
+      testTGZ.getAbsolutePath())).toString();
+    conf.set(MRJobConfig.CLASSPATH_ARCHIVES, testTGZQualifiedPath);
+    conf.set(MRJobConfig.CACHE_ARCHIVES, testTGZQualifiedPath + "#testTGZ");
     Map<String, String> environment = new HashMap<String, String>();
     MRApps.setClasspath(environment, conf);
-    assertTrue(environment.get("CLASSPATH").startsWith("$PWD:"));
+    assertTrue(environment.get("CLASSPATH").startsWith(
+      ApplicationConstants.Environment.PWD.$() + File.pathSeparator));
     String confClasspath = job.getConfiguration().get(YarnConfiguration.YARN_APPLICATION_CLASSPATH);
     if (confClasspath != null) {
-      confClasspath = confClasspath.replaceAll(",\\s*", ":").trim();
+      confClasspath = confClasspath.replaceAll(",\\s*", File.pathSeparator)
+        .trim();
     }
     assertTrue(environment.get("CLASSPATH").contains(confClasspath));
     assertTrue(environment.get("CLASSPATH").contains("testTGZ"));
@@ -213,8 +221,12 @@ public class TestMRApps {
       fail("Got exception while setting classpath");
     }
     String env_str = env.get("CLASSPATH");
-    assertSame("MAPREDUCE_JOB_USER_CLASSPATH_FIRST set, but not taking effect!",
-      env_str.indexOf("$PWD:job.jar/job.jar:job.jar/classes/:job.jar/lib/*:$PWD/*"), 0);
+    String expectedClasspath = StringUtils.join(File.pathSeparator,
+      Arrays.asList(ApplicationConstants.Environment.PWD.$(), "job.jar/job.jar",
+        "job.jar/classes/", "job.jar/lib/*",
+        ApplicationConstants.Environment.PWD.$() + "/*"));
+    assertTrue("MAPREDUCE_JOB_USER_CLASSPATH_FIRST set, but not taking effect!",
+      env_str.startsWith(expectedClasspath));
   }
 
   @Test public void testSetClasspathWithNoUserPrecendence() {
@@ -227,12 +239,13 @@ public class TestMRApps {
       fail("Got exception while setting classpath");
     }
     String env_str = env.get("CLASSPATH");
-    int index = 
-         env_str.indexOf("job.jar/job.jar:job.jar/classes/:job.jar/lib/*:$PWD/*");
-    assertNotSame("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, and job.jar is not"
-            + " in the classpath!", index, -1);
-    assertNotSame("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, but taking effect!",
-      index, 0);
+    String expectedClasspath = StringUtils.join(File.pathSeparator,
+      Arrays.asList("job.jar/job.jar", "job.jar/classes/", "job.jar/lib/*",
+        ApplicationConstants.Environment.PWD.$() + "/*"));
+    assertTrue("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, and job.jar is not in"
+      + " the classpath!", env_str.contains(expectedClasspath));
+    assertFalse("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, but taking effect!",
+      env_str.startsWith(expectedClasspath));
   }
   
   @Test public void testSetClasspathWithJobClassloader() throws IOException {
@@ -242,13 +255,16 @@ public class TestMRApps {
     MRApps.setClasspath(env, conf);
     String cp = env.get("CLASSPATH");
     String appCp = env.get("APP_CLASSPATH");
-    assertSame("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is"
-        + " in the classpath!", cp.indexOf("jar:job"), -1);
-    assertSame("MAPREDUCE_JOB_CLASSLOADER true, but PWD is"
-        + " in the classpath!", cp.indexOf("PWD"), -1);
-    assertEquals("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is not"
-        + " in the app classpath!",
-        "$PWD:job.jar/job.jar:job.jar/classes/:job.jar/lib/*:$PWD/*", appCp);
+    assertFalse("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is in the"
+      + " classpath!", cp.contains("jar" + File.pathSeparator + "job"));
+    assertFalse("MAPREDUCE_JOB_CLASSLOADER true, but PWD is in the classpath!",
+      cp.contains("PWD"));
+    String expectedAppClasspath = StringUtils.join(File.pathSeparator,
+      Arrays.asList(ApplicationConstants.Environment.PWD.$(), "job.jar/job.jar",
+        "job.jar/classes/", "job.jar/lib/*",
+        ApplicationConstants.Environment.PWD.$() + "/*"));
+    assertEquals("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is not in the app"
+      + " classpath!", expectedAppClasspath, appCp);
   }
   
   @Test

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapProgress.java

@@ -61,8 +61,12 @@ import org.apache.hadoop.util.ReflectionUtils;
  */
 public class TestMapProgress extends TestCase {
   public static final Log LOG = LogFactory.getLog(TestMapProgress.class);
-  private static String TEST_ROOT_DIR = new File(System.getProperty(
-           "test.build.data", "/tmp")).getAbsolutePath() + "/mapPahseprogress";
+  private static String TEST_ROOT_DIR;
+  static {
+    String root = new File(System.getProperty("test.build.data", "/tmp"))
+      .getAbsolutePath();
+    TEST_ROOT_DIR = new Path(root, "mapPhaseprogress").toString();
+  }
 
   static class FakeUmbilical implements TaskUmbilicalProtocol {
 

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java

@@ -25,6 +25,8 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.net.URI;
 import java.security.PrivilegedExceptionAction;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.jar.JarOutputStream;
 import java.util.zip.ZipEntry;
 import org.apache.commons.io.FileUtils;
@@ -103,6 +105,8 @@ public class TestMRJobs {
   private static Path TEST_ROOT_DIR = new Path("target",
       TestMRJobs.class.getName() + "-tmpDir").makeQualified(localFs);
   static Path APP_JAR = new Path(TEST_ROOT_DIR, "MRAppJar.jar");
+  private static final String OUTPUT_ROOT_DIR = "/tmp/" +
+    TestMRJobs.class.getSimpleName();
 
   @BeforeClass
   public static void setup() throws IOException {
@@ -226,8 +230,7 @@ public class TestMRJobs {
     mrCluster.getConfig().set(RandomTextWriterJob.TOTAL_BYTES, "3072");
     mrCluster.getConfig().set(RandomTextWriterJob.BYTES_PER_MAP, "1024");
     Job job = randomWriterJob.createJob(mrCluster.getConfig());
-    Path outputDir =
-        new Path(mrCluster.getTestWorkDir().getAbsolutePath(), "random-output");
+    Path outputDir = new Path(OUTPUT_ROOT_DIR, "random-output");
     FileOutputFormat.setOutputPath(job, outputDir);
     job.setSpeculativeExecution(false);
     job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
@@ -342,9 +345,8 @@ public class TestMRJobs {
     job.setMapperClass(FailingMapper.class);
     job.setNumReduceTasks(0);
     
-    FileOutputFormat.setOutputPath(job,
-        new Path(mrCluster.getTestWorkDir().getAbsolutePath(),
-        "failmapper-output"));
+    FileOutputFormat.setOutputPath(job, new Path(OUTPUT_ROOT_DIR,
+      "failmapper-output"));
     job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
     job.submit();
     String trackingUrl = job.getTrackingURL();
@@ -425,14 +427,22 @@ public class TestMRJobs {
       Assert.assertEquals(2, archives.length);
 
       // Check lengths of the files
-      Assert.assertEquals(1, localFs.getFileStatus(files[1]).getLen());
-      Assert.assertTrue(localFs.getFileStatus(files[2]).getLen() > 1);
+      Map<String, Path> filesMap = pathsToMap(files);
+      Assert.assertTrue(filesMap.containsKey("distributed.first.symlink"));
+      Assert.assertEquals(1, localFs.getFileStatus(
+        filesMap.get("distributed.first.symlink")).getLen());
+      Assert.assertTrue(filesMap.containsKey("distributed.second.jar"));
+      Assert.assertTrue(localFs.getFileStatus(
+        filesMap.get("distributed.second.jar")).getLen() > 1);
 
       // Check extraction of the archive
-      Assert.assertTrue(localFs.exists(new Path(archives[0],
-          "distributed.jar.inside3")));
-      Assert.assertTrue(localFs.exists(new Path(archives[1],
-          "distributed.jar.inside4")));
+      Map<String, Path> archivesMap = pathsToMap(archives);
+      Assert.assertTrue(archivesMap.containsKey("distributed.third.jar"));
+      Assert.assertTrue(localFs.exists(new Path(
+        archivesMap.get("distributed.third.jar"), "distributed.jar.inside3")));
+      Assert.assertTrue(archivesMap.containsKey("distributed.fourth.jar"));
+      Assert.assertTrue(localFs.exists(new Path(
+        archivesMap.get("distributed.fourth.jar"), "distributed.jar.inside4")));
 
       // Check the class loaders
       LOG.info("Java Classpath: " + System.getProperty("java.class.path"));
@@ -460,6 +470,23 @@ public class TestMRJobs {
       Assert.assertTrue(FileUtils.isSymlink(jobJarDir));
       Assert.assertTrue(jobJarDir.isDirectory());
     }
+
+    /**
+     * Returns a mapping of the final component of each path to the corresponding
+     * Path instance.  This assumes that every given Path has a unique string in
+     * the final path component, which is true for these tests.
+     * 
+     * @param paths Path[] to map
+     * @return Map<String, Path> mapping the final component of each path to the
+     *   corresponding Path instance
+     */
+    private static Map<String, Path> pathsToMap(Path[] paths) {
+      Map<String, Path> map = new HashMap<String, Path>();
+      for (Path path: paths) {
+        map.put(path.getName(), path);
+      }
+      return map;
+    }
   }
 
   public void _testDistributedCache(String jobJarPath) throws Exception {