HADOOP-3570. Includes user specified libjar files in the client side classpath path. Contributed by Sharad Agarwal.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/core/trunk@695844 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -559,6 +559,9 @@ Trunk (unreleased changes)
     HADOOP-4099. Fix null pointer when using HFTP from an 0.18 server.
     (dhruba via omalley)
 
+    HADOOP-3570. Includes user specified libjar files in the client side 
+    classpath path. (Sharad Agarwal via ddas)
+
 Release 0.18.1 - 2008-09-17
 
   IMPROVEMENTS

src/core/org/apache/hadoop/util/GenericOptionsParser.java

@@ -21,6 +21,8 @@ import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.PrintStream;
 import java.net.URI;
+import java.net.URL;
+import java.net.URLClassLoader;
 
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.CommandLineParser;
@@ -228,6 +230,11 @@ public class GenericOptionsParser {
       if (line.hasOption("libjars")) {
         conf.set("tmpjars", 
                  validateFiles(line.getOptionValue("libjars"), conf));
+        //setting libjars in client classpath
+        ClassLoader loader = new URLClassLoader(getLibJars(conf), 
+            conf.getClassLoader());
+        Thread.currentThread().setContextClassLoader(loader);
+        conf.setClassLoader(loader);
       }
       if (line.hasOption("files")) {
         conf.set("tmpfiles", 
@@ -249,6 +256,26 @@ public class GenericOptionsParser {
     }
     conf.setBoolean("mapred.used.genericoptionsparser", true);
   }
+  
+  /**
+   * If libjars are set in the conf, parse the libjars.
+   * @param conf
+   * @return libjar urls
+   * @throws IOException
+   */
+  public static URL[] getLibJars(Configuration conf) throws IOException {
+    String jars = conf.get("tmpjars");
+    if(jars==null) {
+      return null;
+    }
+    String[] files = jars.split(",");
+    URL[] cp = new URL[files.length];
+    for (int i=0;i<cp.length;i++) {
+      Path tmp = new Path(files[i]);
+      cp[i] = FileSystem.getLocal(conf).pathToFile(tmp).toURI().toURL();
+    }
+    return cp;
+  }
 
   /**
    * takes input as a comma separated list of files

src/core/org/apache/hadoop/util/RunJar.java

@@ -27,6 +27,7 @@ import java.util.*;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.mapred.JobClient;
 
 /** Run a Hadoop job jar. */
 public class RunJar {
@@ -141,6 +142,15 @@ public class RunJar {
         classPath.add(libs[i].toURL());
       }
     }
+    
+    //adding libjars to the classpath
+    Configuration conf = JobClient.getCommandLineConfig();
+    URL[] libJars = GenericOptionsParser.getLibJars(conf);
+    if(libJars!=null) {
+      for(URL url : libJars){
+        classPath.add(url);
+      }
+    }
     ClassLoader loader =
       new URLClassLoader(classPath.toArray(new URL[0]));
 

src/test/org/apache/hadoop/mapred/TestCommandLineJobSubmission.java

@@ -68,8 +68,15 @@ public class TestCommandLineJobSubmission extends TestCase {
       args[3] = "build/test/testjar/testjob.jar";
       args[4] = input.toString();
       args[5] = output.toString();
-      int ret = ToolRunner.run(mr.createJobConf(),
+      
+      JobConf jobConf = mr.createJobConf();
+      //before running the job, verify that libjar is not in client classpath
+      assertTrue("libjar not in client classpath", loadLibJar(jobConf)==null);
+      int ret = ToolRunner.run(jobConf,
                                new testshell.ExternalMapReduce(), args);
+      //after running the job, verify that libjar is in the client classpath
+      assertTrue("libjar added to client classpath", loadLibJar(jobConf)!=null);
+      
       assertTrue("not failed ", ret != -1);
       f.delete();
       thisbuildDir.delete();
@@ -78,4 +85,13 @@ public class TestCommandLineJobSubmission extends TestCase {
       if (mr != null) {mr.shutdown();};
     }
   }
+  
+  @SuppressWarnings("unchecked")
+  private Class loadLibJar(JobConf jobConf) {
+    try {
+      return jobConf.getClassByName("testjar.ClassWordCount");
+    } catch (ClassNotFoundException e) {
+      return null;
+    }
+  }
 }

src/test/testshell/ExternalMapReduce.java

@@ -104,6 +104,16 @@ public class ExternalMapReduce extends Configured implements Tool {
     Path outDir = new Path(argv[1]);
     Path input = new Path(argv[0]);
     JobConf testConf = new JobConf(getConf(), ExternalMapReduce.class);
+    
+    //try to load a class from libjar
+    try {
+      testConf.getClassByName("testjar.ClassWordCount");
+    } catch (ClassNotFoundException e) {
+      System.out.println("Could not find class from libjar");
+      return -1;
+    }
+    
+    
     testConf.setJobName("external job");
     FileInputFormat.setInputPaths(testConf, input);
     FileOutputFormat.setOutputPath(testConf, outDir);