HADOOP-184. Restructure some test code to better support testing on a cluster. Contributed by Mahadev.

git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk@399077 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -158,6 +158,9 @@ Trunk (unreleased)
 41. HADOOP-190.  If a child process hangs after it has reported
     completion, its output should not be lost.  (Stack via cutting)
 
+42. HADOOP-184. Re-structure some test code to better support testing
+    on a cluster.  (Mahadev Konar via cutting)
+
 
 Release 0.1.1 - 2006-04-08
 

build.xml

@@ -182,7 +182,7 @@
   <!-- ================================================================== -->
   <!-- Compile test code                                                  --> 
   <!-- ================================================================== -->
-  <target name="compile-test" depends="compile">
+  <target name="compile-test" depends="compile, jar">
     <javac 
      encoding="${build.encoding}" 
      srcdir="${test.src.dir}"
@@ -195,6 +195,13 @@
      deprecation="${javac.deprecation}">
       <classpath refid="test.classpath"/>
     </javac>    
+    <jar jarfile="${build.dir}/${final.name}-test.jar"
+         basedir="${test.build.classes}">
+         <manifest>
+           <attribute name="Main-Class"
+                      value="org/apache/hadoop/test/AllTestDriver"/>
+         </manifest>
+    </jar>
   </target>
 
   <!-- ================================================================== -->
@@ -290,6 +297,8 @@
     <copy file="${build.dir}/${final.name}.jar" todir="${dist.dir}"/>
 
     <copy file="${build.dir}/${final.name}-examples.jar" todir="${dist.dir}"/>
+    
+    <copy file="${build.dir}/${final.name}-test.jar" todir="${dist.dir}"/>
 
     <copy todir="${dist.dir}/bin">
       <fileset dir="bin"/>

src/examples/org/apache/hadoop/examples/ExampleDriver.java

@@ -15,7 +15,7 @@
  */
 
 package org.apache.hadoop.examples;
-
+import org.apache.hadoop.util.ProgramDriver;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.util.TreeMap;
@@ -29,107 +29,21 @@ public class ExampleDriver {
    * A description of an example program based on its class and a 
    * human-readable description.
    * @author Owen O'Malley
-   * @date feb 2006
+   * @date april 2006
    */
-  static private class ProgramDescription {
-    
-    static final Class[] paramTypes = new Class[] {String[].class};
-    
-    /**
-     * Create a description of an example program.
-     * @param mainClass the class with the main for the example program
-     * @param description a string to display to the user in help messages
-     * @throws SecurityException if we can't use reflection
-     * @throws NoSuchMethodException if the class doesn't have a main method
-     */
-    public ProgramDescription(Class mainClass, 
-                              String description)
-    throws SecurityException, NoSuchMethodException {
-      this.main = mainClass.getMethod("main", paramTypes);
-      this.description = description;
-    }
-    
-    /**
-     * Invoke the example application with the given arguments
-     * @param args the arguments for the application
-     * @throws Throwable The exception thrown by the invoked method
-     */
-    public void invoke(String[] args)
-    throws Throwable {
-      try {
-        main.invoke(null, new Object[]{args});
-      } catch (InvocationTargetException except) {
-        throw except.getCause();
-      }
-    }
     
-    public String getDescription() {
-      return description;
+    public static void main(String argv[]){
+        ProgramDriver pgd = new ProgramDriver();
+        try {
+	    pgd.addClass("wordcount", WordCount.class, 
+			 "A map/reduce program that counts the words in the input files.");
+	    pgd.addClass("grep", Grep.class, 
+			 "A map/reduce program that counts the matches of a regex in the input.");
+	    pgd.driver(argv);
+	}
+	catch(Throwable e){
+	    e.printStackTrace();
+	}
     }
-    
-    private Method main;
-    private String description;
-  }
-  
-  private static void printUsage(Map programs) {
-    System.out.println("Valid program names are:");
-    for(Iterator itr=programs.entrySet().iterator(); itr.hasNext();) {
-      Map.Entry item = (Entry) itr.next();
-      System.out.println("  " + (String) item.getKey() + ": " +
-          ((ProgramDescription) item.getValue()).getDescription());
-    }   
-  }
-  
-  /**
-   * This is a driver for the example programs.
-   * It looks at the first command line argument and tries to find an
-   * example program with that name.
-   * If it is found, it calls the main method in that class with the rest 
-   * of the command line arguments.
-   * @param args The argument from the user. args[0] is the command to run.
-   * @throws NoSuchMethodException 
-   * @throws SecurityException 
-   * @throws IllegalAccessException 
-   * @throws IllegalArgumentException 
-   * @throws Throwable Anything thrown by the example program's main
-   */
-  public static void main(String[] args) 
-  throws Throwable 
-  {
-    Map programs = new TreeMap();
-    
-    // Add new programs to this list
-    programs.put("wordcount", new ProgramDescription(WordCount.class,
-    "A map/reduce program that counts the words in the input files."));
-    programs.put("grep", new ProgramDescription(Grep.class,
-    "A map/reduce program that counts the matches of a regex in the input."));
-    programs.put("sort", new ProgramDescription(Sort.class,
-        "Sort binary keys and values."));
-    programs.put("writer", new ProgramDescription(RandomWriter.class,
-        "Write random binary key/value pairs"));
-    
-    // Make sure they gave us a program name.
-    if (args.length == 0) {
-      System.out.println("An example program must be given as the" + 
-          " first argument.");
-      printUsage(programs);
-      return;
-    }
-    
-    // And that it is good.
-    ProgramDescription pgm = (ProgramDescription) programs.get(args[0]);
-    if (pgm == null) {
-      System.out.println("Unknown program '" + args[0] + "' chosen.");
-      printUsage(programs);
-      return;
-    }
-    
-    // Remove the leading argument and call main
-    String[] new_args = new String[args.length - 1];
-    for(int i=1; i < args.length; ++i) {
-      new_args[i-1] = args[i];
-    }
-    pgm.invoke(new_args);
-  }
-  
 }
+	

src/java/org/apache/hadoop/util/ProgramDriver.java

@@ -0,0 +1,144 @@
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.TreeMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/** A driver that is used to run programs added to it
+ */
+
+public class ProgramDriver {
+    
+    /**
+     * A description of a program based on its class and a 
+     * human-readable description.
+     * @author Owen O'Malley
+     * @date april 2006
+     */
+     Map programs;
+     
+     public ProgramDriver(){
+        programs = new TreeMap();
+     }
+     
+    static private class ProgramDescription {
+	
+	static final Class[] paramTypes = new Class[] {String[].class};
+	
+	/**
+	 * Create a description of an example program.
+	 * @param mainClass the class with the main for the example program
+	 * @param description a string to display to the user in help messages
+	 * @throws SecurityException if we can't use reflection
+	 * @throws NoSuchMethodException if the class doesn't have a main method
+	 */
+	public ProgramDescription(Class mainClass, 
+				  String description)
+	    throws SecurityException, NoSuchMethodException {
+	    this.main = mainClass.getMethod("main", paramTypes);
+	    this.description = description;
+	}
+	
+	/**
+	 * Invoke the example application with the given arguments
+	 * @param args the arguments for the application
+	 * @throws Throwable The exception thrown by the invoked method
+	 */
+	public void invoke(String[] args)
+	    throws Throwable {
+	    try {
+		main.invoke(null, new Object[]{args});
+	    } catch (InvocationTargetException except) {
+		throw except.getCause();
+	    }
+	}
+	
+	public String getDescription() {
+	    return description;
+	}
+	
+	private Method main;
+	private String description;
+    }
+    
+    private static void printUsage(Map programs) {
+	System.out.println("Valid program names are:");
+	for(Iterator itr=programs.entrySet().iterator(); itr.hasNext();) {
+	    Map.Entry item = (Entry) itr.next();
+	    System.out.println("  " + (String) item.getKey() + ": " +
+			       ((ProgramDescription) item.getValue()).getDescription());
+	}   
+    }
+    
+    /**
+     * This is the method that adds the classed to the repository
+     * @param name The name of the string you want the class instance to be called with
+     * @param mainClass The class that you want to add to the repository
+     * @param description The description of the class
+     * @throws NoSuchMethodException 
+     * @throws SecurityException 
+     */
+    public void addClass (String name, Class mainClass, String description) throws Throwable {
+	programs.put(name , new ProgramDescription(mainClass, description));
+    }
+    
+    /**
+     * This is a driver for the example programs.
+     * It looks at the first command line argument and tries to find an
+     * example program with that name.
+     * If it is found, it calls the main method in that class with the rest 
+     * of the command line arguments.
+     * @param args The argument from the user. args[0] is the command to run.
+     * @throws NoSuchMethodException 
+     * @throws SecurityException 
+     * @throws IllegalAccessException 
+     * @throws IllegalArgumentException 
+     * @throws Throwable Anything thrown by the example program's main
+     */
+    public void driver(String[] args) 
+	throws Throwable 
+    {
+	// Make sure they gave us a program name.
+	if (args.length == 0) {
+	    System.out.println("An example program must be given as the" + 
+			       " first argument.");
+	    printUsage(programs);
+	    return;
+	}
+	
+	// And that it is good.
+	ProgramDescription pgm = (ProgramDescription) programs.get(args[0]);
+	if (pgm == null) {
+	    System.out.println("Unknown program '" + args[0] + "' chosen.");
+	    printUsage(programs);
+	    return;
+	}
+	
+	// Remove the leading argument and call main
+	String[] new_args = new String[args.length - 1];
+	for(int i=1; i < args.length; ++i) {
+	    new_args[i-1] = args[i];
+	}
+	pgm.invoke(new_args);
+    }
+    
+}

src/test/org/apache/hadoop/fs/TestFileSystem.java

@@ -59,6 +59,10 @@ public class TestFileSystem extends TestCase {
     writeTest(fs, false);
     readTest(fs, false);
     seekTest(fs, false);
+    fs.delete(CONTROL_DIR);
+    fs.delete(DATA_DIR);
+    fs.delete(WRITE_DIR);
+    fs.delete(READ_DIR);
   }
 
   public static void createControlFile(FileSystem fs,

src/test/org/apache/hadoop/mapred/MapredLoadTest.java → src/test/org/apache/hadoop/mapred/TestMapRed.java

@@ -18,10 +18,11 @@ package org.apache.hadoop.mapred;
 import org.apache.hadoop.fs.*;
 import org.apache.hadoop.io.*;
 import org.apache.hadoop.conf.*;
-
+import junit.framework.TestCase;
 import java.io.*;
 import java.util.*;
 
+
 /**********************************************************
  * MapredLoadTest generates a bunch of work that exercises
  * a Hadoop Map-Reduce system (and DFS, too).  It goes through
@@ -55,8 +56,9 @@ import java.util.*;
  * 7) A mapred job integrates all the count files into a single one.
  *
  **********************************************************/
-public class MapredLoadTest {
+public class TestMapRed extends TestCase {
     /**
+     * Modified to make it a junit test.
      * The RandomGen Job does the actual work of creating
      * a huge file of assorted numbers.  It receives instructions
      * as to how many times each number should be counted.  Then
@@ -197,24 +199,27 @@ public class MapredLoadTest {
         }
     }
 
-    int range;
-    int counts;
-    Random r = new Random();
-    Configuration conf;
+    private static int range = 10;
+    private static int counts = 100;
+    private static Random r = new Random();
+    private static Configuration conf = new Configuration();
 
     /**
-     * MapredLoadTest
-     */
-    public MapredLoadTest(int range, int counts, Configuration conf) throws IOException {
-        this.range = range;
-        this.counts = counts;
-        this.conf = conf;
+       public TestMapRed(int range, int counts, Configuration conf) throws IOException {
+       this.range = range;
+       this.counts = counts;
+       this.conf = conf;
+       }
+    **/
+
+    public void testMapred() throws Exception {
+	launch();
     }
 
     /**
      * 
      */
-    public void launch() throws IOException {
+    public static void launch() throws Exception {
         //
         // Generate distribution of ints.  This is the answer key.
         //
@@ -268,7 +273,7 @@ public class MapredLoadTest {
         // file of random numbers.
         //
         Path randomOuts = new Path(testdir, "genouts");
-        fs.mkdirs(randomOuts);
+        //fs.mkdirs(randomOuts);
 
 
         JobConf genJob = new JobConf(conf);
@@ -317,7 +322,7 @@ public class MapredLoadTest {
         //
         int intermediateReduces = 10;
         Path intermediateOuts = new Path(testdir, "intermediateouts");
-        fs.mkdirs(intermediateOuts);
+        //fs.mkdirs(intermediateOuts);
         JobConf checkJob = new JobConf(conf);
         checkJob.setInputPath(randomOuts);
         checkJob.setInputKeyClass(LongWritable.class);
@@ -342,7 +347,7 @@ public class MapredLoadTest {
         // all the files.
         //
         Path finalOuts = new Path(testdir, "finalouts");        
-        fs.mkdirs(finalOuts);
+        //fs.mkdirs(finalOuts);
         JobConf mergeJob = new JobConf(conf);
         mergeJob.setInputPath(intermediateOuts);
         mergeJob.setInputKeyClass(IntWritable.class);
@@ -415,6 +420,7 @@ public class MapredLoadTest {
         } finally {
             bw.close();
         }
+	fs.delete(testdir);
     }
 
     /**
@@ -422,7 +428,7 @@ public class MapredLoadTest {
      */
     public static void main(String[] argv) throws Exception {
         if (argv.length < 2) {
-            System.err.println("Usage: MapredLoadTest <range> <counts>");
+            System.err.println("Usage: TestMapRed <range> <counts>");
             System.err.println();
             System.err.println("Note: a good test will have a <counts> value that is substantially larger than the <range>");
             return;
@@ -431,8 +437,6 @@ public class MapredLoadTest {
         int i = 0;
         int range = Integer.parseInt(argv[i++]);
         int counts = Integer.parseInt(argv[i++]);
-
-        MapredLoadTest mlt = new MapredLoadTest(range, counts, new Configuration());
-        mlt.launch();
+	launch();
     }
 }

src/test/org/apache/hadoop/test/AllTestDriver.java

@@ -0,0 +1,63 @@
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.test;
+
+import org.apache.hadoop.util.ProgramDriver;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.TreeMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import org.apache.hadoop.mapred.TestMapRed;
+import org.apache.hadoop.mapred.TestTextInputFormat;
+import org.apache.hadoop.mapred.TestSequenceFileInputFormat;
+import org.apache.hadoop.dfs.ClusterTestDFS;
+import org.apache.hadoop.fs.TestFileSystem;
+import org.apache.hadoop.io.TestArrayFile;
+import org.apache.hadoop.io.TestSetFile;
+import org.apache.hadoop.io.TestSequenceFile;
+import org.apache.hadoop.ipc.TestIPC;
+import org.apache.hadoop.ipc.TestRPC;
+
+public class AllTestDriver {
+  
+  /**
+   * A description of the test program for running all the tests using jar file
+   * @date April 2006
+   */
+    
+    public static void main(String argv[]){
+	ProgramDriver pgd = new ProgramDriver();
+	try {
+	    pgd.addClass("mapredtest", TestMapRed.class, "A map/reduce test check.");
+	    pgd.addClass("clustertestdfs", ClusterTestDFS.class, "A pseudo distributed test for DFS.");
+	    pgd.addClass("testfilesystem", TestFileSystem.class, "A test for FileSystem read/write.");
+	    pgd.addClass("testsequencefile", TestSequenceFile.class, "A test for flat files of binary key value pairs.");
+	    pgd.addClass("testsetfile", TestSetFile.class, "A test for flat files of binary key/value pairs.");
+	    pgd.addClass("testarrayfile", TestArrayFile.class, "A test for flat files of binary key/value pairs.");
+	    pgd.addClass("testrpc", TestRPC.class, "A test for rpc.");
+	    pgd.addClass("testipc", TestIPC.class, "A test for ipc.");
+	    pgd.addClass("testsequencefileinputformat", TestSequenceFileInputFormat.class, "A test for sequence file input format.");
+	    pgd.addClass("testtextinputformat", TestTextInputFormat.class, "A test for text input format.");
+	    pgd.driver(argv);
+	}
+	catch(Throwable e){
+	    e.printStackTrace();
+	}
+    }
+}