|
|
@@ -0,0 +1,175 @@
|
|
|
+/**
|
|
|
+ * Licensed to the Apache Software Foundation (ASF) under one
|
|
|
+ * or more contributor license agreements. See the NOTICE file
|
|
|
+ * distributed with this work for additional information
|
|
|
+ * regarding copyright ownership. The ASF licenses this file
|
|
|
+ * to you under the Apache License, Version 2.0 (the
|
|
|
+ * "License"); you may not use this file except in compliance
|
|
|
+ * with the License. You may obtain a copy of the License at
|
|
|
+ *
|
|
|
+ * http://www.apache.org/licenses/LICENSE-2.0
|
|
|
+ *
|
|
|
+ * Unless required by applicable law or agreed to in writing, software
|
|
|
+ * distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
+ * See the License for the specific language governing permissions and
|
|
|
+ * limitations under the License.
|
|
|
+ */
|
|
|
+
|
|
|
+package org.apache.hadoop.mapred;
|
|
|
+
|
|
|
+import java.io.File;
|
|
|
+import java.io.IOException;
|
|
|
+import java.net.URI;
|
|
|
+
|
|
|
+import junit.framework.TestCase;
|
|
|
+
|
|
|
+import org.apache.commons.logging.Log;
|
|
|
+import org.apache.commons.logging.LogFactory;
|
|
|
+import org.apache.hadoop.fs.FileStatus;
|
|
|
+import org.apache.hadoop.fs.FileSystem;
|
|
|
+import org.apache.hadoop.fs.Path;
|
|
|
+import org.apache.hadoop.io.IntWritable;
|
|
|
+import org.apache.hadoop.io.Text;
|
|
|
+import org.apache.hadoop.mapred.lib.IdentityMapper;
|
|
|
+import org.apache.hadoop.mapred.lib.IdentityReducer;
|
|
|
+
|
|
|
+/**
|
|
|
+ * A JUnit test to test Map-Reduce empty jobs.
|
|
|
+ */
|
|
|
+public class TestEmptyJob extends TestCase {
|
|
|
+ private static final Log LOG =
|
|
|
+ LogFactory.getLog(TestEmptyJob.class.getName());
|
|
|
+
|
|
|
+ private static String TEST_ROOT_DIR =
|
|
|
+ new File(System.getProperty("test.build.data", "/tmp")).toURI()
|
|
|
+ .toString().replace(' ', '+');
|
|
|
+
|
|
|
+ MiniMRCluster mr = null;
|
|
|
+
|
|
|
+ /**
|
|
|
+ * Simple method running a MapReduce job with no input data. Used to test that
|
|
|
+ * such a job is successful.
|
|
|
+ *
|
|
|
+ * @param fileSys
|
|
|
+ * @param numMaps
|
|
|
+ * @param numReduces
|
|
|
+ * @return true if the MR job is successful, otherwise false
|
|
|
+ * @throws IOException
|
|
|
+ */
|
|
|
+ private boolean launchEmptyJob(URI fileSys, int numMaps, int numReduces)
|
|
|
+ throws IOException {
|
|
|
+ // create an empty input dir
|
|
|
+ final Path inDir = new Path(TEST_ROOT_DIR, "testing/empty/input");
|
|
|
+ final Path outDir = new Path(TEST_ROOT_DIR, "testing/empty/output");
|
|
|
+ JobConf conf = mr.createJobConf();
|
|
|
+ FileSystem fs = FileSystem.get(fileSys, conf);
|
|
|
+ fs.delete(outDir, true);
|
|
|
+ if (!fs.mkdirs(inDir)) {
|
|
|
+ LOG.warn("Can't create " + inDir);
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+
|
|
|
+ // use WordCount example
|
|
|
+ FileSystem.setDefaultUri(conf, fileSys);
|
|
|
+ conf.setJobName("empty");
|
|
|
+ // use an InputFormat which returns no split
|
|
|
+ conf.setInputFormat(EmptyInputFormat.class);
|
|
|
+ conf.setOutputKeyClass(Text.class);
|
|
|
+ conf.setOutputValueClass(IntWritable.class);
|
|
|
+ conf.setMapperClass(IdentityMapper.class);
|
|
|
+ conf.setReducerClass(IdentityReducer.class);
|
|
|
+ FileInputFormat.setInputPaths(conf, inDir);
|
|
|
+ FileOutputFormat.setOutputPath(conf, outDir);
|
|
|
+ conf.setNumMapTasks(numMaps);
|
|
|
+ conf.setNumReduceTasks(numReduces);
|
|
|
+
|
|
|
+ // run job and wait for completion
|
|
|
+ JobClient jc = new JobClient(conf);
|
|
|
+ RunningJob runningJob = jc.submitJob(conf);
|
|
|
+ while (true) {
|
|
|
+ try {
|
|
|
+ Thread.sleep(100);
|
|
|
+ } catch (InterruptedException e) {
|
|
|
+ }
|
|
|
+ if (runningJob.isComplete()) {
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ assertTrue(runningJob.isComplete());
|
|
|
+ assertTrue(runningJob.isSuccessful());
|
|
|
+ JobID jobID = runningJob.getID();
|
|
|
+
|
|
|
+ TaskReport[] jobSetupTasks = jc.getSetupTaskReports(jobID);
|
|
|
+ assertTrue("Number of job-setup tips is not 2!", jobSetupTasks.length == 2);
|
|
|
+ assertTrue("Setup progress is " + runningJob.setupProgress()
|
|
|
+ + " and not 1.0", runningJob.setupProgress() == 1.0);
|
|
|
+ assertTrue("Setup task is not finished!", mr.getJobTrackerRunner()
|
|
|
+ .getJobTracker().getJob(jobID).isSetupFinished());
|
|
|
+
|
|
|
+ assertTrue("Number of maps is not zero!", jc.getMapTaskReports(runningJob
|
|
|
+ .getID()).length == 0);
|
|
|
+ assertTrue(
|
|
|
+ "Map progress is " + runningJob.mapProgress() + " and not 1.0!",
|
|
|
+ runningJob.mapProgress() == 1.0);
|
|
|
+
|
|
|
+ assertTrue("Reduce progress is " + runningJob.reduceProgress()
|
|
|
+ + " and not 1.0!", runningJob.reduceProgress() == 1.0);
|
|
|
+ assertTrue("Number of reduces is not " + numReduces, jc
|
|
|
+ .getReduceTaskReports(runningJob.getID()).length == numReduces);
|
|
|
+
|
|
|
+ TaskReport[] jobCleanupTasks = jc.getCleanupTaskReports(jobID);
|
|
|
+ assertTrue("Number of job-cleanup tips is not 2!",
|
|
|
+ jobCleanupTasks.length == 2);
|
|
|
+ assertTrue("Cleanup progress is " + runningJob.cleanupProgress()
|
|
|
+ + " and not 1.0", runningJob.cleanupProgress() == 1.0);
|
|
|
+
|
|
|
+ assertTrue("Job output directory doesn't exit!", fs.exists(outDir));
|
|
|
+ FileStatus[] list = fs.listStatus(outDir, new OutputLogFilter());
|
|
|
+ assertTrue("Number of part-files is " + list.length + " and not "
|
|
|
+ + numReduces, list.length == numReduces);
|
|
|
+
|
|
|
+ // cleanup
|
|
|
+ fs.delete(outDir, true);
|
|
|
+
|
|
|
+ // return job result
|
|
|
+ LOG.info("job is complete: " + runningJob.isSuccessful());
|
|
|
+ return (runningJob.isSuccessful());
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * Test that a job with no input data (and thus with no input split and no map
|
|
|
+ * task to execute) is successful.
|
|
|
+ *
|
|
|
+ * @throws IOException
|
|
|
+ */
|
|
|
+ public void testEmptyJob()
|
|
|
+ throws IOException {
|
|
|
+ FileSystem fileSys = null;
|
|
|
+ try {
|
|
|
+ final int taskTrackers = 1;
|
|
|
+ JobConf conf = new JobConf();
|
|
|
+ fileSys = FileSystem.get(conf);
|
|
|
+
|
|
|
+ conf.set("mapred.job.tracker.handler.count", "1");
|
|
|
+ conf.set("mapred.job.tracker", "127.0.0.1:0");
|
|
|
+ conf.set("mapred.job.tracker.http.address", "127.0.0.1:0");
|
|
|
+ conf.set("mapred.task.tracker.http.address", "127.0.0.1:0");
|
|
|
+
|
|
|
+ mr =
|
|
|
+ new MiniMRCluster(taskTrackers, fileSys.getUri().toString(), 1,
|
|
|
+ null, null, conf);
|
|
|
+
|
|
|
+ assertTrue(launchEmptyJob(fileSys.getUri(), 3, 1));
|
|
|
+ assertTrue(launchEmptyJob(fileSys.getUri(), 0, 0));
|
|
|
+ } finally {
|
|
|
+ if (fileSys != null) {
|
|
|
+ fileSys.close();
|
|
|
+ }
|
|
|
+ if (mr != null) {
|
|
|
+ mr.shutdown();
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+}
|
Devaraj Das