HADOOP-5719. Remove jobs that failed initialization from the waiting queue in the capacity scheduler. Contributed by Sreekanth Ramakrishnan.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/core/branches/branch-0.20@772060 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -46,6 +46,9 @@ Release 0.20.1 - Unreleased
     HADOOP-5718. Remove the check for the default queue in capacity scheduler.
     (Sreekanth Ramakrishnan via yhemanth)
 
+    HADOOP-5719. Remove jobs that failed initialization from the waiting queue
+    in the capacity scheduler. (Sreekanth Ramakrishnan via yhemanth)
+
 Release 0.20.0 - 2009-04-15
 
   INCOMPATIBLE CHANGES

src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/JobInitializationPoller.java

@@ -20,7 +20,6 @@ package org.apache.hadoop.mapred;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Set;
 import java.util.TreeMap;
@@ -149,8 +148,8 @@ public class JobInitializationPoller extends Thread {
           } catch (Throwable t) {
             LOG.info("Job initialization failed:\n"
                 + StringUtils.stringifyException(t));
-            if (job != null)
-              job.fail();
+            jobQueueManager.removeJobFromWaitingQueue(job);
+            job.fail(); 
           }
         }
       }

src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacityScheduler.java

@@ -248,6 +248,24 @@ public class TestCapacityScheduler extends TestCase {
     
   }
   
+  static class FakeFailingJobInProgress extends FakeJobInProgress {
+
+    public FakeFailingJobInProgress(JobID id, JobConf jobConf,
+        FakeTaskTrackerManager taskTrackerManager, String user) {
+      super(id, jobConf, taskTrackerManager, user);
+    }
+    
+    @Override
+    public synchronized void initTasks() throws IOException {
+      throw new IOException("Failed Initalization");
+    }
+    
+    @Override
+    synchronized void fail() {
+      this.status.setRunState(JobStatus.FAILED);
+    }
+  }
+  
   static class FakeTaskInProgress extends TaskInProgress {
     private boolean isMap;
     private FakeJobInProgress fakeJob;
@@ -2009,6 +2027,39 @@ public class TestCapacityScheduler extends TestCase {
     t = checkAssignment("tt1", "attempt_test_0001_r_000001_0 on tt1");
   }
   
+  public void testFailedJobInitalizations() throws Exception {
+    String[] qs = {"default"};
+    taskTrackerManager.addQueues(qs);
+    ArrayList<FakeQueueInfo> queues = new ArrayList<FakeQueueInfo>();
+    queues.add(new FakeQueueInfo("default", 100.0f, true, 100));
+    resConf.setFakeQueues(queues);
+    scheduler.setResourceManagerConf(resConf);
+    scheduler.start();
+
+    JobQueuesManager mgr = scheduler.jobQueuesManager;
+    
+    //Submit a job whose initialization would fail always.
+    FakeJobInProgress job =
+      new FakeFailingJobInProgress(new JobID("test", ++jobCounter),
+          new JobConf(), taskTrackerManager,"u1");
+    job.getStatus().setRunState(JobStatus.PREP);
+    taskTrackerManager.submitJob(job);
+    //check if job is present in waiting list.
+    assertEquals("Waiting job list does not contain submitted job",
+        1, mgr.getWaitingJobCount("default"));
+    assertTrue("Waiting job does not contain submitted job", 
+        mgr.getWaitingJobs("default").contains(job));
+    //initialization should fail now.
+    controlledInitializationPoller.selectJobsToInitialize();
+    //Check if the job has been properly cleaned up.
+    assertEquals("Waiting job list contains submitted job",
+        0, mgr.getWaitingJobCount("default"));
+    assertFalse("Waiting job contains submitted job", 
+        mgr.getWaitingJobs("default").contains(job));
+    assertFalse("Waiting job contains submitted job", 
+        mgr.getRunningJobQueue("default").contains(job));
+  }
+  
   private void checkRunningJobMovementAndCompletion() throws IOException {
     
     JobQueuesManager mgr = scheduler.jobQueuesManager;

src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestJobInitialization.java

@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapred;
+
+import java.util.Properties;
+import org.apache.hadoop.mapred.ControlledMapReduceJob.ControlledMapReduceJobRunner;
+
+public class TestJobInitialization extends ClusterWithCapacityScheduler {
+ 
+  public void testFailingJobInitalization() throws Exception {
+    Properties schedulerProps = new Properties();
+    schedulerProps.put(
+        "mapred.capacity-scheduler.queue.default.capacity", "100");
+    Properties clusterProps = new Properties();
+    clusterProps
+        .put("mapred.tasktracker.map.tasks.maximum", String.valueOf(1));
+    clusterProps.put("mapred.tasktracker.reduce.tasks.maximum", String
+        .valueOf(1));
+    clusterProps.put("mapred.jobtracker.maxtasks.per.job", String
+        .valueOf(1));
+    // cluster capacity 1 maps, 1 reduces
+    startCluster(1, clusterProps, schedulerProps);
+    ControlledMapReduceJobRunner jobRunner =
+      ControlledMapReduceJobRunner.getControlledMapReduceJobRunner(
+          getJobConf(), 3, 3);
+    jobRunner.start();
+    JobID myJobID = jobRunner.getJobID();
+    JobInProgress myJob = getJobTracker().getJob(myJobID);
+    while(!myJob.isComplete()) {
+      Thread.sleep(1000);
+    }
+    assertTrue("The submitted job successfully completed", 
+        myJob.status.getRunState() == JobStatus.FAILED);
+    CapacityTaskScheduler scheduler = (CapacityTaskScheduler) getJobTracker().getTaskScheduler();
+    JobQueuesManager mgr = scheduler.jobQueuesManager;
+    assertEquals("Failed job present in Waiting queue", 
+        0, mgr.getWaitingJobCount("default"));
+    assertFalse("Failed job present in Waiting queue", 
+        mgr.getWaitingJobs("default").contains(myJob));
+  }
+}