commit 58507691b6c5af531543929118dcce6f7cd8fa62

Author: Balaji Rajagopalan <balajirg@yahoo-inc.com>
Date:   Tue Jul 27 18:47:43 2010 -0700

    MAPREDUCE-1964 from https://issues.apache.org/jira/secure/ManageAttachments.jspa?id=12469985


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.20-security-patches@1077602 13f79535-47bb-0310-9956-ffa450edef68

src/test/system/java/org/apache/hadoop/mapred/HighRamJobHelper.java

@@ -0,0 +1,55 @@
+package org.apache.hadoop.mapred;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.examples.SleepJob;
+import org.apache.hadoop.mapreduce.test.system.JTProtocol;
+import org.apache.hadoop.mapreduce.test.system.JobInfo;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.junit.Assert;
+
+/**
+ * This is a helper class that is used by test cases to run a high ram job
+ * the intention behind creatint this class is reuse code. 
+ *
+ */
+public class HighRamJobHelper {
+
+  public  HighRamJobHelper () {
+    
+  }
+  
+  /**
+   * The method runs the high ram job
+   * @param conf configuration for unning the job
+   * @param jobClient instance
+   * @param remoteJTClient instance
+   * @return the job id of the high ram job
+   * @throws Exception is thrown when the method fails to run the high ram job
+   */
+  public JobID runHighRamJob (Configuration conf, JobClient jobClient, 
+      JTProtocol remoteJTClient,String assertMessage) throws Exception {
+    SleepJob job = new SleepJob();
+    String jobArgs []= {"-D","mapred.cluster.max.map.memory.mb=2048", 
+                        "-D","mapred.cluster.max.reduce.memory.mb=2048", 
+                        "-D","mapred.cluster.map.memory.mb=1024", 
+                        "-D","mapred.cluster.reduce.memory.mb=1024",
+                        "-m", "6", 
+                        "-r", "2", 
+                        "-mt", "2000", 
+                        "-rt", "2000",
+                        "-recordt","100"};
+    JobConf jobConf = new JobConf(conf);
+    jobConf.setMemoryForMapTask(2048);
+    jobConf.setMemoryForReduceTask(2048);
+    int exitCode = ToolRunner.run(jobConf, job, jobArgs);
+    Assert.assertEquals("Exit Code:", 0, exitCode);
+    UtilsForTests.waitFor(1000); 
+    JobID jobId = jobClient.getAllJobs()[0].getJobID();
+    JobInfo jInfo = remoteJTClient.getJobInfo(jobId);
+    Assert.assertEquals(assertMessage, 
+        jInfo.getStatus().getRunState(), JobStatus.SUCCEEDED);
+    return jobId;
+  }
+  
+}

src/test/system/java/org/apache/hadoop/mapred/TestHealthScriptError.java

@@ -17,7 +17,7 @@ public class TestHealthScriptError {
   public static MRCluster cluster;
   public static HealthScriptHelper helper;
   public static String remotePath;
-  public String healthScriptError="healthScriptError";
+  public static String healthScriptError="healthScriptError";
   public static String remoteHSPath = "test.system.hdrc.healthscript.path";
   static final Log LOG = LogFactory.getLog(TestHealthScriptError.class);
   

src/test/system/java/org/apache/hadoop/mapred/TestHiRamJobWithBlackListTT.java

@@ -0,0 +1,96 @@
+package org.apache.hadoop.mapred;
+
+import java.io.File;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.test.system.JTProtocol;
+import org.apache.hadoop.mapreduce.test.system.MRCluster;
+import org.apache.hadoop.mapreduce.test.system.TTClient;
+import org.apache.hadoop.mapreduce.test.system.MRCluster.Role;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import java.util.List;
+import java.util.ArrayList;
+
+public class TestHiRamJobWithBlackListTT {
+  static final Log LOG = LogFactory.getLog(TestHiRamJobWithBlackListTT.class);
+  private static HealthScriptHelper bListHelper = null;
+  public static String remotePath;
+  public static MRCluster cluster;
+  
+  @BeforeClass
+  public static void setUp() throws java.lang.Exception {
+    String [] expExcludeList = new String[2];
+    expExcludeList[0] = "java.net.ConnectException";
+    expExcludeList[1] = "java.io.IOException";
+    cluster = MRCluster.createCluster(new Configuration());
+    cluster.setExcludeExpList(expExcludeList);
+    cluster.setUp();
+    bListHelper = new HealthScriptHelper();
+    remotePath = cluster.getConf().get(TestHealthScriptError.remoteHSPath);
+  }
+  
+  /** Black List more than 25 % of task trackers , run the high ram
+   * job and make sure that no exception is thrown. 
+   * @throws Exception If fails to blacklist TT or run high ram high
+   */
+  @Test
+  public void testHiRamJobBlackListedTaskTrackers() throws Exception {
+    final HighRamJobHelper hRamHelper = new HighRamJobHelper();
+    List<TTClient> bListedTT = new ArrayList<TTClient>();
+    List<TTClient> tClient = cluster.getTTClients();
+    int count = tClient.size();
+    int moreThan25Per = count / 4 +1;
+    LOG.info ("More than 25 % of TTclient is "+moreThan25Per);
+    for (int i=0; i < moreThan25Per ; ++i) {
+      TTClient client = tClient.get(i);
+      bListedTT.add(client);
+      blackListTT(client);
+    }
+    //Now run the high ram job
+    JobClient jobClient = cluster.getJTClient().getClient();
+    JTProtocol remoteJTClient = cluster.getJTClient().getProxy();
+    Configuration conf = remoteJTClient.getDaemonConf();    
+    hRamHelper.runHighRamJob(conf, jobClient, remoteJTClient,
+        "Job did not succeed");
+    //put the task tracker back in healthy state
+    for( int i =0; i < bListedTT.size() ; ++i) {
+      unBlackListTT(bListedTT.get(i));
+    }
+  }
+  
+  @AfterClass
+  public static void tearDown() throws Exception {    
+    cluster.tearDown();
+  }
+  
+  private void unBlackListTT (TTClient client) throws Exception{
+    //Now put back the task tracker in a healthy state
+    cluster.restart(client, Role.TT);
+    bListHelper.deleteFileOnRemoteHost(remotePath + File.separator +
+        TestHealthScriptError.healthScriptError, client.getHostName());
+  }
+  
+  private void blackListTT(TTClient client) throws Exception {
+    Configuration tConf= client.getProxy().getDaemonConf();    
+    tConf.set("mapred.task.tracker.report.address",
+        cluster.getConf().get("mapred.task.tracker.report.address"));
+    String defaultHealthScript = tConf.get("mapred.healthChecker.script.path");
+    Assert.assertTrue("Health script was not set", defaultHealthScript != null);        
+    tConf.set("mapred.healthChecker.script.path", remotePath+File.separator+
+        TestHealthScriptError.healthScriptError);
+    tConf.setInt("mapred.healthChecker.interval", 1000);
+    bListHelper.copyFileToRemoteHost(TestHealthScriptError.healthScriptError, 
+        client.getHostName(), remotePath, cluster);
+    cluster.restartDaemonWithNewConfig(client, "mapred-site.xml", tConf, 
+        Role.TT);
+    //make sure the TT is blacklisted
+    bListHelper.verifyTTBlackList(tConf, client,
+        "ERROR Task Tracker status is fatal", cluster);
+  }
+
+}

src/test/system/java/org/apache/hadoop/mapred/TestJobSummary.java

@@ -60,6 +60,7 @@ public class TestJobSummary {
   private static Path inputDir = new Path("input");
   private static Path outputDir = new Path("output");
   private static String queueName = null;
+  
   @BeforeClass
   public static void before() throws Exception {
     String [] expExcludeList = {"java.net.ConnectException",
@@ -176,26 +177,10 @@ public class TestJobSummary {
   @Test
   public void testJobSummaryInfoOfHighMemoryJob() throws IOException,
       Exception {
-    SleepJob job = new SleepJob();
-    String jobArgs []= {"-D","mapred.cluster.max.map.memory.mb=2048", 
-                        "-D","mapred.cluster.max.reduce.memory.mb=2048", 
-                        "-D","mapred.cluster.map.memory.mb=1024", 
-                        "-D","mapred.cluster.reduce.memory.mb=1024",
-                        "-m", "6", 
-                        "-r", "2", 
-                        "-mt", "2000", 
-                        "-rt", "2000",
-                        "-recordt","100"};
-    JobConf jobConf = new JobConf(conf);
-    jobConf.setMemoryForMapTask(2048);
-    jobConf.setMemoryForReduceTask(2048);
-    int exitCode = runTool(jobConf,job,jobArgs);
-    Assert.assertEquals("Exit Code:", 0, exitCode);
-    UtilsForTests.waitFor(1000); 
-    JobID jobId = jobClient.getAllJobs()[0].getJobID();
+    final HighRamJobHelper helper = new HighRamJobHelper();
+    JobID jobId = helper.runHighRamJob(conf, jobClient, remoteJTClient,
+        "Job did not succeed");
     JobInfo jInfo = remoteJTClient.getJobInfo(jobId);
-    Assert.assertEquals("Job has not been succeeded", 
-          jInfo.getStatus().getRunState(), JobStatus.SUCCEEDED);
     verifyJobSummaryInfo(jInfo,jobId);
   }
 
@@ -239,11 +224,7 @@ public class TestJobSummary {
      verifyJobSummaryInfo(jInfo,jobId);  
   }
   
-  private int runTool(Configuration job, Tool tool, 
-    String[] jobArgs) throws Exception {
-    int returnStatus = ToolRunner.run(job, tool, jobArgs);
-    return returnStatus;
-  }
+
   private void verifyJobSummaryInfo(JobInfo jInfo, JobID id) 
       throws IOException {
     java.util.HashMap<String,String> map = jtClient.getJobSummary(id);