|
|
@@ -28,6 +28,7 @@ import static org.mockito.Mockito.when;
|
|
|
import static org.mockito.Mockito.never;
|
|
|
|
|
|
import java.io.File;
|
|
|
+import java.io.FileOutputStream;
|
|
|
import java.io.IOException;
|
|
|
|
|
|
import org.junit.Assert;
|
|
|
@@ -35,8 +36,13 @@ import org.junit.Assert;
|
|
|
import org.apache.commons.logging.Log;
|
|
|
import org.apache.commons.logging.LogFactory;
|
|
|
import org.apache.hadoop.conf.Configuration;
|
|
|
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
|
|
|
import org.apache.hadoop.fs.FileContext;
|
|
|
+import org.apache.hadoop.fs.FileSystem;
|
|
|
+import org.apache.hadoop.fs.LocalFileSystem;
|
|
|
import org.apache.hadoop.fs.Path;
|
|
|
+import org.apache.hadoop.hdfs.HdfsConfiguration;
|
|
|
+import org.apache.hadoop.hdfs.MiniDFSCluster;
|
|
|
import org.apache.hadoop.mapreduce.Counters;
|
|
|
import org.apache.hadoop.mapreduce.JobID;
|
|
|
import org.apache.hadoop.mapreduce.MRJobConfig;
|
|
|
@@ -52,6 +58,10 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
|
|
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
|
|
import org.apache.hadoop.yarn.api.records.ContainerId;
|
|
|
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
|
|
|
+import org.junit.After;
|
|
|
+import org.junit.AfterClass;
|
|
|
+import static org.junit.Assert.assertFalse;
|
|
|
+import org.junit.BeforeClass;
|
|
|
import org.junit.Test;
|
|
|
import org.mockito.Mockito;
|
|
|
|
|
|
@@ -60,6 +70,26 @@ public class TestJobHistoryEventHandler {
|
|
|
|
|
|
private static final Log LOG = LogFactory
|
|
|
.getLog(TestJobHistoryEventHandler.class);
|
|
|
+ private static MiniDFSCluster dfsCluster = null;
|
|
|
+ private static String coreSitePath;
|
|
|
+
|
|
|
+ @BeforeClass
|
|
|
+ public static void setUpClass() throws Exception {
|
|
|
+ coreSitePath = "." + File.separator + "target" + File.separator +
|
|
|
+ "test-classes" + File.separator + "core-site.xml";
|
|
|
+ Configuration conf = new HdfsConfiguration();
|
|
|
+ dfsCluster = new MiniDFSCluster.Builder(conf).build();
|
|
|
+ }
|
|
|
+
|
|
|
+ @AfterClass
|
|
|
+ public static void cleanUpClass() throws Exception {
|
|
|
+ dfsCluster.shutdown();
|
|
|
+ }
|
|
|
+
|
|
|
+ @After
|
|
|
+ public void cleanTest() throws Exception {
|
|
|
+ new File(coreSitePath).delete();
|
|
|
+ }
|
|
|
|
|
|
@Test (timeout=50000)
|
|
|
public void testFirstFlushOnCompletionEvent() throws Exception {
|
|
|
@@ -325,6 +355,50 @@ public class TestJobHistoryEventHandler {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+ @Test (timeout=50000)
|
|
|
+ public void testDefaultFsIsUsedForHistory() throws Exception {
|
|
|
+ // Create default configuration pointing to the minicluster
|
|
|
+ Configuration conf = new Configuration();
|
|
|
+ conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
|
|
|
+ dfsCluster.getURI().toString());
|
|
|
+ FileOutputStream os = new FileOutputStream(coreSitePath);
|
|
|
+ conf.writeXml(os);
|
|
|
+ os.close();
|
|
|
+
|
|
|
+ // simulate execution under a non-default namenode
|
|
|
+ conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
|
|
|
+ "file:///");
|
|
|
+
|
|
|
+ TestParams t = new TestParams();
|
|
|
+ conf.set(MRJobConfig.MR_AM_STAGING_DIR, t.dfsWorkDir);
|
|
|
+
|
|
|
+ JHEvenHandlerForTest realJheh =
|
|
|
+ new JHEvenHandlerForTest(t.mockAppContext, 0, false);
|
|
|
+ JHEvenHandlerForTest jheh = spy(realJheh);
|
|
|
+ jheh.init(conf);
|
|
|
+
|
|
|
+ try {
|
|
|
+ jheh.start();
|
|
|
+ handleEvent(jheh, new JobHistoryEvent(t.jobId, new AMStartedEvent(
|
|
|
+ t.appAttemptId, 200, t.containerId, "nmhost", 3000, 4000)));
|
|
|
+
|
|
|
+ handleEvent(jheh, new JobHistoryEvent(t.jobId, new JobFinishedEvent(
|
|
|
+ TypeConverter.fromYarn(t.jobId), 0, 0, 0, 0, 0, new Counters(),
|
|
|
+ new Counters(), new Counters())));
|
|
|
+
|
|
|
+ // If we got here then event handler worked but we don't know with which
|
|
|
+ // file system. Now we check that history stuff was written to minicluster
|
|
|
+ FileSystem dfsFileSystem = dfsCluster.getFileSystem();
|
|
|
+ assertTrue("Minicluster contains some history files",
|
|
|
+ dfsFileSystem.globStatus(new Path(t.dfsWorkDir + "/*")).length != 0);
|
|
|
+ FileSystem localFileSystem = LocalFileSystem.get(conf);
|
|
|
+ assertFalse("No history directory on non-default file system",
|
|
|
+ localFileSystem.exists(new Path(t.dfsWorkDir)));
|
|
|
+ } finally {
|
|
|
+ jheh.stop();
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
private void queueEvent(JHEvenHandlerForTest jheh, JobHistoryEvent event) {
|
|
|
jheh.handle(event);
|
|
|
}
|
|
|
@@ -372,6 +446,7 @@ public class TestJobHistoryEventHandler {
|
|
|
private class TestParams {
|
|
|
boolean isLastAMRetry;
|
|
|
String workDir = setupTestWorkDir();
|
|
|
+ String dfsWorkDir = "/" + this.getClass().getCanonicalName();
|
|
|
ApplicationId appId = ApplicationId.newInstance(200, 1);
|
|
|
ApplicationAttemptId appAttemptId =
|
|
|
ApplicationAttemptId.newInstance(appId, 1);
|
|
|
@@ -451,10 +526,16 @@ public class TestJobHistoryEventHandler {
|
|
|
class JHEvenHandlerForTest extends JobHistoryEventHandler {
|
|
|
|
|
|
private EventWriter eventWriter;
|
|
|
+ private boolean mockHistoryProcessing = true;
|
|
|
public JHEvenHandlerForTest(AppContext context, int startCount) {
|
|
|
super(context, startCount);
|
|
|
}
|
|
|
|
|
|
+ public JHEvenHandlerForTest(AppContext context, int startCount, boolean mockHistoryProcessing) {
|
|
|
+ super(context, startCount);
|
|
|
+ this.mockHistoryProcessing = mockHistoryProcessing;
|
|
|
+ }
|
|
|
+
|
|
|
@Override
|
|
|
protected void serviceStart() {
|
|
|
}
|
|
|
@@ -462,7 +543,12 @@ class JHEvenHandlerForTest extends JobHistoryEventHandler {
|
|
|
@Override
|
|
|
protected EventWriter createEventWriter(Path historyFilePath)
|
|
|
throws IOException {
|
|
|
- this.eventWriter = mock(EventWriter.class);
|
|
|
+ if (mockHistoryProcessing) {
|
|
|
+ this.eventWriter = mock(EventWriter.class);
|
|
|
+ }
|
|
|
+ else {
|
|
|
+ this.eventWriter = super.createEventWriter(historyFilePath);
|
|
|
+ }
|
|
|
return this.eventWriter;
|
|
|
}
|
|
|
|
|
|
@@ -475,8 +561,13 @@ class JHEvenHandlerForTest extends JobHistoryEventHandler {
|
|
|
}
|
|
|
|
|
|
@Override
|
|
|
- protected void processDoneFiles(JobId jobId){
|
|
|
- // do nothing
|
|
|
+ protected void processDoneFiles(JobId jobId) throws IOException {
|
|
|
+ if (!mockHistoryProcessing) {
|
|
|
+ super.processDoneFiles(jobId);
|
|
|
+ }
|
|
|
+ else {
|
|
|
+ // do nothing
|
|
|
+ }
|
|
|
}
|
|
|
}
|
|
|
|
Zhijie Shen