svn merge -c 1362618 FIXES: MAPREDUCE-4448. Fix NM crash during app cleanup if aggregation didn't init. (Jason Lowe via daryn)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1362625 13f79535-47bb-0310-9956-ffa450edef68

hadoop-mapreduce-project/CHANGES.txt

@@ -340,6 +340,9 @@ Release 0.23.3 - UNRELEASED
     MAPREDUCE-4283. Display tail of aggregated logs by default (Jason Lowe via
     bobby)
 
+    MAPREDUCE-4448. Fix NM crash during app cleanup if aggregation didn't
+    init. (Jason Lowe via daryn)
+
 Release 0.23.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java

@@ -342,14 +342,14 @@ public class LogAggregationService extends AbstractService implements
     // A container is complete. Put this containers' logs up for aggregation if
     // this containers' logs are needed.
 
-    if (!this.appLogAggregators.containsKey(
-        containerId.getApplicationAttemptId().getApplicationId())) {
-      throw new YarnException("Application is not initialized yet for "
-          + containerId);
+    AppLogAggregator aggregator = this.appLogAggregators.get(
+        containerId.getApplicationAttemptId().getApplicationId());
+    if (aggregator == null) {
+      LOG.warn("Log aggregation is not initialized for " + containerId
+          + ", did it fail to start?");
+      return;
     }
-    this.appLogAggregators.get(
-        containerId.getApplicationAttemptId().getApplicationId())
-        .startContainerLogAggregation(containerId, exitCode == 0);
+    aggregator.startContainerLogAggregation(containerId, exitCode == 0);
   }
 
   private void stopApp(ApplicationId appId) {
@@ -357,11 +357,13 @@ public class LogAggregationService extends AbstractService implements
     // App is complete. Finish up any containers' pending log aggregation and
     // close the application specific logFile.
 
-    if (!this.appLogAggregators.containsKey(appId)) {
-      throw new YarnException("Application is not initialized yet for "
-          + appId);
+    AppLogAggregator aggregator = this.appLogAggregators.get(appId);
+    if (aggregator == null) {
+      LOG.warn("Log aggregation is not initialized for " + appId
+          + ", did it fail to start?");
+      return;
     }
-    this.appLogAggregators.get(appId).finishLogAggregation();
+    aggregator.finishLogAggregation();
   }
 
   @Override

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java

@@ -380,7 +380,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
   
   @Test
   @SuppressWarnings("unchecked")
-  public void testLogAggregationInitFailsWithoutKillingNM() throws Exception {
+  public void testLogAggregationFailsWithoutKillingNM() throws Exception {
     
     this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
     this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
@@ -412,7 +412,16 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
         new ApplicationFinishEvent(appId, "Application failed to init aggregation: KABOOM!")
     };
     checkEvents(appEventHandler, expectedEvents, false,
-        "getType", "getApplicationID", "getDiagnostic");    
+        "getType", "getApplicationID", "getDiagnostic");
+
+    // verify trying to collect logs for containers/apps we don't know about
+    // doesn't blow up and tear down the NM
+    logAggregationService.handle(new LogHandlerContainerFinishedEvent(
+        BuilderUtils.newContainerId(4, 1, 1, 1), 0));
+    dispatcher.await();
+    logAggregationService.handle(new LogHandlerAppFinishedEvent(
+        BuilderUtils.newApplicationId(1, 5)));
+    dispatcher.await();
   }
   
   private void writeContainerLogs(File appLogDir, ContainerId containerId)