YARN-1206. Fixed AM container log to show on NM web page after application finishes if log-aggregation is disabled. Contributed by Rohith Sharmaks

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1578614 13f79535-47bb-0310-9956-ffa450edef68

hadoop-yarn-project/CHANGES.txt

@@ -498,6 +498,9 @@ Release 2.4.0 - UNRELEASED
     YARN-1685. Fixed few bugs related to handling of containers' log-URLs on
     ResourceManager and history-service. (Zhijie Shen via vinodkv)
 
+    YARN-1206. Fixed AM container log to show on NM web page after application
+    finishes if log-aggregation is disabled. (Rohith Sharmaks via jianhe)
+
 Release 2.3.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsUtils.java

@@ -56,13 +56,18 @@ public class ContainerLogsUtils {
   public static List<File> getContainerLogDirs(ContainerId containerId,
       String remoteUser, Context context) throws YarnException {
     Container container = context.getContainers().get(containerId);
-    if (container == null) {
-      throw new YarnException("Container does not exist.");
-    }
 
     Application application = getApplicationForContainer(containerId, context);
     checkAccess(remoteUser, application, context);
-    checkState(container.getContainerState());
+    // It is not required to have null check for container ( container == null )
+    // and throw back exception.Because when container is completed, NodeManager
+    // remove container information from its NMContext.Configuring log
+    // aggregation to false, container log view request is forwarded to NM. NM
+    // does not have completed container information,but still NM serve request for
+    // reading container logs. 
+    if (container != null) {
+      checkState(container.getContainerState());
+    }
     
     return getContainerLogDirs(containerId, context.getLocalDirsHandler());
   }
@@ -91,14 +96,12 @@ public class ContainerLogsUtils {
   public static File getContainerLogFile(ContainerId containerId,
       String fileName, String remoteUser, Context context) throws YarnException {
     Container container = context.getContainers().get(containerId);
-    if (container == null) {
-      throw new NotFoundException("Container with id " + containerId
-          + " not found.");
-    }
     
     Application application = getApplicationForContainer(containerId, context);
     checkAccess(remoteUser, application, context);
-    checkState(container.getContainerState());
+    if (container != null) {
+      checkState(container.getContainerState());
+    }
     
     try {
       LocalDirsHandlerService dirsHandler = context.getLocalDirsHandler();

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java

@@ -49,10 +49,13 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.server.nodemanager.Context;
 import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
 import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService;
+import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
+import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
 import org.apache.hadoop.yarn.server.nodemanager.webapp.ContainerLogsPage.ContainersLogsBlock;
+import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
 import org.apache.hadoop.yarn.webapp.YarnWebParams;
 import org.apache.hadoop.yarn.webapp.test.WebAppTests;
@@ -74,6 +77,7 @@ public class TestContainerLogsPage {
     NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
     healthChecker.init(conf);
     LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler();
+    NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler, new ApplicationACLsManager(conf));
     // Add an application and the corresponding containers
     RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(conf);
     String user = "nobody";
@@ -87,9 +91,21 @@ public class TestContainerLogsPage {
         appId, 1);
     ContainerId container1 = BuilderUtils.newContainerId(recordFactory, appId,
         appAttemptId, 0);
+    nmContext.getApplications().put(appId, app);
+
+    MockContainer container =
+        new MockContainer(appAttemptId, new AsyncDispatcher(), conf, user,
+            appId, 1);
+    container.setState(ContainerState.RUNNING);
+    nmContext.getContainers().put(container1, container);   
     List<File> files = null;
-    files = ContainerLogsUtils.getContainerLogDirs(
-        container1, dirsHandler);
+    files = ContainerLogsUtils.getContainerLogDirs(container1, user, nmContext);
+    Assert.assertTrue(!(files.get(0).toString().contains("file:")));
+    
+    // After container is completed, it is removed from nmContext
+    nmContext.getContainers().remove(container1);
+    Assert.assertNull(nmContext.getContainers().get(container1));
+    files = ContainerLogsUtils.getContainerLogDirs(container1, user, nmContext);
     Assert.assertTrue(!(files.get(0).toString().contains("file:")));
   }
   

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java

@@ -352,6 +352,16 @@ public class TestNMWebServices extends JerseyTest {
     Assert.assertEquals(Status.NOT_FOUND.getStatusCode(), response.getStatus());
     responseText = response.getEntity(String.class);
     assertTrue(responseText.contains("Cannot find this log on the local disk."));
+    
+    // After container is completed, it is removed from nmContext
+    nmContext.getContainers().remove(containerId);
+    Assert.assertNull(nmContext.getContainers().get(containerId));
+    response =
+        r.path("ws").path("v1").path("node").path("containerlogs")
+            .path(containerIdStr).path(filename).accept(MediaType.TEXT_PLAIN)
+            .get(ClientResponse.class);
+    responseText = response.getEntity(String.class);
+    assertEquals(logMessage, responseText);
   }
 
   public void verifyNodesXML(NodeList nodes) throws JSONException, Exception {