YARN-1685. Fixed few bugs related to handling of containers' log-URLs on ResourceManager and history-service. Contributed by Zhijie Shen.

svn merge --ignore-ancestry -c 1578602 ../../trunk/


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1578603 13f79535-47bb-0310-9956-ffa450edef68

hadoop-yarn-project/CHANGES.txt

@@ -480,6 +480,9 @@ Release 2.4.0 - UNRELEASED
     YARN-1830. Fixed TestRMRestart#testQueueMetricsOnRMRestart failure due to
     race condition when app is submitted. (Zhijie Shen via jianhe)
 
+    YARN-1685. Fixed few bugs related to handling of containers' log-URLs on
+    ResourceManager and history-service. (Zhijie Shen via vinodkv)
+
 Release 2.3.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/application_history_server.proto

@@ -90,9 +90,8 @@ message ContainerHistoryDataProto {
   optional int64 start_time = 5;
   optional int64 finish_time = 6;
   optional string diagnostics_info = 7;
-  optional string log_url = 8;
-  optional int32 container_exit_status = 9;
-  optional ContainerStateProto container_state = 10;
+  optional int32 container_exit_status = 8;
+  optional ContainerStateProto container_state = 9;
 }
 
 message ContainerStartDataProto {
@@ -107,7 +106,6 @@ message ContainerFinishDataProto {
   optional ContainerIdProto container_id = 1;
   optional int64 finish_time = 2;
   optional string diagnostics_info = 3;
-  optional string log_url = 4;
-  optional int32 container_exit_status = 5;
-  optional ContainerStateProto container_state = 6;
+  optional int32 container_exit_status = 4;
+  optional ContainerStateProto container_state = 5;
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java

@@ -178,8 +178,5 @@ public final class StringHelper {
   public static String percent(double value) {
     return String.format("%.2f", value * 100);
   }
-  
-  public static String getPartUrl(String url, String part) {
-    return url.substring(url.indexOf(part));
-  }
+
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java

@@ -17,7 +17,7 @@
 */
 package org.apache.hadoop.yarn.webapp.util;
 
-import static org.apache.hadoop.yarn.util.StringHelper.join;
+import static org.apache.hadoop.yarn.util.StringHelper.PATH_JOINER;
 
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
@@ -29,9 +29,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.http.HttpConfig.Policy;
 import org.apache.hadoop.http.HttpServer2;
 import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.util.ConverterUtils;
 
 @Private
 @Evolving
@@ -169,18 +167,35 @@ public class WebAppUtils {
     }
   }
   
-  public static String getLogUrl(String nodeHttpAddress, String allocatedNode,
-      ContainerId containerId, String user) {
-    return join("//", nodeHttpAddress, "/logs", "/",
-        allocatedNode, "/", ConverterUtils.toString(containerId), "/",
-        ConverterUtils.toString(containerId), "/", user);
+  public static String getRunningLogURL(
+      String nodeHttpAddress, String containerId, String user) {
+    if (nodeHttpAddress == null || nodeHttpAddress.isEmpty() ||
+        containerId == null || containerId.isEmpty() ||
+        user == null || user.isEmpty()) {
+      return null;
+    }
+    return PATH_JOINER.join(
+        nodeHttpAddress, "node", "containerlogs", containerId, user);
+  }
+
+  public static String getAggregatedLogURL(String serverHttpAddress,
+      String allocatedNode, String containerId, String entity, String user) {
+    if (serverHttpAddress == null || serverHttpAddress.isEmpty() ||
+        allocatedNode == null || allocatedNode.isEmpty() ||
+        containerId == null || containerId.isEmpty() ||
+        entity == null || entity.isEmpty() ||
+        user == null || user.isEmpty()) {
+      return null;
+    }
+    return PATH_JOINER.join(serverHttpAddress, "applicationhistory", "logs",
+        allocatedNode, containerId, entity, user);
   }
 
   /**
    * Choose which scheme (HTTP or HTTPS) to use when generating a URL based on
    * the configuration.
    * 
-   * @return the schmeme (HTTP / HTTPS)
+   * @return the scheme (HTTP / HTTPS)
    */
   public static String getHttpSchemePrefix(Configuration conf) {
     return YarnConfiguration.useHttps(conf) ? HTTPS_PREFIX : HTTP_PREFIX;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java

@@ -39,6 +39,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerHistoryData;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 
 import com.google.common.annotations.VisibleForTesting;
 
@@ -49,6 +50,7 @@ public class ApplicationHistoryManagerImpl extends AbstractService implements
   private static final String UNAVAILABLE = "N/A";
 
   private ApplicationHistoryStore historyStore;
+  private String serverHttpAddress;
 
   public ApplicationHistoryManagerImpl() {
     super(ApplicationHistoryManagerImpl.class.getName());
@@ -59,6 +61,8 @@ public class ApplicationHistoryManagerImpl extends AbstractService implements
     LOG.info("ApplicationHistory Init");
     historyStore = createApplicationHistoryStore(conf);
     historyStore.init(conf);
+    serverHttpAddress = WebAppUtils.getHttpSchemePrefix(conf) +
+        WebAppUtils.getAHSWebAppURLWithoutScheme(conf);
     super.serviceInit(conf);
   }
 
@@ -87,7 +91,10 @@ public class ApplicationHistoryManagerImpl extends AbstractService implements
   @Override
   public ContainerReport getAMContainer(ApplicationAttemptId appAttemptId)
       throws IOException {
-    return convertToContainerReport(historyStore.getAMContainer(appAttemptId));
+    ApplicationReport app =
+        getApplication(appAttemptId.getApplicationId());
+    return convertToContainerReport(historyStore.getAMContainer(appAttemptId),
+        app == null ? null : app.getUser());
   }
 
   @Override
@@ -187,16 +194,26 @@ public class ApplicationHistoryManagerImpl extends AbstractService implements
   @Override
   public ContainerReport getContainer(ContainerId containerId)
       throws IOException {
-    return convertToContainerReport(historyStore.getContainer(containerId));
+    ApplicationReport app =
+        getApplication(containerId.getApplicationAttemptId().getApplicationId());
+    return convertToContainerReport(historyStore.getContainer(containerId),
+        app == null ? null: app.getUser());
   }
 
   private ContainerReport convertToContainerReport(
-      ContainerHistoryData containerHistory) {
+      ContainerHistoryData containerHistory, String user) {
+    // If the container has the aggregated log, add the server root url
+    String logUrl = WebAppUtils.getAggregatedLogURL(
+        serverHttpAddress,
+        containerHistory.getAssignedNode().toString(),
+        containerHistory.getContainerId().toString(),
+        containerHistory.getContainerId().toString(),
+        user);
     return ContainerReport.newInstance(containerHistory.getContainerId(),
       containerHistory.getAllocatedResource(),
       containerHistory.getAssignedNode(), containerHistory.getPriority(),
       containerHistory.getStartTime(), containerHistory.getFinishTime(),
-      containerHistory.getDiagnosticsInfo(), containerHistory.getLogURL(),
+      containerHistory.getDiagnosticsInfo(), logUrl,
       containerHistory.getContainerExitStatus(),
       containerHistory.getContainerState());
   }
@@ -204,13 +221,16 @@ public class ApplicationHistoryManagerImpl extends AbstractService implements
   @Override
   public Map<ContainerId, ContainerReport> getContainers(
       ApplicationAttemptId appAttemptId) throws IOException {
+    ApplicationReport app =
+        getApplication(appAttemptId.getApplicationId());
     Map<ContainerId, ContainerHistoryData> histData =
         historyStore.getContainers(appAttemptId);
     HashMap<ContainerId, ContainerReport> containersReport =
         new HashMap<ContainerId, ContainerReport>();
     for (Entry<ContainerId, ContainerHistoryData> entry : histData.entrySet()) {
       containersReport.put(entry.getKey(),
-        convertToContainerReport(entry.getValue()));
+        convertToContainerReport(entry.getValue(),
+            app == null ? null : app.getUser()));
     }
     return containersReport;
   }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java

@@ -315,7 +315,7 @@ public class FileSystemApplicationHistoryStore extends AbstractService
       ContainerHistoryData historyData =
           ContainerHistoryData
             .newInstance(containerId, null, null, null, Long.MIN_VALUE,
-              Long.MAX_VALUE, null, null, Integer.MAX_VALUE, null);
+              Long.MAX_VALUE, null, Integer.MAX_VALUE, null);
       while ((!readStartData || !readFinishData) && hfReader.hasNext()) {
         HistoryFileReader.Entry entry = hfReader.next();
         if (entry.key.id.equals(containerId.toString())) {
@@ -382,7 +382,7 @@ public class FileSystemApplicationHistoryStore extends AbstractService
             if (historyData == null) {
               historyData = ContainerHistoryData.newInstance(
                   containerId, null, null, null, Long.MIN_VALUE,
-                  Long.MAX_VALUE, null, null, Integer.MAX_VALUE, null);
+                  Long.MAX_VALUE, null, Integer.MAX_VALUE, null);
               historyDataMap.put(containerId, historyData);
             }
             if (entry.key.suffix.equals(START_DATA_SUFFIX)) {
@@ -632,7 +632,6 @@ public class FileSystemApplicationHistoryStore extends AbstractService
       ContainerHistoryData historyData, ContainerFinishData finishData) {
     historyData.setFinishTime(finishData.getFinishTime());
     historyData.setDiagnosticsInfo(finishData.getDiagnosticsInfo());
-    historyData.setLogURL(finishData.getLogURL());
     historyData.setContainerExitStatus(finishData.getContainerExitStatus());
     historyData.setContainerState(finishData.getContainerState());
   }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/MemoryApplicationHistoryStore.java

@@ -233,7 +233,7 @@ public class MemoryApplicationHistoryStore extends AbstractService implements
           ContainerHistoryData.newInstance(containerStart.getContainerId(),
             containerStart.getAllocatedResource(),
             containerStart.getAssignedNode(), containerStart.getPriority(),
-            containerStart.getStartTime(), Long.MAX_VALUE, null, null,
+            containerStart.getStartTime(), Long.MAX_VALUE, null,
             Integer.MAX_VALUE, null));
     if (oldData != null) {
       throw new IOException("The start information of container "
@@ -260,7 +260,6 @@ public class MemoryApplicationHistoryStore extends AbstractService implements
     }
     data.setFinishTime(containerFinish.getFinishTime());
     data.setDiagnosticsInfo(containerFinish.getDiagnosticsInfo());
-    data.setLogURL(containerFinish.getLogURL());
     data.setContainerExitStatus(containerFinish.getContainerExitStatus());
     data.setContainerState(containerFinish.getContainerState());
   }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerFinishData.java

@@ -35,14 +35,13 @@ public abstract class ContainerFinishData {
   @Public
   @Unstable
   public static ContainerFinishData newInstance(ContainerId containerId,
-      long finishTime, String diagnosticsInfo, String logURL,
-      int containerExitCode, ContainerState containerState) {
+      long finishTime, String diagnosticsInfo, int containerExitCode,
+      ContainerState containerState) {
     ContainerFinishData containerFD =
         Records.newRecord(ContainerFinishData.class);
     containerFD.setContainerId(containerId);
     containerFD.setFinishTime(finishTime);
     containerFD.setDiagnosticsInfo(diagnosticsInfo);
-    containerFD.setLogURL(logURL);
     containerFD.setContainerExitStatus(containerExitCode);
     containerFD.setContainerState(containerState);
     return containerFD;
@@ -72,14 +71,6 @@ public abstract class ContainerFinishData {
   @Unstable
   public abstract void setDiagnosticsInfo(String diagnosticsInfo);
 
-  @Public
-  @Unstable
-  public abstract String getLogURL();
-
-  @Public
-  @Unstable
-  public abstract void setLogURL(String logURL);
-
   @Public
   @Unstable
   public abstract int getContainerExitStatus();

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerHistoryData.java

@@ -48,8 +48,6 @@ public class ContainerHistoryData {
 
   private String diagnosticsInfo;
 
-  private String logURL;
-
   private int containerExitStatus;
 
   private ContainerState containerState;
@@ -58,7 +56,7 @@ public class ContainerHistoryData {
   @Unstable
   public static ContainerHistoryData newInstance(ContainerId containerId,
       Resource allocatedResource, NodeId assignedNode, Priority priority,
-      long startTime, long finishTime, String diagnosticsInfo, String logURL,
+      long startTime, long finishTime, String diagnosticsInfo,
       int containerExitCode, ContainerState containerState) {
     ContainerHistoryData containerHD = new ContainerHistoryData();
     containerHD.setContainerId(containerId);
@@ -68,7 +66,6 @@ public class ContainerHistoryData {
     containerHD.setStartTime(startTime);
     containerHD.setFinishTime(finishTime);
     containerHD.setDiagnosticsInfo(diagnosticsInfo);
-    containerHD.setLogURL(logURL);
     containerHD.setContainerExitStatus(containerExitCode);
     containerHD.setContainerState(containerState);
     return containerHD;
@@ -158,18 +155,6 @@ public class ContainerHistoryData {
     this.diagnosticsInfo = diagnosticsInfo;
   }
 
-  @Public
-  @Unstable
-  public String getLogURL() {
-    return logURL;
-  }
-
-  @Public
-  @Unstable
-  public void setLogURL(String logURL) {
-    this.logURL = logURL;
-  }
-
   @Public
   @Unstable
   public int getContainerExitStatus() {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ContainerFinishDataPBImpl.java

@@ -101,25 +101,6 @@ public class ContainerFinishDataPBImpl extends ContainerFinishData {
     builder.setDiagnosticsInfo(diagnosticsInfo);
   }
 
-  @Override
-  public String getLogURL() {
-    ContainerFinishDataProtoOrBuilder p = viaProto ? proto : builder;
-    if (!p.hasLogUrl()) {
-      return null;
-    }
-    return p.getLogUrl();
-  }
-
-  @Override
-  public void setLogURL(String logURL) {
-    maybeInitBuilder();
-    if (logURL == null) {
-      builder.clearLogUrl();
-      return;
-    }
-    builder.setLogUrl(logURL);
-  }
-
   @Override
   public int getContainerExitStatus() {
     ContainerFinishDataProtoOrBuilder p = viaProto ? proto : builder;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStoreTestUtils.java

@@ -78,8 +78,7 @@ public class ApplicationHistoryStoreTestUtils {
   protected void writeContainerFinishData(ContainerId containerId)
       throws IOException {
     store.containerFinished(ContainerFinishData.newInstance(containerId, 0,
-      containerId.toString(), "http://localhost:0/log", 0,
-      ContainerState.COMPLETE));
+      containerId.toString(), 0, ContainerState.COMPLETE));
   }
 
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java

@@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerReport;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -52,11 +53,16 @@ public class TestApplicationHistoryClientService extends
     ApplicationHistoryStoreTestUtils {
 
   ApplicationHistoryServer historyServer = null;
+  String expectedLogUrl = null;
 
   @Before
   public void setup() {
     historyServer = new ApplicationHistoryServer();
     Configuration config = new YarnConfiguration();
+    expectedLogUrl = WebAppUtils.getHttpSchemePrefix(config) +
+        WebAppUtils.getAHSWebAppURLWithoutScheme(config) +
+        "/applicationhistory/logs/localhost:0/container_0_0001_01_000001/" +
+        "container_0_0001_01_000001/test user";
     config.setClass(YarnConfiguration.APPLICATION_HISTORY_STORE,
       MemoryApplicationHistoryStore.class, ApplicationHistoryStore.class);
     historyServer.init(config);
@@ -156,11 +162,13 @@ public class TestApplicationHistoryClientService extends
   @Test
   public void testContainerReport() throws IOException, YarnException {
     ApplicationId appId = ApplicationId.newInstance(0, 1);
+    writeApplicationStartData(appId);
     ApplicationAttemptId appAttemptId =
         ApplicationAttemptId.newInstance(appId, 1);
     ContainerId containerId = ContainerId.newInstance(appAttemptId, 1);
     writeContainerStartData(containerId);
     writeContainerFinishData(containerId);
+    writeApplicationFinishData(appId);
     GetContainerReportRequest request =
         GetContainerReportRequest.newInstance(containerId);
     GetContainerReportResponse response =
@@ -169,11 +177,13 @@ public class TestApplicationHistoryClientService extends
     ContainerReport container = response.getContainerReport();
     Assert.assertNotNull(container);
     Assert.assertEquals(containerId, container.getContainerId());
+    Assert.assertEquals(expectedLogUrl, container.getLogUrl());
   }
 
   @Test
   public void testContainers() throws IOException, YarnException {
     ApplicationId appId = ApplicationId.newInstance(0, 1);
+    writeApplicationStartData(appId);
     ApplicationAttemptId appAttemptId =
         ApplicationAttemptId.newInstance(appId, 1);
     ContainerId containerId = ContainerId.newInstance(appAttemptId, 1);
@@ -182,6 +192,7 @@ public class TestApplicationHistoryClientService extends
     writeContainerFinishData(containerId);
     writeContainerStartData(containerId1);
     writeContainerFinishData(containerId1);
+    writeApplicationFinishData(appId);
     GetContainersRequest request =
         GetContainersRequest.newInstance(appAttemptId);
     GetContainersResponse response =

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java

@@ -25,6 +25,7 @@ import javax.ws.rs.core.MediaType;
 
 import org.junit.Assert;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -34,12 +35,14 @@ import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.api.ApplicationContext;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryManager;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryStore;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.MemoryApplicationHistoryStore;
 import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
 import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 import org.codehaus.jettison.json.JSONArray;
 import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
@@ -287,7 +290,12 @@ public class TestAHSWebServices extends JerseyTest {
       container.getString("assignedNodeId"));
     assertEquals(Priority.newInstance(containerId.getId()).toString(),
       container.getString("priority"));
-    assertEquals("http://localhost:0/log", container.getString("logUrl"));
+    Configuration conf = new YarnConfiguration();
+    assertEquals(WebAppUtils.getHttpSchemePrefix(conf) +
+        WebAppUtils.getAHSWebAppURLWithoutScheme(conf) +
+        "/applicationhistory/logs/localhost:0/container_0_0001_01_000001/" +
+        "container_0_0001_01_000001/test user",
+        container.getString("logUrl"));
     assertEquals(ContainerState.COMPLETE.toString(),
       container.getString("containerState"));
   }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java

@@ -18,7 +18,6 @@
 package org.apache.hadoop.yarn.server.webapp;
 
 import static org.apache.hadoop.yarn.util.StringHelper.join;
-import static org.apache.hadoop.yarn.util.StringHelper.getPartUrl;
 import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPLICATION_ATTEMPT_ID;
 
 import java.io.IOException;
@@ -127,8 +126,6 @@ public class AppAttemptBlock extends HtmlBlock {
 
     StringBuilder containersTableData = new StringBuilder("[\n");
     for (ContainerReport containerReport : containers) {
-      String logURL = containerReport.getLogUrl();
-      logURL = getPartUrl(logURL, "log");
       ContainerInfo container = new ContainerInfo(containerReport);
       // ConatinerID numerical value parsed by parseHadoopID in
       // yarn.dt.plugins.js
@@ -144,8 +141,10 @@ public class AppAttemptBlock extends HtmlBlock {
           StringEscapeUtils.escapeJavaScript(StringEscapeUtils
             .escapeHtml(container.getAssignedNodeId()))).append("</a>\",\"")
         .append(container.getContainerExitStatus()).append("\",\"<a href='")
-        .append(logURL == null ? "#" : url(logURL)).append("'>")
-        .append(logURL == null ? "N/A" : "Logs").append("</a>\"],\n");
+        .append(container.getLogUrl() == null ?
+            "#" : container.getLogUrl()).append("'>")
+        .append(container.getLogUrl() == null ?
+            "N/A" : "Logs").append("</a>\"],\n");
     }
     if (containersTableData.charAt(containersTableData.length() - 2) == ',') {
       containersTableData.delete(containersTableData.length() - 2,

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java

@@ -19,14 +19,12 @@
 package org.apache.hadoop.yarn.server.webapp;
 
 import static org.apache.hadoop.yarn.util.StringHelper.join;
-import static org.apache.hadoop.yarn.util.StringHelper.getPartUrl;
 import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPLICATION_ID;
 
 import java.io.IOException;
 import java.util.Collection;
 
 import org.apache.commons.lang.StringEscapeUtils;
-import org.apache.hadoop.http.HttpConfig;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -145,7 +143,6 @@ public class AppBlock extends HtmlBlock {
         ContainerInfo container = new ContainerInfo(containerReport);
         startTime = container.getStartedTime();
         logsLink = containerReport.getLogUrl();
-        logsLink = getPartUrl(logsLink, "log");
       }
       String nodeLink = null;
       if (appAttempt.getHost() != null && appAttempt.getRpcPort() >= 0
@@ -169,8 +166,8 @@ public class AppBlock extends HtmlBlock {
           nodeLink == null ? "N/A" : StringEscapeUtils
             .escapeJavaScript(StringEscapeUtils.escapeHtml(nodeLink)))
         .append("</a>\",\"<a href='")
-        .append(logsLink == null ? "#" : url(logsLink)).append("'>")
-        .append(nodeLink == null ? "N/A" : "Logs").append("</a>\"],\n");
+        .append(logsLink == null ? "#" : logsLink).append("'>")
+        .append(logsLink == null ? "N/A" : "Logs").append("</a>\"],\n");
     }
     if (attemptsTableData.charAt(attemptsTableData.length() - 2) == ',') {
       attemptsTableData.delete(attemptsTableData.length() - 2,

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java

@@ -18,7 +18,6 @@
 package org.apache.hadoop.yarn.server.webapp;
 
 import static org.apache.hadoop.yarn.util.StringHelper.join;
-import static org.apache.hadoop.yarn.util.StringHelper.getPartUrl;
 import static org.apache.hadoop.yarn.webapp.YarnWebParams.CONTAINER_ID;
 
 import java.io.IOException;
@@ -79,8 +78,6 @@ public class ContainerBlock extends HtmlBlock {
     }
 
     ContainerInfo container = new ContainerInfo(containerReport);
-    String logURL = containerReport.getLogUrl();
-    logURL = getPartUrl(logURL, "log");
     setTitle(join("Container ", containerid));
 
     info("Container Overview")
@@ -97,7 +94,8 @@ public class ContainerBlock extends HtmlBlock {
         "Resource:",
         container.getAllocatedMB() + " Memory, "
             + container.getAllocatedVCores() + " VCores")
-      ._("Logs:", logURL == null ? "#" : url(logURL), "Logs")
+      ._("Logs:", container.getLogUrl() == null ? "#" : container.getLogUrl(),
+          container.getLogUrl() == null ? "N/A" : "Logs")
       ._("Diagnostics:", container.getDiagnosticsInfo());
 
     html._(InfoBlock.class);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java

@@ -280,7 +280,7 @@ public class RMApplicationHistoryWriter extends CompositeService {
         new WritingContainerFinishEvent(container.getContainerId(),
           ContainerFinishData.newInstance(container.getContainerId(),
             container.getFinishTime(), container.getDiagnosticsInfo(),
-            container.getLogURL(), container.getContainerExitStatus(),
+            container.getContainerExitStatus(),
             container.getContainerState())));
     }
   }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java

@@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.state.InvalidStateTransitonException;
 import org.apache.hadoop.yarn.state.SingleArcTransition;
 import org.apache.hadoop.yarn.state.StateMachine;
 import org.apache.hadoop.yarn.state.StateMachineFactory;
+import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 
 @SuppressWarnings({"unchecked", "rawtypes"})
@@ -148,7 +149,6 @@ public class RMContainerImpl implements RMContainer {
   private Priority reservedPriority;
   private long startTime;
   private long finishTime;
-  private String logURL;
   private ContainerStatus finishedStatus;
 
 
@@ -264,7 +264,8 @@ public class RMContainerImpl implements RMContainer {
   public String getLogURL() {
     try {
       readLock.lock();
-      return logURL;
+      return WebAppUtils.getRunningLogURL("//" + container.getNodeHttpAddress(),
+          ConverterUtils.toString(containerId), user);
     } finally {
       readLock.unlock();
     }
@@ -380,11 +381,6 @@ public class RMContainerImpl implements RMContainer {
 
     @Override
     public void transition(RMContainerImpl container, RMContainerEvent event) {
-      // The logs of running containers should be found on NM webUI
-      // The logs should be accessible after the container is launched
-      container.logURL = WebAppUtils.getLogUrl(container.container
-          .getNodeHttpAddress(), container.getAllocatedNode().toString(),
-          container.containerId, container.user);
       // Unregister from containerAllocationExpirer.
       container.containerAllocationExpirer.unregister(container
           .getContainerId());
@@ -399,9 +395,6 @@ public class RMContainerImpl implements RMContainer {
 
       container.finishTime = System.currentTimeMillis();
       container.finishedStatus = finishedEvent.getRemoteContainerStatus();
-      // TODO: when AHS webUI is ready, logURL should be updated to point to
-      // the web page that will show the aggregated logs
-
       // Inform AppAttempt
       container.eventHandler.handle(new RMAppAttemptContainerFinishedEvent(
           container.appAttemptId, finishedEvent.getRemoteContainerStatus()));
@@ -415,7 +408,6 @@ public class RMContainerImpl implements RMContainer {
       FinishedTransition {
     @Override
     public void transition(RMContainerImpl container, RMContainerEvent event) {
-
       // Unregister from containerAllocationExpirer.
       container.containerAllocationExpirer.unregister(container
           .getContainerId());

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java

@@ -17,8 +17,6 @@
  */
 package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
 
-import static org.apache.hadoop.yarn.util.StringHelper.join;
-
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
 import javax.xml.bind.annotation.XmlRootElement;
@@ -26,6 +24,7 @@ import javax.xml.bind.annotation.XmlRootElement;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
 import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 
 @XmlRootElement(name = "appAttempt")
 @XmlAccessorType(XmlAccessType.FIELD)
@@ -55,10 +54,9 @@ public class AppAttemptInfo {
         this.containerId = masterContainer.getId().toString();
         this.nodeHttpAddress = masterContainer.getNodeHttpAddress();
         this.nodeId = masterContainer.getNodeId().toString();
-        this.logsLink = join("//",
-            masterContainer.getNodeHttpAddress(),
-            "/node", "/containerlogs/",
-            ConverterUtils.toString(masterContainer.getId()), "/", user);
+        this.logsLink =
+            WebAppUtils.getRunningLogURL("//" + masterContainer.getNodeHttpAddress(),
+                ConverterUtils.toString(masterContainer.getId()), user);
       }
     }
   }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java

@@ -17,14 +17,11 @@
  */
 package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
 
-import static org.apache.hadoop.yarn.util.StringHelper.join;
-
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
 import javax.xml.bind.annotation.XmlRootElement;
 import javax.xml.bind.annotation.XmlTransient;
 
-import com.google.common.base.Joiner;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
 import org.apache.hadoop.yarn.api.records.Container;
@@ -38,6 +35,8 @@ import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.util.Times;
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 
+import com.google.common.base.Joiner;
+
 @XmlRootElement(name = "app")
 @XmlAccessorType(XmlAccessType.FIELD)
 public class AppInfo {
@@ -131,12 +130,10 @@ public class AppInfo {
           Container masterContainer = attempt.getMasterContainer();
           if (masterContainer != null) {
             this.amContainerLogsExist = true;
-            String url = join(schemePrefix,
-                masterContainer.getNodeHttpAddress(),
-                "/node", "/containerlogs/",
+            this.amContainerLogs = WebAppUtils.getRunningLogURL(
+                schemePrefix + masterContainer.getNodeHttpAddress(),
                 ConverterUtils.toString(masterContainer.getId()),
-                "/", app.getUser());
-            this.amContainerLogs = url;
+                app.getUser());
             this.amHostHttpAddress = masterContainer.getNodeHttpAddress();
           }
           

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java

@@ -296,7 +296,6 @@ public class TestRMApplicationHistoryWriter {
     }
     Assert.assertEquals("test diagnostics info",
       containerHD.getDiagnosticsInfo());
-    Assert.assertEquals("test log url", containerHD.getLogURL());
     Assert.assertEquals(-1, containerHD.getContainerExitStatus());
     Assert.assertEquals(ContainerState.COMPLETE,
       containerHD.getContainerState());

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java

@@ -104,8 +104,7 @@ public class TestRMContainerImpl {
         RMContainerEventType.LAUNCHED));
     drainDispatcher.await();
     assertEquals(RMContainerState.RUNNING, rmContainer.getState());
-    assertEquals(
-        "//host:3465/logs/host:3425/container_1_0001_01_000001/container_1_0001_01_000001/user",
+    assertEquals("//host:3465/node/containerlogs/container_1_0001_01_000001/user",
         rmContainer.getLogURL());
 
     // In RUNNING state. Verify RELEASED and associated actions.
@@ -191,8 +190,7 @@ public class TestRMContainerImpl {
         RMContainerEventType.LAUNCHED));
     drainDispatcher.await();
     assertEquals(RMContainerState.RUNNING, rmContainer.getState());
-    assertEquals(
-        "//host:3465/logs/host:3425/container_1_0001_01_000001/container_1_0001_01_000001/user",
+    assertEquals("//host:3465/node/containerlogs/container_1_0001_01_000001/user",
         rmContainer.getLogURL());
 
     // In RUNNING state. Verify EXPIRE and associated actions.