Revert "YARN-4920. ATS/NM should support a link to dowload/get the logs in text format. Contributed by Xuan Gong."

This reverts commit 22ac37615a933f9cee8cf19ad0182586a037b690.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java

@@ -18,11 +18,6 @@
 
 package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp;
 
-import java.io.DataInputStream;
-import java.io.EOFException;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.nio.charset.Charset;
 import java.util.Collections;
 import java.util.Set;
 
@@ -33,30 +28,13 @@ import javax.ws.rs.Path;
 import javax.ws.rs.PathParam;
 import javax.ws.rs.Produces;
 import javax.ws.rs.QueryParam;
-import javax.ws.rs.WebApplicationException;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.StreamingOutput;
-import javax.ws.rs.core.Response.ResponseBuilder;
-import javax.ws.rs.core.Response.Status;
 
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileContext;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.api.ApplicationBaseProtocol;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat;
-import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils;
-import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey;
 import org.apache.hadoop.yarn.server.webapp.WebServices;
 import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo;
 import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptsInfo;
@@ -64,10 +42,9 @@ import org.apache.hadoop.yarn.server.webapp.dao.AppInfo;
 import org.apache.hadoop.yarn.server.webapp.dao.AppsInfo;
 import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo;
 import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo;
-import org.apache.hadoop.yarn.util.Times;
 import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
 import org.apache.hadoop.yarn.webapp.BadRequestException;
-import com.google.common.base.Joiner;
+
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
 
@@ -75,17 +52,9 @@ import com.google.inject.Singleton;
 @Path("/ws/v1/applicationhistory")
 public class AHSWebServices extends WebServices {
 
-  private static final String NM_DOWNLOAD_URI_STR =
-      "/ws/v1/node/containerlogs";
-  private static final Joiner JOINER = Joiner.on("");
-  private static final Joiner DOT_JOINER = Joiner.on(". ");
-  private final Configuration conf;
-
   @Inject
-  public AHSWebServices(ApplicationBaseProtocol appBaseProt,
-      Configuration conf) {
+  public AHSWebServices(ApplicationBaseProtocol appBaseProt) {
     super(appBaseProt);
-    this.conf = conf;
   }
 
   @GET
@@ -203,236 +172,4 @@ public class AHSWebServices extends WebServices {
       }
     }
   }
-
-  @GET
-  @Path("/containerlogs/{containerid}/{filename}")
-  @Produces({ MediaType.TEXT_PLAIN })
-  @Public
-  @Unstable
-  public Response getLogs(@Context HttpServletRequest req,
-      @Context HttpServletResponse res,
-      @PathParam("containerid") String containerIdStr,
-      @PathParam("filename") String filename,
-      @QueryParam("download") String download) {
-    init(res);
-    ContainerId containerId;
-    try {
-      containerId = ContainerId.fromString(containerIdStr);
-    } catch (IllegalArgumentException ex) {
-      return createBadResponse(Status.NOT_FOUND,
-          "Invalid ContainerId: " + containerIdStr);
-    }
-
-    boolean downloadFile = parseBooleanParam(download);
-
-    ApplicationId appId = containerId.getApplicationAttemptId()
-        .getApplicationId();
-    AppInfo appInfo;
-    try {
-      appInfo = super.getApp(req, res, appId.toString());
-    } catch (Exception ex) {
-      // directly find logs from HDFS.
-      return sendStreamOutputResponse(appId, null, null, containerIdStr,
-          filename, downloadFile);
-    }
-    String appOwner = appInfo.getUser();
-
-    ContainerInfo containerInfo;
-    try {
-      containerInfo = super.getContainer(
-          req, res, appId.toString(),
-          containerId.getApplicationAttemptId().toString(),
-          containerId.toString());
-    } catch (Exception ex) {
-      if (isFinishedState(appInfo.getAppState())) {
-        // directly find logs from HDFS.
-        return sendStreamOutputResponse(appId, appOwner, null, containerIdStr,
-            filename, downloadFile);
-      }
-      return createBadResponse(Status.INTERNAL_SERVER_ERROR,
-          "Can not get ContainerInfo for the container: " + containerId);
-    }
-    String nodeId = containerInfo.getNodeId();
-    if (isRunningState(appInfo.getAppState())) {
-      String nodeHttpAddress = containerInfo.getNodeHttpAddress();
-      String uri = "/" + containerId.toString() + "/" + filename;
-      String resURI = JOINER.join(nodeHttpAddress, NM_DOWNLOAD_URI_STR, uri);
-      String query = req.getQueryString();
-      if (query != null && !query.isEmpty()) {
-        resURI += "?" + query;
-      }
-      ResponseBuilder response = Response.status(
-          HttpServletResponse.SC_TEMPORARY_REDIRECT);
-      response.header("Location", resURI);
-      return response.build();
-    } else if (isFinishedState(appInfo.getAppState())) {
-      return sendStreamOutputResponse(appId, appOwner, nodeId,
-          containerIdStr, filename, downloadFile);
-    } else {
-      return createBadResponse(Status.NOT_FOUND,
-          "The application is not at Running or Finished State.");
-    }
-  }
-
-  private boolean isRunningState(YarnApplicationState appState) {
-    return appState == YarnApplicationState.RUNNING;
-  }
-
-  private boolean isFinishedState(YarnApplicationState appState) {
-    return appState == YarnApplicationState.FINISHED
-        || appState == YarnApplicationState.FAILED
-        || appState == YarnApplicationState.KILLED;
-  }
-
-  private Response createBadResponse(Status status, String errMessage) {
-    Response response = Response.status(status)
-        .entity(DOT_JOINER.join(status.toString(), errMessage)).build();
-    return response;
-  }
-
-  private boolean parseBooleanParam(String param) {
-    return ("true").equalsIgnoreCase(param);
-  }
-
-  private Response sendStreamOutputResponse(ApplicationId appId,
-      String appOwner, String nodeId, String containerIdStr,
-      String fileName, boolean downloadFile) {
-    StreamingOutput stream = null;
-    try {
-      stream = getStreamingOutput(appId, appOwner, nodeId,
-          containerIdStr, fileName);
-    } catch (Exception ex) {
-      return createBadResponse(Status.INTERNAL_SERVER_ERROR,
-          ex.getMessage());
-    }
-    if (stream == null) {
-      return createBadResponse(Status.INTERNAL_SERVER_ERROR,
-          "Can not get log for container: " + containerIdStr);
-    }
-    ResponseBuilder response = Response.ok(stream);
-    if (downloadFile) {
-      response.header("Content-Type", "application/octet-stream");
-      response.header("Content-Disposition", "attachment; filename="
-          + fileName);
-    }
-    return response.build();
-  }
-
-  private StreamingOutput getStreamingOutput(ApplicationId appId,
-      String appOwner, final String nodeId, final String containerIdStr,
-      final String logFile) throws IOException{
-    String suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(conf);
-    org.apache.hadoop.fs.Path remoteRootLogDir = new org.apache.hadoop.fs.Path(
-        conf.get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
-            YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR));
-    org.apache.hadoop.fs.Path qualifiedRemoteRootLogDir =
-        FileContext.getFileContext(conf).makeQualified(remoteRootLogDir);
-    FileContext fc = FileContext.getFileContext(
-        qualifiedRemoteRootLogDir.toUri(), conf);
-    org.apache.hadoop.fs.Path remoteAppDir = null;
-    if (appOwner == null) {
-      org.apache.hadoop.fs.Path toMatch = LogAggregationUtils
-          .getRemoteAppLogDir(remoteRootLogDir, appId, "*", suffix);
-      FileStatus[] matching  = fc.util().globStatus(toMatch);
-      if (matching == null || matching.length != 1) {
-        return null;
-      }
-      remoteAppDir = matching[0].getPath();
-    } else {
-      remoteAppDir = LogAggregationUtils
-          .getRemoteAppLogDir(remoteRootLogDir, appId, appOwner, suffix);
-    }
-    final RemoteIterator<FileStatus> nodeFiles;
-    nodeFiles = fc.listStatus(remoteAppDir);
-    if (!nodeFiles.hasNext()) {
-      return null;
-    }
-
-    StreamingOutput stream = new StreamingOutput() {
-
-      @Override
-      public void write(OutputStream os) throws IOException,
-          WebApplicationException {
-        byte[] buf = new byte[65535];
-        boolean findLogs = false;
-        while (nodeFiles.hasNext()) {
-          final FileStatus thisNodeFile = nodeFiles.next();
-          String nodeName = thisNodeFile.getPath().getName();
-          if ((nodeId == null || nodeName.contains(LogAggregationUtils
-              .getNodeString(nodeId))) && !nodeName.endsWith(
-              LogAggregationUtils.TMP_FILE_SUFFIX)) {
-            AggregatedLogFormat.LogReader reader =
-                new AggregatedLogFormat.LogReader(conf,
-                    thisNodeFile.getPath());
-            DataInputStream valueStream;
-            LogKey key = new LogKey();
-            valueStream = reader.next(key);
-            while (valueStream != null && !key.toString()
-                .equals(containerIdStr)) {
-              // Next container
-              key = new LogKey();
-              valueStream = reader.next(key);
-            }
-            if (valueStream == null) {
-              continue;
-            }
-            while (true) {
-              try {
-                String fileType = valueStream.readUTF();
-                String fileLengthStr = valueStream.readUTF();
-                long fileLength = Long.parseLong(fileLengthStr);
-                if (fileType.equalsIgnoreCase(logFile)) {
-                  StringBuilder sb = new StringBuilder();
-                  sb.append("LogType:");
-                  sb.append(fileType + "\n");
-                  sb.append("Log Upload Time:");
-                  sb.append(Times.format(System.currentTimeMillis()) + "\n");
-                  sb.append("LogLength:");
-                  sb.append(fileLengthStr + "\n");
-                  sb.append("Log Contents:\n");
-                  byte[] b = sb.toString().getBytes(Charset.forName("UTF-8"));
-                  os.write(b, 0, b.length);
-
-                  long curRead = 0;
-                  long pendingRead = fileLength - curRead;
-                  int toRead = pendingRead > buf.length ? buf.length
-                      : (int) pendingRead;
-                  int len = valueStream.read(buf, 0, toRead);
-                  while (len != -1 && curRead < fileLength) {
-                    os.write(buf, 0, len);
-                    curRead += len;
-
-                    pendingRead = fileLength - curRead;
-                    toRead = pendingRead > buf.length ? buf.length
-                        : (int) pendingRead;
-                    len = valueStream.read(buf, 0, toRead);
-                  }
-                  sb = new StringBuilder();
-                  sb.append("\nEnd of LogType:" + fileType + "\n");
-                  b = sb.toString().getBytes(Charset.forName("UTF-8"));
-                  os.write(b, 0, b.length);
-                  findLogs = true;
-                } else {
-                  long totalSkipped = 0;
-                  long currSkipped = 0;
-                  while (currSkipped != -1 && totalSkipped < fileLength) {
-                    currSkipped = valueStream.skip(fileLength - totalSkipped);
-                    totalSkipped += currSkipped;
-                  }
-                }
-              } catch (EOFException eof) {
-                break;
-              }
-            }
-          }
-        }
-        os.flush();
-        if (!findLogs) {
-          throw new IOException("Can not find logs for container:"
-              + containerIdStr);
-        }
-      }
-    };
-    return stream;
-  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java

@@ -80,11 +80,9 @@ public class TestApplicationHistoryManagerOnTimelineStore {
     store = createStore(SCALE);
     TimelineEntities entities = new TimelineEntities();
     entities.addEntity(createApplicationTimelineEntity(
-        ApplicationId.newInstance(0, SCALE + 1), true, true, false, false,
-        YarnApplicationState.FINISHED));
+        ApplicationId.newInstance(0, SCALE + 1), true, true, false, false));
     entities.addEntity(createApplicationTimelineEntity(
-        ApplicationId.newInstance(0, SCALE + 2), true, false, true, false,
-        YarnApplicationState.FINISHED));
+        ApplicationId.newInstance(0, SCALE + 2), true, false, true, false));
     store.put(entities);
   }
 
@@ -142,10 +140,10 @@ public class TestApplicationHistoryManagerOnTimelineStore {
       ApplicationId appId = ApplicationId.newInstance(0, i);
       if (i == 2) {
         entities.addEntity(createApplicationTimelineEntity(
-            appId, true, false, false, true, YarnApplicationState.FINISHED));
+            appId, true, false, false, true));
       } else {
         entities.addEntity(createApplicationTimelineEntity(
-            appId, false, false, false, false, YarnApplicationState.FINISHED));
+            appId, false, false, false, false));
       }
       store.put(entities);
       for (int j = 1; j <= scale; ++j) {
@@ -162,16 +160,6 @@ public class TestApplicationHistoryManagerOnTimelineStore {
         }
       }
     }
-    TimelineEntities entities = new TimelineEntities();
-    ApplicationId appId = ApplicationId.newInstance(1234, 1);
-    ApplicationAttemptId appAttemptId =
-        ApplicationAttemptId.newInstance(appId, 1);
-    ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1);
-    entities.addEntity(createApplicationTimelineEntity(
-        appId, true, false, false, false, YarnApplicationState.RUNNING));
-    entities.addEntity(createAppAttemptTimelineEntity(appAttemptId));
-    entities.addEntity(createContainerEntity(containerId));
-    store.put(entities);
   }
 
   @Test
@@ -367,7 +355,7 @@ public class TestApplicationHistoryManagerOnTimelineStore {
         historyManager.getApplications(Long.MAX_VALUE, 0L, Long.MAX_VALUE)
           .values();
     Assert.assertNotNull(apps);
-    Assert.assertEquals(SCALE + 2, apps.size());
+    Assert.assertEquals(SCALE + 1, apps.size());
     ApplicationId ignoredAppId = ApplicationId.newInstance(0, SCALE + 2);
     for (ApplicationReport app : apps) {
       Assert.assertNotEquals(ignoredAppId, app.getApplicationId());
@@ -479,8 +467,7 @@ public class TestApplicationHistoryManagerOnTimelineStore {
 
   private static TimelineEntity createApplicationTimelineEntity(
       ApplicationId appId, boolean emptyACLs, boolean noAttemptId,
-      boolean wrongAppId, boolean enableUpdateEvent,
-      YarnApplicationState state) {
+      boolean wrongAppId, boolean enableUpdateEvent) {
     TimelineEntity entity = new TimelineEntity();
     entity.setEntityType(ApplicationMetricsConstants.ENTITY_TYPE);
     if (wrongAppId) {
@@ -530,7 +517,7 @@ public class TestApplicationHistoryManagerOnTimelineStore {
     eventInfo.put(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO,
         FinalApplicationStatus.UNDEFINED.toString());
     eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO,
-        state.toString());
+        YarnApplicationState.FINISHED.toString());
     if (!noAttemptId) {
       eventInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO,
           ApplicationAttemptId.newInstance(appId, 1));
@@ -635,8 +622,6 @@ public class TestApplicationHistoryManagerOnTimelineStore {
     entityInfo.put(ContainerMetricsConstants.ALLOCATED_PORT_ENTITY_INFO, 100);
     entityInfo
         .put(ContainerMetricsConstants.ALLOCATED_PRIORITY_ENTITY_INFO, -1);
-    entityInfo.put(ContainerMetricsConstants
-        .ALLOCATED_HOST_HTTP_ADDRESS_ENTITY_INFO, "http://test:1234");
     entity.setOtherInfo(entityInfo);
     TimelineEvent tEvent = new TimelineEvent();
     tEvent.setEventType(ContainerMetricsConstants.CREATED_EVENT_TYPE);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java

@@ -19,30 +19,17 @@
 package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
-import java.io.File;
-import java.io.FileWriter;
-import java.io.Writer;
-import java.net.HttpURLConnection;
-import java.net.URI;
-import java.net.URL;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.List;
 import java.util.Properties;
 
 import javax.servlet.FilterConfig;
 import javax.servlet.ServletException;
-import javax.servlet.http.HttpServletResponse;
 import javax.ws.rs.core.MediaType;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
 import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler;
 import org.apache.hadoop.yarn.api.ApplicationBaseProtocol;
@@ -55,8 +42,6 @@ import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat;
-import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryClientService;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryManagerOnTimelineStore;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.TestApplicationHistoryManagerOnTimelineStore;
@@ -96,17 +81,12 @@ import com.sun.jersey.test.framework.WebAppDescriptor;
 public class TestAHSWebServices extends JerseyTestBase {
 
   private static ApplicationHistoryClientService historyClientService;
-  private static AHSWebServices ahsWebservice;
   private static final String[] USERS = new String[] { "foo" , "bar" };
   private static final int MAX_APPS = 5;
-  private static Configuration conf;
-  private static FileSystem fs;
-  private static final String remoteLogRootDir = "target/logs/";
-  private static final String rootLogDir = "target/LocalLogs";
 
   @BeforeClass
   public static void setupClass() throws Exception {
-    conf = new YarnConfiguration();
+    Configuration conf = new YarnConfiguration();
     TimelineStore store =
         TestApplicationHistoryManagerOnTimelineStore.createStore(MAX_APPS);
     TimelineACLsManager aclsManager = new TimelineACLsManager(conf);
@@ -115,8 +95,6 @@ public class TestAHSWebServices extends JerseyTestBase {
         new TimelineDataManager(store, aclsManager);
     conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
     conf.set(YarnConfiguration.YARN_ADMIN_ACL, "foo");
-    conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
-    conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
     dataManager.init(conf);
     ApplicationACLsManager appAclsManager = new ApplicationACLsManager(conf);
     ApplicationHistoryManagerOnTimelineStore historyManager =
@@ -130,8 +108,6 @@ public class TestAHSWebServices extends JerseyTestBase {
     };
     historyClientService.init(conf);
     historyClientService.start();
-    ahsWebservice = new AHSWebServices(historyClientService, conf);
-    fs = FileSystem.get(conf);
   }
 
   @AfterClass
@@ -139,8 +115,6 @@ public class TestAHSWebServices extends JerseyTestBase {
     if (historyClientService != null) {
       historyClientService.stop();
     }
-    fs.delete(new Path(remoteLogRootDir), true);
-    fs.delete(new Path(rootLogDir), true);
   }
 
   @Parameterized.Parameters
@@ -153,7 +127,7 @@ public class TestAHSWebServices extends JerseyTestBase {
     @Override
     protected void configureServlets() {
       bind(JAXBContextResolver.class);
-      bind(AHSWebServices.class).toInstance(ahsWebservice);;
+      bind(AHSWebServices.class);
       bind(GenericExceptionHandler.class);
       bind(ApplicationBaseProtocol.class).toInstance(historyClientService);
       serve("/*").with(GuiceContainer.class);
@@ -497,177 +471,4 @@ public class TestAHSWebServices extends JerseyTestBase {
     assertEquals(ContainerState.COMPLETE.toString(),
       container.getString("containerState"));
   }
-
-  @Test(timeout = 10000)
-  public void testContainerLogsForFinishedApps() throws Exception {
-    String fileName = "syslog";
-    String user = "user1";
-    UserGroupInformation ugi = UserGroupInformation.createRemoteUser("user1");
-    NodeId nodeId = NodeId.newInstance("test host", 100);
-    NodeId nodeId2 = NodeId.newInstance("host2", 1234);
-    //prepare the logs for remote directory
-    ApplicationId appId = ApplicationId.newInstance(0, 1);
-    // create local logs
-    List<String> rootLogDirList = new ArrayList<String>();
-    rootLogDirList.add(rootLogDir);
-    Path rootLogDirPath = new Path(rootLogDir);
-    if (fs.exists(rootLogDirPath)) {
-      fs.delete(rootLogDirPath, true);
-    }
-    assertTrue(fs.mkdirs(rootLogDirPath));
-
-    Path appLogsDir = new Path(rootLogDirPath, appId.toString());
-    if (fs.exists(appLogsDir)) {
-      fs.delete(appLogsDir, true);
-    }
-    assertTrue(fs.mkdirs(appLogsDir));
-
-    // create container logs in local log file dir
-    // create two container log files. We can get containerInfo
-    // for container1 from AHS, but can not get such info for
-    // container100
-    ApplicationAttemptId appAttemptId =
-        ApplicationAttemptId.newInstance(appId, 1);
-    ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 1);
-    ContainerId containerId100 = ContainerId.newContainerId(appAttemptId, 100);
-    createContainerLogInLocalDir(appLogsDir, containerId1, fs, fileName,
-        ("Hello." + containerId1));
-    createContainerLogInLocalDir(appLogsDir, containerId100, fs, fileName,
-        ("Hello." + containerId100));
-
-    // upload container logs to remote log dir
-    Path path = new Path(conf.get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR) +
-        user + "/logs/" + appId.toString());
-    if (fs.exists(path)) {
-      fs.delete(path, true);
-    }
-    assertTrue(fs.mkdirs(path));
-    uploadContainerLogIntoRemoteDir(ugi, conf, rootLogDirList, nodeId,
-        containerId1, path, fs);
-    uploadContainerLogIntoRemoteDir(ugi, conf, rootLogDirList, nodeId2,
-        containerId100, path, fs);
-
-    // test whether we can find container log from remote diretory if
-    // the containerInfo for this container could be fetched from AHS.
-    WebResource r = resource();
-    ClientResponse response = r.path("ws").path("v1")
-        .path("applicationhistory").path("containerlogs")
-        .path(containerId1.toString()).path(fileName)
-        .queryParam("user.name", user)
-        .accept(MediaType.TEXT_PLAIN)
-        .get(ClientResponse.class);
-    String responseText = response.getEntity(String.class);
-    assertTrue(responseText.contains("Hello." + containerId1));
-
-    // test whether we can find container log from remote diretory if
-    // the containerInfo for this container could not be fetched from AHS.
-    r = resource();
-    response = r.path("ws").path("v1")
-        .path("applicationhistory").path("containerlogs")
-        .path(containerId100.toString()).path(fileName)
-        .queryParam("user.name", user)
-        .accept(MediaType.TEXT_PLAIN)
-        .get(ClientResponse.class);
-    responseText = response.getEntity(String.class);
-    assertTrue(responseText.contains("Hello." + containerId100));
-
-    // create an application which can not be found from AHS
-    ApplicationId appId100 = ApplicationId.newInstance(0, 100);
-    appLogsDir = new Path(rootLogDirPath, appId100.toString());
-    if (fs.exists(appLogsDir)) {
-      fs.delete(appLogsDir, true);
-    }
-    assertTrue(fs.mkdirs(appLogsDir));
-    ApplicationAttemptId appAttemptId100 =
-        ApplicationAttemptId.newInstance(appId100, 1);
-    ContainerId containerId1ForApp100 = ContainerId
-        .newContainerId(appAttemptId100, 1);
-    createContainerLogInLocalDir(appLogsDir, containerId1ForApp100, fs,
-        fileName, ("Hello." + containerId1ForApp100));
-    path = new Path(conf.get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR) +
-        user + "/logs/" + appId100.toString());
-    if (fs.exists(path)) {
-      fs.delete(path, true);
-    }
-    assertTrue(fs.mkdirs(path));
-    uploadContainerLogIntoRemoteDir(ugi, conf, rootLogDirList, nodeId2,
-        containerId1ForApp100, path, fs);
-    r = resource();
-    response = r.path("ws").path("v1")
-        .path("applicationhistory").path("containerlogs")
-        .path(containerId1ForApp100.toString()).path(fileName)
-        .queryParam("user.name", user)
-        .accept(MediaType.TEXT_PLAIN)
-        .get(ClientResponse.class);
-    responseText = response.getEntity(String.class);
-    assertTrue(responseText.contains("Hello." + containerId1ForApp100));
-  }
-
-  private static void createContainerLogInLocalDir(Path appLogsDir,
-      ContainerId containerId, FileSystem fs, String fileName, String content)
-      throws Exception {
-    Path containerLogsDir = new Path(appLogsDir, containerId.toString());
-    if (fs.exists(containerLogsDir)) {
-      fs.delete(containerLogsDir, true);
-    }
-    assertTrue(fs.mkdirs(containerLogsDir));
-    Writer writer =
-        new FileWriter(new File(containerLogsDir.toString(), fileName));
-    writer.write(content);
-    writer.close();
-  }
-
-  private static void uploadContainerLogIntoRemoteDir(UserGroupInformation ugi,
-      Configuration configuration, List<String> rootLogDirs, NodeId nodeId,
-      ContainerId containerId, Path appDir, FileSystem fs) throws Exception {
-    Path path =
-        new Path(appDir, LogAggregationUtils.getNodeString(nodeId));
-    AggregatedLogFormat.LogWriter writer =
-        new AggregatedLogFormat.LogWriter(configuration, path, ugi);
-    writer.writeApplicationOwner(ugi.getUserName());
-
-    writer.append(new AggregatedLogFormat.LogKey(containerId),
-        new AggregatedLogFormat.LogValue(rootLogDirs, containerId,
-        ugi.getShortUserName()));
-    writer.close();
-  }
-
-  @Test(timeout = 10000)
-  public void testContainerLogsForRunningApps() throws Exception {
-    String fileName = "syslog";
-    String user = "user1";
-    ApplicationId appId = ApplicationId.newInstance(
-        1234, 1);
-    ApplicationAttemptId appAttemptId =
-        ApplicationAttemptId.newInstance(appId, 1);
-    ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 1);
-    WebResource r = resource();
-    URI requestURI = r.path("ws").path("v1")
-        .path("applicationhistory").path("containerlogs")
-        .path(containerId1.toString()).path(fileName)
-        .queryParam("user.name", user).getURI();
-    String redirectURL = getRedirectURL(requestURI.toString());
-    assertTrue(redirectURL != null);
-    assertTrue(redirectURL.contains("test:1234"));
-    assertTrue(redirectURL.contains("ws/v1/node/containerlogs"));
-    assertTrue(redirectURL.contains(containerId1.toString()));
-    assertTrue(redirectURL.contains("user.name=" + user));
-  }
-
-  private static String getRedirectURL(String url) {
-    String redirectUrl = null;
-    try {
-      HttpURLConnection conn = (HttpURLConnection) new URL(url)
-          .openConnection();
-      // do not automatically follow the redirection
-      // otherwise we get too many redirections exception
-      conn.setInstanceFollowRedirects(false);
-      if(conn.getResponseCode() == HttpServletResponse.SC_TEMPORARY_REDIRECT) {
-        redirectUrl = conn.getHeaderField("Location");
-      }
-    } catch (Exception e) {
-      // throw new RuntimeException(e);
-    }
-    return redirectUrl;
-  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java

@@ -48,7 +48,6 @@ public class ContainerInfo {
   protected int containerExitStatus;
   protected ContainerState containerState;
   protected String nodeHttpAddress;
-  protected String nodeId;
 
   public ContainerInfo() {
     // JAXB needs this
@@ -72,7 +71,6 @@ public class ContainerInfo {
     containerExitStatus = container.getContainerExitStatus();
     containerState = container.getContainerState();
     nodeHttpAddress = container.getNodeHttpAddress();
-    nodeId = container.getAssignedNode().toString();
   }
 
   public String getContainerId() {
@@ -126,8 +124,4 @@ public class ContainerInfo {
   public String getNodeHttpAddress() {
     return nodeHttpAddress;
   }
-
-  public String getNodeId() {
-    return nodeId;
-  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java

@@ -33,7 +33,6 @@ import javax.ws.rs.QueryParam;
 import javax.ws.rs.WebApplicationException;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
-import javax.ws.rs.core.Response.ResponseBuilder;
 import javax.ws.rs.core.Response.Status;
 import javax.ws.rs.core.StreamingOutput;
 import javax.ws.rs.core.UriInfo;
@@ -216,8 +215,7 @@ public class NMWebServices {
   @Public
   @Unstable
   public Response getLogs(@PathParam("containerid") String containerIdStr,
-      @PathParam("filename") String filename,
-      @QueryParam("download") String download) {
+      @PathParam("filename") String filename) {
     ContainerId containerId;
     try {
       containerId = ConverterUtils.toContainerId(containerIdStr);
@@ -234,7 +232,7 @@ public class NMWebServices {
     } catch (YarnException ex) {
       return Response.serverError().entity(ex.getMessage()).build();
     }
-    boolean downloadFile = parseBooleanParam(download);
+    
     try {
       final FileInputStream fis = ContainerLogsUtils.openLogFileForRead(
           containerIdStr, logFile, nmContext);
@@ -252,22 +250,10 @@ public class NMWebServices {
           os.flush();
         }
       };
-      ResponseBuilder resp = Response.ok(stream);
-      if (downloadFile) {
-        resp.header("Content-Type", "application/octet-stream");
-        resp.header("Content-Disposition", "attachment; filename="
-            + logFile.getName());
-      }
-      return resp.build();
+      
+      return Response.ok(stream).build();
     } catch (IOException ex) {
       return Response.serverError().entity(ex.getMessage()).build();
     }
   }
-
-  private boolean parseBooleanParam(String param) {
-    if (param != null) {
-      return ("true").equalsIgnoreCase(param);
-    }
-    return false;
-  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java

@@ -50,6 +50,7 @@ import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService;
 import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
 import org.apache.hadoop.yarn.server.nodemanager.ResourceView;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationImpl;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
 import org.apache.hadoop.yarn.server.nodemanager.webapp.WebServer.NMWebApp;
@@ -351,16 +352,7 @@ public class TestNMWebServices extends JerseyTestBase {
         .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
     String responseText = response.getEntity(String.class);
     assertEquals(logMessage, responseText);
-
-    // ask and download it
-    response = r.path("ws").path("v1").path("node").path("containerlogs")
-        .path(containerIdStr).path(filename).queryParam("download", "true")
-        .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
-    responseText = response.getEntity(String.class);
-    assertEquals(logMessage, responseText);
-    assertEquals(200, response.getStatus());
-    assertEquals("application/octet-stream", response.getType().toString());
-
+    
     // ask for file that doesn't exist
     response = r.path("ws").path("v1").path("node")
         .path("containerlogs").path(containerIdStr).path("uhhh")