YARN-4149. yarn logs -am should provide an option to fetch all the log

files. Contributed by Varun Vasudev

hadoop-yarn-project/CHANGES.txt

@@ -439,6 +439,9 @@ Release 2.8.0 - UNRELEASED
     YARN-3717. Expose app/am/queue's node-label-expression to RM web UI / 
     CLI / REST-API. (Naganarasimha G R via wangda)
 
+    YARN-4149. yarn logs -am should provide an option to fetch all the log files
+    (Varun Vasudev via xgong)
+
   OPTIMIZATIONS
 
     YARN-3339. TestDockerContainerExecutor should pull a single image and not

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java

@@ -19,12 +19,15 @@
 package org.apache.hadoop.yarn.client.cli;
 
 import java.io.IOException;
+import java.io.StringReader;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 
 import javax.ws.rs.core.MediaType;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
 
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.CommandLineParser;
@@ -61,6 +64,9 @@ import com.sun.jersey.api.client.ClientHandlerException;
 import com.sun.jersey.api.client.ClientResponse;
 import com.sun.jersey.api.client.UniformInterfaceException;
 import com.sun.jersey.api.client.WebResource;
+import org.w3c.dom.Document;
+import org.w3c.dom.NodeList;
+import org.xml.sax.InputSource;
 
 @Public
 @Evolving
@@ -105,7 +111,8 @@ public class LogsCLI extends Configured implements Tool {
     opts.addOption(amOption);
     Option logFileOpt = new Option(CONTAINER_LOG_FILES, true,
       "Work with -am/-containerId and specify comma-separated value "
-      + "to get specified Container log files");
+        + "to get specified container log files. Use \"ALL\" to fetch all the "
+        + "log files for the container.");
     logFileOpt.setValueSeparator(',');
     logFileOpt.setArgs(Option.UNLIMITED_VALUES);
     logFileOpt.setArgName("Log File Name");
@@ -248,8 +255,8 @@ public class LogsCLI extends Configured implements Tool {
             logFiles, logCliHelper, appOwner, true);
         } else {
           System.out
-            .println("Can not get AMContainers logs for the application:"
-                + appId);
+            .println(
+                "Can not get AMContainers logs for the application:" + appId);
           System.out.println("This application:" + appId + " is finished."
               + " Please enable the application history service. Or Using "
               + "yarn logs -applicationId <appId> -containerId <containerId> "
@@ -264,9 +271,18 @@ public class LogsCLI extends Configured implements Tool {
       // if we provide the node address and the application is in the final
       // state, we could directly get logs from HDFS.
       if (nodeAddress != null && isApplicationFinished(appState)) {
+        // if user specified "ALL" as the logFiles param, pass null
+        // to logCliHelper so that it fetches all the logs
+        List<String> logs;
+        if (logFiles == null) {
+          logs = null;
+        } else if (fetchAllLogFiles(logFiles)) {
+          logs = null;
+        } else {
+          logs = Arrays.asList(logFiles);
+        }
         return logCliHelper.dumpAContainersLogsForALogType(appIdStr,
-            containerIdStr, nodeAddress, appOwner, logFiles == null ? null
-                : Arrays.asList(logFiles));
+            containerIdStr, nodeAddress, appOwner, logs);
       }
       try {
         // If the nodeAddress is not provided, we will try to get
@@ -288,10 +304,14 @@ public class LogsCLI extends Configured implements Tool {
             containerIdStr, nodeHttpAddress, nodeId, logFiles, logCliHelper,
             appOwner);
         } else {
+          String [] requestedLogFiles = logFiles;
+          if(fetchAllLogFiles(logFiles)) {
+            requestedLogFiles = null;
+          }
           // If the application is in the final state, we will directly
           // get the container logs from HDFS.
           printContainerLogsForFinishedApplication(appIdStr, containerIdStr,
-            nodeId, logFiles, logCliHelper, appOwner);
+            nodeId, requestedLogFiles, logCliHelper, appOwner);
         }
         return resultCode;
       } catch (IOException | YarnException ex) {
@@ -401,15 +421,69 @@ public class LogsCLI extends Configured implements Tool {
     return amContainersList;
   }
 
+  private boolean fetchAllLogFiles(String[] logFiles) {
+    if(logFiles != null) {
+      List<String> logs = Arrays.asList(logFiles);
+      if(logs.contains("ALL")) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  private String[] getContainerLogFiles(Configuration conf,
+      String containerIdStr, String nodeHttpAddress) throws IOException {
+    List<String> logFiles = new ArrayList<>();
+    Client webServiceClient = Client.create();
+    try {
+      WebResource webResource = webServiceClient
+          .resource(WebAppUtils.getHttpSchemePrefix(conf) + nodeHttpAddress);
+      ClientResponse response =
+          webResource.path("ws").path("v1").path("node").path("containers")
+              .path(containerIdStr).accept(MediaType.APPLICATION_XML)
+              .get(ClientResponse.class);
+      if (response.getClientResponseStatus().equals(ClientResponse.Status.OK)) {
+        try {
+          String xml = response.getEntity(String.class);
+          DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+          DocumentBuilder db = dbf.newDocumentBuilder();
+          InputSource is = new InputSource();
+          is.setCharacterStream(new StringReader(xml));
+          Document dom = db.parse(is);
+          NodeList elements = dom.getElementsByTagName("containerLogFiles");
+          for (int i = 0; i < elements.getLength(); i++) {
+            logFiles.add(elements.item(i).getTextContent());
+          }
+        } catch (Exception e) {
+          System.out.println("Unable to parse xml from webservice. Error:");
+          System.out.println(e.getMessage());
+          throw new IOException(e);
+        }
+      }
+
+    } catch (ClientHandlerException | UniformInterfaceException ex) {
+      System.out.println("Unable to fetch log files list");
+      throw new IOException(ex);
+    }
+    return logFiles.toArray(new String[0]);
+  }
+
   private void printContainerLogsFromRunningApplication(Configuration conf,
       String appId, String containerIdStr, String nodeHttpAddress,
       String nodeId, String[] logFiles, LogCLIHelpers logCliHelper,
       String appOwner) throws IOException {
+    String [] requestedLogFiles = logFiles;
+    // fetch all the log files for the container
+    if (fetchAllLogFiles(logFiles)) {
+      requestedLogFiles =
+          getContainerLogFiles(getConf(), containerIdStr, nodeHttpAddress);
+    }
     Client webServiceClient = Client.create();
     String containerString = "\n\nContainer: " + containerIdStr;
     System.out.println(containerString);
     System.out.println(StringUtils.repeat("=", containerString.length()));
-    for (String logFile : logFiles) {
+
+    for (String logFile : requestedLogFiles) {
       System.out.println("LogType:" + logFile);
       System.out.println("Log Upload Time:"
           + Times.format(System.currentTimeMillis()));
@@ -432,7 +506,7 @@ public class LogsCLI extends Configured implements Tool {
     }
     // for the case, we have already uploaded partial logs in HDFS
     logCliHelper.dumpAContainersLogsForALogType(appId, containerIdStr, nodeId,
-      appOwner, Arrays.asList(logFiles));
+      appOwner, Arrays.asList(requestedLogFiles));
   }
 
   private void printContainerLogsForFinishedApplication(String appId,
@@ -551,15 +625,25 @@ public class LogsCLI extends Configured implements Tool {
           }
         }
         if (nodeId != null && !nodeId.isEmpty()) {
+          String [] requestedLogFilesList = null;
+          if(!fetchAllLogFiles(logFiles)) {
+            requestedLogFilesList = logFiles;
+          }
           printContainerLogsForFinishedApplication(appId, containerId, nodeId,
-            logFiles, logCliHelper, appOwner);
+            requestedLogFilesList, logCliHelper, appOwner);
         }
       }
     } else {
       if (nodeHttpAddress != null && containerId != null
           && !nodeHttpAddress.isEmpty() && !containerId.isEmpty()) {
+        String [] requestedLogFiles = logFiles;
+        // fetch all the log files for the AM
+        if (fetchAllLogFiles(logFiles)) {
+          requestedLogFiles =
+              getContainerLogFiles(getConf(), containerId, nodeHttpAddress);
+        }
         printContainerLogsFromRunningApplication(conf, appId, containerId,
-          nodeHttpAddress, nodeId, logFiles, logCliHelper, appOwner);
+          nodeHttpAddress, nodeId, requestedLogFiles, logCliHelper, appOwner);
       }
     }
   }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java

@@ -168,7 +168,8 @@ public class TestLogsCLI {
     pw.println(" -help                           Displays help for all commands.");
     pw.println(" -logFiles <Log File Name>       Work with -am/-containerId and specify");
     pw.println("                                 comma-separated value to get specified");
-    pw.println("                                 Container log files");
+    pw.println("                                 container log files. Use \"ALL\" to fetch");
+    pw.println("                                 all the log files for the container.");
     pw.println(" -nodeAddress <Node Address>     NodeAddress in the format nodename:port");
     pw.close();
     String appReportStr = baos.toString("UTF-8");

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java

@@ -129,7 +129,7 @@ public class NMWebServices {
           String msg = "Error: You must specify a non-empty string for the user";
           throw new BadRequestException(msg);
         }
-        if (!appInfo.getUser().toString().equals(userQuery)) {
+        if (!appInfo.getUser().equals(userQuery)) {
           continue;
         }
       }
@@ -158,7 +158,8 @@ public class NMWebServices {
   @GET
   @Path("/containers")
   @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
-  public ContainersInfo getNodeContainers() {
+  public ContainersInfo getNodeContainers(@javax.ws.rs.core.Context
+      HttpServletRequest hsr) {
     init();
     ContainersInfo allContainers = new ContainersInfo();
     for (Entry<ContainerId, Container> entry : this.nmContext.getContainers()
@@ -168,7 +169,7 @@ public class NMWebServices {
         continue;
       }
       ContainerInfo info = new ContainerInfo(this.nmContext, entry.getValue(),
-          uriInfo.getBaseUri().toString(), webapp.name());
+          uriInfo.getBaseUri().toString(), webapp.name(), hsr.getRemoteUser());
       allContainers.add(info);
     }
     return allContainers;
@@ -177,7 +178,8 @@ public class NMWebServices {
   @GET
   @Path("/containers/{containerid}")
   @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
-  public ContainerInfo getNodeContainer(@PathParam("containerid") String id) {
+  public ContainerInfo getNodeContainer(@javax.ws.rs.core.Context
+      HttpServletRequest hsr, @PathParam("containerid") String id) {
     ContainerId containerId = null;
     init();
     try {
@@ -191,7 +193,7 @@ public class NMWebServices {
       throw new NotFoundException("container with id, " + id + ", not found");
     }
     return new ContainerInfo(this.nmContext, container, uriInfo.getBaseUri()
-        .toString(), webapp.name());
+        .toString(), webapp.name(), hsr.getRemoteUser());
 
   }
   

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java

@@ -21,16 +21,19 @@ package org.apache.hadoop.yarn.server.nodemanager.webapp.dao;
 import static org.apache.hadoop.yarn.util.StringHelper.join;
 import static org.apache.hadoop.yarn.util.StringHelper.ujoin;
 
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlRootElement;
-import javax.xml.bind.annotation.XmlTransient;
+import javax.xml.bind.annotation.*;
 
 import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
+import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.server.nodemanager.Context;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.webapp.ContainerLogsUtils;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
 
 @XmlRootElement(name = "container")
 @XmlAccessorType(XmlAccessType.FIELD)
@@ -50,15 +53,18 @@ public class ContainerInfo {
   @XmlTransient
   protected String exitStatus;
 
+  @XmlElementWrapper
+  protected List<String> containerLogFiles;
+
   public ContainerInfo() {
   } // JAXB needs this
 
   public ContainerInfo(final Context nmContext, final Container container) {
-    this(nmContext, container, "", "");
+    this(nmContext, container, "", "", "");
   }
 
   public ContainerInfo(final Context nmContext, final Container container,
-       String requestUri, String pathPrefix) {
+       String requestUri, String pathPrefix, String remoteUser) {
 
     this.id = container.getContainerId().toString();
     this.nodeId = nmContext.getNodeId().toString();
@@ -90,6 +96,8 @@ public class ContainerInfo {
     }
     this.containerLogsLink = join(requestUri, pathPrefix,
         this.containerLogsShortLink);
+    this.containerLogFiles =
+        getContainerLogFiles(container.getContainerId(), remoteUser, nmContext);
   }
 
   public String getId() {
@@ -136,4 +144,30 @@ public class ContainerInfo {
     return this.totalVCoresNeeded;
   }
 
+  public List<String> getContainerLogFiles() {
+    return this.containerLogFiles;
+  }
+
+  private List<String> getContainerLogFiles(ContainerId id, String remoteUser,
+      Context nmContext) {
+    List<String> logFiles = new ArrayList<>();
+    try {
+      List<File> logDirs =
+          ContainerLogsUtils.getContainerLogDirs(id, remoteUser, nmContext);
+      for (File containerLogsDir : logDirs) {
+        File[] logs = containerLogsDir.listFiles();
+        if (logs != null) {
+          for (File log : logs) {
+            if (log.isFile()) {
+              logFiles.add(log.getName());
+            }
+          }
+        }
+      }
+    } catch (Exception ye) {
+      return logFiles;
+    }
+    return logFiles;
+  }
+
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java

@@ -32,6 +32,7 @@ import javax.ws.rs.core.MediaType;
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
 
+import com.sun.jersey.api.client.filter.LoggingFilter;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.util.NodeHealthScriptRunner;
@@ -245,6 +246,7 @@ public class TestNMWebServicesContainers extends JerseyTestBase {
     Application app2 = new MockApp(2);
     nmContext.getApplications().put(app2.getAppId(), app2);
     addAppContainers(app2);
+    client().addFilter(new LoggingFilter());
 
     ClientResponse response = r.path("ws").path("v1").path("node").path(path)
         .accept(media).get(ClientResponse.class);
@@ -412,6 +414,7 @@ public class TestNMWebServicesContainers extends JerseyTestBase {
     Application app2 = new MockApp(2);
     nmContext.getApplications().put(app2.getAppId(), app2);
     addAppContainers(app2);
+    client().addFilter(new LoggingFilter(System.out));
 
     for (String id : hash.keySet()) {
       ClientResponse response = r.path("ws").path("v1").path("node")
@@ -471,12 +474,16 @@ public class TestNMWebServicesContainers extends JerseyTestBase {
           WebServicesTestUtils.getXmlInt(element, "totalMemoryNeededMB"),
           WebServicesTestUtils.getXmlInt(element, "totalVCoresNeeded"),
           WebServicesTestUtils.getXmlString(element, "containerLogsLink"));
+      // verify that the container log files element exists
+      assertTrue("containerLogFiles missing",
+          WebServicesTestUtils.getXmlString(element, "containerLogFiles")
+              != null);
     }
   }
 
   public void verifyNodeContainerInfo(JSONObject info, Container cont)
       throws JSONException, Exception {
-    assertEquals("incorrect number of elements", 9, info.length());
+    assertEquals("incorrect number of elements", 10, info.length());
 
     verifyNodeContainerInfoGeneric(cont, info.getString("id"),
         info.getString("state"), info.getString("user"),
@@ -484,6 +491,9 @@ public class TestNMWebServicesContainers extends JerseyTestBase {
         info.getString("nodeId"), info.getInt("totalMemoryNeededMB"),
         info.getInt("totalVCoresNeeded"),
         info.getString("containerLogsLink"));
+    // verify that the container log files element exists
+    assertTrue("containerLogFiles missing",
+        info.getJSONArray("containerLogFiles") != null);
   }
 
   public void verifyNodeContainerInfoGeneric(Container cont, String id,
@@ -514,5 +524,4 @@ public class TestNMWebServicesContainers extends JerseyTestBase {
             cont.getUser());
     assertTrue("containerLogsLink wrong", logsLink.contains(shortLink));
   }
-
 }