YARN-781. Exposing LOGDIR in all containers' environment which should be used by containers for logging purposes. Contributed by Jian He.

svn merge --ignore-ancestry -c 1493428 ../../trunk/


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1493429 13f79535-47bb-0310-9956-ffa450edef68

hadoop-yarn-project/CHANGES.txt

@@ -156,6 +156,9 @@ Release 2.1.0-beta - UNRELEASED
     YARN-398. Make it possible to specify hard locality constraints in resource
     requests for CapacityScheduler. (acmurthy)
 
+    YARN-781. Exposing LOGDIR in all containers' environment which should be used
+    by containers for logging purposes. (Jian He via vinodkv)
+
   IMPROVEMENTS
 
     YARN-365. Change NM heartbeat handling to not generate a scheduler event

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java

@@ -177,7 +177,15 @@ public interface ApplicationConstants {
      * $LOCAL_DIRS
      * Final, exported by NodeManager and non-modifiable by users.
      */
-    LOCAL_DIRS("LOCAL_DIRS");
+    LOCAL_DIRS("LOCAL_DIRS"),
+
+    /**
+     * $LOG_DIRS
+     * Final, exported by NodeManager and non-modifiable by users.
+     * Comma separate list of directories that the container should use for
+     * logging.
+     */
+    LOG_DIRS("LOG_DIRS");
 
     private final String variable;
     private Environment(String variable) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java

@@ -132,9 +132,10 @@ public class ContainerLaunch implements Callable<Integer> {
       // Before the container script gets written out.
       List<String> newCmds = new ArrayList<String>(command.size());
       String appIdStr = app.getAppId().toString();
+      String relativeContainerLogDir = ContainerLaunch
+          .getRelativeContainerLogDir(appIdStr, containerIdStr);
       Path containerLogDir =
-          dirsHandler.getLogPathForWrite(ContainerLaunch
-              .getRelativeContainerLogDir(appIdStr, containerIdStr), false);
+          dirsHandler.getLogPathForWrite(relativeContainerLogDir, false);
       for (String str : command) {
         // TODO: Should we instead work via symlinks without this grammar?
         newCmds.add(str.replace(ApplicationConstants.LOG_DIR_EXPANSION_VAR,
@@ -189,6 +190,11 @@ public class ContainerLaunch implements Callable<Integer> {
       List<String> localDirs = dirsHandler.getLocalDirs();
       List<String> logDirs = dirsHandler.getLogDirs();
 
+      List<String> containerLogDirs = new ArrayList<String>();
+      for( String logDir : logDirs) {
+        containerLogDirs.add(logDir + Path.SEPARATOR + relativeContainerLogDir);
+      }
+
       if (!dirsHandler.areDisksHealthy()) {
         ret = ContainerExitStatus.DISKS_FAILED;
         throw new IOException("Most of the disks failed. "
@@ -215,7 +221,8 @@ public class ContainerLaunch implements Callable<Integer> {
                 FINAL_CONTAINER_TOKENS_FILE).toUri().getPath());
 
         // Sanitize the container's environment
-        sanitizeEnv(environment, containerWorkDir, appDirs, localResources);
+        sanitizeEnv(environment, containerWorkDir, appDirs, containerLogDirs,
+          localResources);
         
         // Write out the environment
         writeLaunchEnv(containerScriptOutStream, environment, localResources,
@@ -543,9 +550,9 @@ public class ContainerLaunch implements Callable<Integer> {
     }
   }
   
-  public void sanitizeEnv(Map<String, String> environment, 
-      Path pwd, List<Path> appDirs, Map<Path, List<String>> resources)
-      throws IOException {
+  public void sanitizeEnv(Map<String, String> environment, Path pwd,
+      List<Path> appDirs, List<String> containerLogDirs,
+      Map<Path, List<String>> resources) throws IOException {
     /**
      * Non-modifiable environment variables
      */
@@ -565,6 +572,9 @@ public class ContainerLaunch implements Callable<Integer> {
     environment.put(Environment.LOCAL_DIRS.name(),
         StringUtils.join(",", appDirs));
 
+    environment.put(Environment.LOG_DIRS.name(),
+      StringUtils.join(",", containerLogDirs));
+
     putEnvIfNotNull(environment, Environment.USER.name(), container.getUser());
     
     putEnvIfNotNull(environment, 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java

@@ -250,6 +250,8 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
 
     // Now verify the contents of the file
     List<String> localDirs = dirsHandler.getLocalDirs();
+    List<String> logDirs = dirsHandler.getLogDirs();
+
     List<Path> appDirs = new ArrayList<Path>(localDirs.size());
     for (String localDir : localDirs) {
       Path usersdir = new Path(localDir, ContainerLocalizer.USERCACHE);
@@ -257,6 +259,12 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
       Path appsdir = new Path(userdir, ContainerLocalizer.APPCACHE);
       appDirs.add(new Path(appsdir, appId.toString()));
     }
+    List<String> containerLogDirs = new ArrayList<String>();
+    String relativeContainerLogDir = ContainerLaunch
+        .getRelativeContainerLogDir(appId.toString(), cId.toString());
+    for(String logDir : logDirs){
+      containerLogDirs.add(logDir + Path.SEPARATOR + relativeContainerLogDir);
+    }
     BufferedReader reader =
         new BufferedReader(new FileReader(processStartFile));
     Assert.assertEquals(cId.toString(), reader.readLine());
@@ -276,6 +284,9 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
       .getEnvironment().get(Environment.NM_HTTP_PORT.name()));
     Assert.assertEquals(StringUtils.join(",", appDirs), containerLaunchContext
         .getEnvironment().get(Environment.LOCAL_DIRS.name()));
+    Assert.assertEquals(StringUtils.join(",", containerLogDirs),
+      containerLaunchContext.getEnvironment().get(Environment.LOG_DIRS.name()));
+
     // Get the pid of the process
     String pid = reader.readLine().trim();
     // No more lines