YARN-5298. Mount usercache and NM filecache directories into Docker container. Contributed by Sidharta Seethana.

(cherry-picked from commit 58e18508018081b5b5aa7c12cc5af386146cd26b)

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java

@@ -240,6 +240,12 @@ public interface ApplicationConstants {
      */
     LOCAL_DIRS("LOCAL_DIRS"),
 
+    /**
+     * $LOCAL_USER_DIRS
+     * Final, exported by NodeManager and non-modifiable by users.
+     */
+    LOCAL_USER_DIRS("LOCAL_USER_DIRS"),
+
     /**
      * $LOG_DIRS
      * Final, exported by NodeManager and non-modifiable by users.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java

@@ -52,7 +52,6 @@ import org.apache.hadoop.yarn.server.nodemanager.executor.LocalizerStartContext;
 import org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler;
 import org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler;
 import org.apache.hadoop.yarn.server.nodemanager.util.LCEResourcesHandler;
-import org.apache.hadoop.yarn.util.ConverterUtils;
 
 import java.io.File;
 import java.io.IOException;
@@ -311,6 +310,8 @@ public class LinuxContainerExecutor extends ContainerExecutor {
     Path containerWorkDir = ctx.getContainerWorkDir();
     List<String> localDirs = ctx.getLocalDirs();
     List<String> logDirs = ctx.getLogDirs();
+    List<String> filecacheDirs = ctx.getFilecacheDirs();
+    List<String> userLocalDirs = ctx.getUserLocalDirs();
     List<String> containerLocalDirs = ctx.getContainerLocalDirs();
     List<String> containerLogDirs = ctx.getContainerLogDirs();
     Map<Path, List<String>> localizedResources = ctx.getLocalizedResources();
@@ -397,6 +398,8 @@ public class LinuxContainerExecutor extends ContainerExecutor {
             .setExecutionAttribute(PID_FILE_PATH, pidFilePath)
             .setExecutionAttribute(LOCAL_DIRS, localDirs)
             .setExecutionAttribute(LOG_DIRS, logDirs)
+            .setExecutionAttribute(FILECACHE_DIRS, filecacheDirs)
+            .setExecutionAttribute(USER_LOCAL_DIRS, userLocalDirs)
             .setExecutionAttribute(CONTAINER_LOCAL_DIRS, containerLocalDirs)
             .setExecutionAttribute(CONTAINER_LOG_DIRS, containerLogDirs)
             .setExecutionAttribute(RESOURCES_OPTIONS, resourcesOptions);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java

@@ -230,6 +230,8 @@ public class ContainerLaunch implements Callable<Integer> {
       pidFilePath = dirsHandler.getLocalPathForWrite(pidFileSubpath);
       List<String> localDirs = dirsHandler.getLocalDirs();
       List<String> logDirs = dirsHandler.getLogDirs();
+      List<String> filecacheDirs = getNMFilecacheDirs(localDirs);
+      List<String> userLocalDirs = getUserLocalDirs(localDirs);
       List<String> containerLocalDirs = getContainerLocalDirs(localDirs);
       List<String> containerLogDirs = getContainerLogDirs(logDirs);
 
@@ -242,6 +244,7 @@ public class ContainerLaunch implements Callable<Integer> {
       try {
         // /////////// Write out the container-script in the nmPrivate space.
         List<Path> appDirs = new ArrayList<Path>(localDirs.size());
+
         for (String localDir : localDirs) {
           Path usersdir = new Path(localDir, ContainerLocalizer.USERCACHE);
           Path userdir = new Path(usersdir, user);
@@ -258,7 +261,8 @@ public class ContainerLaunch implements Callable<Integer> {
             new Path(containerWorkDir, 
                 FINAL_CONTAINER_TOKENS_FILE).toUri().getPath());
         // Sanitize the container's environment
-        sanitizeEnv(environment, containerWorkDir, appDirs, containerLogDirs,
+        sanitizeEnv(environment, containerWorkDir, appDirs, userLocalDirs,
+            containerLogDirs,
           localResources, nmPrivateClasspathJarDir);
 
         // Write out the environment
@@ -288,6 +292,8 @@ public class ContainerLaunch implements Callable<Integer> {
           .setContainerWorkDir(containerWorkDir)
           .setLocalDirs(localDirs)
           .setLogDirs(logDirs)
+          .setFilecacheDirs(filecacheDirs)
+          .setUserLocalDirs(userLocalDirs)
           .setContainerLocalDirs(containerLocalDirs)
           .setContainerLogDirs(containerLogDirs)
           .build());
@@ -353,6 +359,35 @@ public class ContainerLaunch implements Callable<Integer> {
     return containerLocalDirs;
   }
 
+  protected List<String> getUserLocalDirs(List<String> localDirs) {
+    List<String> userLocalDirs = new ArrayList<>(localDirs.size());
+    String user = container.getUser();
+
+    for (String localDir : localDirs) {
+      String userLocalDir = localDir + Path.SEPARATOR +
+          ContainerLocalizer.USERCACHE + Path.SEPARATOR + user
+          + Path.SEPARATOR;
+
+      userLocalDirs.add(userLocalDir);
+    }
+
+    return userLocalDirs;
+  }
+
+  protected List<String> getNMFilecacheDirs(List<String> localDirs) {
+    List<String> filecacheDirs = new ArrayList<>(localDirs.size());
+
+    for (String localDir : localDirs) {
+      String filecacheDir = localDir + Path.SEPARATOR +
+          ContainerLocalizer.FILECACHE;
+
+      filecacheDirs.add(filecacheDir);
+    }
+
+    return filecacheDirs;
+  }
+
+
   protected Map<Path, List<String>> getLocalizedResources()
       throws YarnException {
     Map<Path, List<String>> localResources = container.getLocalizedResources();
@@ -990,7 +1025,8 @@ public class ContainerLaunch implements Callable<Integer> {
   }
   
   public void sanitizeEnv(Map<String, String> environment, Path pwd,
-      List<Path> appDirs, List<String> containerLogDirs,
+      List<Path> appDirs, List<String> userLocalDirs, List<String>
+      containerLogDirs,
       Map<Path, List<String>> resources,
       Path nmPrivateClasspathJarDir) throws IOException {
     /**
@@ -1012,6 +1048,9 @@ public class ContainerLaunch implements Callable<Integer> {
     environment.put(Environment.LOCAL_DIRS.name(),
         StringUtils.join(",", appDirs));
 
+    environment.put(Environment.LOCAL_USER_DIRS.name(), StringUtils.join(",",
+        userLocalDirs));
+
     environment.put(Environment.LOG_DIRS.name(),
       StringUtils.join(",", containerLogDirs));
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java

@@ -318,6 +318,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
     @SuppressWarnings("unchecked")
     List<String> logDirs = ctx.getExecutionAttribute(LOG_DIRS);
     @SuppressWarnings("unchecked")
+    List<String> filecacheDirs = ctx.getExecutionAttribute(FILECACHE_DIRS);
+    @SuppressWarnings("unchecked")
     List<String> containerLocalDirs = ctx.getExecutionAttribute(
         CONTAINER_LOCAL_DIRS);
     @SuppressWarnings("unchecked")
@@ -326,6 +328,9 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
     @SuppressWarnings("unchecked")
     Map<Path, List<String>> localizedResources = ctx.getExecutionAttribute(
         LOCALIZED_RESOURCES);
+    @SuppressWarnings("unchecked")
+    List<String> userLocalDirs = ctx.getExecutionAttribute(USER_LOCAL_DIRS);
+
     Set<String> capabilities = new HashSet<>(Arrays.asList(conf.getStrings(
         YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES,
         YarnConfiguration.DEFAULT_NM_DOCKER_CONTAINER_CAPABILITIES)));
@@ -340,8 +345,10 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
         .addMountLocation("/etc/passwd", "/etc/password:ro");
     List<String> allDirs = new ArrayList<>(containerLocalDirs);
 
+    allDirs.addAll(filecacheDirs);
     allDirs.add(containerWorkDir.toString());
     allDirs.addAll(containerLogDirs);
+    allDirs.addAll(userLocalDirs);
     for (String dir: allDirs) {
       runCommand.addMountLocation(dir, dir);
     }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java

@@ -55,6 +55,10 @@ public final class LinuxContainerRuntimeConstants {
       List.class, "local_dirs");
   public static final Attribute<List> LOG_DIRS = Attribute.attribute(
       List.class, "log_dirs");
+  public static final Attribute<List> FILECACHE_DIRS = Attribute.attribute(
+      List.class, "filecache_dirs");
+  public static final Attribute<List> USER_LOCAL_DIRS = Attribute.attribute(
+      List.class, "user_local_dirs");
   public static final Attribute<List> CONTAINER_LOCAL_DIRS = Attribute
       .attribute(List.class, "container_local_dirs");
   public static final Attribute<List> CONTAINER_LOG_DIRS = Attribute.attribute(

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerStartContext.java

@@ -45,6 +45,8 @@ public final class ContainerStartContext {
   private final Path containerWorkDir;
   private final List<String> localDirs;
   private final List<String> logDirs;
+  private final List<String> filecacheDirs;
+  private final List<String> userLocalDirs;
   private final List<String> containerLocalDirs;
   private final List<String> containerLogDirs;
 
@@ -58,6 +60,8 @@ public final class ContainerStartContext {
     private Path containerWorkDir;
     private List<String> localDirs;
     private List<String> logDirs;
+    private List<String> filecacheDirs;
+    private List<String> userLocalDirs;
     private List<String> containerLocalDirs;
     private List<String> containerLogDirs;
 
@@ -111,6 +115,16 @@ public final class ContainerStartContext {
       return this;
     }
 
+    public Builder setFilecacheDirs(List<String> filecacheDirs) {
+      this.filecacheDirs = filecacheDirs;
+      return this;
+    }
+
+    public Builder setUserLocalDirs(List<String> userLocalDirs) {
+      this.userLocalDirs = userLocalDirs;
+      return this;
+    }
+
     public Builder setContainerLocalDirs(List<String> containerLocalDirs) {
       this.containerLocalDirs = containerLocalDirs;
       return this;
@@ -136,6 +150,8 @@ public final class ContainerStartContext {
     this.containerWorkDir = builder.containerWorkDir;
     this.localDirs = builder.localDirs;
     this.logDirs = builder.logDirs;
+    this.filecacheDirs = builder.filecacheDirs;
+    this.userLocalDirs = builder.userLocalDirs;
     this.containerLocalDirs = builder.containerLocalDirs;
     this.containerLogDirs = builder.containerLogDirs;
   }
@@ -180,11 +196,20 @@ public final class ContainerStartContext {
     return Collections.unmodifiableList(this.logDirs);
   }
 
+  public List<String> getFilecacheDirs() {
+    return Collections.unmodifiableList(this.filecacheDirs);
+  }
+
+  public List<String> getUserLocalDirs() {
+    return Collections.unmodifiableList(this.userLocalDirs);
+  }
+
   public List<String> getContainerLocalDirs() {
-    return this.containerLocalDirs;
+    return Collections.unmodifiableList(this.containerLocalDirs);
   }
 
   public List<String> getContainerLogDirs() {
-    return this.containerLogDirs;
+    return Collections.unmodifiableList(this
+        .containerLogDirs);
   }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java

@@ -180,6 +180,10 @@ public class TestLinuxContainerExecutorWithMocks {
         .setContainerWorkDir(workDir)
         .setLocalDirs(dirsHandler.getLocalDirs())
         .setLogDirs(dirsHandler.getLogDirs())
+        .setFilecacheDirs(new ArrayList<String>())
+        .setUserLocalDirs(new ArrayList<String>())
+        .setContainerLocalDirs(new ArrayList<String>())
+        .setContainerLogDirs(new ArrayList<String>())
         .build());
     assertEquals(0, ret);
     assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
@@ -345,6 +349,10 @@ public class TestLinuxContainerExecutorWithMocks {
         .setContainerWorkDir(workDir)
         .setLocalDirs(dirsHandler.getLocalDirs())
         .setLogDirs(dirsHandler.getLogDirs())
+        .setFilecacheDirs(new ArrayList<String>())
+        .setUserLocalDirs(new ArrayList<String>())
+        .setContainerLocalDirs(new ArrayList<String>())
+        .setContainerLogDirs(new ArrayList<String>())
         .build());
 
     Assert.assertNotSame(0, ret);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java

@@ -454,6 +454,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
         "target/test-dir");
     Path pwd = new Path(testDir);
     List<Path> appDirs = new ArrayList<Path>();
+    List<String> userLocalDirs = new ArrayList<>();
     List<String> containerLogs = new ArrayList<String>();
 
     Map<Path, List<String>> resources = new HashMap<Path, List<String>>();
@@ -464,8 +465,8 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
 
     Path nmp = new Path(testDir);
 
-    launch.sanitizeEnv(
-      userSetEnv, pwd, appDirs, containerLogs, resources, nmp);
+    launch.sanitizeEnv(userSetEnv, pwd, appDirs, userLocalDirs, containerLogs,
+        resources, nmp);
 
     List<String> result =
       getJarManifestClasspath(userSetEnv.get(Environment.CLASSPATH.name()));
@@ -483,8 +484,8 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
     launch = new ContainerLaunch(distContext, conf,
         dispatcher, exec, null, container, dirsHandler, containerManager);
 
-    launch.sanitizeEnv(
-      userSetEnv, pwd, appDirs, containerLogs, resources, nmp);
+    launch.sanitizeEnv(userSetEnv, pwd, appDirs, userLocalDirs, containerLogs,
+        resources, nmp);
 
     result =
       getJarManifestClasspath(userSetEnv.get(Environment.CLASSPATH.name()));

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java

@@ -82,6 +82,8 @@ public class TestDockerContainerRuntime {
   private Path pidFilePath;
   private List<String> localDirs;
   private List<String> logDirs;
+  private List<String> filecacheDirs;
+  private List<String> userLocalDirs;
   private List<String> containerLocalDirs;
   private List<String> containerLogDirs;
   private Map<Path, List<String>> localizedResources;
@@ -126,13 +128,17 @@ public class TestDockerContainerRuntime {
     pidFilePath = new Path("/test_pid_file_path");
     localDirs = new ArrayList<>();
     logDirs = new ArrayList<>();
+    filecacheDirs = new ArrayList<>();
     resourcesOptions = "cgroups=none";
+    userLocalDirs = new ArrayList<>();
     containerLocalDirs = new ArrayList<>();
     containerLogDirs = new ArrayList<>();
     localizedResources = new HashMap<>();
 
     localDirs.add("/test_local_dir");
     logDirs.add("/test_log_dir");
+    filecacheDirs.add("/test_filecache_dir");
+    userLocalDirs.add("/test_user_local_dir");
     containerLocalDirs.add("/test_container_local_dir");
     containerLogDirs.add("/test_container_log_dir");
     localizedResources.put(new Path("/test_local_dir/test_resource_file"),
@@ -156,6 +162,8 @@ public class TestDockerContainerRuntime {
         .setExecutionAttribute(PID_FILE_PATH, pidFilePath)
         .setExecutionAttribute(LOCAL_DIRS, localDirs)
         .setExecutionAttribute(LOG_DIRS, logDirs)
+        .setExecutionAttribute(FILECACHE_DIRS, filecacheDirs)
+        .setExecutionAttribute(USER_LOCAL_DIRS, userLocalDirs)
         .setExecutionAttribute(CONTAINER_LOCAL_DIRS, containerLocalDirs)
         .setExecutionAttribute(CONTAINER_LOG_DIRS, containerLogDirs)
         .setExecutionAttribute(LOCALIZED_RESOURCES, localizedResources)
@@ -262,12 +270,15 @@ public class TestDockerContainerRuntime {
         .append("-v %4$s:%4$s ")
         .append("-v %5$s:%5$s ")
         .append("-v %6$s:%6$s ")
-        .append("%7$s ")
-        .append("bash %8$s/launch_container.sh");
+        .append("-v %7$s:%7$s ")
+        .append("-v %8$s:%8$s ").append("%9$s ")
+        .append("bash %10$s/launch_container.sh");
 
-    String expectedCommand = String.format(expectedCommandTemplate.toString(),
-        containerId, runAsUser, containerWorkDir, containerLocalDirs.get(0),
-        containerWorkDir, containerLogDirs.get(0), image, containerWorkDir);
+    String expectedCommand = String
+        .format(expectedCommandTemplate.toString(), containerId, runAsUser,
+            containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0),
+            containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0),
+            image, containerWorkDir);
 
     List<String> dockerCommands = Files.readAllLines(Paths.get
             (dockerCommandFile), Charset.forName("UTF-8"));
@@ -363,13 +374,15 @@ public class TestDockerContainerRuntime {
             .append(getExpectedTestCapabilitiesArgumentString())
             .append("-v /etc/passwd:/etc/password:ro ")
             .append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
-            .append("-v %6$s:%6$s ").append("%7$s ")
-            .append("bash %8$s/launch_container.sh");
+            .append("-v %6$s:%6$s ").append("-v %7$s:%7$s ")
+            .append("-v %8$s:%8$s ").append("%9$s ")
+            .append("bash %10$s/launch_container.sh");
 
     String expectedCommand = String
         .format(expectedCommandTemplate.toString(), containerId, runAsUser,
-            containerWorkDir, containerLocalDirs.get(0), containerWorkDir,
-            containerLogDirs.get(0), image, containerWorkDir);
+            containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0),
+            containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0),
+            image, containerWorkDir);
 
     List<String> dockerCommands = Files
         .readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
@@ -416,13 +429,15 @@ public class TestDockerContainerRuntime {
             .append(getExpectedTestCapabilitiesArgumentString())
             .append("-v /etc/passwd:/etc/password:ro ")
             .append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
-            .append("-v %6$s:%6$s ").append("%7$s ")
-            .append("bash %8$s/launch_container.sh");
+            .append("-v %6$s:%6$s ").append("-v %7$s:%7$s ")
+            .append("-v %8$s:%8$s ").append("%9$s ")
+            .append("bash %10$s/launch_container.sh");
 
     String expectedCommand = String
         .format(expectedCommandTemplate.toString(), containerId, runAsUser,
-            containerWorkDir, containerLocalDirs.get(0), containerWorkDir,
-            containerLogDirs.get(0), image, containerWorkDir);
+            containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0),
+            containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0),
+            image, containerWorkDir);
 
     List<String> dockerCommands = Files
         .readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
@@ -451,14 +466,15 @@ public class TestDockerContainerRuntime {
             .append(getExpectedTestCapabilitiesArgumentString())
             .append("-v /etc/passwd:/etc/password:ro ")
             .append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
-            .append("-v %6$s:%6$s ").append("%7$s ")
-            .append("bash %8$s/launch_container.sh");
+            .append("-v %6$s:%6$s ").append("-v %7$s:%7$s ")
+            .append("-v %8$s:%8$s ").append("%9$s ")
+            .append("bash %10$s/launch_container.sh");
 
     expectedCommand = String
         .format(expectedCommandTemplate.toString(), containerId, runAsUser,
-            containerWorkDir, containerLocalDirs.get(0), containerWorkDir,
-            containerLogDirs.get(0), image, containerWorkDir);
-
+            containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0),
+            containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0),
+            image, containerWorkDir);
     dockerCommands = Files
         .readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));