YARN-8962. Add ability to use interactive shell with normal yarn container. Contributed by Eric Yang

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperation.java

@@ -44,7 +44,7 @@ public class PrivilegedOperation {
     INITIALIZE_CONTAINER(""), //no CLI switch supported yet
     LAUNCH_CONTAINER(""), //no CLI switch supported yet
     SIGNAL_CONTAINER(""), //no CLI switch supported yet
-    EXEC_CONTAINER("--run-docker"), //no CLI switch supported yet
+    EXEC_CONTAINER("--exec-container"), //no CLI switch supported yet
     DELETE_AS_USER(""), //no CLI switch supported yet
     LAUNCH_DOCKER_CONTAINER(""), //no CLI switch supported yet
     TC_MODIFY_STATE("--tc-modify-state"),

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java

@@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntime;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeConstants;
@@ -41,6 +42,14 @@ import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerExecContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.io.Writer;
+import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -57,6 +66,8 @@ import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.r
 public class DefaultLinuxContainerRuntime implements LinuxContainerRuntime {
   private static final Logger LOG =
       LoggerFactory.getLogger(DefaultLinuxContainerRuntime.class);
+  private static final String TMP_FILE_PREFIX = "yarn.";
+  private static final String TMP_FILE_SUFFIX = ".cmd";
   private final PrivilegedOperationExecutor privilegedOperationExecutor;
   private Configuration conf;
 
@@ -198,8 +209,93 @@ public class DefaultLinuxContainerRuntime implements LinuxContainerRuntime {
   }
 
   @Override
-  public IOStreamPair execContainer(ContainerExecContext containerExecContext)
+  public IOStreamPair execContainer(ContainerExecContext ctx)
       throws ContainerExecutionException {
-    throw new ContainerExecutionException("Unsupported operation.");
+    IOStreamPair output;
+    try {
+      PrivilegedOperation privOp = new PrivilegedOperation(
+          PrivilegedOperation.OperationType.EXEC_CONTAINER);
+      String commandFile = writeCommandToTempFile(ctx);
+      privOp.appendArgs(commandFile);
+      privOp.disableFailureLogging();
+      output =
+          privilegedOperationExecutor.executePrivilegedInteractiveOperation(
+              null, privOp);
+    } catch (PrivilegedOperationException e) {
+      throw new ContainerExecutionException(
+          "Execute container interactive shell failed", e.getExitCode(),
+          e.getOutput(), e.getErrorOutput());
+    } catch (InterruptedException ie) {
+      LOG.warn("InterruptedException executing command: ", ie);
+      throw new ContainerExecutionException(ie.getMessage());
+    }
+    return output;
+  }
+
+  private String writeCommandToTempFile(ContainerExecContext ctx)
+      throws ContainerExecutionException {
+    Container container = ctx.getContainer();
+    File cmdDir = null;
+    String appId = container.getContainerId().getApplicationAttemptId()
+        .getApplicationId().toString();
+    String containerId = container.getContainerId().toString();
+    String filePrefix = containerId.toString();
+    try {
+      String cmdDirPath = ctx.getLocalDirsHandlerService().getLocalPathForWrite(
+          ResourceLocalizationService.NM_PRIVATE_DIR + Path.SEPARATOR +
+          appId + Path.SEPARATOR + filePrefix + Path.SEPARATOR).toString();
+      cmdDir = new File(cmdDirPath);
+      if (!cmdDir.mkdirs() && !cmdDir.exists()) {
+        throw new IOException("Cannot create container private directory "
+            + cmdDir);
+      }
+      File commandFile = File.createTempFile(TMP_FILE_PREFIX + filePrefix,
+          TMP_FILE_SUFFIX, cmdDir);
+      try (
+          Writer writer = new OutputStreamWriter(
+              new FileOutputStream(commandFile.toString()), "UTF-8");
+          PrintWriter printWriter = new PrintWriter(writer);
+      ) {
+        Map<String, List<String>> cmd = new HashMap<String, List<String>>();
+        // command = exec
+        List<String> exec = new ArrayList<String>();
+        exec.add("exec");
+        cmd.put("command", exec);
+        // user = foobar
+        List<String> user = new ArrayList<String>();
+        user.add(container.getUser());
+        cmd.put("user", user);
+        // launch-command = bash,-i
+        List<String> commands = new ArrayList<String>();
+        commands.add("/bin/bash");
+        commands.add("-ir");
+        cmd.put("launch-command", commands);
+        // workdir = ../nm-local-dir/usercache/appcache/appid/containerid
+        List<String> workdir = new ArrayList<String>();
+        workdir.add(container.getWorkDir());
+        cmd.put("workdir", workdir);
+        // generate cmd file
+        printWriter.println("[command-execution]");
+        for (Map.Entry<String, List<String>> entry :
+            cmd.entrySet()) {
+          if (entry.getKey().contains("=")) {
+            throw new ContainerExecutionException(
+                "'=' found in entry for docker command file, key = " + entry
+                    .getKey() + "; value = " + entry.getValue());
+          }
+          if (entry.getValue().contains("\n")) {
+            throw new ContainerExecutionException(
+                "'\\n' found in entry for docker command file, key = " + entry
+                    .getKey() + "; value = " + entry.getValue());
+          }
+          printWriter.println("  " + entry.getKey() + "=" + StringUtils
+              .join(",", entry.getValue()));
+        }
+        return commandFile.toString();
+      }
+    } catch (IOException e) {
+      LOG.warn("Unable to write command to " + cmdDir);
+      throw new ContainerExecutionException(e);
+    }
   }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerExecContext.java

@@ -22,6 +22,7 @@ package org.apache.hadoop.yarn.server.nodemanager.executor;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
 
 /**
@@ -34,6 +35,7 @@ public final class ContainerExecContext {
   private final String user;
   private final String appId;
   private final Container container;
+  private final LocalDirsHandlerService localDirsHandler;
 
   /**
    *  Builder for ContainerExecContext.
@@ -42,6 +44,7 @@ public final class ContainerExecContext {
     private String user;
     private String appId;
     private Container container;
+    private LocalDirsHandlerService localDirsHandler;
 
     public Builder() {
     }
@@ -64,12 +67,19 @@ public final class ContainerExecContext {
     public ContainerExecContext build() {
       return new ContainerExecContext(this);
     }
+
+    public Builder setNMLocalPath(
+        LocalDirsHandlerService ldhs) {
+      this.localDirsHandler = ldhs;
+      return this;
+    }
   }
 
   private ContainerExecContext(Builder builder) {
     this.container = builder.container;
     this.user = builder.user;
     this.appId = builder.appId;
+    this.localDirsHandler = builder.localDirsHandler;
   }
 
   public String getUser() {
@@ -83,4 +93,8 @@ public final class ContainerExecContext {
   public Container getContainer() {
     return this.container;
   }
+
+  public LocalDirsHandlerService getLocalDirsHandlerService() {
+    return this.localDirsHandler;
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerShellWebSocket.java

@@ -112,6 +112,7 @@ public class ContainerShellWebSocket {
       ContainerExecContext execContext = new ContainerExecContext
           .Builder()
           .setContainer(container)
+          .setNMLocalPath(nmContext.getLocalDirsHandler())
           .build();
       pair = exec.execContainer(execContext);
     } catch (Exception e) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c

@@ -1374,28 +1374,12 @@ char **construct_docker_command(const char *command_file) {
 }
 
 int run_docker(const char *command_file) {
-  struct configuration command_config = {0, NULL};
-
-  int ret = read_config(command_file, &command_config);
-  if (ret != 0) {
-    free_configuration(&command_config);
-    return INVALID_COMMAND_FILE;
-  }
-  char *value = get_configuration_value("docker-command", DOCKER_COMMAND_FILE_SECTION, &command_config);
-  if (value != NULL && strcasecmp(value, "exec") == 0) {
-    free(value);
-    free_configuration(&command_config);
-    return run_docker_with_pty(command_file);
-  }
-  free_configuration(&command_config);
-  free(value);
-
   char **args = construct_docker_command(command_file);
-  char* docker_binary = get_docker_binary(&CFG);
+  char *docker_binary = get_docker_binary(&CFG);
 
   int exit_code = -1;
   if (execvp(docker_binary, args) != 0) {
-    fprintf(ERRORFILE, "Couldn't execute the container launch with args %s - %s",
+    fprintf(ERRORFILE, "Couldn't execute the container launch with args %s - %s\n",
               docker_binary, strerror(errno));
     fflush(LOGFILE);
     fflush(ERRORFILE);
@@ -1409,12 +1393,55 @@ int run_docker(const char *command_file) {
   return exit_code;
 }
 
-int run_docker_with_pty(const char *command_file) {
+int exec_container(const char *command_file) {
   int exit_code = -1;
-  char **args = construct_docker_command(command_file);
-  char* docker_binary = get_docker_binary(&CFG);
+  struct configuration command_config = {0, NULL};
+  char **args = NULL;
+  char **env = NULL;
+  char *workdir = NULL;
+  char *docker_binary = get_docker_binary(&CFG);
+  char *binary = NULL;
   int fdm, fds, rc;
   char input[4000];
+  int docker = 0;
+  char *user = NULL;
+
+  int ret = read_config(command_file, &command_config);
+  if (ret != 0) {
+    free_configuration(&command_config);
+    return INVALID_COMMAND_FILE;
+  }
+
+  char *value = get_configuration_value("docker-command", DOCKER_COMMAND_FILE_SECTION, &command_config);
+  if (value != NULL && strcasecmp(value, "exec") == 0) {
+    args = construct_docker_command(command_file);
+    binary = docker_binary;
+    docker = 1;
+  } else {
+    value = get_configuration_value("command", COMMAND_FILE_SECTION, &command_config);
+    if (value != NULL && strcasecmp(value, "exec") == 0) {
+      args = get_configuration_values_delimiter("launch-command",
+                                                COMMAND_FILE_SECTION, &command_config, ",");
+      if (args == NULL) {
+        goto cleanup;
+      }
+      binary = strdup(args[0]);
+      workdir = get_configuration_value("workdir", COMMAND_FILE_SECTION, &command_config);
+      if (workdir == NULL) {
+        goto cleanup;
+      }
+      env = (char **) alloc_and_clear_memory(3, sizeof(char *));
+      env[0] = make_string("PWD=%s", workdir);
+      env[1] = make_string("TERM=%s", "xterm-256color");
+      env[2] = NULL;
+      user = get_configuration_value("user", COMMAND_FILE_SECTION, &command_config);
+      if (user == NULL) {
+        goto cleanup;
+      }
+    } else {
+      goto cleanup;
+    }
+  }
 
   fdm = posix_openpt(O_RDWR);
   if (fdm < 0) {
@@ -1513,6 +1540,9 @@ int run_docker_with_pty(const char *command_file) {
     // Set raw mode on the slave side of the PTY
     new_term_settings = slave_orig_term_settings;
     cfmakeraw (&new_term_settings);
+    if (!docker) {
+      new_term_settings.c_lflag |= ECHO;
+    }
     tcsetattr (fds, TCSANOW, &new_term_settings);
 
     // The slave side of the PTY becomes the standard input and outputs of the child process
@@ -1537,24 +1567,48 @@ int run_docker_with_pty(const char *command_file) {
     close(fds);
 
     // Make the current process a new session leader
-    setsid();
+    if (docker) {
+      setsid();
+    } else {
+      exit_code = set_user(user);
+      if (exit_code!=0) {
+        goto cleanup;
+      }
+    }
 
     // As the child is a session leader, set the controlling terminal to be the slave side of the PTY
     // (Mandatory for programs like the shell to make them manage correctly their outputs)
     ioctl(0, TIOCSCTTY, 1);
-
-    if (execvp(docker_binary, args) != 0) {
-      fprintf(ERRORFILE, "Couldn't execute the container launch with args %s - %s",
-              docker_binary, strerror(errno));
-      free(docker_binary);
-      free_values(args);
+    if (docker) {
+      ret = execvp(binary, args);
+    } else {
+      if (change_user(user_detail->pw_uid, user_detail->pw_gid) != 0) {
+        exit_code = DOCKER_EXEC_FAILED;
+        goto cleanup;
+      }
+      ret = chdir(workdir);
+      if (ret != 0) {
+        exit_code = DOCKER_EXEC_FAILED;
+        goto cleanup;
+      }
+      ret = execve(binary, args, env);
+    }
+    if (ret != 0) {
+      fprintf(ERRORFILE, "Couldn't execute the container launch with args %s - %s\n",
+            binary, strerror(errno));
       exit_code = DOCKER_EXEC_FAILED;
     } else {
-      free_values(args);
       exit_code = 0;
     }
   }
 
+cleanup:
+  free(docker_binary);
+  free(binary);
+  free(user);
+  free(workdir);
+  free_values(args);
+  free_configuration(&command_config);
   return exit_code;
 }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h

@@ -51,7 +51,8 @@ enum operations {
   RUN_AS_USER_LIST = 12,
   REMOVE_DOCKER_CONTAINER = 13,
   INSPECT_DOCKER_CONTAINER = 14,
-  RUN_AS_USER_SYNC_YARN_SYSFS = 15
+  RUN_AS_USER_SYNC_YARN_SYSFS = 15,
+  EXEC_CONTAINER = 16
 };
 
 #define NM_GROUP_KEY "yarn.nodemanager.linux-container-executor.group"
@@ -71,6 +72,7 @@ enum operations {
 #define MOUNT_CGROUP_SUPPORT_ENABLED_KEY "feature.mount-cgroup.enabled"
 #define YARN_SYSFS_SUPPORT_ENABLED_KEY "feature.yarn.sysfs.enabled"
 #define TMP_DIR "tmp"
+#define COMMAND_FILE_SECTION "command-execution"
 
 extern struct passwd *user_detail;
 
@@ -296,6 +298,11 @@ int run_docker_with_pty(const char *command_file);
  */
 int exec_docker_command(char *docker_command, char **argv, int argc);
 
+/**
+ * Exec a container terminal.
+ */
+int exec_container(const char *command_file);
+
 /** Check if yarn sysfs is enabled in configuration. */
 int is_yarn_sysfs_support_enabled();
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c

@@ -54,12 +54,14 @@ static void display_usage(FILE *stream) {
   if(is_docker_support_enabled()) {
     fprintf(stream,
       "       container-executor --run-docker <command-file>\n"
+      "       container-executor --exec-container <command-file>\n"
       "       container-executor --remove-docker-container [hierarchy] "
       "<container_id>\n"
       "       container-executor --inspect-docker-container <container_id>\n");
   } else {
     fprintf(stream,
       "[DISABLED] container-executor --run-docker <command-file>\n"
+      "[DISABLED] container-executor --exec-container <command-file>\n"
       "[DISABLED] container-executor --remove-docker-container [hierarchy] "
       "<container_id>\n"
       "[DISABLED] container-executor --inspect-docker-container "
@@ -247,7 +249,7 @@ static struct {
   const char *target_dir;
   int container_pid;
   int signal;
-  const char *docker_command_file;
+  const char *command_file;
 } cmd_input;
 
 static int validate_run_as_user_commands(int argc, char **argv, int *operation);
@@ -348,6 +350,22 @@ static int validate_arguments(int argc, char **argv , int *operation) {
     }
   }
 
+  if (strcmp("--exec-container", argv[1]) == 0) {
+    if(is_docker_support_enabled()) {
+      if (argc != 3) {
+        display_usage(stdout);
+        return INVALID_ARGUMENT_NUMBER;
+      }
+      optind++;
+      cmd_input.command_file = argv[optind++];
+      *operation = EXEC_CONTAINER;
+      return 0;
+    } else {
+        display_feature_disabled_message("docker");
+        return FEATURE_DISABLED;
+    }
+  }
+
   if (strcmp("--run-docker", argv[1]) == 0) {
     if(is_docker_support_enabled()) {
       if (argc != 3) {
@@ -355,7 +373,7 @@ static int validate_arguments(int argc, char **argv , int *operation) {
         return INVALID_ARGUMENT_NUMBER;
       }
       optind++;
-      cmd_input.docker_command_file = argv[optind++];
+      cmd_input.command_file = argv[optind++];
       *operation = RUN_DOCKER;
       return 0;
     } else {
@@ -469,7 +487,7 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation)
       cmd_input.local_dirs = argv[optind++];
       // good log dirs as a comma separated list
       cmd_input.log_dirs = argv[optind++];
-      cmd_input.docker_command_file = argv[optind++];
+      cmd_input.command_file = argv[optind++];
       //network isolation through tc
       if ((argc == 15 && !cmd_input.https) || (argc == 17 && cmd_input.https)) {
         if(is_tc_support_enabled()) {
@@ -624,8 +642,11 @@ int main(int argc, char **argv) {
   case TRAFFIC_CONTROL_READ_STATS:
     exit_code = traffic_control_read_stats(cmd_input.traffic_control_command_file);
     break;
+  case EXEC_CONTAINER:
+    exit_code = exec_container(cmd_input.command_file);
+    break;
   case RUN_DOCKER:
-    exit_code = run_docker(cmd_input.docker_command_file);
+    exit_code = run_docker(cmd_input.command_file);
     break;
   case REMOVE_DOCKER_CONTAINER:
     exit_code = remove_docker_container(argv + optind, argc - optind);
@@ -674,7 +695,7 @@ int main(int argc, char **argv) {
                       cmd_input.pid_file,
                       split(cmd_input.local_dirs),
                       split(cmd_input.log_dirs),
-                      cmd_input.docker_command_file);
+                      cmd_input.command_file);
       break;
   case RUN_AS_USER_LAUNCH_CONTAINER:
     if (cmd_input.traffic_control_command_file != NULL) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c

@@ -1472,6 +1472,53 @@ void test_cleaning_docker_cgroups() {
   }
 }
 
+void test_exec_container() {
+  int ret = -1;
+  char* filename = TEST_ROOT "/exec_container.cmd";
+  FILE *file = fopen(filename, "w");
+  if (file == NULL) {
+    printf("FAIL: Could not write to command file: %s\n", filename);
+    exit(1);
+  }
+  // Test missing user
+  fprintf(file, "[command-execution]\n");
+  fprintf(file, "workdir=/tmp/container_1541184499854_0001_01_000001\n");
+  fprintf(file, "launch-command=/bin/bash,-ir\n");
+  fprintf(file, "command=exec\n");
+  fclose(file);
+  ret = exec_container(filename);
+  if (ret!=-1) {
+    printf("FAIL: broken command file should not pass.\n");
+    exit(1);
+  }
+
+  // Test missing workdir
+  file = fopen(filename, "w");
+  fprintf(file, "[command-execution]\n");
+  fprintf(file, "launch-command=/bin/bash,-ir\n");
+  fprintf(file, "user=test\n");
+  fprintf(file, "command=exec\n");
+  fclose(file);
+  ret = exec_container(filename);
+  if (ret!=-1) {
+    printf("FAIL: broken command file should not pass.\n");
+    exit(1);
+  }
+
+  // Test missing launch-command
+  file = fopen(filename, "w");
+  fprintf(file, "[command-execution]\n");
+  fprintf(file, "workdir=/tmp/container_1541184499854_0001_01_000001\n");
+  fprintf(file, "user=test\n");
+  fprintf(file, "command=exec\n");
+  fclose(file);
+  ret = exec_container(filename);
+  if (ret!=-1) {
+    printf("FAIL: broken command file should not pass.\n");
+    exit(1);
+  }
+}
+
 // This test is expected to be executed either by a regular
 // user or by root. If executed by a regular user it doesn't
 // test all the functions that would depend on changing the
@@ -1612,6 +1659,9 @@ int main(int argc, char **argv) {
   printf("\nTesting yarn sysfs\n");
   test_yarn_sysfs();
 
+  printf("\nTesting exec_container()\n");
+  test_exec_container();
+
   test_check_user(0);
 
   test_cleaning_docker_cgroups();