YARN-7516. Add security check for trusted docker images. Contributed by Eric Yang

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c

@@ -74,6 +74,48 @@ static int add_param_to_command(const struct configuration *command_config, cons
   return ret;
 }
 
+int check_trusted_image(const struct configuration *command_config, const struct configuration *conf) {
+  int found = 0;
+  int i = 0;
+  int ret = 0;
+  char *image_name = get_configuration_value("image", DOCKER_COMMAND_FILE_SECTION, command_config);
+  char **privileged_registry = get_configuration_values_delimiter("docker.privileged-containers.registries", CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, conf, ",");
+  char *registry_ptr = NULL;
+  if (image_name == NULL) {
+    ret = INVALID_DOCKER_IMAGE_NAME;
+    goto free_and_exit;
+  }
+  if (privileged_registry != NULL) {
+    for (i = 0; privileged_registry[i] != NULL; i++) {
+      int len = strlen(privileged_registry[i]);
+      if (privileged_registry[i][len - 1] != '/') {
+        registry_ptr = (char *) alloc_and_clear_memory(len + 2, sizeof(char));
+        strncpy(registry_ptr, privileged_registry[i], len);
+        registry_ptr[len] = '/';
+        registry_ptr[len + 1] = '\0';
+      } else {
+        registry_ptr = strdup(privileged_registry[i]);
+      }
+      if (strncmp(image_name, registry_ptr, strlen(registry_ptr))==0) {
+        fprintf(ERRORFILE, "image: %s is trusted in %s registry.\n", image_name, privileged_registry[i]);
+        found=1;
+        free(registry_ptr);
+        break;
+      }
+      free(registry_ptr);
+    }
+  }
+  if (found==0) {
+    fprintf(ERRORFILE, "image: %s is not trusted.\n", image_name);
+    ret = INVALID_DOCKER_IMAGE_TRUST;
+  }
+  free(image_name);
+
+  free_and_exit:
+  free(privileged_registry);
+  return ret;
+}
+
 static int add_param_to_command_if_allowed(const struct configuration *command_config,
                                            const struct configuration *executor_cfg,
                                            const char *key, const char *allowed_key, const char *param,
@@ -100,6 +142,14 @@ static int add_param_to_command_if_allowed(const struct configuration *command_c
   }
 
   if (values != NULL) {
+    // Disable capabilities, devices if image is not trusted.
+    if (strcmp(key, "net") != 0) {
+      if (check_trusted_image(command_config, executor_cfg) != 0) {
+        fprintf(ERRORFILE, "Disable %s for untrusted image\n", key);
+        return INVALID_DOCKER_IMAGE_TRUST;
+      }
+    }
+
     if (permitted_values != NULL) {
       for (i = 0; values[i] != NULL; ++i) {
         memset(tmp_buffer, 0, tmp_buffer_size);
@@ -222,6 +272,8 @@ const char *get_docker_error_message(const int error_code) {
       return "Host pid namespace is disabled";
     case INVALID_PID_NAMESPACE:
       return "Invalid pid namespace";
+    case INVALID_DOCKER_IMAGE_TRUST:
+      return "Docker image is not trusted";
     default:
       return "Unknown error";
   }
@@ -840,14 +892,22 @@ static int set_capabilities(const struct configuration *command_config,
   if (ret != 0) {
     return BUFFER_TOO_SMALL;
   }
+
   ret = add_param_to_command_if_allowed(command_config, conf, "cap-add",
                                         "docker.allowed.capabilities",
                                         "--cap-add=", 1, 0,
                                         out, outlen);
-  if (ret != 0) {
-    fprintf(ERRORFILE, "Invalid docker capability requested\n");
-    ret = INVALID_DOCKER_CAPABILITY;
-    memset(out, 0, outlen);
+  switch (ret) {
+    case 0:
+      break;
+    case INVALID_DOCKER_IMAGE_TRUST:
+      fprintf(ERRORFILE, "Docker capability disabled for untrusted image\n");
+      ret = 0;
+      break;
+    default:
+      fprintf(ERRORFILE, "Invalid docker capability requested\n");
+      ret = INVALID_DOCKER_CAPABILITY;
+      memset(out, 0, outlen);
   }
 
   return ret;
@@ -999,6 +1059,19 @@ static int add_mounts(const struct configuration *command_config, const struct c
   }
 
   if (values != NULL) {
+    // Disable mount volumes if image is not trusted.
+    if (check_trusted_image(command_config, conf) != 0) {
+      fprintf(ERRORFILE, "Disable mount volume for untrusted image\n");
+      // YARN will implicitly bind node manager local directory to
+      // docker image.  This can create file system security holes,
+      // if docker container has binary to escalate privileges.
+      // For untrusted image, we drop mounting without reporting
+      // INVALID_DOCKER_MOUNT messages to allow running untrusted
+      // image in a sandbox.
+      ret = 0;
+      goto free_and_exit;
+    }
+
     ret = normalize_mounts(permitted_ro_mounts);
     ret |= normalize_mounts(permitted_rw_mounts);
     if (ret != 0) {
@@ -1100,6 +1173,12 @@ static int set_privileged(const struct configuration *command_config, const stru
     if (privileged_container_enabled != NULL) {
       if (strcmp(privileged_container_enabled, "1") == 0 ||
           strcasecmp(privileged_container_enabled, "True") == 0) {
+        // Disable set privileged if image is not trusted.
+        if (check_trusted_image(command_config, conf) != 0) {
+          fprintf(ERRORFILE, "Privileged containers are disabled from untrusted source\n");
+          ret = PRIVILEGED_CONTAINERS_DISABLED;
+          goto free_and_exit;
+        }
         ret = add_to_buffer(out, outlen, "--privileged ");
         if (ret != 0) {
           ret = BUFFER_TOO_SMALL;
@@ -1251,6 +1330,11 @@ int get_docker_run_command(const char *command_file, const struct configuration
 
   launch_command = get_configuration_values_delimiter("launch-command", DOCKER_COMMAND_FILE_SECTION, &command_config,
                                                       ",");
+
+  if (check_trusted_image(&command_config, conf) != 0) {
+    launch_command = NULL;
+  }
+
   if (launch_command != NULL) {
     for (i = 0; launch_command[i] != NULL; ++i) {
       memset(tmp_buffer, 0, tmp_buffer_size);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h

@@ -57,7 +57,8 @@ enum docker_error_codes {
     INVALID_DOCKER_VOLUME_NAME,
     INVALID_DOCKER_VOLUME_COMMAND,
     PID_HOST_DISABLED,
-    INVALID_PID_NAMESPACE
+    INVALID_PID_NAMESPACE,
+    INVALID_DOCKER_IMAGE_TRUST
 };
 
 /**

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc

@@ -590,16 +590,20 @@ namespace ContainerExecutor {
     const int buff_len = 1024;
     char buff[buff_len];
     int ret = 0;
-    std::string container_executor_cfg_contents[] = {"[docker]\n  docker.privileged-containers.enabled=1",
-                                                     "[docker]\n  docker.privileged-containers.enabled=true",
-                                                     "[docker]\n  docker.privileged-containers.enabled=True",
+    std::string container_executor_cfg_contents[] = {"[docker]\n  docker.privileged-containers.enabled=1\n  docker.privileged-containers.registries=hadoop",
+                                                     "[docker]\n  docker.privileged-containers.enabled=true\n  docker.privileged-containers.registries=hadoop",
+                                                     "[docker]\n  docker.privileged-containers.enabled=True\n  docker.privileged-containers.registries=hadoop",
                                                      "[docker]\n  docker.privileged-containers.enabled=0",
                                                      "[docker]\n  docker.privileged-containers.enabled=false",
                                                      "[docker]\n"};
     std::vector<std::pair<std::string, std::string> > file_cmd_vec;
     std::vector<std::pair<std::string, std::string> >::const_iterator itr;
     file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
-        "[docker-command-execution]\n  docker-command=run\n  privileged=true", "--privileged "));
+        "[docker-command-execution]\n  docker-command=run\n  privileged=true\n  image=hadoop/image", "--privileged "));
+    file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+        "[docker-command-execution]\n  docker-command=run\n  privileged=false\n image=hadoop/image", ""));
+    file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+        "[docker-command-execution]\n  docker-command=run\n  privileged=false\n image=nothadoop/image", ""));
     file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
         "[docker-command-execution]\n  docker-command=run\n  privileged=false", ""));
     file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
@@ -622,6 +626,14 @@ namespace ContainerExecutor {
         ASSERT_EQ(0, ret);
         ASSERT_STREQ(itr->second.c_str(), buff);
       }
+      write_command_file("[docker-command-execution]\n docker-command=run\n  privileged=true\n image=nothadoop/image");
+      ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+      if (ret != 0) {
+        FAIL();
+      }
+      ret = set_privileged(&cmd_cfg, &container_cfg, buff, buff_len);
+      ASSERT_EQ(PRIVILEGED_CONTAINERS_DISABLED, ret);
+      ASSERT_EQ(0, strlen(buff));
     }
 
 
@@ -664,15 +676,25 @@ namespace ContainerExecutor {
     const int buff_len = 1024;
     char buff[buff_len];
     int ret = 0;
-    std::string container_executor_cfg_contents = "[docker]\n  docker.allowed.capabilities=CHROOT,MKNOD";
+    std::string container_executor_cfg_contents = "[docker]\n"
+        "  docker.allowed.capabilities=CHROOT,MKNOD\n"
+        "  docker.privileged-containers.registries=hadoop\n";
     std::vector<std::pair<std::string, std::string> > file_cmd_vec;
     file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
-        "[docker-command-execution]\n  docker-command=run\n  cap-add=CHROOT,MKNOD",
+        "[docker-command-execution]\n  docker-command=run\n  image=hadoop/docker-image\n  cap-add=CHROOT,MKNOD",
         "--cap-drop='ALL' --cap-add='CHROOT' --cap-add='MKNOD' "));
     file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
-        "[docker-command-execution]\n  docker-command=run\n  cap-add=CHROOT", "--cap-drop='ALL' --cap-add='CHROOT' "));
+        "[docker-command-execution]\n  docker-command=run\n image=nothadoop/docker-image\n  cap-add=CHROOT,MKNOD",
+        "--cap-drop='ALL' "));
     file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
-        "[docker-command-execution]\n  docker-command=run", "--cap-drop='ALL' "));
+        "[docker-command-execution]\n  docker-command=run\n  image=hadoop/docker-image\n  cap-add=CHROOT",
+        "--cap-drop='ALL' --cap-add='CHROOT' "));
+    file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+        "[docker-command-execution]\n  docker-command=run\n  image=hadoop/docker-image\n",
+        "--cap-drop='ALL' "));
+    file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+        "[docker-command-execution]\n  docker-command=run\n image=nothadoop/docker-image\n",
+        "--cap-drop='ALL' "));
     write_container_executor_cfg(container_executor_cfg_contents);
     ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
 
@@ -691,7 +713,7 @@ namespace ContainerExecutor {
       ASSERT_EQ(0, ret);
       ASSERT_STREQ(itr->second.c_str(), buff);
     }
-    write_command_file("[docker-command-execution]\n  docker-command=run\n  cap-add=SETGID");
+    write_command_file("[docker-command-execution]\n  docker-command=run\n  image=hadoop/docker-image\n  cap-add=SETGID");
     ret = read_config(docker_command_file.c_str(), &cmd_cfg);
     if (ret != 0) {
       FAIL();
@@ -701,7 +723,7 @@ namespace ContainerExecutor {
     ASSERT_EQ(INVALID_DOCKER_CAPABILITY, ret);
     ASSERT_EQ(0, strlen(buff));
 
-    container_executor_cfg_contents = "[docker]\n";
+    container_executor_cfg_contents = "[docker]\n  docker.privileged-containers.registries=hadoop\n";
     write_container_executor_cfg(container_executor_cfg_contents);
     ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
     if (ret != 0) {
@@ -718,20 +740,22 @@ namespace ContainerExecutor {
     const int buff_len = 1024;
     char buff[buff_len];
     int ret = 0;
-    std::string container_executor_cfg_contents = "[docker]\n  docker.allowed.devices=/dev/test-device,/dev/device2";
+    std::string container_executor_cfg_contents = "[docker]\n"
+        "  docker.privileged-containers.registries=hadoop\n"
+        "  docker.allowed.devices=/dev/test-device,/dev/device2";
     std::vector<std::pair<std::string, std::string> > file_cmd_vec;
     file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
-        "[docker-command-execution]\n  docker-command=run\n  devices=/dev/test-device:/dev/test-device",
+        "[docker-command-execution]\n  docker-command=run\n  image=hadoop/image\n  devices=/dev/test-device:/dev/test-device",
         "--device='/dev/test-device:/dev/test-device' "));
     file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
-        "[docker-command-execution]\n  docker-command=run\n  devices=/dev/device2:/dev/device2",
+        "[docker-command-execution]\n  docker-command=run\n  image=hadoop/image\n  devices=/dev/device2:/dev/device2",
         "--device='/dev/device2:/dev/device2' "));
     file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
-        "[docker-command-execution]\n  docker-command=run\n  "
-            "devices=/dev/test-device:/dev/test-device,/dev/device2:/dev/device2",
+        "[docker-command-execution]\n  docker-command=run\n  image=hadoop/image\n"
+            "  devices=/dev/test-device:/dev/test-device,/dev/device2:/dev/device2",
         "--device='/dev/test-device:/dev/test-device' --device='/dev/device2:/dev/device2' "));
     file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
-        "[docker-command-execution]\n  docker-command=run\n", ""));
+        "[docker-command-execution]\n  docker-command=run\n  image=hadoop/image", ""));
     write_container_executor_cfg(container_executor_cfg_contents);
     ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
 
@@ -750,7 +774,7 @@ namespace ContainerExecutor {
       ASSERT_EQ(0, ret);
       ASSERT_STREQ(itr->second.c_str(), buff);
     }
-    write_command_file("[docker-command-execution]\n  docker-command=run\n  devices=/dev/device3:/dev/device3");
+    write_command_file("[docker-command-execution]\n  docker-command=run\n image=nothadoop/image\n  devices=/dev/test-device:/dev/test-device");
     ret = read_config(docker_command_file.c_str(), &cmd_cfg);
     if (ret != 0) {
       FAIL();
@@ -760,7 +784,17 @@ namespace ContainerExecutor {
     ASSERT_EQ(INVALID_DOCKER_DEVICE, ret);
     ASSERT_EQ(0, strlen(buff));
 
-    write_command_file("[docker-command-execution]\n  docker-command=run\n  devices=/dev/device1");
+    write_command_file("[docker-command-execution]\n  docker-command=run\n  image=hadoop/image\n  devices=/dev/device3:/dev/device3");
+    ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+    if (ret != 0) {
+      FAIL();
+    }
+    strcpy(buff, "test string");
+    ret = set_devices(&cmd_cfg, &container_cfg, buff, buff_len);
+    ASSERT_EQ(INVALID_DOCKER_DEVICE, ret);
+    ASSERT_EQ(0, strlen(buff));
+
+    write_command_file("[docker-command-execution]\n  docker-command=run\n  image=hadoop/image\n  devices=/dev/device1");
     ret = read_config(docker_command_file.c_str(), &cmd_cfg);
     if (ret != 0) {
       FAIL();
@@ -788,21 +822,32 @@ namespace ContainerExecutor {
     const int buff_len = 1024;
     char buff[buff_len];
     int ret = 0;
-    std::string container_executor_cfg_contents = "[docker]\n  docker.allowed.rw-mounts=/opt,/var,/usr/bin/cut,..\n  "
+    std::string container_executor_cfg_contents = "[docker]\n  docker.privileged-containers.registries=hadoop\n  "
+                                                              "docker.allowed.rw-mounts=/opt,/var,/usr/bin/cut,..\n  "
                                                               "docker.allowed.ro-mounts=/etc/passwd";
     std::vector<std::pair<std::string, std::string> > file_cmd_vec;
     file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
-        "[docker-command-execution]\n  docker-command=run\n  rw-mounts=/var:/var", "-v '/var:/var' "));
+        "[docker-command-execution]\n  docker-command=run\n  image=hadoop/image\n  rw-mounts=/var:/var", "-v '/var:/var' "));
+    file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+        "[docker-command-execution]\n  docker-command=run\n image=nothadoop/image\n  rw-mounts=/var:/var", ""));
+    file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+        "[docker-command-execution]\n  docker-command=run\n  image=hadoop/image\n  rw-mounts=/var/:/var/", "-v '/var/:/var/' "));
     file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
-        "[docker-command-execution]\n  docker-command=run\n  rw-mounts=/var/:/var/", "-v '/var/:/var/' "));
+        "[docker-command-execution]\n  docker-command=run\n  image=hadoop/image\n  rw-mounts=/usr/bin/cut:/usr/bin/cut",
+        "-v '/usr/bin/cut:/usr/bin/cut' "));
     file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
-        "[docker-command-execution]\n  docker-command=run\n  rw-mounts=/usr/bin/cut:/usr/bin/cut",
-         "-v '/usr/bin/cut:/usr/bin/cut' "));
+        "[docker-command-execution]\n  docker-command=run\n image=nothadoop/image\n  rw-mounts=/lib:/lib",
+        ""));
     file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
-        "[docker-command-execution]\n  docker-command=run\n  rw-mounts=/opt:/mydisk1,/var/log/:/mydisk2",
+        "[docker-command-execution]\n  docker-command=run\n  image=hadoop/image\n  rw-mounts=/opt:/mydisk1,/var/log/:/mydisk2",
         "-v '/opt:/mydisk1' -v '/var/log/:/mydisk2' "));
     file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
-        "[docker-command-execution]\n  docker-command=run\n", ""));
+        "[docker-command-execution]\n  docker-command=run\n image=nothadoop/image\n  rw-mounts=/opt:/mydisk1,/var/log/:/mydisk2",
+        ""));
+    file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+        "[docker-command-execution]\n  docker-command=run\n  image=hadoop/image\n", ""));
+    file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+        "[docker-command-execution]\n  docker-command=run\n image=nothadoop/image\n", ""));
     write_container_executor_cfg(container_executor_cfg_contents);
     ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
     if (ret != 0) {
@@ -831,13 +876,13 @@ namespace ContainerExecutor {
 
     std::vector<std::pair<std::string, int> > bad_file_cmds_vec;
     bad_file_cmds_vec.push_back(std::make_pair<std::string, int>(
-        "[docker-command-execution]\n  docker-command=run\n  rw-mounts=/lib:/lib",
+        "[docker-command-execution]\n  docker-command=run\n  image=hadoop/image\n  rw-mounts=/lib:/lib",
         static_cast<int>(INVALID_DOCKER_RW_MOUNT)));
     bad_file_cmds_vec.push_back(std::make_pair<std::string, int>(
-        "[docker-command-execution]\n  docker-command=run\n  rw-mounts=/usr/bin/:/usr/bin",
+        "[docker-command-execution]\n  docker-command=run\n  image=hadoop/image\n  rw-mounts=/usr/bin/:/usr/bin",
         static_cast<int>(INVALID_DOCKER_RW_MOUNT)));
     bad_file_cmds_vec.push_back(std::make_pair<std::string, int>(
-        "[docker-command-execution]\n  docker-command=run\n  rw-mounts=/blah:/blah",
+        "[docker-command-execution]\n  docker-command=run\n  image=hadoop/image\n  rw-mounts=/blah:/blah",
         static_cast<int>(INVALID_DOCKER_MOUNT)));
 
     std::vector<std::pair<std::string, int> >::const_iterator itr2;
@@ -858,7 +903,7 @@ namespace ContainerExecutor {
     // verify that you can't mount any directory in the container-executor.cfg path
     char *ce_path = realpath("../etc/hadoop/container-executor.cfg", NULL);
     while (strlen(ce_path) != 0) {
-      std::string cmd_file_contents = "[docker-command-execution]\n  docker-command=run\n  rw-mounts=";
+      std::string cmd_file_contents = "[docker-command-execution]\n  docker-command=run\n  image=hadoop/image\n  rw-mounts=";
       cmd_file_contents.append(ce_path).append(":").append("/etc/hadoop");
       memset(buff, 0, buff_len);
       write_command_file(cmd_file_contents);
@@ -877,6 +922,8 @@ namespace ContainerExecutor {
     }
     free(ce_path);
 
+    // For untrusted image, container add_rw_mounts will pass through
+    // without mounting or report error code.
     container_executor_cfg_contents = "[docker]\n";
     write_container_executor_cfg(container_executor_cfg_contents);
     ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
@@ -885,8 +932,8 @@ namespace ContainerExecutor {
     }
     strcpy(buff, "test string");
     ret = add_rw_mounts(&cmd_cfg, &container_cfg, buff, buff_len);
-    ASSERT_EQ(INVALID_DOCKER_RW_MOUNT, ret);
-    ASSERT_EQ(0, strlen(buff));
+    ASSERT_EQ(0, ret);
+    ASSERT_EQ(11, strlen(buff));
   }
 
   TEST_F(TestDockerUtil, test_add_ro_mounts) {
@@ -895,31 +942,36 @@ namespace ContainerExecutor {
     char buff[buff_len];
     int ret = 0;
 
-    std::string container_executor_cfg_contents = "[docker]\n  docker.allowed.rw-mounts=/home/,/var,/usr/bin/cut,..\n  "
+    std::string container_executor_cfg_contents = "[docker]\n  docker.privileged-containers.registries=hadoop\n  "
+                                                              "docker.allowed.rw-mounts=/home/,/var,/usr/bin/cut,..\n  "
                                                               "docker.allowed.ro-mounts=/etc/passwd,/etc/group";
     std::vector<std::pair<std::string, std::string> > file_cmd_vec;
     file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
-        "[docker-command-execution]\n  docker-command=run\n  ro-mounts=/var:/var", "-v '/var:/var:ro' "));
+        "[docker-command-execution]\n  docker-command=run\n  image=hadoop/image\n  ro-mounts=/var:/var", "-v '/var:/var:ro' "));
+    file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+        "[docker-command-execution]\n  docker-command=run\n image=nothadoop/image\n  ro-mounts=/var:/var", ""));
     file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
-        "[docker-command-execution]\n  docker-command=run\n  ro-mounts=/var/:/var/", "-v '/var/:/var/:ro' "));
+        "[docker-command-execution]\n  docker-command=run\n image=nothadoop/image\n  ro-mounts=/etc:/etc", ""));
     file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
-        "[docker-command-execution]\n  docker-command=run\n  ro-mounts=/home:/home", "-v '/home:/home:ro' "));
+        "[docker-command-execution]\n  docker-command=run\n  image=hadoop/image\n  ro-mounts=/var/:/var/", "-v '/var/:/var/:ro' "));
     file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
-        "[docker-command-execution]\n  docker-command=run\n  ro-mounts=/home/:/home", "-v '/home/:/home:ro' "));
+        "[docker-command-execution]\n  docker-command=run\n  image=hadoop/image\n  ro-mounts=/home:/home", "-v '/home:/home:ro' "));
     file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
-        "[docker-command-execution]\n  docker-command=run\n  ro-mounts=/usr/bin/cut:/usr/bin/cut",
+        "[docker-command-execution]\n  docker-command=run\n  image=hadoop/image\n  ro-mounts=/home/:/home", "-v '/home/:/home:ro' "));
+    file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+        "[docker-command-execution]\n  docker-command=run\n  image=hadoop/image\n  ro-mounts=/usr/bin/cut:/usr/bin/cut",
         "-v '/usr/bin/cut:/usr/bin/cut:ro' "));
     file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
-        "[docker-command-execution]\n  docker-command=run\n  ro-mounts=/etc/group:/etc/group",
+        "[docker-command-execution]\n  docker-command=run\n  image=hadoop/image\n  ro-mounts=/etc/group:/etc/group",
         "-v '/etc/group:/etc/group:ro' "));
     file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
-        "[docker-command-execution]\n  docker-command=run\n  ro-mounts=/etc/passwd:/etc/passwd",
+        "[docker-command-execution]\n  docker-command=run\n  image=hadoop/image\n  ro-mounts=/etc/passwd:/etc/passwd",
         "-v '/etc/passwd:/etc/passwd:ro' "));
     file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
-        "[docker-command-execution]\n  docker-command=run\n  ro-mounts=/var/log:/mydisk1,/etc/passwd:/etc/passwd",
+        "[docker-command-execution]\n  docker-command=run\n  image=hadoop/image\n  ro-mounts=/var/log:/mydisk1,/etc/passwd:/etc/passwd",
         "-v '/var/log:/mydisk1:ro' -v '/etc/passwd:/etc/passwd:ro' "));
     file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
-        "[docker-command-execution]\n  docker-command=run\n", ""));
+        "[docker-command-execution]\n  docker-command=run\n  image=hadoop/image\n", ""));
     write_container_executor_cfg(container_executor_cfg_contents);
     ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
     if (ret != 0) {
@@ -948,10 +1000,10 @@ namespace ContainerExecutor {
 
     std::vector<std::pair<std::string, int> > bad_file_cmds_vec;
     bad_file_cmds_vec.push_back(std::make_pair<std::string, int>(
-        "[docker-command-execution]\n  docker-command=run\n  ro-mounts=/etc:/etc",
+        "[docker-command-execution]\n  docker-command=run\n  image=hadoop/image\n  ro-mounts=/etc:/etc",
         static_cast<int>(INVALID_DOCKER_RO_MOUNT)));
     bad_file_cmds_vec.push_back(std::make_pair<std::string, int>(
-        "[docker-command-execution]\n  docker-command=run\n  ro-mounts=/blah:/blah",
+        "[docker-command-execution]\n  docker-command=run\n  image=hadoop/image\n  ro-mounts=/blah:/blah",
         static_cast<int>(INVALID_DOCKER_MOUNT)));
 
     std::vector<std::pair<std::string, int> >::const_iterator itr2;
@@ -969,13 +1021,13 @@ namespace ContainerExecutor {
       ASSERT_STREQ("", buff);
     }
 
-    container_executor_cfg_contents = "[docker]\n";
+    container_executor_cfg_contents = "[docker]\n  docker.privileged-containers.registries=hadoop\n";
     write_container_executor_cfg(container_executor_cfg_contents);
     ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
     if (ret != 0) {
       FAIL();
     }
-    write_command_file("[docker-command-execution]\n  docker-command=run\n  ro-mounts=/home:/home");
+    write_command_file("[docker-command-execution]\n  docker-command=run\n  image=hadoop/image\n  ro-mounts=/home:/home");
     strcpy(buff, "test string");
     ret = add_ro_mounts(&cmd_cfg, &container_cfg, buff, buff_len);
     ASSERT_EQ(INVALID_DOCKER_RO_MOUNT, ret);
@@ -987,7 +1039,7 @@ namespace ContainerExecutor {
     std::string container_executor_contents = "[docker]\n  docker.allowed.ro-mounts=/var,/etc,/usr/bin/cut\n"
         "  docker.allowed.rw-mounts=/tmp\n  docker.allowed.networks=bridge\n "
         "  docker.privileged-containers.enabled=1\n  docker.allowed.capabilities=CHOWN,SETUID\n"
-        "  docker.allowed.devices=/dev/test";
+        "  docker.allowed.devices=/dev/test\n  docker.privileged-containers.registries=hadoop\n";
     write_file(container_executor_cfg_file, container_executor_contents);
     int ret = read_config(container_executor_cfg_file.c_str(), &container_executor_cfg);
     if (ret != 0) {
@@ -1001,53 +1053,74 @@ namespace ContainerExecutor {
 
     std::vector<std::pair<std::string, std::string> > file_cmd_vec;
     file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
-        "[docker-command-execution]\n  docker-command=run\n  name=container_e1_12312_11111_02_000001\n  image=docker-image\n  user=test",
-        "run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'docker-image' "));
+        "[docker-command-execution]\n  docker-command=run\n  name=container_e1_12312_11111_02_000001\n  image=hadoop/docker-image\n  user=test",
+        "run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'hadoop/docker-image' "));
     file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
-        "[docker-command-execution]\n  docker-command=run\n  name=container_e1_12312_11111_02_000001\n  image=docker-image\n  user=test\n"
+        "[docker-command-execution]\n  docker-command=run\n name=container_e1_12312_11111_02_000001\n image=nothadoop/docker-image\n  user=test",
+        "run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'nothadoop/docker-image' "));
+    file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+        "[docker-command-execution]\n  docker-command=run\n  name=container_e1_12312_11111_02_000001\n  image=hadoop/docker-image\n  user=test\n"
             "  launch-command=bash,test_script.sh,arg1,arg2",
-        "run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'docker-image' 'bash' 'test_script.sh' 'arg1' 'arg2' "));
+        "run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'hadoop/docker-image' 'bash' 'test_script.sh' 'arg1' 'arg2' "));
 
     file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
         "[docker-command-execution]\n"
-            "  docker-command=run\n  name=container_e1_12312_11111_02_000001\n  image=docker-image\n  user=test\n  hostname=host-id\n"
+            "  docker-command=run\n  name=container_e1_12312_11111_02_000001\n  image=hadoop/docker-image\n  user=test\n  hostname=host-id\n"
             "  ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n  rw-mounts=/tmp:/tmp\n"
             "  network=bridge\n  devices=/dev/test:/dev/test\n"
             "  cap-add=CHOWN,SETUID\n  cgroup-parent=ctr-cgroup\n  detach=true\n  rm=true\n"
             "  launch-command=bash,test_script.sh,arg1,arg2",
         "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'"
             " -v '/usr/bin/cut:/usr/bin/cut:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --cap-add='CHOWN'"
-            " --cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'docker-image' 'bash' "
+            " --cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'hadoop/docker-image' 'bash' "
             "'test_script.sh' 'arg1' 'arg2' "));
+    file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+        "[docker-command-execution]\n"
+            "  docker-command=run\n  name=container_e1_12312_11111_02_000001\n image=nothadoop/docker-image\n  user=test\n  hostname=host-id\n"
+            "  ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n  rw-mounts=/tmp:/tmp\n"
+            "  network=bridge\n"
+            "  cap-add=CHOWN,SETUID\n  cgroup-parent=ctr-cgroup\n  detach=true\n  rm=true\n"
+            "  launch-command=bash,test_script.sh,arg1,arg2",
+        "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm"
+            " --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --hostname='host-id' 'nothadoop/docker-image' "));
 
     file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
         "[docker-command-execution]\n"
-            "  docker-command=run\n  name=container_e1_12312_11111_02_000001\n  image=docker-image\n  user=test\n  hostname=host-id\n"
+            "  docker-command=run\n  name=container_e1_12312_11111_02_000001\n  image=hadoop/docker-image\n  user=test\n  hostname=host-id\n"
             "  ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n  rw-mounts=/tmp:/tmp\n"
             "  network=bridge\n  devices=/dev/test:/dev/test\n  net=bridge\n"
             "  cap-add=CHOWN,SETUID\n  cgroup-parent=ctr-cgroup\n  detach=true\n  rm=true\n"
             "  launch-command=bash,test_script.sh,arg1,arg2",
         "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge' -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'"
             " -v '/usr/bin/cut:/usr/bin/cut:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --cap-add='CHOWN' "
-            "--cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'docker-image' 'bash'"
+            "--cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'hadoop/docker-image' 'bash'"
             " 'test_script.sh' 'arg1' 'arg2' "));
+    file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+        "[docker-command-execution]\n"
+            "  docker-command=run\n  name=container_e1_12312_11111_02_000001\n image=nothadoop/docker-image\n  user=test\n  hostname=host-id\n"
+            "  ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n  rw-mounts=/tmp:/tmp\n"
+            "  network=bridge\n  net=bridge\n"
+            "  cap-add=CHOWN,SETUID\n  cgroup-parent=ctr-cgroup\n  detach=true\n  rm=true\n"
+            "  launch-command=bash,test_script.sh,arg1,arg2",
+        "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge'"
+            " --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --hostname='host-id' 'nothadoop/docker-image' "));
 
     file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
         "[docker-command-execution]\n"
-            "  docker-command=run\n  name=container_e1_12312_11111_02_000001\n  image=docker-image\n  user=test\n  hostname=host-id\n"
+            "  docker-command=run\n  name=container_e1_12312_11111_02_000001\n  image=hadoop/docker-image\n  user=test\n  hostname=host-id\n"
             "  ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n  rw-mounts=/tmp:/tmp\n"
             "  network=bridge\n  devices=/dev/test:/dev/test\n  net=bridge\n  privileged=true\n"
             "  cap-add=CHOWN,SETUID\n  cgroup-parent=ctr-cgroup\n  detach=true\n  rm=true\n"
             "  launch-command=bash,test_script.sh,arg1,arg2",
         "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge' -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'"
             " -v '/usr/bin/cut:/usr/bin/cut:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --privileged --cap-drop='ALL' "
-            "--cap-add='CHOWN' --cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'docker-image' "
+            "--cap-add='CHOWN' --cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'hadoop/docker-image' "
             "'bash' 'test_script.sh' 'arg1' 'arg2' "));
 
 
     file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
         "[docker-command-execution]\n"
-            "  docker-command=run\n  name=container_e1_12312_11111_02_000001\n  image=docker-image\n  user=test\n  hostname=host-id\n"
+            "  docker-command=run\n  name=container_e1_12312_11111_02_000001\n  image=hadoop/docker-image\n  user=test\n  hostname=host-id\n"
             "  ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n  rw-mounts=/tmp:/tmp\n"
             "  network=bridge\n  devices=/dev/test:/dev/test\n  net=bridge\n  privileged=true\n"
             "  cap-add=CHOWN,SETUID\n  cgroup-parent=ctr-cgroup\n  detach=true\n  rm=true\n  group-add=1000,1001\n"
@@ -1055,25 +1128,42 @@ namespace ContainerExecutor {
         "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge' -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'"
             " -v '/usr/bin/cut:/usr/bin/cut:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --privileged --cap-drop='ALL' "
             "--cap-add='CHOWN' --cap-add='SETUID' --hostname='host-id' --group-add '1000' --group-add '1001' "
-            "--device='/dev/test:/dev/test' 'docker-image' 'bash' 'test_script.sh' 'arg1' 'arg2' "));
+            "--device='/dev/test:/dev/test' 'hadoop/docker-image' 'bash' 'test_script.sh' 'arg1' 'arg2' "));
 
+    file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+        "[docker-command-execution]\n"
+            "  docker-command=run\n  name=container_e1_12312_11111_02_000001\n  image=docker-image\n  user=test\n  hostname=host-id\n"
+            "  network=bridge\n  net=bridge\n"
+            "  detach=true\n  rm=true\n  group-add=1000,1001\n"
+            "  launch-command=bash,test_script.sh,arg1,arg2",
+        "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge' --cap-drop='ALL' "
+            "--hostname='host-id' --group-add '1000' --group-add '1001' "
+            "'docker-image' "));
 
     std::vector<std::pair<std::string, int> > bad_file_cmd_vec;
 
     bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
-        "[docker-command-execution]\n  docker-command=run\n  image=docker-image\n  user=test",
+        "[docker-command-execution]\n  docker-command=run\n  image=hadoop/docker-image\n  user=test",
         static_cast<int>(INVALID_DOCKER_CONTAINER_NAME)));
     bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
         "[docker-command-execution]\n  docker-command=run\n  name=container_e1_12312_11111_02_000001\n  user=test\n",
         static_cast<int>(INVALID_DOCKER_IMAGE_NAME)));
     bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
-        "[docker-command-execution]\n  docker-command=run\n  name=container_e1_12312_11111_02_000001\n  image=docker-image\n",
+        "[docker-command-execution]\n  docker-command=run\n  name=container_e1_12312_11111_02_000001\n  image=hadoop/docker-image\n",
         static_cast<int>(INVALID_DOCKER_USER_NAME)));
+    bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+        "[docker-command-execution]\n"
+            "  docker-command=run\n  name=container_e1_12312_11111_02_000001\n image=nothadoop/docker-image\n  user=test\n  hostname=host-id\n"
+            "  ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n  rw-mounts=/tmp:/tmp\n"
+            "  network=bridge\n  net=bridge\n  privileged=true\n"
+            "  cap-add=CHOWN,SETUID\n  cgroup-parent=ctr-cgroup\n  detach=true\n  rm=true\n  group-add=1000,1001\n"
+            "  launch-command=bash,test_script.sh,arg1,arg2",
+        PRIVILEGED_CONTAINERS_DISABLED));
 
     // invalid rw mount
     bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
         "[docker-command-execution]\n"
-            "  docker-command=run\n  name=container_e1_12312_11111_02_000001\n  image=docker-image\n  user=test\n  hostname=host-id\n"
+            "  docker-command=run\n  name=container_e1_12312_11111_02_000001\n  image=hadoop/docker-image\n  user=test\n  hostname=host-id\n"
             "  ro-mounts=/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n  rw-mounts=/var/log:/var/log\n"
             "  network=bridge\n  devices=/dev/test:/dev/test\n"
             "  cap-add=CHOWN,SETUID\n  cgroup-parent=ctr-cgroup\n  detach=true\n  rm=true\n"
@@ -1083,7 +1173,7 @@ namespace ContainerExecutor {
     // invalid ro mount
     bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
         "[docker-command-execution]\n"
-            "  docker-command=run\n  name=container_e1_12312_11111_02_000001\n  image=docker-image\n  user=test\n  hostname=host-id\n"
+            "  docker-command=run\n  name=container_e1_12312_11111_02_000001\n  image=hadoop/docker-image\n  user=test\n  hostname=host-id\n"
             "  ro-mounts=/bin:/bin,/usr/bin/cut:/usr/bin/cut\n  rw-mounts=/tmp:/tmp\n"
             "  network=bridge\n  devices=/dev/test:/dev/test\n"
             "  cap-add=CHOWN,SETUID\n  cgroup-parent=ctr-cgroup\n  detach=true\n  rm=true\n"
@@ -1093,7 +1183,7 @@ namespace ContainerExecutor {
     // invalid capability
     bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
         "[docker-command-execution]\n"
-            "  docker-command=run\n  name=container_e1_12312_11111_02_000001\n  image=docker-image\n  user=test\n  hostname=host-id\n"
+            "  docker-command=run\n  name=container_e1_12312_11111_02_000001\n  image=hadoop/docker-image\n  user=test\n  hostname=host-id\n"
             "  ro-mounts=/usr/bin/cut:/usr/bin/cut\n  rw-mounts=/tmp:/tmp\n"
             "  network=bridge\n  devices=/dev/test:/dev/test\n"
             "  cap-add=CHOWN,SETUID,SETGID\n  cgroup-parent=ctr-cgroup\n  detach=true\n  rm=true\n"
@@ -1103,7 +1193,7 @@ namespace ContainerExecutor {
     // invalid device
     bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
         "[docker-command-execution]\n"
-            "  docker-command=run\n  name=container_e1_12312_11111_02_000001\n  image=docker-image\n  user=test\n  hostname=host-id\n"
+            "  docker-command=run\n  name=container_e1_12312_11111_02_000001\n  image=hadoop/docker-image\n  user=test\n  hostname=host-id\n"
             "  ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n  rw-mounts=/tmp:/tmp\n"
             "  network=bridge\n  devices=/dev/dev1:/dev/dev1\n  privileged=true\n"
             "  cap-add=CHOWN,SETUID\n  cgroup-parent=ctr-cgroup\n  detach=true\n  rm=true\n"
@@ -1113,7 +1203,7 @@ namespace ContainerExecutor {
     // invalid network
     bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
         "[docker-command-execution]\n"
-            "  docker-command=run\n  name=container_e1_12312_11111_02_000001\n  image=docker-image\n  user=test\n  hostname=host-id\n"
+            "  docker-command=run\n  name=container_e1_12312_11111_02_000001\n  image=hadoop/docker-image\n  user=test\n  hostname=host-id\n"
             "  ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n  rw-mounts=/tmp:/tmp\n"
             "  network=bridge\n  devices=/dev/test:/dev/test\n  privileged=true\n  net=host\n"
             "  cap-add=CHOWN,SETUID\n  cgroup-parent=ctr-cgroup\n  detach=true\n  rm=true\n"
@@ -1126,11 +1216,13 @@ namespace ContainerExecutor {
   TEST_F(TestDockerUtil, test_docker_run_no_privileged) {
 
     std::string container_executor_contents[] = {"[docker]\n  docker.allowed.ro-mounts=/var,/etc,/usr/bin/cut\n"
-                                                     "  docker.allowed.rw-mounts=/tmp\n  docker.allowed.networks=bridge\n "
+                                                     "  docker.privileged-containers.registries=hadoop\n"
+                                                     "  docker.allowed.rw-mounts=/tmp\n  docker.allowed.networks=bridge\n"
                                                      "  docker.allowed.capabilities=CHOWN,SETUID\n"
                                                      "  docker.allowed.devices=/dev/test",
                                                  "[docker]\n  docker.allowed.ro-mounts=/var,/etc,/usr/bin/cut\n"
-                                                     "  docker.allowed.rw-mounts=/tmp\n  docker.allowed.networks=bridge\n "
+                                                     "  docker.privileged-containers.registries=hadoop\n"
+                                                     "  docker.allowed.rw-mounts=/tmp\n  docker.allowed.networks=bridge\n"
                                                      "  docker.allowed.capabilities=CHOWN,SETUID\n"
                                                      "  privileged=0\n"
                                                      "  docker.allowed.devices=/dev/test"};
@@ -1153,36 +1245,54 @@ namespace ContainerExecutor {
       file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
           "[docker-command-execution]\n  docker-command=run\n  name=container_e1_12312_11111_02_000001\n  image=docker-image\n"
               "  user=test\n  launch-command=bash,test_script.sh,arg1,arg2",
-          "run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'docker-image' 'bash' 'test_script.sh' 'arg1' 'arg2' "));
+          "run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'docker-image' "));
 
       file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
           "[docker-command-execution]\n"
-              "  docker-command=run\n  name=container_e1_12312_11111_02_000001\n  image=docker-image\n  user=test\n  hostname=host-id\n"
+              "  docker-command=run\n  name=container_e1_12312_11111_02_000001\n  image=hadoop/docker-image\n  user=test\n  hostname=host-id\n"
               "  ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n  rw-mounts=/tmp:/tmp\n"
               "  network=bridge\n  devices=/dev/test:/dev/test\n"
               "  cap-add=CHOWN,SETUID\n  cgroup-parent=ctr-cgroup\n  detach=true\n  rm=true\n"
               "  launch-command=bash,test_script.sh,arg1,arg2",
           "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'"
               " -v '/usr/bin/cut:/usr/bin/cut:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --cap-add='CHOWN'"
-              " --cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'docker-image' 'bash' "
+              " --cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'hadoop/docker-image' 'bash' "
               "'test_script.sh' 'arg1' 'arg2' "));
+      file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+          "[docker-command-execution]\n"
+              "  docker-command=run\n  name=container_e1_12312_11111_02_000001\n image=nothadoop/docker-image\n  user=test\n  hostname=host-id\n"
+              "  ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n  rw-mounts=/tmp:/tmp\n"
+              "  network=bridge\n"
+              "  cap-add=CHOWN,SETUID\n  cgroup-parent=ctr-cgroup\n  detach=true\n  rm=true\n"
+              "  launch-command=bash,test_script.sh,arg1,arg2",
+          "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm"
+              " --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --hostname='host-id' 'nothadoop/docker-image' "));
 
       file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
           "[docker-command-execution]\n"
-              "  docker-command=run\n  name=container_e1_12312_11111_02_000001\n  image=docker-image\n  user=test\n  hostname=host-id\n"
+              "  docker-command=run\n  name=container_e1_12312_11111_02_000001\n  image=hadoop/docker-image\n  user=test\n  hostname=host-id\n"
               "  ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n  rw-mounts=/tmp:/tmp\n"
               "  network=bridge\n  devices=/dev/test:/dev/test\n  net=bridge\n"
               "  cap-add=CHOWN,SETUID\n  cgroup-parent=ctr-cgroup\n  detach=true\n  rm=true\n"
               "  launch-command=bash,test_script.sh,arg1,arg2",
           "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge' -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'"
               " -v '/usr/bin/cut:/usr/bin/cut:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --cap-add='CHOWN' "
-              "--cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'docker-image' 'bash'"
+              "--cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'hadoop/docker-image' 'bash'"
               " 'test_script.sh' 'arg1' 'arg2' "));
+      file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+          "[docker-command-execution]\n"
+              "  docker-command=run\n  name=container_e1_12312_11111_02_000001\n image=nothadoop/docker-image\n  user=test\n  hostname=host-id\n"
+              "  ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n  rw-mounts=/tmp:/tmp\n"
+              "  network=bridge\n  net=bridge\n"
+              "  cap-add=CHOWN,SETUID\n  cgroup-parent=ctr-cgroup\n  detach=true\n  rm=true\n"
+              "  launch-command=bash,test_script.sh,arg1,arg2",
+          "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge'"
+              " --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --hostname='host-id' 'nothadoop/docker-image' "));
 
       std::vector<std::pair<std::string, int> > bad_file_cmd_vec;
       bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
           "[docker-command-execution]\n"
-              "  docker-command=run\n  name=container_e1_12312_11111_02_000001\n  image=docker-image\n  user=test\n  hostname=host-id\n"
+              "  docker-command=run\n  name=container_e1_12312_11111_02_000001\n  image=hadoop/docker-image\n  user=test\n  hostname=host-id\n"
               "  ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n  rw-mounts=/tmp:/tmp\n"
               "  network=bridge\n  devices=/dev/test:/dev/test\n  net=bridge\n  privileged=true\n"
               "  cap-add=CHOWN,SETUID\n  cgroup-parent=ctr-cgroup\n  detach=true\n  rm=true\n"

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md

@@ -206,6 +206,7 @@ are allowed. It contains the following properties:
 | `docker.allowed.rw-mounts` | Comma separated directories that containers are allowed to mount in read-write mode. By default, no directories are allowed to mounted. |
 | `docker.host-pid-namespace.enabled` | Set to "true" or "false" to enable or disable using the host's PID namespace. Default value is "false". |
 | `docker.privileged-containers.enabled` | Set to "true" or "false" to enable or disable launching privileged containers. Default value is "false". |
+| `docker.privileged-containers.registries` | Comma separated list of trusted docker registries for running trusted privileged docker containers.  By default, no registries are defined. |
 
 Please note that if you wish to run Docker containers that require access to the YARN local directories, you must add them to the docker.allowed.rw-mounts list.
 
@@ -226,6 +227,8 @@ Part of a container-executor.cfg which allows Docker containers to be launched i
 yarn.nodemanager.linux-container-executor.group=yarn
 [docker]
   module.enabled=true
+  docker.privileged-containers.enabled=true
+  docker.privileged-containers.registries=centos
   docker.allowed.capabilities=SYS_CHROOT,MKNOD,SETFCAP,SETPCAP,FSETID,CHOWN,AUDIT_WRITE,SETGID,NET_RAW,FOWNER,SETUID,DAC_OVERRIDE,KILL,NET_BIND_SERVICE
   docker.allowed.networks=bridge,host,none
   docker.allowed.ro-mounts=/sys/fs/cgroup
@@ -361,6 +364,19 @@ the environment variable would be set to "/sys/fs/cgroup:/sys/fs/cgroup:ro".
 The destination path is not restricted, "/sys/fs/cgroup:/cgroup:ro" would also
 be valid given the example admin whitelist.
 
+Privileged Container Security Consideration
+-------------------------------------------
+
+Privileged docker container can interact with host system devices.  This can cause harm to host operating system without proper care.  In order to mitigate risk of allowing privileged container to run on Hadoop cluster, we implemented a controlled process to sandbox unauthorized privileged docker images.
+
+The default behavior is disallow any privileged docker containers.  When `docker.privileged-containers.enabled` is set to enabled, docker image can run with root privileges in the docker container, but access to host level devices are disabled.  This allows developer and tester to run docker images from internet without causing harm to host operating system.
+
+When docker images have been certified by developers and testers to be trustworthy.  The trusted image can be promoted to trusted docker registry.  System administrator can define `docker.privileged-containers.registries`, and setup private docker registry server to promote trusted images.
+
+Trusted images are allowed to mount external devices such as HDFS via NFS gateway, or host level Hadoop configuration.  If system administrators allow writing to external volumes using `docker.allow.rw-mounts directive`, privileged docker container can have full control of host level files in the predefined volumes.
+
+For [YARN Service HTTPD example](./yarn-service/Examples.html), container-executor.cfg must define centos docker registry to be trusted for the example to run.
+
 Connecting to a Secure Docker Repository
 ----------------------------------------
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md

@@ -19,6 +19,10 @@ This document describes some example service definitions (`Yarnfile`).
 <!-- MACRO{toc|fromDepth=0|toDepth=3} -->
 
 ## Apache web server - httpd (with registry DNS)
+
+For this example to work, centos/httpd-24-centos7 image must be included in `docker.privileged-containers.registries`.
+For server side configuration, please refer to [Running Applications in Docker Containers](../DockerContainers.html) document.
+
 Below is the `Yarnfile` for a service called `httpd-service` with two `httpd` instances.
 There is also an httpd proxy instance (httpd-proxy-0) that proxies between the other two httpd instances (httpd-0 and httpd-1).