YARN-6302. Fail the node if Linux Container Executor is not configured properly

(Contributed by Miklos Szegedi via Daniel Templeton)

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ConfigurationException.java

@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.exceptions;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
+
+/**
+ * This exception is thrown on unrecoverable configuration errors.
+ * An example is container launch error due to configuration.
+ */
+@Public
+@Evolving
+public class ConfigurationException extends YarnException {
+
+  private static final long serialVersionUID = 0x9801a7a0f8e3L;
+
+  public ConfigurationException(Throwable cause) {
+    super(cause);
+  }
+
+  public ConfigurationException(String message) {
+    super(message);
+  }
+
+  public ConfigurationException(String message, Throwable cause) {
+    super(message, cause);
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java

@@ -48,6 +48,7 @@ import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.ConfigurationException;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
@@ -156,9 +157,10 @@ public abstract class ContainerExecutor implements Configurable {
    * @param ctx Encapsulates information necessary for launching containers.
    * @return the return status of the launch
    * @throws IOException if the container launch fails
+   * @throws ConfigurationException if config error was found
    */
   public abstract int launchContainer(ContainerStartContext ctx) throws
-      IOException;
+      IOException, ConfigurationException;
 
   /**
    * Signal container with the specified signal.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java

@@ -51,6 +51,7 @@ import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.ConfigurationException;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent;
@@ -209,7 +210,8 @@ public class DefaultContainerExecutor extends ContainerExecutor {
   }
 
   @Override
-  public int launchContainer(ContainerStartContext ctx) throws IOException {
+  public int launchContainer(ContainerStartContext ctx)
+      throws IOException, ConfigurationException {
     Container container = ctx.getContainer();
     Path nmPrivateContainerScriptPath = ctx.getNmPrivateContainerScriptPath();
     Path nmPrivateTokensPath = ctx.getNmPrivateTokensPath();
@@ -291,8 +293,7 @@ public class DefaultContainerExecutor extends ContainerExecutor {
       
       if (isContainerActive(containerId)) {
         shExec.execute();
-      }
-      else {
+      } else {
         LOG.info("Container " + containerIdStr +
             " was marked as inactive. Returning terminated error");
         return ExitCode.TERMINATED.getExitCode();

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java

@@ -30,6 +30,7 @@ import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.ConfigurationException;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
@@ -109,6 +110,58 @@ public class LinuxContainerExecutor extends ContainerExecutor {
   private ResourceHandler resourceHandlerChain;
   private LinuxContainerRuntime linuxContainerRuntime;
 
+  /**
+   * The container exit code.
+   */
+  public enum ExitCode {
+    SUCCESS(0),
+    INVALID_ARGUMENT_NUMBER(1),
+    INVALID_COMMAND_PROVIDED(3),
+    INVALID_NM_ROOT_DIRS(5),
+    SETUID_OPER_FAILED(6),
+    UNABLE_TO_EXECUTE_CONTAINER_SCRIPT(7),
+    UNABLE_TO_SIGNAL_CONTAINER(8),
+    INVALID_CONTAINER_PID(9),
+    OUT_OF_MEMORY(18),
+    INITIALIZE_USER_FAILED(20),
+    PATH_TO_DELETE_IS_NULL(21),
+    INVALID_CONTAINER_EXEC_PERMISSIONS(22),
+    INVALID_CONFIG_FILE(24),
+    SETSID_OPER_FAILED(25),
+    WRITE_PIDFILE_FAILED(26),
+    WRITE_CGROUP_FAILED(27),
+    TRAFFIC_CONTROL_EXECUTION_FAILED(28),
+    DOCKER_RUN_FAILED(29),
+    ERROR_OPENING_DOCKER_FILE(30),
+    ERROR_READING_DOCKER_FILE(31),
+    FEATURE_DISABLED(32),
+    COULD_NOT_CREATE_SCRIPT_COPY(33),
+    COULD_NOT_CREATE_CREDENTIALS_FILE(34),
+    COULD_NOT_CREATE_WORK_DIRECTORIES(35),
+    COULD_NOT_CREATE_APP_LOG_DIRECTORIES(36),
+    COULD_NOT_CREATE_TMP_DIRECTORIES(37),
+    ERROR_CREATE_CONTAINER_DIRECTORIES_ARGUMENTS(38);
+
+    private final int code;
+
+    ExitCode(int exitCode) {
+      this.code = exitCode;
+    }
+
+    /**
+     * Get the exit code as an int.
+     * @return the exit code as an int
+     */
+    public int getExitCode() {
+      return code;
+    }
+
+    @Override
+    public String toString() {
+      return String.valueOf(code);
+    }
+  }
+
   /**
    * Default constructor to allow for creation through reflection.
    */
@@ -363,7 +416,8 @@ public class LinuxContainerExecutor extends ContainerExecutor {
   }
 
   @Override
-  public int launchContainer(ContainerStartContext ctx) throws IOException {
+  public int launchContainer(ContainerStartContext ctx)
+      throws IOException, ConfigurationException {
     Container container = ctx.getContainer();
     Path nmPrivateContainerScriptPath = ctx.getNmPrivateContainerScriptPath();
     Path nmPrivateTokensPath = ctx.getNmPrivateTokensPath();
@@ -473,7 +527,7 @@ public class LinuxContainerExecutor extends ContainerExecutor {
       } else {
         LOG.info(
             "Container was marked as inactive. Returning terminated error");
-        return ExitCode.TERMINATED.getExitCode();
+        return ContainerExecutor.ExitCode.TERMINATED.getExitCode();
       }
     } catch (ContainerExecutionException e) {
       int exitCode = e.getExitCode();
@@ -481,8 +535,8 @@ public class LinuxContainerExecutor extends ContainerExecutor {
       // 143 (SIGTERM) and 137 (SIGKILL) exit codes means the container was
       // terminated/killed forcefully. In all other cases, log the
       // output
-      if (exitCode != ExitCode.FORCE_KILLED.getExitCode()
-          && exitCode != ExitCode.TERMINATED.getExitCode()) {
+      if (exitCode != ContainerExecutor.ExitCode.FORCE_KILLED.getExitCode()
+          && exitCode != ContainerExecutor.ExitCode.TERMINATED.getExitCode()) {
         LOG.warn("Exception from container-launch with container ID: "
             + containerId + " and exit code: " + exitCode, e);
 
@@ -502,6 +556,23 @@ public class LinuxContainerExecutor extends ContainerExecutor {
         logOutput(diagnostics);
         container.handle(new ContainerDiagnosticsUpdateEvent(containerId,
             diagnostics));
+        if (exitCode ==
+                ExitCode.INVALID_CONTAINER_EXEC_PERMISSIONS.getExitCode() ||
+            exitCode ==
+                ExitCode.INVALID_CONFIG_FILE.getExitCode() ||
+            exitCode ==
+                ExitCode.COULD_NOT_CREATE_SCRIPT_COPY.getExitCode() ||
+            exitCode ==
+                ExitCode.COULD_NOT_CREATE_CREDENTIALS_FILE.getExitCode() ||
+            exitCode ==
+                ExitCode.COULD_NOT_CREATE_WORK_DIRECTORIES.getExitCode() ||
+            exitCode ==
+                ExitCode.COULD_NOT_CREATE_APP_LOG_DIRECTORIES.getExitCode() ||
+            exitCode ==
+                ExitCode.COULD_NOT_CREATE_TMP_DIRECTORIES.getExitCode()) {
+          throw new ConfigurationException(
+              "Linux Container Executor reached unrecoverable exception", e);
+        }
       } else {
         container.handle(new ContainerDiagnosticsUpdateEvent(containerId,
             "Container killed on request. Exit code is " + exitCode));

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeHealthCheckerService.java

@@ -18,10 +18,15 @@
 
 package org.apache.hadoop.yarn.server.nodemanager;
 
+import com.google.common.base.Joiner;
+import com.google.common.base.Strings;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.service.CompositeService;
 import org.apache.hadoop.util.NodeHealthScriptRunner;
 
+import java.util.Arrays;
+import java.util.Collections;
+
 /**
  * The class which provides functionality of checking the health of the node and
  * reporting back to the service for which the health checker has been asked to
@@ -31,6 +36,8 @@ public class NodeHealthCheckerService extends CompositeService {
 
   private NodeHealthScriptRunner nodeHealthScriptRunner;
   private LocalDirsHandlerService dirsHandler;
+  private Exception nodeHealthException;
+  private long nodeHealthExceptionReportTime;
 
   static final String SEPARATOR = ";";
 
@@ -39,6 +46,8 @@ public class NodeHealthCheckerService extends CompositeService {
     super(NodeHealthCheckerService.class.getName());
     nodeHealthScriptRunner = scriptRunner;
     dirsHandler = dirHandlerService;
+    nodeHealthException = null;
+    nodeHealthExceptionReportTime = 0;
   }
 
   @Override
@@ -54,33 +63,39 @@ public class NodeHealthCheckerService extends CompositeService {
    * @return the reporting string of health of the node
    */
   String getHealthReport() {
-    String scriptReport = (nodeHealthScriptRunner == null) ? ""
-        : nodeHealthScriptRunner.getHealthReport();
-    if (scriptReport.equals("")) {
-      return dirsHandler.getDisksHealthReport(false);
-    } else {
-      return scriptReport.concat(SEPARATOR + dirsHandler.getDisksHealthReport(false));
-    }
+    String scriptReport = Strings.emptyToNull(
+        nodeHealthScriptRunner == null ? null :
+        nodeHealthScriptRunner.getHealthReport());
+    String discReport =
+        Strings.emptyToNull(
+            dirsHandler.getDisksHealthReport(false));
+    String exceptionReport = Strings.emptyToNull(
+        nodeHealthException == null ? null :
+        nodeHealthException.getMessage());
+
+    return Joiner.on(SEPARATOR).skipNulls()
+        .join(scriptReport, discReport, exceptionReport);
   }
 
   /**
    * @return <em>true</em> if the node is healthy
    */
   boolean isHealthy() {
-    boolean scriptHealthStatus = (nodeHealthScriptRunner == null) ? true
-        : nodeHealthScriptRunner.isHealthy();
-    return scriptHealthStatus && dirsHandler.areDisksHealthy();
+    boolean scriptHealthy = nodeHealthScriptRunner == null ||
+        nodeHealthScriptRunner.isHealthy();
+    return nodeHealthException == null &&
+        scriptHealthy && dirsHandler.areDisksHealthy();
   }
 
   /**
    * @return when the last time the node health status is reported
    */
   long getLastHealthReportTime() {
-    long diskCheckTime = dirsHandler.getLastDisksCheckTime();
-    long lastReportTime = (nodeHealthScriptRunner == null)
-        ? diskCheckTime
-        : Math.max(nodeHealthScriptRunner.getLastReportedTime(), diskCheckTime);
-    return lastReportTime;
+    return Collections.max(Arrays.asList(
+        dirsHandler.getLastDisksCheckTime(),
+        nodeHealthScriptRunner == null ? 0 :
+            nodeHealthScriptRunner.getLastReportedTime(),
+        nodeHealthExceptionReportTime));
   }
 
   /**
@@ -96,4 +111,13 @@ public class NodeHealthCheckerService extends CompositeService {
   NodeHealthScriptRunner getNodeHealthScriptRunner() {
     return nodeHealthScriptRunner;
   }
+
+  /**
+   * Report an exception to mark the node as unhealthy.
+   * @param ex the exception that makes the node unhealthy
+   */
+  void reportException(Exception ex) {
+    nodeHealthException = ex;
+    nodeHealthExceptionReportTime = System.currentTimeMillis();
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdater.java

@@ -53,4 +53,10 @@ public interface NodeStatusUpdater extends Service {
    * Clear the list of recently completed containers
    */
   public void clearFinishedContainersFromCache();
+
+  /**
+   * Report an unrecoverable exception.
+   * @param ex exception that makes the node unhealthy
+   */
+  void reportException(Exception ex);
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java

@@ -943,6 +943,12 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
     return false;
   }
 
+  @Override
+  public void reportException(Exception ex) {
+    healthChecker.reportException(ex);
+    sendOutofBandHeartBeat();
+  }
+
   private List<LogAggregationReport> getLogAggregationReportsForApps(
       ConcurrentLinkedQueue<LogAggregationReport> lastestLogAggregationStatus) {
     LogAggregationReport status;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java

@@ -58,6 +58,7 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
 import org.apache.hadoop.yarn.api.records.SignalContainerCommand;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.exceptions.ConfigurationException;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.ipc.RPCUtil;
 import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
@@ -198,34 +199,30 @@ public class ContainerLaunch implements Callable<Integer> {
 
       FileContext lfs = FileContext.getLocalFSFileContext();
 
-      Path nmPrivateContainerScriptPath =
-          dirsHandler.getLocalPathForWrite(
-              getContainerPrivateDir(appIdStr, containerIdStr) + Path.SEPARATOR
-                  + CONTAINER_SCRIPT);
-      Path nmPrivateTokensPath =
-          dirsHandler.getLocalPathForWrite(
-              getContainerPrivateDir(appIdStr, containerIdStr)
-                  + Path.SEPARATOR
-                  + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT,
-                      containerIdStr));
-      Path nmPrivateClasspathJarDir = 
-          dirsHandler.getLocalPathForWrite(
-              getContainerPrivateDir(appIdStr, containerIdStr));
+      Path nmPrivateContainerScriptPath = dirsHandler.getLocalPathForWrite(
+          getContainerPrivateDir(appIdStr, containerIdStr) + Path.SEPARATOR
+              + CONTAINER_SCRIPT);
+      Path nmPrivateTokensPath = dirsHandler.getLocalPathForWrite(
+          getContainerPrivateDir(appIdStr, containerIdStr) + Path.SEPARATOR
+              + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT,
+              containerIdStr));
+      Path nmPrivateClasspathJarDir = dirsHandler.getLocalPathForWrite(
+          getContainerPrivateDir(appIdStr, containerIdStr));
       DataOutputStream containerScriptOutStream = null;
       DataOutputStream tokensOutStream = null;
 
       // Select the working directory for the container
       Path containerWorkDir =
           dirsHandler.getLocalPathForWrite(ContainerLocalizer.USERCACHE
-              + Path.SEPARATOR + user + Path.SEPARATOR
-              + ContainerLocalizer.APPCACHE + Path.SEPARATOR + appIdStr
-              + Path.SEPARATOR + containerIdStr,
+                  + Path.SEPARATOR + user + Path.SEPARATOR
+                  + ContainerLocalizer.APPCACHE + Path.SEPARATOR + appIdStr
+                  + Path.SEPARATOR + containerIdStr,
               LocalDirAllocator.SIZE_UNKNOWN, false);
       recordContainerWorkDir(containerID, containerWorkDir.toString());
 
       String pidFileSubpath = getPidFileSubpath(appIdStr, containerIdStr);
 
-      // pid file should be in nm private dir so that it is not 
+      // pid file should be in nm private dir so that it is not
       // accessible by users
       pidFilePath = dirsHandler.getLocalPathForWrite(pidFileSubpath);
       List<String> localDirs = dirsHandler.getLocalDirs();
@@ -252,13 +249,13 @@ public class ContainerLaunch implements Callable<Integer> {
           appDirs.add(new Path(appsdir, appIdStr));
         }
         containerScriptOutStream =
-          lfs.create(nmPrivateContainerScriptPath,
-              EnumSet.of(CREATE, OVERWRITE));
+            lfs.create(nmPrivateContainerScriptPath,
+                EnumSet.of(CREATE, OVERWRITE));
 
         // Set the token location too.
         environment.put(
-            ApplicationConstants.CONTAINER_TOKEN_FILE_ENV_NAME, 
-            new Path(containerWorkDir, 
+            ApplicationConstants.CONTAINER_TOKEN_FILE_ENV_NAME,
+            new Path(containerWorkDir,
                 FINAL_CONTAINER_TOKENS_FILE).toUri().getPath());
         // Sanitize the container's environment
         sanitizeEnv(environment, containerWorkDir, appDirs, userLocalDirs,
@@ -267,7 +264,7 @@ public class ContainerLaunch implements Callable<Integer> {
 
         // Write out the environment
         exec.writeLaunchEnv(containerScriptOutStream, environment,
-          localResources, launchContext.getCommands(),
+            localResources, launchContext.getCommands(),
             new Path(containerLogDirs.get(0)), user);
 
         // /////////// End of writing out container-script
@@ -297,6 +294,14 @@ public class ContainerLaunch implements Callable<Integer> {
           .setContainerLocalDirs(containerLocalDirs)
           .setContainerLogDirs(containerLogDirs)
           .build());
+    } catch (ConfigurationException e) {
+      LOG.error("Failed to launch container due to configuration error.", e);
+      dispatcher.getEventHandler().handle(new ContainerExitEvent(
+          containerID, ContainerEventType.CONTAINER_EXITED_WITH_FAILURE, ret,
+          e.getMessage()));
+      // Mark the node as unhealthy
+      context.getNodeStatusUpdater().reportException(e);
+      return ret;
     } catch (Throwable e) {
       LOG.warn("Failed to launch container.", e);
       dispatcher.getEventHandler().handle(new ContainerExitEvent(
@@ -400,7 +405,8 @@ public class ContainerLaunch implements Callable<Integer> {
   }
 
   @SuppressWarnings("unchecked")
-  protected int launchContainer(ContainerStartContext ctx) throws IOException {
+  protected int launchContainer(ContainerStartContext ctx)
+      throws IOException, ConfigurationException {
     ContainerId containerId = container.getContainerId();
     if (container.isMarkedForKilling()) {
       LOG.info("Container " + containerId + " not launched as it has already "

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerRelaunch.java

@@ -25,6 +25,7 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.exceptions.ConfigurationException;
 import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
 import org.apache.hadoop.yarn.server.nodemanager.Context;
 import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
@@ -115,6 +116,14 @@ public class ContainerRelaunch extends ContainerLaunch {
           .setContainerLocalDirs(containerLocalDirs)
           .setContainerLogDirs(containerLogDirs)
           .build());
+    } catch (ConfigurationException e) {
+      LOG.error("Failed to launch container due to configuration error.", e);
+      dispatcher.getEventHandler().handle(new ContainerExitEvent(
+          containerId, ContainerEventType.CONTAINER_EXITED_WITH_FAILURE, ret,
+          e.getMessage()));
+      // Mark the node as unhealthy
+      getContext().getNodeStatusUpdater().reportException(e);
+      return ret;
     } catch (Throwable e) {
       LOG.warn("Failed to relaunch container.", e);
       dispatcher.getEventHandler().handle(new ContainerExitEvent(

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c

@@ -649,19 +649,21 @@ static int create_container_directories(const char* user, const char *app_id,
     const char *container_id, char* const* local_dir, char* const* log_dir, const char *work_dir) {
   // create dirs as 0750
   const mode_t perms = S_IRWXU | S_IRGRP | S_IXGRP;
-  if (app_id == NULL || container_id == NULL || user == NULL || user_detail == NULL || user_detail->pw_name == NULL) {
+  if (user == NULL || app_id == NULL || container_id == NULL ||
+      local_dir == NULL || log_dir == NULL || work_dir == NULL ||
+      user_detail == NULL || user_detail->pw_name == NULL) {
     fprintf(LOGFILE,
             "Either app_id, container_id or the user passed is null.\n");
-    return -1;
+    return ERROR_CREATE_CONTAINER_DIRECTORIES_ARGUMENTS;
   }
 
-  int result = -1;
+  int result = COULD_NOT_CREATE_WORK_DIRECTORIES;
   char* const* local_dir_ptr;
   for(local_dir_ptr = local_dir; *local_dir_ptr != NULL; ++local_dir_ptr) {
     char *container_dir = get_container_work_directory(*local_dir_ptr, user, app_id,
                                                 container_id);
     if (container_dir == NULL) {
-      return -1;
+      return OUT_OF_MEMORY;
     }
     if (mkdirs(container_dir, perms) == 0) {
       result = 0;
@@ -674,12 +676,12 @@ static int create_container_directories(const char* user, const char *app_id,
     return result;
   }
 
-  result = -1;
+  result = COULD_NOT_CREATE_APP_LOG_DIRECTORIES;
   // also make the directory for the container logs
   char *combined_name = malloc(strlen(app_id) + strlen(container_id) + 2);
   if (combined_name == NULL) {
     fprintf(LOGFILE, "Malloc of combined name failed\n");
-    result = -1;
+    result = OUT_OF_MEMORY;
   } else {
     sprintf(combined_name, "%s/%s", app_id, container_id);
 
@@ -688,7 +690,7 @@ static int create_container_directories(const char* user, const char *app_id,
       char *container_log_dir = get_app_log_directory(*log_dir_ptr, combined_name);
       if (container_log_dir == NULL) {
         free(combined_name);
-        return -1;
+        return OUT_OF_MEMORY;
       } else if (mkdirs(container_log_dir, perms) != 0) {
     	free(container_log_dir);
       } else {
@@ -703,12 +705,12 @@ static int create_container_directories(const char* user, const char *app_id,
     return result;
   }
 
-  result = -1;
+  result = COULD_NOT_CREATE_TMP_DIRECTORIES;
   // also make the tmp directory
   char *tmp_dir = get_tmp_directory(work_dir);
 
   if (tmp_dir == NULL) {
-    return -1;
+    return OUT_OF_MEMORY;
   }
   if (mkdirs(tmp_dir, perms) == 0) {
     result = 0;
@@ -1145,12 +1147,12 @@ char* parse_docker_command_file(const char* command_file) {
    fprintf(ERRORFILE, "Cannot open file %s - %s",
                  command_file, strerror(errno));
    fflush(ERRORFILE);
-   exit(ERROR_OPENING_FILE);
+   exit(ERROR_OPENING_DOCKER_FILE);
   }
   if ((read = getline(&line, &len, stream)) == -1) {
      fprintf(ERRORFILE, "Error reading command_file %s\n", command_file);
      fflush(ERRORFILE);
-     exit(ERROR_READING_FILE);
+     exit(ERROR_READING_DOCKER_FILE);
   }
   fclose(stream);
 
@@ -1263,25 +1265,27 @@ int create_local_dirs(const char * user, const char *app_id,
   // Create container specific directories as user. If there are no resources
   // to localize for this container, app-directories and log-directories are
   // also created automatically as part of this call.
-  if (create_container_directories(user, app_id, container_id, local_dirs,
-                                   log_dirs, work_dir) != 0) {
+  int directory_create_result = create_container_directories(user, app_id,
+    container_id, local_dirs, log_dirs, work_dir);
+  if (directory_create_result != 0) {
     fprintf(ERRORFILE, "Could not create container dirs");
     fflush(ERRORFILE);
+    exit_code = directory_create_result;
     goto cleanup;
   }
 
-  // 700
+  // Copy script file with permissions 700
   if (copy_file(container_file_source, script_name, script_file_dest,S_IRWXU) != 0) {
     fprintf(ERRORFILE, "Could not create copy file %d %s\n", container_file_source, script_file_dest);
     fflush(ERRORFILE);
-    exit_code = INVALID_COMMAND_PROVIDED;
+    exit_code = COULD_NOT_CREATE_SCRIPT_COPY;
     goto cleanup;
   }
 
-  // 600
+  // Copy credential file to permissions 600
   if (copy_file(cred_file_source, cred_file, cred_file_dest,
         S_IRUSR | S_IWUSR) != 0) {
-    exit_code = UNABLE_TO_EXECUTE_CONTAINER_SCRIPT;
+    exit_code = COULD_NOT_CREATE_CREDENTIALS_FILE;
     fprintf(ERRORFILE, "Could not copy file");
     fflush(ERRORFILE);
     goto cleanup;
@@ -1866,7 +1870,7 @@ static int delete_path(const char *full_path,
   /* Return an error if the path is null. */
   if (full_path == NULL) {
     fprintf(LOGFILE, "Path is null\n");
-    return UNABLE_TO_BUILD_PATH;
+    return PATH_TO_DELETE_IS_NULL;
   }
   ret = recursive_unlink_children(full_path);
   if (ret == ENOENT) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h

@@ -37,8 +37,8 @@ enum command {
 
 enum errorcodes {
   INVALID_ARGUMENT_NUMBER = 1,
-  INVALID_USER_NAME, //2
-  INVALID_COMMAND_PROVIDED, //3
+  //INVALID_USER_NAME 2
+  INVALID_COMMAND_PROVIDED = 3,
   // SUPER_USER_NOT_ALLOWED_TO_RUN_TASKS (NOT USED) 4
   INVALID_NM_ROOT_DIRS = 5,
   SETUID_OPER_FAILED, //6
@@ -56,18 +56,24 @@ enum errorcodes {
   OUT_OF_MEMORY = 18,
   // INITIALIZE_DISTCACHEFILE_FAILED (NOT USED) 19
   INITIALIZE_USER_FAILED = 20,
-  UNABLE_TO_BUILD_PATH, //21
+  PATH_TO_DELETE_IS_NULL, //21
   INVALID_CONTAINER_EXEC_PERMISSIONS, //22
   // PREPARE_JOB_LOGS_FAILED (NOT USED) 23
-  INVALID_CONFIG_FILE =  24,
+  INVALID_CONFIG_FILE = 24,
   SETSID_OPER_FAILED = 25,
   WRITE_PIDFILE_FAILED = 26,
   WRITE_CGROUP_FAILED = 27,
   TRAFFIC_CONTROL_EXECUTION_FAILED = 28,
-  DOCKER_RUN_FAILED=29,
-  ERROR_OPENING_FILE = 30,
-  ERROR_READING_FILE = 31,
-  FEATURE_DISABLED = 32
+  DOCKER_RUN_FAILED = 29,
+  ERROR_OPENING_DOCKER_FILE = 30,
+  ERROR_READING_DOCKER_FILE = 31,
+  FEATURE_DISABLED = 32,
+  COULD_NOT_CREATE_SCRIPT_COPY = 33,
+  COULD_NOT_CREATE_CREDENTIALS_FILE = 34,
+  COULD_NOT_CREATE_WORK_DIRECTORIES = 35,
+  COULD_NOT_CREATE_APP_LOG_DIRECTORIES = 36,
+  COULD_NOT_CREATE_TMP_DIRECTORIES = 37,
+  ERROR_CREATE_CONTAINER_DIRECTORIES_ARGUMENTS = 38,
 };
 
 enum operations {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java

@@ -61,6 +61,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.ConfigurationException;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.nodemanager.api.LocalizationProtocol;
 import org.apache.hadoop.yarn.server.nodemanager.api.ResourceLocalizationSpec;
@@ -226,7 +227,7 @@ public class TestDefaultContainerExecutor {
 
   @Test
   public void testContainerLaunchError()
-      throws IOException, InterruptedException {
+      throws IOException, InterruptedException, ConfigurationException {
 
     if (Shell.WINDOWS) {
       BASE_TMP_PATH =

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java

@@ -56,6 +56,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.ConfigurationException;
 import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.Signal;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
@@ -305,11 +306,13 @@ public class TestLinuxContainerExecutor {
     return cId;
   }
 
-  private int runAndBlock(String... cmd) throws IOException {
+  private int runAndBlock(String... cmd)
+      throws IOException, ConfigurationException {
     return runAndBlock(getNextContainerId(), cmd);
   }
 
-  private int runAndBlock(ContainerId cId, String... cmd) throws IOException {
+  private int runAndBlock(ContainerId cId, String... cmd)
+      throws IOException, ConfigurationException {
     String appId = "APP_" + getNextId();
     Container container = mock(Container.class);
     ContainerLaunchContext context = mock(ContainerLaunchContext.class);
@@ -448,7 +451,7 @@ public class TestLinuxContainerExecutor {
       public void run() {
         try {
           runAndBlock(sleepId, "sleep", "100");
-        } catch (IOException e) {
+        } catch (IOException|ConfigurationException e) {
           LOG.warn("Caught exception while running sleep", e);
         }
       };

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java

@@ -19,6 +19,7 @@
 package org.apache.hadoop.yarn.server.nodemanager;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assume.assumeTrue;
 import static org.mockito.Matchers.any;
@@ -49,6 +50,7 @@ import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.ConfigurationException;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
@@ -78,6 +80,8 @@ public class TestLinuxContainerExecutorWithMocks {
       "./src/test/resources/mock-container-executor";
   private static final String MOCK_EXECUTOR_WITH_ERROR =
       "./src/test/resources/mock-container-executer-with-error";
+  private static final String MOCK_EXECUTOR_WITH_CONFIG_ERROR =
+      "./src/test/resources/mock-container-executer-with-configuration-error";
 
   private String tmpMockExecutor;
   private LinuxContainerExecutor mockExec = null;
@@ -147,7 +151,8 @@ public class TestLinuxContainerExecutorWithMocks {
   }
   
   @Test
-  public void testContainerLaunch() throws IOException {
+  public void testContainerLaunch()
+      throws IOException, ConfigurationException {
     String appSubmitter = "nobody";
     String cmd = String.valueOf(
         PrivilegedOperation.RunAsUserCommand.LAUNCH_CONTAINER.getValue());
@@ -198,7 +203,8 @@ public class TestLinuxContainerExecutorWithMocks {
   }
 
   @Test (timeout = 5000)
-  public void testContainerLaunchWithPriority() throws IOException {
+  public void testContainerLaunchWithPriority()
+      throws IOException, ConfigurationException {
 
     // set the scheduler priority to make sure still works with nice -n prio
     Configuration conf = new Configuration();
@@ -272,102 +278,121 @@ public class TestLinuxContainerExecutorWithMocks {
   public void testContainerLaunchError()
       throws IOException, ContainerExecutionException {
 
-    // reinitialize executer
-    Configuration conf = new Configuration();
-    setupMockExecutor(MOCK_EXECUTOR_WITH_ERROR, conf);
-    conf.set(YarnConfiguration.NM_LOCAL_DIRS, "file:///bin/echo");
-    conf.set(YarnConfiguration.NM_LOG_DIRS, "file:///dev/null");
-
-
-    LinuxContainerExecutor exec;
-    LinuxContainerRuntime linuxContainerRuntime = new
-        DefaultLinuxContainerRuntime(PrivilegedOperationExecutor.getInstance
-        (conf));
-
-    linuxContainerRuntime.initialize(conf);
-    exec = new LinuxContainerExecutor(linuxContainerRuntime);
-
-    mockExec = spy(exec);
-    doAnswer(
-        new Answer() {
-          @Override
-          public Object answer(InvocationOnMock invocationOnMock)
-              throws Throwable {
-             String diagnostics = (String) invocationOnMock.getArguments()[0];
-            assertTrue("Invalid Diagnostics message: " + diagnostics,
-                diagnostics.contains("badcommand"));
-            return null;
+    final String[] expecetedMessage = {"badcommand", "Exit code: 24"};
+    final String[] executor = {
+        MOCK_EXECUTOR_WITH_ERROR,
+        MOCK_EXECUTOR_WITH_CONFIG_ERROR
+    };
+
+    for (int i = 0; i < expecetedMessage.length; ++i) {
+      final int j = i;
+      // reinitialize executer
+      Configuration conf = new Configuration();
+      setupMockExecutor(executor[j], conf);
+      conf.set(YarnConfiguration.NM_LOCAL_DIRS, "file:///bin/echo");
+      conf.set(YarnConfiguration.NM_LOG_DIRS, "file:///dev/null");
+
+
+      LinuxContainerExecutor exec;
+      LinuxContainerRuntime linuxContainerRuntime = new
+          DefaultLinuxContainerRuntime(PrivilegedOperationExecutor.getInstance(
+              conf));
+
+      linuxContainerRuntime.initialize(conf);
+      exec = new LinuxContainerExecutor(linuxContainerRuntime);
+
+      mockExec = spy(exec);
+      doAnswer(
+          new Answer() {
+            @Override
+            public Object answer(InvocationOnMock invocationOnMock)
+                throws Throwable {
+              String diagnostics = (String) invocationOnMock.getArguments()[0];
+              assertTrue("Invalid Diagnostics message: " + diagnostics,
+                  diagnostics.contains(expecetedMessage[j]));
+              return null;
+            }
           }
-        }
-    ).when(mockExec).logOutput(any(String.class));
-    dirsHandler = new LocalDirsHandlerService();
-    dirsHandler.init(conf);
-    mockExec.setConf(conf);
-
-    String appSubmitter = "nobody";
-    String cmd = String
-        .valueOf(PrivilegedOperation.RunAsUserCommand.LAUNCH_CONTAINER.getValue());
-    String appId = "APP_ID";
-    String containerId = "CONTAINER_ID";
-    Container container = mock(Container.class);
-    ContainerId cId = mock(ContainerId.class);
-    ContainerLaunchContext context = mock(ContainerLaunchContext.class);
-    HashMap<String, String> env = new HashMap<String, String>();
-
-    when(container.getContainerId()).thenReturn(cId);
-    when(container.getLaunchContext()).thenReturn(context);
-    doAnswer(
-        new Answer() {
-          @Override
-          public Object answer(InvocationOnMock invocationOnMock)
-              throws Throwable {
-            ContainerDiagnosticsUpdateEvent event =
-                (ContainerDiagnosticsUpdateEvent) invocationOnMock
-                    .getArguments()[0];
-            assertTrue("Invalid Diagnostics message: " +
-                event.getDiagnosticsUpdate(),
-                event.getDiagnosticsUpdate().contains("badcommand"));
-            return null;
+      ).when(mockExec).logOutput(any(String.class));
+      dirsHandler = new LocalDirsHandlerService();
+      dirsHandler.init(conf);
+      mockExec.setConf(conf);
+
+      String appSubmitter = "nobody";
+      String cmd = String
+          .valueOf(PrivilegedOperation.RunAsUserCommand.LAUNCH_CONTAINER.
+              getValue());
+      String appId = "APP_ID";
+      String containerId = "CONTAINER_ID";
+      Container container = mock(Container.class);
+      ContainerId cId = mock(ContainerId.class);
+      ContainerLaunchContext context = mock(ContainerLaunchContext.class);
+      HashMap<String, String> env = new HashMap<String, String>();
+
+      when(container.getContainerId()).thenReturn(cId);
+      when(container.getLaunchContext()).thenReturn(context);
+      doAnswer(
+          new Answer() {
+            @Override
+            public Object answer(InvocationOnMock invocationOnMock)
+                throws Throwable {
+              ContainerDiagnosticsUpdateEvent event =
+                  (ContainerDiagnosticsUpdateEvent) invocationOnMock
+                      .getArguments()[0];
+              assertTrue("Invalid Diagnostics message: " +
+                      event.getDiagnosticsUpdate(),
+                  event.getDiagnosticsUpdate().contains(expecetedMessage[j]));
+              return null;
+            }
           }
-        }
-    ).when(container).handle(any(ContainerDiagnosticsUpdateEvent.class));
-    
-    when(cId.toString()).thenReturn(containerId);
-
-    when(context.getEnvironment()).thenReturn(env);
-
-    Path scriptPath = new Path("file:///bin/echo");
-    Path tokensPath = new Path("file:///dev/null");
-    Path workDir = new Path("/tmp");
-    Path pidFile = new Path(workDir, "pid.txt");
-
-    mockExec.activateContainer(cId, pidFile);
-
-    int ret = mockExec.launchContainer(new ContainerStartContext.Builder()
-        .setContainer(container)
-        .setNmPrivateContainerScriptPath(scriptPath)
-        .setNmPrivateTokensPath(tokensPath)
-        .setUser(appSubmitter)
-        .setAppId(appId)
-        .setContainerWorkDir(workDir)
-        .setLocalDirs(dirsHandler.getLocalDirs())
-        .setLogDirs(dirsHandler.getLogDirs())
-        .setFilecacheDirs(new ArrayList<String>())
-        .setUserLocalDirs(new ArrayList<String>())
-        .setContainerLocalDirs(new ArrayList<String>())
-        .setContainerLogDirs(new ArrayList<String>())
-        .build());
-
-    Assert.assertNotSame(0, ret);
-    assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
-        appSubmitter, cmd, appId, containerId,
-        workDir.toString(), "/bin/echo", "/dev/null", pidFile.toString(),
-        StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
-            dirsHandler.getLocalDirs()),
-        StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
-            dirsHandler.getLogDirs()),
-        "cgroups=none"), readMockParams());
-
+      ).when(container).handle(any(ContainerDiagnosticsUpdateEvent.class));
+
+      when(cId.toString()).thenReturn(containerId);
+
+      when(context.getEnvironment()).thenReturn(env);
+
+      Path scriptPath = new Path("file:///bin/echo");
+      Path tokensPath = new Path("file:///dev/null");
+      Path workDir = new Path("/tmp");
+      Path pidFile = new Path(workDir, "pid.txt");
+
+      mockExec.activateContainer(cId, pidFile);
+
+      try {
+        int ret = mockExec.launchContainer(new ContainerStartContext.Builder()
+            .setContainer(container)
+            .setNmPrivateContainerScriptPath(scriptPath)
+            .setNmPrivateTokensPath(tokensPath)
+            .setUser(appSubmitter)
+            .setAppId(appId)
+            .setContainerWorkDir(workDir)
+            .setLocalDirs(dirsHandler.getLocalDirs())
+            .setLogDirs(dirsHandler.getLogDirs())
+            .setFilecacheDirs(new ArrayList<String>())
+            .setUserLocalDirs(new ArrayList<String>())
+            .setContainerLocalDirs(new ArrayList<String>())
+            .setContainerLogDirs(new ArrayList<String>())
+            .build());
+
+        Assert.assertNotSame(0, ret);
+        assertEquals(Arrays.asList(YarnConfiguration.
+                DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
+            appSubmitter, cmd, appId, containerId,
+            workDir.toString(), "/bin/echo", "/dev/null", pidFile.toString(),
+            StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
+                dirsHandler.getLocalDirs()),
+            StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
+                dirsHandler.getLogDirs()),
+            "cgroups=none"), readMockParams());
+
+        assertNotEquals("Expected YarnRuntimeException",
+            MOCK_EXECUTOR_WITH_CONFIG_ERROR, executor[i]);
+      } catch (ConfigurationException ex) {
+        assertEquals(MOCK_EXECUTOR_WITH_CONFIG_ERROR, executor[i]);
+        Assert.assertEquals("Linux Container Executor reached unrecoverable " +
+            "exception", ex.getMessage());
+      }
+    }
   }
   
   @Test

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeHealthService.java

@@ -23,6 +23,8 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.PrintWriter;
 
+import com.google.common.base.Joiner;
+import com.google.common.base.Strings;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -165,8 +167,11 @@ public class TestNodeHealthService {
         healthStatus.getIsNodeHealthy());
     Assert.assertTrue("Node script time out message not propagated",
         healthStatus.getHealthReport().equals(
-            NodeHealthScriptRunner.NODE_HEALTH_SCRIPT_TIMED_OUT_MSG
-            + NodeHealthCheckerService.SEPARATOR
-            + nodeHealthChecker.getDiskHandler().getDisksHealthReport(false)));
+            Joiner.on(NodeHealthCheckerService.SEPARATOR).skipNulls().join(
+                NodeHealthScriptRunner.NODE_HEALTH_SCRIPT_TIMED_OUT_MSG,
+                Strings.emptyToNull(
+                    nodeHealthChecker.getDiskHandler()
+                        .getDisksHealthReport(false))
+            )));
   }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java

@@ -21,7 +21,11 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.fail;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.atLeastOnce;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 import java.io.BufferedReader;
@@ -42,6 +46,7 @@ import java.util.StringTokenizer;
 import java.util.jar.JarFile;
 import java.util.jar.Manifest;
 
+import com.google.common.collect.Lists;
 import org.apache.commons.codec.binary.Base64;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileUtil;
@@ -77,13 +82,14 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.Dispatcher;
 import org.apache.hadoop.yarn.event.Event;
 import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.exceptions.ConfigurationException;
 import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
 import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
 import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
 import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
-import org.apache.hadoop.yarn.server.nodemanager.Context;
 import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
 import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
+import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
@@ -92,6 +98,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerExitEvent;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.ShellScriptBuilder;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
+import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
 import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService;
 import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
 import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
@@ -107,11 +114,12 @@ import org.junit.Assert;
 import org.junit.Assume;
 import org.junit.Before;
 import org.junit.Test;
+import org.mockito.Mockito;
 
 public class TestContainerLaunch extends BaseContainerManagerTest {
 
   private static final String INVALID_JAVA_HOME = "/no/jvm/here";
-  protected Context distContext =
+  private NMContext distContext =
       new NMContext(new NMContainerTokenSecretManager(conf),
           new NMTokenSecretManagerInNM(), null,
           new ApplicationACLsManager(conf), new NMNullStateStoreService(),
@@ -1437,4 +1445,55 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
       }
     }
   }
+
+  /**
+   * Test container launch fault.
+   * @throws Exception
+   */
+  @Test
+  public void testContainerLaunchOnConfigurationError() throws Exception {
+    Dispatcher dispatcher = mock(Dispatcher.class);
+    EventHandler handler = mock(EventHandler.class);
+    when(dispatcher.getEventHandler()).thenReturn(handler);
+    Application app = mock(Application.class);
+    ApplicationId appId = mock(ApplicationId.class);
+    when(appId.toString()).thenReturn("1");
+    when(app.getAppId()).thenReturn(appId);
+    Container container = mock(Container.class);
+    ContainerId id = mock(ContainerId.class);
+    when(id.toString()).thenReturn("1");
+    when(container.getContainerId()).thenReturn(id);
+    when(container.getUser()).thenReturn("user");
+    ContainerLaunchContext clc = mock(ContainerLaunchContext.class);
+    List<String> ret = Lists.newArrayList();
+    when(clc.getCommands()).thenReturn(ret);
+    when(container.getLaunchContext()).thenReturn(clc);
+    Credentials credentials = mock(Credentials.class);
+    when(container.getCredentials()).thenReturn(credentials);
+
+    // Configuration errors should result in node shutdown...
+    ContainerExecutor returnConfigError = mock(ContainerExecutor.class);
+    when(returnConfigError.launchContainer((ContainerStartContext)any())).
+        thenThrow(new ConfigurationException("Mock configuration error"));
+    ContainerLaunch launchConfigError = new ContainerLaunch(
+        distContext, conf, dispatcher,
+        returnConfigError, app, container, dirsHandler, containerManager);
+    NodeStatusUpdater updater = mock(NodeStatusUpdater.class);
+    distContext.setNodeStatusUpdater(updater);
+    launchConfigError.call();
+    verify(updater, atLeastOnce()).reportException((Exception)any());
+
+    // ... any other error should continue.
+    ContainerExecutor returnOtherError = mock(ContainerExecutor.class);
+    when(returnOtherError.launchContainer((ContainerStartContext)any())).
+        thenThrow(new IOException("Mock configuration error"));
+    ContainerLaunch launchOtherError = new ContainerLaunch(
+        distContext, conf, dispatcher,
+        returnOtherError, app, container, dirsHandler, containerManager);
+    NodeStatusUpdater updaterNoCall = mock(NodeStatusUpdater.class);
+    distContext.setNodeStatusUpdater(updaterNoCall);
+    launchOtherError.call();
+    verify(updaterNoCall, never()).reportException((Exception)any());
+
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java

@@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.ResourceUtilization;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.AsyncDispatcher;
 import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.exceptions.ConfigurationException;
 import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
 import org.apache.hadoop.yarn.server.nodemanager.Context;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
@@ -77,7 +78,7 @@ public class TestContainersMonitorResourceChange {
     }
     @Override
     public int launchContainer(ContainerStartContext ctx) throws
-        IOException {
+        IOException, ConfigurationException {
       return 0;
     }
     @Override

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java

@@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
 import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.ExecutionType;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.ConfigurationException;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.security.NMTokenIdentifier;
 import org.apache.hadoop.yarn.server.api.records.ContainerQueuingLimit;
@@ -121,7 +122,8 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
   protected ContainerExecutor createContainerExecutor() {
     DefaultContainerExecutor exec = new DefaultContainerExecutor() {
       @Override
-      public int launchContainer(ContainerStartContext ctx) throws IOException {
+      public int launchContainer(ContainerStartContext ctx)
+          throws IOException, ConfigurationException {
         if (delayContainers) {
           try {
             Thread.sleep(10000);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/mock-container-executer-with-configuration-error

@@ -0,0 +1,20 @@
+#!/bin/sh
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+for PARAM in "$@"
+do
+    echo $PARAM;
+done > params.txt
+
+exit 24
+