YARN-2194. Fix bug causing CGroups functionality to fail on RHEL7.

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java

@@ -872,6 +872,11 @@ public class StringUtils {
     return sb.toString();
   }
 
+  public static String join(char separator, Iterable<?> strings) {
+    return join(separator + "", strings);
+  }
+
+
   /**
    * Concatenates strings, using a separator.
    *
@@ -894,6 +899,10 @@ public class StringUtils {
     return sb.toString();
   }
 
+  public static String join(char separator, String[] strings) {
+    return join(separator + "", strings);
+  }
+
   /**
    * Convert SOME_STUFF to SomeStuff
    *

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java

@@ -277,8 +277,12 @@ public class TestStringUtils extends UnitTestcaseTimeLimit {
     s.add("c");
     assertEquals("", StringUtils.join(":", s.subList(0, 0)));
     assertEquals("a", StringUtils.join(":", s.subList(0, 1)));
+    assertEquals("", StringUtils.join(':', s.subList(0, 0)));
+    assertEquals("a", StringUtils.join(':', s.subList(0, 1)));
     assertEquals("a:b", StringUtils.join(":", s.subList(0, 2)));
     assertEquals("a:b:c", StringUtils.join(":", s.subList(0, 3)));
+    assertEquals("a:b", StringUtils.join(':', s.subList(0, 2)));
+    assertEquals("a:b:c", StringUtils.join(':', s.subList(0, 3)));
   }
   
   @Test (timeout = 30000)

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java

@@ -53,6 +53,8 @@ public class LinuxContainerExecutor extends ContainerExecutor {
   private static final Log LOG = LogFactory
       .getLog(LinuxContainerExecutor.class);
 
+  public final static char LINUX_FILE_PATH_SEPARATOR = '%';
+
   private String nonsecureLocalUser;
   private Pattern nonsecureLocalUserPattern;
   private String containerExecutorExe;
@@ -214,8 +216,10 @@ public class LinuxContainerExecutor extends ContainerExecutor {
                    Integer.toString(Commands.INITIALIZE_CONTAINER.getValue()),
                    appId,
                    nmPrivateContainerTokensPath.toUri().getPath().toString(),
-                   StringUtils.join(",", localDirs),
-                   StringUtils.join(",", logDirs)));
+                   StringUtils.join(LINUX_FILE_PATH_SEPARATOR,
+                       localDirs),
+                   StringUtils.join(LINUX_FILE_PATH_SEPARATOR,
+                       logDirs)));
 
     File jvm =                                  // use same jvm as parent
       new File(new File(System.getProperty("java.home"), "bin"), "java");
@@ -286,8 +290,10 @@ public class LinuxContainerExecutor extends ContainerExecutor {
             nmPrivateCotainerScriptPath.toUri().getPath().toString(),
             nmPrivateTokensPath.toUri().getPath().toString(),
             pidFilePath.toString(),
-            StringUtils.join(",", localDirs),
-            StringUtils.join(",", logDirs),
+            StringUtils.join(LINUX_FILE_PATH_SEPARATOR,
+                localDirs),
+            StringUtils.join(LINUX_FILE_PATH_SEPARATOR,
+                logDirs),
             resourcesOptions));
         String[] commandArray = command.toArray(new String[command.size()]);
         shExec = new ShellCommandExecutor(commandArray, null, // NM's cwd

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java

@@ -409,10 +409,11 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler {
 
     if (isCpuWeightEnabled()) {
       sb.append(pathForCgroup(CONTROLLER_CPU, containerName) + "/tasks");
-      sb.append(",");
+      sb.append(LinuxContainerExecutor.LINUX_FILE_PATH_SEPARATOR);
     }
 
-    if (sb.charAt(sb.length() - 1) == ',') {
+    if (sb.charAt(sb.length() - 1) ==
+        LinuxContainerExecutor.LINUX_FILE_PATH_SEPARATOR) {
       sb.deleteCharAt(sb.length() - 1);
     }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c

@@ -286,14 +286,14 @@ char ** extract_values_delim(char *value, const char *delim) {
   //first allocate any array of 10
   if(value != NULL) {
     toPass = (char **) malloc(sizeof(char *) * toPassSize);
-    tempTok = strtok_r((char *)value, ",", &tempstr);
+    tempTok = strtok_r((char *)value, "%", &tempstr);
     while (tempTok != NULL) {
       toPass[size++] = tempTok;
       if(size == toPassSize) {
         toPassSize += MAX_SIZE;
         toPass = (char **) realloc(toPass,(sizeof(char *) * toPassSize));
       }
-      tempTok = strtok_r(NULL, ",", &tempstr);
+      tempTok = strtok_r(NULL, "%", &tempstr);
     }
   }
   if (toPass != NULL) {
@@ -303,7 +303,7 @@ char ** extract_values_delim(char *value, const char *delim) {
 }
 
 /**
- * Extracts array of values from the comma separated list of values.
+ * Extracts array of values from the '%' separated list of values.
  */
 char ** extract_values(char *value) {
   return extract_values_delim(value, ",");
@@ -311,7 +311,7 @@ char ** extract_values(char *value) {
 
 /**
  * Function to return an array of values for a key.
- * Value delimiter is assumed to be a comma.
+ * Value delimiter is assumed to be a '%'.
  */
 char ** get_values(const char * key, struct configuration *cfg) {
   char *value = get_value(key, cfg);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c

@@ -42,8 +42,8 @@
 #define TEST_ROOT TMPDIR "/test-container-executor"
 
 #define DONT_TOUCH_FILE "dont-touch-me"
-#define NM_LOCAL_DIRS       TEST_ROOT "/local-1," TEST_ROOT "/local-2," \
-               TEST_ROOT "/local-3," TEST_ROOT "/local-4," TEST_ROOT "/local-5"
+#define NM_LOCAL_DIRS       TEST_ROOT "/local-1%" TEST_ROOT "/local-2%" \
+               TEST_ROOT "/local-3%" TEST_ROOT "/local-4%" TEST_ROOT "/local-5"
 #define NM_LOG_DIRS         TEST_ROOT "/logs/userlogs"
 #define ARRAY_SIZE 1000
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java

@@ -137,8 +137,10 @@ public class TestLinuxContainerExecutorWithMocks {
     assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
         appSubmitter, cmd, appId, containerId,
         workDir.toString(), "/bin/echo", "/dev/null", pidFile.toString(),
-        StringUtils.join(",", dirsHandler.getLocalDirs()),
-        StringUtils.join(",", dirsHandler.getLogDirs()), "cgroups=none"),
+        StringUtils.join(LinuxContainerExecutor.LINUX_FILE_PATH_SEPARATOR,
+            dirsHandler.getLocalDirs()),
+        StringUtils.join(LinuxContainerExecutor.LINUX_FILE_PATH_SEPARATOR,
+            dirsHandler.getLogDirs()), "cgroups=none"),
         readMockParams());
     
   }
@@ -283,8 +285,10 @@ public class TestLinuxContainerExecutorWithMocks {
     assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
         appSubmitter, cmd, appId, containerId,
         workDir.toString(), "/bin/echo", "/dev/null", pidFile.toString(),
-        StringUtils.join(",", dirsHandler.getLocalDirs()),
-        StringUtils.join(",", dirsHandler.getLogDirs()),
+        StringUtils.join(LinuxContainerExecutor.LINUX_FILE_PATH_SEPARATOR,
+            dirsHandler.getLocalDirs()),
+        StringUtils.join(LinuxContainerExecutor.LINUX_FILE_PATH_SEPARATOR,
+            dirsHandler.getLogDirs()),
         "cgroups=none"), readMockParams());
 
   }