YARN-6447. Provide container sandbox policies for groups (gphillips via rkanter)

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java

@@ -1500,6 +1500,10 @@ public class YarnConfiguration extends Configuration {
   public static final String YARN_CONTAINER_SANDBOX_POLICY =
       YARN_CONTAINER_SANDBOX + ".policy";
 
+  /** Prefix for group to policy file mapping.*/
+  public static final String YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX =
+      YARN_CONTAINER_SANDBOX_POLICY + ".group.";
+
   /** The group which will run by default without the java security manager.*/
   public static final String YARN_CONTAINER_SANDBOX_WHITELIST_GROUP =
       YARN_CONTAINER_SANDBOX + ".whitelist-group";

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java

@@ -50,10 +50,13 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import static org.apache.hadoop.fs.Path.SEPARATOR;
 import static org.apache.hadoop.util.Shell.SYSPROP_HADOOP_HOME_DIR;
 import static org.apache.hadoop.yarn.api.ApplicationConstants.Environment.JAVA_HOME;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.YARN_CONTAINER_SANDBOX;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX;
 import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_ID_STR;
 import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_LOCAL_DIRS;
 import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_RUN_CMDS;
@@ -93,13 +96,21 @@ import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.r
  *     Accepts canonical path to a java policy file on the local filesystem.
  *     This file will be loaded as the base policy, any additional container
  *     grants will be appended to this base file.  If not specified, the default
- *     java.policy   file provided with hadoop resources will be used.
+ *     java.policy file provided with hadoop resources will be used.
  *   </li>
  *   <li>
  *     {@value YarnConfiguration#YARN_CONTAINER_SANDBOX_WHITELIST_GROUP} :
  *     Optional setting to specify a YARN queue which will be exempt from the
  *     sand-boxing process.
  *   </li>
+ *   <li>
+ *     {@value
+ *     YarnConfiguration#YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX}$groupName :
+ *     Optional setting to map groups to java policy files.  The value is a path
+ *     to the java policy file for $groupName.  A user which is a member of
+ *     multiple groups with different policies will receive the superset of all
+ *     the permissions across their groups.
+ *   </li>
  * </ul>
  */
 @InterfaceAudience.Private
@@ -138,7 +149,7 @@ public class JavaSandboxLinuxContainerRuntime
     this.configuration = conf;
     this.sandboxMode =
         SandboxMode.get(
-            this.configuration.get(YarnConfiguration.YARN_CONTAINER_SANDBOX,
+            this.configuration.get(YARN_CONTAINER_SANDBOX,
                 YarnConfiguration.DEFAULT_YARN_CONTAINER_SANDBOX));
 
     super.initialize(conf);
@@ -211,8 +222,10 @@ public class JavaSandboxLinuxContainerRuntime
         ctx.getExecutionAttribute(CONTAINER_RUN_CMDS);
     Map<String, String> env =
         ctx.getContainer().getLaunchContext().getEnvironment();
+    String username =
+        ctx.getExecutionAttribute(USER);
 
-    if(!isSandboxContainerWhitelisted(ctx, commands)) {
+    if(!isSandboxContainerWhitelisted(username, commands)) {
       String tmpDirBase = configuration.get("hadoop.tmp.dir");
       if (tmpDirBase == null) {
         throw new ContainerExecutionException("hadoop.tmp.dir not set!");
@@ -223,6 +236,8 @@ public class JavaSandboxLinuxContainerRuntime
         String containerID = ctx.getExecutionAttribute(CONTAINER_ID_STR);
         initializePolicyDir();
 
+        List<String> groupPolicyFiles =
+            getGroupPolicyFiles(configuration, ctx.getExecutionAttribute(USER));
         Path policyFilePath = Files.createFile(
             Paths.get(policyFileDir.toString(),
             containerID + "-" + NMContainerPolicyUtils.POLICY_FILE),
@@ -231,12 +246,12 @@ public class JavaSandboxLinuxContainerRuntime
 
         containerPolicies.put(containerID, policyFilePath);
 
-        NMContainerPolicyUtils.generatePolicyFile(
-            policyOutputStream, localDirs, resources, configuration);
+        NMContainerPolicyUtils.generatePolicyFile(policyOutputStream,
+            localDirs, groupPolicyFiles, resources, configuration);
         NMContainerPolicyUtils.appendSecurityFlags(
             commands, env, policyFilePath, sandboxMode);
 
-      } catch (Exception e) {
+      } catch (IOException e) {
         throw new ContainerExecutionException(e);
       } finally {
         IOUtils.cleanup(LOG, policyOutputStream);
@@ -264,15 +279,32 @@ public class JavaSandboxLinuxContainerRuntime
     return sandboxMode != SandboxMode.disabled;
   }
 
+  private static List<String> getGroupPolicyFiles(Configuration conf,
+      String user) throws ContainerExecutionException {
+    Groups groups = Groups.getUserToGroupsMappingService(conf);
+    List<String> userGroups;
+    try {
+      userGroups = groups.getGroups(user);
+    } catch (IOException e) {
+      throw new ContainerExecutionException("Container user does not exist");
+    }
+
+    return userGroups.stream()
+        .map(group -> conf.get(YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX
+            + group))
+        .filter(groupPolicy -> groupPolicy != null)
+        .collect(Collectors.toList());
+  }
+
   /**
    * Determine if the container should be whitelisted (i.e. exempt from the
    * Java Security Manager).
-   * @param ctx The container runtime context for the requested container
+   * @param username The name of the user running the container
    * @param commands The list of run commands for the container
    * @return boolean value denoting whether the container should be whitelisted.
    * @throws ContainerExecutionException If container user can not be resolved
    */
-  private boolean isSandboxContainerWhitelisted(ContainerRuntimeContext ctx,
+  private boolean isSandboxContainerWhitelisted(String username,
       List<String> commands) throws ContainerExecutionException {
     String whitelistGroup = configuration.get(
         YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP);
@@ -281,7 +313,7 @@ public class JavaSandboxLinuxContainerRuntime
     boolean isWhitelisted = false;
 
     try {
-      userGroups = groups.getGroups(ctx.getExecutionAttribute(USER));
+      userGroups = groups.getGroups(username);
     } catch (IOException e) {
       throw new ContainerExecutionException("Container user does not exist");
     }
@@ -399,8 +431,9 @@ public class JavaSandboxLinuxContainerRuntime
      * base policy file or if it is unable to create a new policy file.
      */
     static void generatePolicyFile(OutputStream policyOutStream,
-        List<String> localDirs, Map<org.apache.hadoop.fs.Path,
-        List<String>> resources, Configuration conf)
+        List<String> localDirs, List<String> groupPolicyPaths,
+        Map<org.apache.hadoop.fs.Path, List<String>> resources,
+        Configuration conf)
         throws IOException {
 
       String policyFilePath =
@@ -414,13 +447,16 @@ public class JavaSandboxLinuxContainerRuntime
         cacheDirs.add(path.getParent().toString());
       }
 
-      if(policyFilePath == null) {
+      if (groupPolicyPaths != null) {
+        for(String policyPath : groupPolicyPaths) {
+          Files.copy(Paths.get(policyPath), policyOutStream);
+        }
+      } else if (policyFilePath == null) {
         IOUtils.copyBytes(
             NMContainerPolicyUtils.class.getResourceAsStream("/" + POLICY_FILE),
             policyOutStream, conf, false);
       } else {
         Files.copy(Paths.get(policyFilePath), policyOutStream);
-        policyOutStream.flush();
       }
 
       Formatter filePermissionFormat = new Formatter(policyOutStream,
@@ -484,7 +520,7 @@ public class JavaSandboxLinuxContainerRuntime
 
     private static boolean validateJavaHome(String containerJavaHome)
         throws ContainerExecutionException{
-      if (System.getenv(JAVA_HOME.name()) == null){
+      if (System.getenv(JAVA_HOME.name()) == null) {
         throw new ContainerExecutionException(
             "JAVA_HOME is not set for NodeManager");
       }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestJavaSandboxLinuxContainerRuntime.java

@@ -37,20 +37,29 @@ import org.junit.rules.ExpectedException;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.FilePermission;
+import java.io.FileWriter;
+import java.io.IOException;
 import java.io.OutputStream;
+import java.net.SocketPermission;
 import java.nio.file.Files;
 import java.nio.file.Paths;
+import java.security.Permission;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Formatter;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Properties;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import static org.apache.hadoop.yarn.api.ApplicationConstants.Environment.JAVA_HOME;
 import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.LOG;
 import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.MULTI_COMMAND_REGEX;
 import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.CLEAN_CMD_REGEX;
 import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.CONTAINS_JAVA_CMD;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.POLICY_APPEND_FLAG;
 import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.POLICY_FILE;
 import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.POLICY_FLAG;
 import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.SECURITY_FLAG;
@@ -76,8 +85,9 @@ import static org.mockito.Mockito.when;
  */
 public class TestJavaSandboxLinuxContainerRuntime {
 
-  private static final String HADOOP_HOME = "hadoop.home.dir";
-  private static String hadoopHomeDir = System.getProperty(HADOOP_HOME);
+  private final static String HADOOP_HOME = "hadoop.home.dir";
+  private final static String HADOOP_HOME_DIR = System.getProperty(HADOOP_HOME);
+  private final Properties baseProps = new Properties(System.getProperties());
 
   @Rule
   public ExpectedException exception = ExpectedException.none();
@@ -101,11 +111,12 @@ public class TestJavaSandboxLinuxContainerRuntime {
   private final static String WHITELIST_GROUP = "captains";
   private final static String CONTAINER_ID = "container_1234567890";
   private final static String APPLICATION_ID = "application_1234567890";
+  private File baseTestDirectory;
 
   @Before
   public void setup() throws Exception {
 
-    File baseTestDirectory = new File(System.getProperty("test.build.data",
+    baseTestDirectory = new File(System.getProperty("test.build.data",
         System.getProperty("java.io.tmpdir", "target")),
         TestJavaSandboxLinuxContainerRuntime.class.getName());
 
@@ -114,10 +125,8 @@ public class TestJavaSandboxLinuxContainerRuntime {
 
     conf = new Configuration();
     conf.set(CommonConfigurationKeys.HADOOP_USER_GROUP_STATIC_OVERRIDES,
-        WHITELIST_USER + "=" + WHITELIST_GROUP + ";"
+        WHITELIST_USER + "=" + WHITELIST_GROUP + "," + NORMAL_GROUP + ";"
             + NORMAL_USER + "=" + NORMAL_GROUP + ";");
-    conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP,
-        WHITELIST_GROUP);
     conf.set("hadoop.tmp.dir", baseTestDirectory.getAbsolutePath());
 
     Files.deleteIfExists(Paths.get(baseTestDirectory.getAbsolutePath(),
@@ -151,21 +160,20 @@ public class TestJavaSandboxLinuxContainerRuntime {
 
     runtimeContextBuilder = createRuntimeContext();
 
-    if (hadoopHomeDir == null) {
+    if (HADOOP_HOME_DIR == null) {
       System.setProperty(HADOOP_HOME, policyFile.getParent());
     }
 
     OutputStream outStream = new FileOutputStream(policyFile);
     JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils
-        .generatePolicyFile(outStream, symLinks, resources, conf);
+        .generatePolicyFile(outStream, symLinks, null, resources, conf);
     outStream.close();
 
     System.setProperty("java.security.policy", policyFile.getCanonicalPath());
     securityManager = new SecurityManager();
-
   }
 
-  public  ContainerRuntimeContext.Builder createRuntimeContext(){
+  public ContainerRuntimeContext.Builder createRuntimeContext(){
 
     Container container = mock(Container.class);
     ContainerLaunchContext  ctx = mock(ContainerLaunchContext.class);
@@ -194,6 +202,71 @@ public class TestJavaSandboxLinuxContainerRuntime {
     return builder;
   }
 
+  public static final String SOCKET_PERMISSION_FORMAT =
+      "grant { \n" +
+      "   permission %1s \"%2s\", \"%3s\";\n" +
+      "};\n";
+  public static final String RUNTIME_PERMISSION_FORMAT =
+      "grant { \n" +
+          "   permission %1s \"%2s\";\n" +
+          "};\n";
+
+  @Test
+  public void testGroupPolicies()
+      throws IOException, ContainerExecutionException {
+    // Generate new policy files each containing one grant
+    File openSocketPolicyFile =
+        File.createTempFile("openSocket", "policy", baseTestDirectory);
+    File classLoaderPolicyFile =
+        File.createTempFile("createClassLoader", "policy", baseTestDirectory);
+    Permission socketPerm = new SocketPermission("localhost:0", "listen");
+    Permission runtimePerm = new RuntimePermission("createClassLoader");
+
+    StringBuilder socketPermString = new StringBuilder();
+    Formatter openSocketPolicyFormatter = new Formatter(socketPermString);
+    openSocketPolicyFormatter.format(SOCKET_PERMISSION_FORMAT,
+        socketPerm.getClass().getName(), socketPerm.getName(),
+        socketPerm.getActions());
+    FileWriter socketPermWriter = new FileWriter(openSocketPolicyFile);
+    socketPermWriter.write(socketPermString.toString());
+    socketPermWriter.close();
+
+    StringBuilder classLoaderPermString = new StringBuilder();
+    Formatter classLoaderPolicyFormatter = new Formatter(classLoaderPermString);
+    classLoaderPolicyFormatter.format(RUNTIME_PERMISSION_FORMAT,
+        runtimePerm.getClass().getName(), runtimePerm.getName());
+    FileWriter classLoaderPermWriter = new FileWriter(classLoaderPolicyFile);
+    classLoaderPermWriter.write(classLoaderPermString.toString());
+    classLoaderPermWriter.close();
+
+    conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX +
+        WHITELIST_GROUP, openSocketPolicyFile.toString());
+    conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX +
+        NORMAL_GROUP, classLoaderPolicyFile.toString());
+
+    String[] inputCommand = {"$JAVA_HOME/bin/java jar MyJob.jar"};
+    List<String> commands = Arrays.asList(inputCommand);
+
+    runtimeContextBuilder.setExecutionAttribute(USER, WHITELIST_USER);
+    runtimeContextBuilder.setExecutionAttribute(CONTAINER_RUN_CMDS, commands);
+
+    runtime.prepareContainer(runtimeContextBuilder.build());
+
+    //pull generated policy from cmd
+    Matcher policyMatches = Pattern.compile(POLICY_APPEND_FLAG + "=?([^ ]+)")
+        .matcher(commands.get(0));
+    policyMatches.find();
+    String generatedPolicy = policyMatches.group(1);
+
+    //Test that generated policy file has included both policies
+    Assert.assertTrue(
+        Files.readAllLines(Paths.get(generatedPolicy)).contains(
+            classLoaderPermString.toString().split("\n")[1]));
+    Assert.assertTrue(
+        Files.readAllLines(Paths.get(generatedPolicy)).contains(
+            socketPermString.toString().split("\n")[1]));
+  }
+
   @Test
   public void testGrant() throws Exception {
     FilePermission grantPermission =
@@ -235,7 +308,6 @@ public class TestJavaSandboxLinuxContainerRuntime {
     JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils
         .appendSecurityFlags(commands, env, policyFilePath,
               JavaSandboxLinuxContainerRuntime.SandboxMode.permissive);
-
   }
 
   @Test
@@ -247,6 +319,9 @@ public class TestJavaSandboxLinuxContainerRuntime {
     };
     List<String> commands = Arrays.asList(inputCommand);
 
+    conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP,
+        WHITELIST_GROUP);
+
     runtimeContextBuilder.setExecutionAttribute(USER, WHITELIST_USER);
     runtimeContextBuilder.setExecutionAttribute(CONTAINER_RUN_CMDS, commands);
     runtime.prepareContainer(runtimeContextBuilder.build());
@@ -254,7 +329,6 @@ public class TestJavaSandboxLinuxContainerRuntime {
     Assert.assertTrue("Command should not be modified when user is " +
             "member of whitelisted group",
         inputCommand[0].equals(commands.get(0)));
-
   }
 
   @Test
@@ -265,6 +339,9 @@ public class TestJavaSandboxLinuxContainerRuntime {
     };
     List<String> commands = Arrays.asList(inputCommand);
 
+    conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP,
+        WHITELIST_GROUP);
+
     runtimeContextBuilder.setExecutionAttribute(USER, WHITELIST_USER);
     runtimeContextBuilder.setExecutionAttribute(CONTAINER_RUN_CMDS, commands);
     runtime.prepareContainer(runtimeContextBuilder.build());
@@ -283,6 +360,9 @@ public class TestJavaSandboxLinuxContainerRuntime {
     };
     List<String> commands = Arrays.asList(inputCommand);
 
+    conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP,
+        WHITELIST_GROUP);
+
     runtimeContextBuilder.setExecutionAttribute(USER, NORMAL_USER);
     runtimeContextBuilder.setExecutionAttribute(CONTAINER_RUN_CMDS, commands);
     runtime.prepareContainer(runtimeContextBuilder.build());
@@ -373,6 +453,6 @@ public class TestJavaSandboxLinuxContainerRuntime {
 
   @After
   public void cleanup(){
-    System.setSecurityManager(null);
+    System.setProperties(baseProps);
   }
 }