YARN-4007. Add support for different network setups when launching the docker container. Contributed by Sidharta Seethana.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java

@@ -1369,6 +1369,30 @@ public class YarnConfiguration extends Configuration {
   /** Default list for users allowed to run privileged containers is empty. */
   public static final String DEFAULT_NM_DOCKER_PRIVILEGED_CONTAINERS_ACL = "";
 
+  /** The set of networks allowed when launching containers using the
+   * DockerContainerRuntime. */
+  public static final String NM_DOCKER_ALLOWED_CONTAINER_NETWORKS =
+      DOCKER_CONTAINER_RUNTIME_PREFIX + "allowed-container-networks";
+
+  /** The set of networks allowed when launching containers using the
+   * DockerContainerRuntime. */
+  public static final String[] DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_NETWORKS =
+      {"host", "none", "bridge"};
+
+  /** The network used when launching containers using the
+   * DockerContainerRuntime when no network is specified in the request. This
+   *  network must be one of the (configurable) set of allowed container
+   *  networks. */
+  public static final String NM_DOCKER_DEFAULT_CONTAINER_NETWORK =
+      DOCKER_CONTAINER_RUNTIME_PREFIX + "default-container-network";
+
+  /** The network used when launching containers using the
+   * DockerContainerRuntime when no network is specified in the request and
+   * no default network is configured.
+   * . */
+  public static final String DEFAULT_NM_DOCKER_DEFAULT_CONTAINER_NETWORK =
+      "host";
+
   /** The path to the Linux container executor.*/
   public static final String NM_LINUX_CONTAINER_EXECUTOR_PATH =
     NM_PREFIX + "linux-container-executor.path";

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml

@@ -1510,6 +1510,22 @@
     <value></value>
   </property>
 
+  <property>
+    <description>The set of networks allowed when launching containers using the
+      DockerContainerRuntime.</description>
+    <name>yarn.nodemanager.runtime.linux.docker.allowed-container-networks</name>
+    <value>host,none,bridge</value>
+  </property>
+
+  <property>
+    <description>The network used when launching containers using the
+      DockerContainerRuntime when no network is specified in the request
+      . This network must be one of the (configurable) set of allowed container
+      networks.</description>
+    <name>yarn.nodemanager.runtime.linux.docker.default-container-network</name>
+    <value>host</value>
+  </property>
+
   <property>
     <description>This flag determines whether memory limit will be set for the Windows Job
     Object of the containers launched by the default container executor.</description>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java

@@ -44,7 +44,6 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.Contai
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeConstants;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext;
 
-
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.util.ArrayList;
@@ -73,6 +72,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
   public static final String ENV_DOCKER_CONTAINER_RUN_OVERRIDE_DISABLE =
       "YARN_CONTAINER_RUNTIME_DOCKER_RUN_OVERRIDE_DISABLE";
   @InterfaceAudience.Private
+  public static final String ENV_DOCKER_CONTAINER_NETWORK =
+      "YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK";
   public static final String ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER =
       "YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER";
   @InterfaceAudience.Private
@@ -82,6 +83,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
   private Configuration conf;
   private DockerClient dockerClient;
   private PrivilegedOperationExecutor privilegedOperationExecutor;
+  private Set<String> allowedNetworks = new HashSet<>();
+  private String defaultNetwork;
   private CGroupsHandler cGroupsHandler;
   private AccessControlList privilegedContainersAcl;
 
@@ -122,6 +125,26 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
       throws ContainerExecutionException {
     this.conf = conf;
     dockerClient = new DockerClient(conf);
+    allowedNetworks.clear();
+    allowedNetworks.addAll(Arrays.asList(
+        conf.getStrings(YarnConfiguration.NM_DOCKER_ALLOWED_CONTAINER_NETWORKS,
+            YarnConfiguration.DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_NETWORKS)));
+    defaultNetwork = conf.get(
+        YarnConfiguration.NM_DOCKER_DEFAULT_CONTAINER_NETWORK,
+        YarnConfiguration.DEFAULT_NM_DOCKER_DEFAULT_CONTAINER_NETWORK);
+
+    if(!allowedNetworks.contains(defaultNetwork)) {
+      String message = "Default network: " + defaultNetwork
+          + " is not in the set of allowed networks: " + allowedNetworks;
+
+      if (LOG.isWarnEnabled()) {
+        LOG.warn(message + ". Please check "
+            + "configuration");
+      }
+
+      throw new ContainerExecutionException(message);
+    }
+
     privilegedContainersAcl = new AccessControlList(conf.get(
         YarnConfiguration.NM_DOCKER_PRIVILEGED_CONTAINERS_ACL,
         YarnConfiguration.DEFAULT_NM_DOCKER_PRIVILEGED_CONTAINERS_ACL));
@@ -133,6 +156,18 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
 
   }
 
+  private void validateContainerNetworkType(String network)
+      throws ContainerExecutionException {
+    if (allowedNetworks.contains(network)) {
+      return;
+    }
+
+    String msg = "Disallowed network:  '" + network
+        + "' specified. Allowed networks: are " + allowedNetworks
+        .toString();
+    throw new ContainerExecutionException(msg);
+  }
+
   public void addCGroupParentIfRequired(String resourcesOptions,
       String containerIdStr, DockerRunCommand runCommand)
       throws ContainerExecutionException {
@@ -260,6 +295,13 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
     Map<String, String> environment = container.getLaunchContext()
         .getEnvironment();
     String imageName = environment.get(ENV_DOCKER_CONTAINER_IMAGE);
+    String network = environment.get(ENV_DOCKER_CONTAINER_NETWORK);
+
+    if(network == null || network.isEmpty()) {
+      network = defaultNetwork;
+    }
+
+    validateContainerNetworkType(network);
 
     if (imageName == null) {
       throw new ContainerExecutionException(ENV_DOCKER_CONTAINER_IMAGE
@@ -293,7 +335,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
         runAsUser, imageName)
         .detachOnRun()
         .setContainerWorkDir(containerWorkDir.toString())
-        .setNetworkType("host")
+        .setNetworkType(network)
         .setCapabilities(capabilities)
         .addMountLocation("/etc/passwd", "/etc/password:ro");
     List<String> allDirs = new ArrayList<>(containerLocalDirs);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java

@@ -47,7 +47,7 @@ import java.io.IOException;
 import java.nio.charset.Charset;
 import java.nio.file.Files;
 import java.nio.file.Paths;
-import java.util.ArrayList;
+import java.util.*;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
@@ -64,37 +64,37 @@ public class TestDockerContainerRuntime {
   private static final Log LOG = LogFactory
       .getLog(TestDockerContainerRuntime.class);
   private Configuration conf;
-  PrivilegedOperationExecutor mockExecutor;
-  CGroupsHandler mockCGroupsHandler;
-  String containerId;
-  Container container;
-  ContainerId cId;
-  ContainerLaunchContext context;
-  HashMap<String, String> env;
-  String image;
-  String runAsUser;
-  String user;
-  String appId;
-  String containerIdStr = containerId;
-  Path containerWorkDir;
-  Path nmPrivateContainerScriptPath;
-  Path nmPrivateTokensPath;
-  Path pidFilePath;
-  List<String> localDirs;
-  List<String> logDirs;
-  List<String> containerLocalDirs;
-  List<String> containerLogDirs;
-  Map<Path,List<String>> localizedResources;
-  String resourcesOptions;
-  ContainerRuntimeContext.Builder builder;
-  String submittingUser = "anakin";
-  String whitelistedUser = "yoda";
+  private PrivilegedOperationExecutor mockExecutor;
+  private CGroupsHandler mockCGroupsHandler;
+  private String containerId;
+  private Container container;
+  private ContainerId cId;
+  private ContainerLaunchContext context;
+  private HashMap<String, String> env;
+  private String image;
+  private String runAsUser;
+  private String user;
+  private String appId;
+  private String containerIdStr = containerId;
+  private Path containerWorkDir;
+  private Path nmPrivateContainerScriptPath;
+  private Path nmPrivateTokensPath;
+  private Path pidFilePath;
+  private List<String> localDirs;
+  private List<String> logDirs;
+  private List<String> containerLocalDirs;
+  private List<String> containerLogDirs;
+  private Map<Path, List<String>> localizedResources;
+  private String resourcesOptions;
+  private ContainerRuntimeContext.Builder builder;
+  private final String submittingUser = "anakin";
+  private final String whitelistedUser = "yoda";
+  private String[] testCapabilities;
 
   @Before
   public void setup() {
     String tmpPath = new StringBuffer(System.getProperty("test.build.data"))
-        .append
-        ('/').append("hadoop.tmp.dir").toString();
+        .append('/').append("hadoop.tmp.dir").toString();
 
     conf = new Configuration();
     conf.set("hadoop.tmp.dir", tmpPath);
@@ -138,6 +138,10 @@ public class TestDockerContainerRuntime {
     localizedResources.put(new Path("/test_local_dir/test_resource_file"),
         Collections.singletonList("test_dir/test_resource_file"));
 
+    testCapabilities = new String[] {"NET_BIND_SERVICE", "SYS_CHROOT"};
+    conf.setStrings(YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES,
+        testCapabilities);
+
     builder = new ContainerRuntimeContext
         .Builder(container);
 
@@ -187,6 +191,10 @@ public class TestDockerContainerRuntime {
         .executePrivilegedOperation(anyList(), opCaptor.capture(), any(
             File.class), any(Map.class), eq(false), eq(false));
 
+    //verification completed. we need to isolate specific invications.
+    // hence, reset mock here
+    Mockito.reset(mockExecutor);
+
     PrivilegedOperation op = opCaptor.getValue();
 
     Assert.assertEquals(PrivilegedOperation.OperationType
@@ -217,6 +225,20 @@ public class TestDockerContainerRuntime {
     return op;
   }
 
+  private String getExpectedTestCapabilitiesArgumentString()  {
+    /* Ordering of capabilities depends on HashSet ordering. */
+    Set<String> capabilitySet = new HashSet<>(Arrays.asList(testCapabilities));
+    StringBuilder expectedCapabilitiesString = new StringBuilder(
+        "--cap-drop=ALL ");
+
+    for(String capability : capabilitySet) {
+      expectedCapabilitiesString.append("--cap-add=").append(capability)
+          .append(" ");
+    }
+
+    return expectedCapabilitiesString.toString();
+  }
+
   @Test
   public void testDockerContainerLaunch()
       throws ContainerExecutionException, PrivilegedOperationException,
@@ -224,33 +246,18 @@ public class TestDockerContainerRuntime {
     DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(
         mockExecutor, mockCGroupsHandler);
     runtime.initialize(conf);
-
-    String[] testCapabilities = {"NET_BIND_SERVICE", "SYS_CHROOT"};
-
-    conf.setStrings(YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES,
-        testCapabilities);
     runtime.launchContainer(builder.build());
 
     PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs();
     List<String> args = op.getArguments();
     String dockerCommandFile = args.get(11);
 
-    /* Ordering of capabilities depends on HashSet ordering. */
-    Set<String> capabilitySet = new HashSet<>(Arrays.asList(testCapabilities));
-    StringBuilder expectedCapabilitiesString = new StringBuilder(
-        "--cap-drop=ALL ");
-
-    for(String capability : capabilitySet) {
-      expectedCapabilitiesString.append("--cap-add=").append(capability)
-          .append(" ");
-    }
-
     //This is the expected docker invocation for this case
     StringBuffer expectedCommandTemplate = new StringBuffer("run --name=%1$s ")
         .append("--user=%2$s -d ")
         .append("--workdir=%3$s ")
         .append("--net=host ")
-        .append(expectedCapabilitiesString)
+        .append(getExpectedTestCapabilitiesArgumentString())
         .append("-v /etc/passwd:/etc/password:ro ")
         .append("-v %4$s:%4$s ")
         .append("-v %5$s:%5$s ")
@@ -269,6 +276,208 @@ public class TestDockerContainerRuntime {
     Assert.assertEquals(expectedCommand, dockerCommands.get(0));
   }
 
+  @Test
+  public void testAllowedNetworksConfiguration() throws
+      ContainerExecutionException {
+    //the default network configuration should cause
+    // no exception should be thrown.
+
+    DockerLinuxContainerRuntime runtime =
+        new DockerLinuxContainerRuntime(mockExecutor, mockCGroupsHandler);
+    runtime.initialize(conf);
+
+    //invalid default network configuration - sdn2 is included in allowed
+    // networks
+
+    String[] networks = {"host", "none", "bridge", "sdn1"};
+    String invalidDefaultNetwork = "sdn2";
+
+    conf.setStrings(YarnConfiguration.NM_DOCKER_ALLOWED_CONTAINER_NETWORKS,
+        networks);
+    conf.set(YarnConfiguration.NM_DOCKER_DEFAULT_CONTAINER_NETWORK,
+        invalidDefaultNetwork);
+
+    try {
+      runtime =
+          new DockerLinuxContainerRuntime(mockExecutor, mockCGroupsHandler);
+      runtime.initialize(conf);
+      Assert.fail("Invalid default network configuration should did not "
+          + "trigger initialization failure.");
+    } catch (ContainerExecutionException e) {
+      LOG.info("Caught expected exception : " + e);
+    }
+
+    //valid default network configuration - sdn1 is included in allowed
+    // networks - no exception should be thrown.
+
+    String validDefaultNetwork = "sdn1";
+
+    conf.set(YarnConfiguration.NM_DOCKER_DEFAULT_CONTAINER_NETWORK,
+        validDefaultNetwork);
+    runtime =
+        new DockerLinuxContainerRuntime(mockExecutor, mockCGroupsHandler);
+    runtime.initialize(conf);
+  }
+
+  @Test
+  @SuppressWarnings("unchecked")
+  public void testContainerLaunchWithNetworkingDefaults()
+      throws ContainerExecutionException, IOException,
+      PrivilegedOperationException {
+    DockerLinuxContainerRuntime runtime =
+        new DockerLinuxContainerRuntime(mockExecutor, mockCGroupsHandler);
+    runtime.initialize(conf);
+
+    Random randEngine = new Random();
+    String disallowedNetwork = "sdn" + Integer.toString(randEngine.nextInt());
+
+    try {
+      env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK",
+          disallowedNetwork);
+      runtime.launchContainer(builder.build());
+      Assert.fail("Network was expected to be disallowed: " +
+          disallowedNetwork);
+    } catch (ContainerExecutionException e) {
+      LOG.info("Caught expected exception: " + e);
+    }
+
+    int size = YarnConfiguration
+        .DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_NETWORKS.length;
+    String allowedNetwork = YarnConfiguration
+        .DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_NETWORKS[randEngine.nextInt(size)];
+    env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK",
+        allowedNetwork);
+
+    //this should cause no failures.
+
+    runtime.launchContainer(builder.build());
+    PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs();
+    List<String> args = op.getArguments();
+    String dockerCommandFile = args.get(11);
+
+    //This is the expected docker invocation for this case
+    StringBuffer expectedCommandTemplate =
+        new StringBuffer("run --name=%1$s ").append("--user=%2$s -d ")
+            .append("--workdir=%3$s ")
+            .append("--net=" + allowedNetwork + " ")
+            .append(getExpectedTestCapabilitiesArgumentString())
+            .append("-v /etc/passwd:/etc/password:ro ")
+            .append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
+            .append("-v %6$s:%6$s ").append("%7$s ")
+            .append("bash %8$s/launch_container.sh");
+
+    String expectedCommand = String
+        .format(expectedCommandTemplate.toString(), containerId, runAsUser,
+            containerWorkDir, containerLocalDirs.get(0), containerWorkDir,
+            containerLogDirs.get(0), image, containerWorkDir);
+
+    List<String> dockerCommands = Files
+        .readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
+
+    Assert.assertEquals(1, dockerCommands.size());
+    Assert.assertEquals(expectedCommand, dockerCommands.get(0));
+  }
+
+  @Test
+  @SuppressWarnings("unchecked")
+  public void testContainerLaunchWithCustomNetworks()
+      throws ContainerExecutionException, IOException,
+      PrivilegedOperationException {
+    DockerLinuxContainerRuntime runtime =
+        new DockerLinuxContainerRuntime(mockExecutor, mockCGroupsHandler);
+
+    String customNetwork1 = "sdn1";
+    String customNetwork2 = "sdn2";
+    String customNetwork3 = "sdn3";
+
+    String[] networks = {"host", "none", "bridge", customNetwork1,
+        customNetwork2};
+
+    //customized set of allowed networks
+    conf.setStrings(YarnConfiguration.NM_DOCKER_ALLOWED_CONTAINER_NETWORKS,
+        networks);
+    //default network is "sdn1"
+    conf.set(YarnConfiguration.NM_DOCKER_DEFAULT_CONTAINER_NETWORK,
+        customNetwork1);
+
+    //this should cause no failures.
+    runtime.initialize(conf);
+    runtime.launchContainer(builder.build());
+    PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs();
+    List<String> args = op.getArguments();
+    String dockerCommandFile = args.get(11);
+
+    //This is the expected docker invocation for this case. customNetwork1
+    // ("sdn1") is the expected network to be used in this case
+    StringBuffer expectedCommandTemplate =
+        new StringBuffer("run --name=%1$s ").append("--user=%2$s -d ")
+            .append("--workdir=%3$s ")
+            .append("--net=" + customNetwork1 + " ")
+            .append(getExpectedTestCapabilitiesArgumentString())
+            .append("-v /etc/passwd:/etc/password:ro ")
+            .append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
+            .append("-v %6$s:%6$s ").append("%7$s ")
+            .append("bash %8$s/launch_container.sh");
+
+    String expectedCommand = String
+        .format(expectedCommandTemplate.toString(), containerId, runAsUser,
+            containerWorkDir, containerLocalDirs.get(0), containerWorkDir,
+            containerLogDirs.get(0), image, containerWorkDir);
+
+    List<String> dockerCommands = Files
+        .readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
+
+    Assert.assertEquals(1, dockerCommands.size());
+    Assert.assertEquals(expectedCommand, dockerCommands.get(0));
+
+
+    //now set an explicit (non-default) allowedNetwork and ensure that it is
+    // used.
+
+    env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK",
+        customNetwork2);
+    runtime.launchContainer(builder.build());
+
+    op = capturePrivilegedOperationAndVerifyArgs();
+    args = op.getArguments();
+    dockerCommandFile = args.get(11);
+
+    //This is the expected docker invocation for this case. customNetwork2
+    // ("sdn2") is the expected network to be used in this case
+    expectedCommandTemplate =
+        new StringBuffer("run --name=%1$s ").append("--user=%2$s -d ")
+            .append("--workdir=%3$s ")
+            .append("--net=" + customNetwork2 + " ")
+            .append(getExpectedTestCapabilitiesArgumentString())
+            .append("-v /etc/passwd:/etc/password:ro ")
+            .append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
+            .append("-v %6$s:%6$s ").append("%7$s ")
+            .append("bash %8$s/launch_container.sh");
+
+    expectedCommand = String
+        .format(expectedCommandTemplate.toString(), containerId, runAsUser,
+            containerWorkDir, containerLocalDirs.get(0), containerWorkDir,
+            containerLogDirs.get(0), image, containerWorkDir);
+
+    dockerCommands = Files
+        .readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
+
+    Assert.assertEquals(1, dockerCommands.size());
+    Assert.assertEquals(expectedCommand, dockerCommands.get(0));
+
+    //disallowed network should trigger a launch failure
+
+    env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK",
+        customNetwork3);
+    try {
+      runtime.launchContainer(builder.build());
+      Assert.fail("Disallowed network : " + customNetwork3
+          + "did not trigger launch failure.");
+    } catch (ContainerExecutionException e) {
+      LOG.info("Caught expected exception : " + e);
+    }
+  }
+
   @Test
   public void testLaunchPrivilegedContainersInvalidEnvVar()
       throws ContainerExecutionException, PrivilegedOperationException,
@@ -342,7 +551,6 @@ public class TestDockerContainerRuntime {
     }
   }
 
-
   @Test
   public void
   testLaunchPrivilegedContainersEnabledAndUserNotInWhitelist()