MAPREDUCE-4669. MRAM web UI does not work with HTTPS. (Contributed by Robert Kanter)

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/KeyStoreTestUtil.java

@@ -32,15 +32,20 @@ import java.io.FileWriter;
 import java.io.IOException;
 import java.io.Writer;
 import java.math.BigInteger;
+import java.net.Socket;
 import java.net.URL;
 import java.security.GeneralSecurityException;
 import java.security.Key;
+import java.security.KeyManagementException;
 import java.security.KeyPair;
 import java.security.KeyPairGenerator;
 import java.security.KeyStore;
 import java.security.NoSuchAlgorithmException;
+import java.security.Principal;
+import java.security.PrivateKey;
 import java.security.SecureRandom;
 import java.security.cert.Certificate;
+import java.security.cert.CertificateException;
 import java.security.cert.X509Certificate;
 import java.util.Date;
 import java.util.HashMap;
@@ -50,8 +55,15 @@ import java.security.InvalidKeyException;
 import java.security.NoSuchProviderException;
 import java.security.SignatureException;
 import java.security.cert.CertificateEncodingException;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.KeyManager;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509KeyManager;
+import javax.net.ssl.X509TrustManager;
 import javax.security.auth.x500.X500Principal;
 
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
 import org.bouncycastle.x509.X509V1CertificateGenerator;
 
 public class KeyStoreTestUtil {
@@ -538,4 +550,97 @@ public class KeyStoreTestUtil {
     sslConf.set(SSLFactory.SSL_CLIENT_CONF_KEY, sslClientConfFile);
     return sslConf;
   }
+
+  /**
+   * Configures the passed in {@link HttpsURLConnection} to allow all SSL
+   * certificates.
+   *
+   * @param httpsConn The HttpsURLConnection to configure
+   * @throws KeyManagementException
+   * @throws NoSuchAlgorithmException
+   */
+  public static void setAllowAllSSL(HttpsURLConnection httpsConn)
+      throws KeyManagementException, NoSuchAlgorithmException {
+    setAllowAllSSL(httpsConn, null);
+  }
+
+  /**
+   * Configures the passed in {@link HttpsURLConnection} to allow all SSL
+   * certificates.  Also presents a client certificate.
+   *
+   * @param httpsConn The HttpsURLConnection to configure
+   * @param clientCert The client certificate to present
+   * @param clientKeyPair The KeyPair for the client certificate
+   * @throws KeyManagementException
+   * @throws NoSuchAlgorithmException
+   */
+  public static void setAllowAllSSL(HttpsURLConnection httpsConn,
+      X509Certificate clientCert, KeyPair clientKeyPair)
+      throws KeyManagementException, NoSuchAlgorithmException {
+    X509KeyManager km = new X509KeyManager() {
+      @Override
+      public String[] getClientAliases(String s, Principal[] principals) {
+        return new String[]{"client"};
+      }
+
+      @Override
+      public String chooseClientAlias(String[] strings,
+          Principal[] principals, Socket socket) {
+        return "client";
+      }
+
+      @Override
+      public String[] getServerAliases(String s, Principal[] principals) {
+        return null;
+      }
+
+      @Override
+      public String chooseServerAlias(String s, Principal[] principals,
+          Socket socket) {
+        return null;
+      }
+
+      @Override
+      public X509Certificate[] getCertificateChain(String s) {
+        return new X509Certificate[]{clientCert};
+      }
+
+      @Override
+      public PrivateKey getPrivateKey(String s) {
+        return clientKeyPair.getPrivate();
+      }
+    };
+    setAllowAllSSL(httpsConn, km);
+  }
+
+  private static void setAllowAllSSL(HttpsURLConnection httpsConn,
+      KeyManager km) throws KeyManagementException, NoSuchAlgorithmException {
+    // Create a TrustManager that trusts anything
+    TrustManager[] trustAllCerts = new TrustManager[] {
+        new X509TrustManager() {
+          @Override
+          public java.security.cert.X509Certificate[] getAcceptedIssuers() {
+            return new X509Certificate[]{};
+          }
+
+          @Override
+          public void checkClientTrusted(
+              java.security.cert.X509Certificate[] certs, String authType)
+              throws CertificateException {
+          }
+
+          @Override
+          public void checkServerTrusted(
+              java.security.cert.X509Certificate[] certs, String authType)
+              throws CertificateException {
+          }
+        }
+    };
+    KeyManager[] kms = (km == null) ? null : new KeyManager[]{km};
+    SSLContext sc = SSLContext.getInstance("SSL");
+    sc.init(kms, trustAllCerts, new SecureRandom());
+    httpsConn.setSSLSocketFactory(sc.getSocketFactory());
+    // Don't check the hostname
+    httpsConn.setHostnameVerifier(new NoopHostnameVerifier());
+  }
 }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml

@@ -108,6 +108,11 @@
       <artifactId>bcpkix-jdk15on</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>com.github.stefanbirkner</groupId>
+      <artifactId>system-rules</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java

@@ -25,6 +25,7 @@ import java.util.Collection;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.http.HttpConfig;
 import org.apache.hadoop.http.HttpConfig.Policy;
 import org.apache.hadoop.ipc.Server;
 import org.apache.hadoop.mapreduce.JobACL;
@@ -136,14 +137,18 @@ public class MRClientService extends AbstractService implements ClientService {
         server.getListenerAddress().getPort());
     LOG.info("Instantiated MRClientService at " + this.bindAddress);
     try {
-      // Explicitly disabling SSL for map reduce task as we can't allow MR users
-      // to gain access to keystore file for opening SSL listener. We can trust
-      // RM/NM to issue SSL certificates but definitely not MR-AM as it is
-      // running in user-land.
+      HttpConfig.Policy httpPolicy = conf.getBoolean(
+          MRJobConfig.MR_AM_WEBAPP_HTTPS_ENABLED,
+          MRJobConfig.DEFAULT_MR_AM_WEBAPP_HTTPS_ENABLED)
+          ? Policy.HTTPS_ONLY : Policy.HTTP_ONLY;
+      boolean needsClientAuth = conf.getBoolean(
+          MRJobConfig.MR_AM_WEBAPP_HTTPS_CLIENT_AUTH,
+          MRJobConfig.DEFAULT_MR_AM_WEBAPP_HTTPS_CLIENT_AUTH);
       webApp =
           WebApps.$for("mapreduce", AppContext.class, appContext, "ws")
-            .withHttpPolicy(conf, Policy.HTTP_ONLY)
+            .withHttpPolicy(conf, httpPolicy)
             .withPortRange(conf, MRJobConfig.MR_AM_WEBAPP_PORT_RANGE)
+            .needsClientAuth(needsClientAuth)
             .start(new AMWebApp());
     } catch (Exception e) {
       LOG.error("Webapps failed to start. Ignoring for now:", e);

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebApp.java

@@ -22,15 +22,23 @@ import static org.apache.hadoop.mapreduce.v2.app.webapp.AMParams.APP_ID;
 import static org.junit.Assert.assertEquals;
 
 import java.io.ByteArrayOutputStream;
+import java.io.File;
 import java.io.InputStream;
 import java.net.HttpURLConnection;
+import java.net.SocketException;
 import java.net.URL;
+import java.security.KeyPair;
+import java.security.cert.Certificate;
+import java.security.cert.X509Certificate;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Map.Entry;
 
+import javax.net.ssl.HttpsURLConnection;
 import javax.net.ssl.SSLException;
 
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
 import org.junit.Assert;
 
 import org.apache.hadoop.conf.Configuration;
@@ -57,13 +65,26 @@ import org.apache.hadoop.yarn.webapp.WebApps;
 import org.apache.hadoop.yarn.webapp.test.WebAppTests;
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 import org.apache.http.HttpStatus;
+import org.junit.After;
+import org.junit.Rule;
 import org.junit.Test;
 
 import com.google.common.net.HttpHeaders;
 import com.google.inject.Injector;
+import org.junit.contrib.java.lang.system.EnvironmentVariables;
 
 public class TestAMWebApp {
 
+  private static final File TEST_DIR = new File(
+      System.getProperty("test.build.data",
+          System.getProperty("java.io.tmpdir")),
+      TestAMWebApp.class.getName());
+
+  @After
+  public void tearDown() {
+    TEST_DIR.delete();
+  }
+
   @Test public void testAppControllerIndex() {
     AppContext ctx = new MockAppContext(0, 1, 1, 1);
     Injector injector = WebAppTests.createMockInjector(AppContext.class, ctx);
@@ -179,7 +200,7 @@ public class TestAMWebApp {
       }
     };
     Configuration conf = new Configuration();
-    // MR is explicitly disabling SSL, even though setting as HTTPS_ONLY
+    // MR is explicitly disabling SSL, even though YARN setting as HTTPS_ONLY
     conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY, Policy.HTTPS_ONLY.name());
     Job job = app.submit(conf);
 
@@ -201,12 +222,143 @@ public class TestAMWebApp {
           (HttpURLConnection) httpsUrl.openConnection();
       httpsConn.getInputStream();
       Assert.fail("https:// is not accessible, expected to fail");
-    } catch (Exception e) {
-      Assert.assertTrue(e instanceof SSLException);
+    } catch (SSLException e) {
+      // expected
+    }
+
+    app.waitForState(job, JobState.SUCCEEDED);
+    app.verifyCompleted();
+  }
+
+  @Rule
+  public final EnvironmentVariables environmentVariables
+      = new EnvironmentVariables();
+
+  @Test
+  public void testMRWebAppSSLEnabled() throws Exception {
+    MRApp app = new MRApp(2, 2, true, this.getClass().getName(), true) {
+      @Override
+      protected ClientService createClientService(AppContext context) {
+        return new MRClientService(context);
+      }
+    };
+    Configuration conf = new Configuration();
+    conf.setBoolean(MRJobConfig.MR_AM_WEBAPP_HTTPS_ENABLED, true);
+
+    KeyPair keyPair = KeyStoreTestUtil.generateKeyPair("RSA");
+    Certificate cert = KeyStoreTestUtil.generateCertificate(
+        "CN=foo", keyPair, 5, "SHA512WITHRSA");
+    File keystoreFile = new File(TEST_DIR, "server.keystore");
+    keystoreFile.getParentFile().mkdirs();
+    KeyStoreTestUtil.createKeyStore(keystoreFile.getAbsolutePath(), "password",
+        "server", keyPair.getPrivate(), cert);
+    environmentVariables.set("KEYSTORE_FILE_LOCATION",
+        keystoreFile.getAbsolutePath());
+    environmentVariables.set("KEYSTORE_PASSWORD", "password");
+
+    Job job = app.submit(conf);
+
+    String hostPort =
+        NetUtils.getHostPortString(((MRClientService) app.getClientService())
+            .getWebApp().getListenerAddress());
+    // https:// should be accessible
+    URL httpsUrl = new URL("https://" + hostPort);
+    HttpsURLConnection httpsConn =
+        (HttpsURLConnection) httpsUrl.openConnection();
+    KeyStoreTestUtil.setAllowAllSSL(httpsConn);
+
+    InputStream in = httpsConn.getInputStream();
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+    IOUtils.copyBytes(in, out, 1024);
+    Assert.assertTrue(out.toString().contains("MapReduce Application"));
+
+    // http:// is not accessible.
+    URL httpUrl = new URL("http://" + hostPort);
+    try {
+      HttpURLConnection httpConn =
+          (HttpURLConnection) httpUrl.openConnection();
+      httpConn.getResponseCode();
+      Assert.fail("http:// is not accessible, expected to fail");
+    } catch (SocketException e) {
+      // expected
     }
 
     app.waitForState(job, JobState.SUCCEEDED);
     app.verifyCompleted();
+
+    keystoreFile.delete();
+  }
+
+  @Test
+  public void testMRWebAppSSLEnabledWithClientAuth() throws Exception {
+    MRApp app = new MRApp(2, 2, true, this.getClass().getName(), true) {
+      @Override
+      protected ClientService createClientService(AppContext context) {
+        return new MRClientService(context);
+      }
+    };
+    Configuration conf = new Configuration();
+    conf.setBoolean(MRJobConfig.MR_AM_WEBAPP_HTTPS_ENABLED, true);
+    conf.setBoolean(MRJobConfig.MR_AM_WEBAPP_HTTPS_CLIENT_AUTH, true);
+
+    KeyPair keyPair = KeyStoreTestUtil.generateKeyPair("RSA");
+    Certificate cert = KeyStoreTestUtil.generateCertificate(
+        "CN=foo", keyPair, 5, "SHA512WITHRSA");
+    File keystoreFile = new File(TEST_DIR, "server.keystore");
+    keystoreFile.getParentFile().mkdirs();
+    KeyStoreTestUtil.createKeyStore(keystoreFile.getAbsolutePath(), "password",
+        "server", keyPair.getPrivate(), cert);
+    environmentVariables.set("KEYSTORE_FILE_LOCATION",
+        keystoreFile.getAbsolutePath());
+    environmentVariables.set("KEYSTORE_PASSWORD", "password");
+
+    KeyPair clientKeyPair = KeyStoreTestUtil.generateKeyPair("RSA");
+    X509Certificate clientCert = KeyStoreTestUtil.generateCertificate(
+        "CN=bar", clientKeyPair, 5, "SHA512WITHRSA");
+    File truststoreFile = new File(TEST_DIR, "client.truststore");
+    truststoreFile.getParentFile().mkdirs();
+    KeyStoreTestUtil.createTrustStore(truststoreFile.getAbsolutePath(),
+        "password", "client", clientCert);
+    environmentVariables.set("TRUSTSTORE_FILE_LOCATION",
+        truststoreFile.getAbsolutePath());
+    environmentVariables.set("TRUSTSTORE_PASSWORD", "password");
+
+    Job job = app.submit(conf);
+
+    String hostPort =
+        NetUtils.getHostPortString(((MRClientService) app.getClientService())
+            .getWebApp().getListenerAddress());
+    // https:// should be accessible
+    URL httpsUrl = new URL("https://" + hostPort);
+    HttpsURLConnection httpsConn =
+        (HttpsURLConnection) httpsUrl.openConnection();
+    KeyStoreTestUtil.setAllowAllSSL(httpsConn, clientCert, clientKeyPair);
+
+    InputStream in = httpsConn.getInputStream();
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+    IOUtils.copyBytes(in, out, 1024);
+    Assert.assertTrue(out.toString().contains("MapReduce Application"));
+
+    // Try with wrong client cert
+    KeyPair otherClientKeyPair = KeyStoreTestUtil.generateKeyPair("RSA");
+    X509Certificate otherClientCert = KeyStoreTestUtil.generateCertificate(
+        "CN=bar", otherClientKeyPair, 5, "SHA512WITHRSA");
+    KeyStoreTestUtil.setAllowAllSSL(httpsConn, otherClientCert, clientKeyPair);
+
+    try {
+      HttpURLConnection httpConn =
+          (HttpURLConnection) httpsUrl.openConnection();
+      httpConn.getResponseCode();
+      Assert.fail("Wrong client certificate, expected to fail");
+    } catch (SSLException e) {
+      // expected
+    }
+
+    app.waitForState(job, JobState.SUCCEEDED);
+    app.verifyCompleted();
+
+    keystoreFile.delete();
+    truststoreFile.delete();
   }
 
   static String webProxyBase = null;

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRWebAppUtil.java

@@ -24,6 +24,7 @@ import org.apache.hadoop.classification.InterfaceStability.Evolving;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.http.HttpConfig;
 import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.TypeConverter;
 import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
 import org.apache.hadoop.net.NetUtils;
@@ -178,6 +179,9 @@ public class MRWebAppUtil {
   }
 
   public static String getAMWebappScheme(Configuration conf) {
-    return "http://";
+    return conf.getBoolean(
+        MRJobConfig.MR_AM_WEBAPP_HTTPS_ENABLED,
+        MRJobConfig.DEFAULT_MR_AM_WEBAPP_HTTPS_ENABLED)
+        ? "https://" : "http://";
   }
 }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java

@@ -762,6 +762,28 @@ public interface MRJobConfig {
    */
   String MR_AM_WEBAPP_PORT_RANGE = MR_AM_PREFIX + "webapp.port-range";
 
+  /**
+   * True if the MR AM should use HTTPS for its webapp.  If
+   * {@link org.apache.hadoop.yarn.conf.YarnConfiguration#RM_APPLICATION_HTTPS_POLICY}
+   * is set to LENIENT or STRICT, the MR AM will automatically use the
+   * keystore provided by YARN with a certificate for the MR AM webapp, unless
+   * provided by the user.
+   */
+  String MR_AM_WEBAPP_HTTPS_ENABLED = MR_AM_PREFIX + "webapp.https.enabled";
+  boolean DEFAULT_MR_AM_WEBAPP_HTTPS_ENABLED = false;
+
+  /**
+   * True if the MR AM webapp should require client HTTPS authentication (i.e.
+   * the proxy server (RM) should present a certificate to the MR AM webapp).
+   * If {@link org.apache.hadoop.yarn.conf.YarnConfiguration#RM_APPLICATION_HTTPS_POLICY}
+   * is set to LENIENT or STRICT, the MR AM will automatically use the
+   * truststore provided by YARN with the RMs certificate, unless provided by
+   * the user.
+   */
+  String MR_AM_WEBAPP_HTTPS_CLIENT_AUTH =
+      MR_AM_PREFIX + "webapp.https.client.auth";
+  boolean DEFAULT_MR_AM_WEBAPP_HTTPS_CLIENT_AUTH = false;
+
   /** Enable blacklisting of nodes in the job.*/
   public static final String MR_AM_JOB_NODE_BLACKLISTING_ENABLE = 
     MR_AM_PREFIX  + "job.node-blacklisting.enable";

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml

@@ -1481,6 +1481,27 @@
     For example 50000-50050,50100-50200</description>
 </property>
 
+<property>
+  <name>yarn.app.mapreduce.am.webapp.https.enabled</name>
+  <value>false</value>
+  <description>True if the MR AM should use HTTPS for its webapp.  If
+    yarn.resourcemanager.application-https.policy is set to LENIENT or STRICT,
+    the MR AM will automatically use the keystore provided by YARN with a
+    certificate for the MR AM webapp, unless provided by the user.
+  </description>
+</property>
+
+<property>
+  <name>yarn.app.mapreduce.am.webapp.https.client.auth</name>
+  <value>false</value>
+  <description>True if the MR AM webapp should require client HTTPS
+    authentication (i.e. the proxy server (RM) should present a certificate to
+    the MR AM webapp). If yarn.resourcemanager.application-https.policy is set
+    to LENIENT or STRICT, the MR AM will automatically use the truststore
+    provided by YARN with the RMs certificate, unless provided by the user.
+  </description>
+</property>
+
 <property>
   <name>yarn.app.mapreduce.am.job.committer.cancel-timeout</name>
   <value>60000</value>

hadoop-project/pom.xml

@@ -1073,6 +1073,11 @@
         <artifactId>junit</artifactId>
         <version>4.11</version>
       </dependency>
+      <dependency>
+        <groupId>com.github.stefanbirkner</groupId>
+        <artifactId>system-rules</artifactId>
+        <version>1.18.0</version>
+      </dependency>
       <dependency>
         <groupId>commons-collections</groupId>
         <artifactId>commons-collections</artifactId>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java

@@ -95,6 +95,7 @@ public class WebApps {
     boolean findPort = false;
     Configuration conf;
     Policy httpPolicy = null;
+    boolean needsClientAuth = false;
     String portRangeConfigKey = null;
     boolean devMode = false;
     private String spnegoPrincipalKey;
@@ -174,6 +175,11 @@ public class WebApps {
       return this;
     }
 
+    public Builder<T> needsClientAuth(boolean needsClientAuth) {
+      this.needsClientAuth = needsClientAuth;
+      return this;
+    }
+
     /**
      * Set port range config key and associated configuration object.
      * @param config configuration.
@@ -335,7 +341,24 @@ public class WebApps {
         }
 
         if (httpScheme.equals(WebAppUtils.HTTPS_PREFIX)) {
-          WebAppUtils.loadSslConfiguration(builder, conf);
+          String amKeystoreLoc = System.getenv("KEYSTORE_FILE_LOCATION");
+          if (amKeystoreLoc != null) {
+            LOG.info("Setting keystore location to " + amKeystoreLoc);
+            String password = System.getenv("KEYSTORE_PASSWORD");
+            builder.keyStore(amKeystoreLoc, password, "jks");
+          } else {
+            LOG.info("Loading standard ssl config");
+            WebAppUtils.loadSslConfiguration(builder, conf);
+          }
+          builder.needsClientAuth(needsClientAuth);
+          if (needsClientAuth) {
+            String amTruststoreLoc = System.getenv("TRUSTSTORE_FILE_LOCATION");
+            if (amTruststoreLoc != null) {
+              LOG.info("Setting truststore location to " + amTruststoreLoc);
+              String password = System.getenv("TRUSTSTORE_PASSWORD");
+              builder.trustStore(amTruststoreLoc, password, "jks");
+            }
+          }
         }
 
         HttpServer2 server = builder.build();