YARN-1085. Modified YARN and MR2 web-apps to do HTTP authentication in secure setup with kerberos. Contributed by Omkar Vinit Joshi.

svn merge --ignore-ancestry -c 1517101 ../../trunk/


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2.1-beta@1517103 13f79535-47bb-0310-9956-ffa450edef68

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java

@@ -80,6 +80,7 @@ import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.ipc.YarnRPC;
@@ -148,8 +149,14 @@ public class HistoryClientService extends AbstractService {
         JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_ADDRESS,
         JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_PORT);
     // NOTE: there should be a .at(InetSocketAddress)
-    WebApps.$for("jobhistory", HistoryClientService.class, this, "ws")
-        .with(conf).at(NetUtils.getHostPortString(bindAddress)).start(webApp);
+    WebApps
+        .$for("jobhistory", HistoryClientService.class, this, "ws")
+        .with(conf)
+        .withHttpSpnegoKeytabKey(
+            YarnConfiguration.JHS_WEBAPP_SPNEGO_KEYTAB_FILE_KEY)
+        .withHttpSpnegoPrincipalKey(
+            YarnConfiguration.JHS_WEBAPP_SPNEGO_USER_NAME_KEY)
+        .at(NetUtils.getHostPortString(bindAddress)).start(webApp);
     conf.updateConnectAddr(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS,
                            webApp.getListenerAddress());
   }

hadoop-yarn-project/CHANGES.txt

@@ -52,6 +52,9 @@ Release 2.1.1-beta - UNRELEASED
     YARN-1082. Create base directories on HDFS after RM login to ensure RM
     recovery doesn't fail in secure mode. (vinodkv via acmurthy)
 
+    YARN-1085. Modified YARN and MR2 web-apps to do HTTP authentication in
+    secure setup with kerberos. (Omkar Vinit Joshi via vinodkv)
+
 Release 2.1.0-beta - 2013-08-22
 
   INCOMPATIBLE CHANGES

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java

@@ -205,6 +205,12 @@ public class YarnConfiguration extends Configuration {
   public static final String RM_KEYTAB = 
     RM_PREFIX + "keytab";
 
+  public static final String RM_WEBAPP_SPNEGO_USER_NAME_KEY =
+      RM_PREFIX + "webapp.spnego-principal";
+  
+  public static final String RM_WEBAPP_SPENGO_KEYTAB_FILE_KEY =
+      RM_PREFIX + "webapp.spengo-keytab-file";
+
   /** How long to wait until a container is considered dead.*/
   public static final String RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS = 
     RM_PREFIX + "rm.container-allocation.expiry-interval-ms";
@@ -599,7 +605,13 @@ public class YarnConfiguration extends Configuration {
 
   public static final String NM_USER_HOME_DIR =
       NM_PREFIX + "user-home-dir";
-
+  
+  public static final String NM_WEBAPP_SPNEGO_USER_NAME_KEY =
+      NM_PREFIX + "webapp.spnego-principal";
+  
+  public static final String NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY =
+      NM_PREFIX + "webapp.spnego-keytab-file";
+  
   public static final String DEFAULT_NM_USER_HOME_DIR= "/home/";
 
   ////////////////////////////////
@@ -729,6 +741,12 @@ public class YarnConfiguration extends Configuration {
   // Other Configs
   ////////////////////////////////
 
+  public static final String JHS_WEBAPP_SPNEGO_USER_NAME_KEY =
+      "jobhistoryserver.webapp.spnego-principal";
+  
+  public static final String JHS_WEBAPP_SPNEGO_KEYTAB_FILE_KEY =
+      "jobhistoryserver.webapp.spnego-keytab-file";
+
   /**
    * The interval of the yarn client's querying application state after
    * application submission. The unit is millisecond.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java

@@ -33,6 +33,8 @@ import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.http.HttpServer;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.security.AdminACLsManager;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -65,7 +67,6 @@ import com.google.inject.servlet.GuiceFilter;
 @InterfaceAudience.LimitedPrivate({"YARN", "MapReduce"})
 public class WebApps {
   static final Logger LOG = LoggerFactory.getLogger(WebApps.class);
-
   public static class Builder<T> {
     static class ServletStruct {
       public Class<? extends HttpServlet> clazz;
@@ -82,6 +83,8 @@ public class WebApps {
     boolean findPort = false;
     Configuration conf;
     boolean devMode = false;
+    private String spnegoPrincipalKey;
+    private String spnegoKeytabKey;
     private final HashSet<ServletStruct> servlets = new HashSet<ServletStruct>();
     private final HashMap<String, Object> attributes = new HashMap<String, Object>();
 
@@ -135,6 +138,16 @@ public class WebApps {
       this.conf = conf;
       return this;
     }
+    
+    public Builder<T> withHttpSpnegoPrincipalKey(String spnegoPrincipalKey) {
+      this.spnegoPrincipalKey = spnegoPrincipalKey;
+      return this;
+    }
+    
+    public Builder<T> withHttpSpnegoKeytabKey(String spnegoKeytabKey) {
+      this.spnegoKeytabKey = spnegoKeytabKey;
+      return this;
+    }
 
     public Builder<T> inDevMode() {
       devMode = true;
@@ -197,8 +210,30 @@ public class WebApps {
           }
         }
         HttpServer server =
-            new HttpServer(name, bindAddress, port, findPort, conf, 
-            new AdminACLsManager(conf).getAdminAcl(), null, webapp.getServePathSpecs());
+            new HttpServer(name, bindAddress, port, findPort, conf,
+                new AdminACLsManager(conf).getAdminAcl(), null,
+                webapp.getServePathSpecs()) {
+
+              {
+                if (UserGroupInformation.isSecurityEnabled()) {
+                  boolean initSpnego = true;
+                  if (spnegoPrincipalKey == null || spnegoPrincipalKey.isEmpty()) {
+                    LOG.warn("Principal for spnego filter is not set");
+                    initSpnego = false;
+                  }
+                  if (spnegoKeytabKey == null || spnegoKeytabKey.isEmpty()) {
+                    LOG.warn("Keytab for spnego filter is not set");
+                    initSpnego = false;
+                  }
+                  if (initSpnego) {
+                    LOG.info("Initializing spnego filter with principal key : "
+                        + spnegoPrincipalKey + " keytab key : "
+                        + spnegoKeytabKey);
+                    initSpnego(conf, spnegoPrincipalKey, spnegoKeytabKey);
+                  }
+                }
+              }
+            };
         for(ServletStruct struct: servlets) {
           server.addServlet(struct.name, struct.spec, struct.clazz);
         }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java

@@ -59,8 +59,15 @@ public class WebServer extends AbstractService {
     LOG.info("Instantiating NMWebApp at " + bindAddress);
     try {
       this.webApp =
-          WebApps.$for("node", Context.class, this.nmContext, "ws")
-              .at(bindAddress).with(getConfig()).start(this.nmWebApp);
+          WebApps
+            .$for("node", Context.class, this.nmContext, "ws")
+            .at(bindAddress)
+            .with(getConfig())
+            .withHttpSpnegoPrincipalKey(
+              YarnConfiguration.NM_WEBAPP_SPNEGO_USER_NAME_KEY)
+            .withHttpSpnegoKeytabKey(
+              YarnConfiguration.NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY)
+            .start(this.nmWebApp);
       this.port = this.webApp.httpServer().getPort();
     } catch (Exception e) {
       String msg = "NMWebapps failed to start.";

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java

@@ -30,6 +30,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.metrics2.source.JvmMetrics;
 import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.service.AbstractService;
 import org.apache.hadoop.service.CompositeService;
 import org.apache.hadoop.service.Service;
@@ -573,9 +574,16 @@ public class ResourceManager extends CompositeService implements Recoverable {
   
   protected void startWepApp() {
     Builder<ApplicationMasterService> builder = 
-      WebApps.$for("cluster", ApplicationMasterService.class, masterService, "ws").at(
-          this.conf.get(YarnConfiguration.RM_WEBAPP_ADDRESS,
-          YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS)); 
+        WebApps
+            .$for("cluster", ApplicationMasterService.class, masterService,
+                "ws")
+            .with(conf)
+            .withHttpSpnegoPrincipalKey(
+                YarnConfiguration.RM_WEBAPP_SPNEGO_USER_NAME_KEY)
+            .withHttpSpnegoKeytabKey(
+                YarnConfiguration.RM_WEBAPP_SPENGO_KEYTAB_FILE_KEY)
+            .at(this.conf.get(YarnConfiguration.RM_WEBAPP_ADDRESS,
+        YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS)); 
     String proxyHostAndPort = YarnConfiguration.getProxyHostAndPort(conf);
     if(YarnConfiguration.getRMWebAppHostAndPort(conf).
         equals(proxyHostAndPort)) {