Merge r1569890 through r1572722 from trunk.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-5535@1572723 13f79535-47bb-0310-9956-ffa450edef68

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -384,6 +384,9 @@ Release 2.4.0 - UNRELEASED
     HADOOP-10368. InputStream is not closed in VersionInfo ctor.
     (Tsuyoshi OZAWA via szetszwo)
 
+    HADOOP-10353. FsUrlStreamHandlerFactory is not thread safe.
+    (Tudor Scurtu via cnauroth)
+
   BREAKDOWN OF HADOOP-10184 SUBTASKS AND RELATED JIRAS
 
     HADOOP-10185. FileSystem API for ACLs. (cnauroth)

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsUrlStreamHandlerFactory.java

@@ -19,8 +19,8 @@ package org.apache.hadoop.fs;
 
 import java.io.IOException;
 import java.net.URLStreamHandlerFactory;
-import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -45,7 +45,8 @@ public class FsUrlStreamHandlerFactory implements
   private Configuration conf;
 
   // This map stores whether a protocol is know or not by FileSystem
-  private Map<String, Boolean> protocols = new HashMap<String, Boolean>();
+  private Map<String, Boolean> protocols =
+      new ConcurrentHashMap<String, Boolean>();
 
   // The URL Stream handler
   private java.net.URLStreamHandler handler;

hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt

@@ -365,6 +365,8 @@ Release 2.4.0 - UNRELEASED
     HDFS-3969. Small bug fixes and improvements for disk locations API.
     (Todd Lipcon and Andrew Wang)
 
+    HDFS-6025. Update findbugsExcludeFile.xml. (szetszwo)
+
   OPTIMIZATIONS
 
     HDFS-5790. LeaseManager.findPath is very slow when many leases need recovery

hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml

@@ -50,103 +50,6 @@
      <Match>
        <Bug code="HRS" />
      </Match>
-     <!--
-       Ignore warnings where child class has the same name as
-       super class. Classes based on Old API shadow names from
-       new API. Should go off after HADOOP-1.0
-     -->
-     <Match>
-       <Class name="~org.apache.hadoop.mapred.*" />
-       <Bug pattern="NM_SAME_SIMPLE_NAME_AS_SUPERCLASS" />
-     </Match>
-     <!--
-       Ignore warnings for usage of System.exit. This is
-       required and have been well thought out
-     -->
-     <Match>
-       <Class name="org.apache.hadoop.mapred.Child$2" />
-       <Method name="run" />
-       <Bug pattern="DM_EXIT" />
-     </Match>
-     <Match>
-       <Class name="org.apache.hadoop.mapred.JobTracker" />
-       <Method name="addHostToNodeMapping" />
-       <Bug pattern="DM_EXIT" />
-     </Match>
-     <Match>
-       <Class name="org.apache.hadoop.mapred.Task" />
-       <Or>
-       <Method name="done" />
-       <Method name="commit" />
-       <Method name="statusUpdate" />
-       </Or>
-       <Bug pattern="DM_EXIT" />
-     </Match>
-     <Match>
-       <Class name="org.apache.hadoop.mapred.Task$TaskReporter" />
-       <Method name="run" />
-       <Bug pattern="DM_EXIT" />
-     </Match>     
-     <!--
-       We need to cast objects between old and new api objects
-     -->
-     <Match>
-       <Class name="org.apache.hadoop.mapred.OutputCommitter" />
-       <Bug pattern="BC_UNCONFIRMED_CAST" />
-     </Match>
-     <!--
-       We intentionally do the get name from the inner class
-     -->
-     <Match>
-       <Class name="org.apache.hadoop.mapred.TaskTracker$MapEventsFetcherThread" />
-       <Method name="run" />
-       <Bug pattern="IA_AMBIGUOUS_INVOCATION_OF_INHERITED_OR_OUTER_METHOD" />
-     </Match>
-     <Match>
-       <Class name="org.apache.hadoop.mapred.FileOutputCommitter" />
-       <Bug pattern="NM_WRONG_PACKAGE_INTENTIONAL" />
-     </Match>
-     <!--
-       Ignoring this warning as resolving this would need a non-trivial change in code 
-     -->
-     <Match>
-       <Class name="org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorBaseDescriptor" />
-       <Method name="configure" />
-       <Field name="maxNumItems" />
-       <Bug pattern="ST_WRITE_TO_STATIC_FROM_INSTANCE_METHOD" />
-     </Match>
-     <!--
-       Comes from org.apache.jasper.runtime.ResourceInjector. Cannot do much.
-     -->
-     <Match>
-       <Class name="org.apache.hadoop.mapred.jobqueue_005fdetails_jsp" />
-       <Field name="_jspx_resourceInjector" />
-       <Bug pattern="SE_BAD_FIELD" />
-     </Match>
-     <!--
-       Storing textInputFormat and then passing it as a parameter. Safe to ignore.
-     -->
-     <Match>
-       <Class name="org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorJob" />
-       <Method name="createValueAggregatorJob" />
-       <Bug pattern="DLS_DEAD_STORE_OF_CLASS_LITERAL" />
-     </Match>
-     <!--
-       Can remove this after the upgrade to findbugs1.3.8
-     -->
-     <Match>
-       <Class name="org.apache.hadoop.mapred.lib.db.DBInputFormat" />
-       <Method name="getSplits" />
-       <Bug pattern="DLS_DEAD_LOCAL_STORE" />
-     </Match>
-    <!--
-      This is a spurious warning. Just ignore
-    -->
-     <Match>
-       <Class name="org.apache.hadoop.mapred.MapTask$MapOutputBuffer" />
-       <Field name="kvindex" />
-       <Bug pattern="IS2_INCONSISTENT_SYNC" />
-     </Match>
 
      <!-- 
         core changes 
@@ -156,68 +59,6 @@
        <Bug code="MS" />
      </Match>
 
-     <Match>
-       <Class name="org.apache.hadoop.fs.FileSystem" />
-       <Method name="checkPath" />
-       <Bug pattern="ES_COMPARING_STRINGS_WITH_EQ" />
-     </Match>
-
-     <Match>
-       <Class name="org.apache.hadoop.fs.kfs.KFSOutputStream" />
-       <Field name="path" />
-       <Bug pattern="URF_UNREAD_FIELD" />
-     </Match>
-
-     <Match>
-       <Class name="org.apache.hadoop.fs.kfs.KosmosFileSystem" />
-       <Method name="initialize" />
-       <Bug pattern="DM_EXIT" />
-     </Match>
-
-     <Match>
-       <Class name="org.apache.hadoop.io.Closeable" />
-       <Bug pattern="NM_SAME_SIMPLE_NAME_AS_INTERFACE" />
-     </Match>
-
-     <Match>
-       <Class name="org.apache.hadoop.security.AccessControlException" />
-       <Bug pattern="NM_SAME_SIMPLE_NAME_AS_SUPERCLASS" />
-     </Match>
-
-     <Match>
-       <Class name="org.apache.hadoop.record.meta.Utils" />
-       <Method name="skip" />
-       <Bug pattern="BC_UNCONFIRMED_CAST" />
-     </Match>
-
-     <!--
-        The compareTo method is actually a dummy method that just
-        throws excpetions. So, no need to override equals. Ignore
-     -->
-     <Match>
-       <Class name="org.apache.hadoop.record.meta.RecordTypeInfo" />
-       <Bug pattern="EQ_COMPARETO_USE_OBJECT_EQUALS" />
-     </Match>
-
-     <Match>
-       <Class name="org.apache.hadoop.util.ProcfsBasedProcessTree" />
-       <Bug pattern="DMI_HARDCODED_ABSOLUTE_FILENAME" />
-     </Match>
-
-     <!--
-       Streaming, Examples
-     -->
-     <Match>
-       <Class name="org.apache.hadoop.streaming.StreamUtil$TaskId" />
-       <Bug pattern="URF_UNREAD_FIELD" />
-     </Match>
-
-     <Match>
-       <Class name="org.apache.hadoop.examples.DBCountPageView" />
-       <Method name="verify" />
-       <Bug pattern="OBL_UNSATISFIED_OBLIGATION" />
-     </Match>
-
      <!--
        getTmpInputStreams is pretty much like a stream constructor.
        The newly created streams are not supposed to be closed in the constructor. So ignore
@@ -233,7 +74,7 @@
       ResponseProccessor is thread that is designed to catch RuntimeException.
      -->
      <Match>
-       <Class name="org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$DataStreamer$ResponseProcessor" />
+       <Class name="org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer$ResponseProcessor" />
        <Method name="run" />
        <Bug pattern="REC_CATCH_EXCEPTION" />
      </Match>
@@ -311,22 +152,6 @@
        <Method name="notifyNamenodeBlockImmediately" />
        <Field name="pendingReceivedRequests" />
        <Bug pattern="VO_VOLATILE_INCREMENT" />
-     </Match>
-     <!-- The "LightWeight" classes are explicitly not thread safe -->
-     <Match>
-       <Class name="org.apache.hadoop.hdfs.util.LightWeightGSet" />
-       <Field name="modification" />
-       <Bug pattern="VO_VOLATILE_INCREMENT" />
-     </Match>
-     <Match>
-       <Class name="org.apache.hadoop.hdfs.util.LightWeightHashSet" />
-       <Field name="modification" />
-       <Bug pattern="VO_VOLATILE_INCREMENT" />
-     </Match>
-     <Match>
-       <Class name="org.apache.hadoop.hdfs.util.LightWeightLinkedSet" />
-       <Field name="modification" />
-       <Bug pattern="VO_VOLATILE_INCREMENT" />
      </Match>
       <!-- Replace System.exit() call with ExitUtil.terminate() -->
      <Match>

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestUrlStreamHandler.java

@@ -35,7 +35,7 @@ import org.apache.hadoop.test.PathUtils;
 import org.junit.Test;
 
 /**
- * Test of the URL stream handler factory.
+ * Test of the URL stream handler.
  */
 public class TestUrlStreamHandler {
 

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestUrlStreamHandlerFactory.java

@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs;
+
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Random;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Test of the URL stream handler factory.
+ */
+public class TestUrlStreamHandlerFactory {
+
+  private static final int RUNS = 20;
+  private static final int THREADS = 10;
+  private static final int TASKS = 200;
+  private static final int TIMEOUT = 30;
+
+  @Test
+  public void testConcurrency() throws Exception {
+    for (int i = 0; i < RUNS; i++) {
+      singleRun();
+    }
+  }
+
+  private void singleRun() throws Exception {
+    final FsUrlStreamHandlerFactory factory = new FsUrlStreamHandlerFactory();
+    final Random random = new Random();
+    ExecutorService executor = Executors.newFixedThreadPool(THREADS);
+    ArrayList<Future<?>> futures = new ArrayList<Future<?>>(TASKS);
+
+    for (int i = 0; i < TASKS ; i++) {
+      final int aux = i;
+      futures.add(executor.submit(new Runnable() {
+        @Override
+        public void run() {
+          int rand = aux + random.nextInt(3);
+          factory.createURLStreamHandler(String.valueOf(rand));
+        }
+      }));
+    }
+
+    executor.shutdown();
+    try {
+      executor.awaitTermination(TIMEOUT, TimeUnit.SECONDS);
+      executor.shutdownNow();
+    } catch (InterruptedException e) {
+      // pass
+    }
+
+    // check for exceptions
+    for (Future future : futures) {
+      if (!future.isDone()) {
+        break; // timed out
+      }
+      future.get();
+    }
+  }
+}

hadoop-mapreduce-project/CHANGES.txt

@@ -195,6 +195,10 @@ Release 2.4.0 - UNRELEASED
     MAPREDUCE-5757. ConcurrentModificationException in JobControl.toList
     (jlowe)
 
+    MAPREDUCE-5770. Fixed MapReduce ApplicationMaster to correctly redirect
+    to the YARN's web-app proxy with the correct scheme prefix. (Jian He via
+    vinodkv)
+
 Release 2.3.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java

@@ -1382,13 +1382,7 @@ public class MRAppMaster extends CompositeService {
       JobConf conf = new JobConf(new YarnConfiguration());
       conf.addResource(new Path(MRJobConfig.JOB_CONF_FILE));
       
-      // Explicitly disabling SSL for map reduce task as we can't allow MR users
-      // to gain access to keystore file for opening SSL listener. We can trust
-      // RM/NM to issue SSL certificates but definitely not MR-AM as it is
-      // running in user-land.
       MRWebAppUtil.initialize(conf);
-      conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY,
-          HttpConfig.Policy.HTTP_ONLY.name());
       // log the system properties
       String systemPropsToLog = MRApps.getSystemPropertiesToLog(conf);
       if (systemPropsToLog != null) {
@@ -1490,4 +1484,7 @@ public class MRAppMaster extends CompositeService {
     LogManager.shutdown();
   }
 
+  public ClientService getClientService() {
+    return clientService;
+  }
 }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java

@@ -27,6 +27,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.http.HttpConfig.Policy;
 import org.apache.hadoop.ipc.Server;
 import org.apache.hadoop.mapreduce.JobACL;
 import org.apache.hadoop.mapreduce.MRJobConfig;
@@ -133,8 +134,13 @@ public class MRClientService extends AbstractService implements ClientService {
     this.bindAddress = NetUtils.getConnectAddress(server);
     LOG.info("Instantiated MRClientService at " + this.bindAddress);
     try {
-      webApp = WebApps.$for("mapreduce", AppContext.class, appContext, "ws").with(conf).
-          start(new AMWebApp());
+      // Explicitly disabling SSL for map reduce task as we can't allow MR users
+      // to gain access to keystore file for opening SSL listener. We can trust
+      // RM/NM to issue SSL certificates but definitely not MR-AM as it is
+      // running in user-land.
+      webApp =
+          WebApps.$for("mapreduce", AppContext.class, appContext, "ws")
+            .withHttpPolicy(conf, Policy.HTTP_ONLY).start(new AMWebApp());
     } catch (Exception e) {
       LOG.error("Webapps failed to start. Ignoring for now:", e);
     }
@@ -412,4 +418,8 @@ public class MRClientService extends AbstractService implements ClientService {
           " token");
     }
   }
+
+  public WebApp getWebApp() {
+    return webApp;
+  }
 }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebApp.java

@@ -21,23 +21,45 @@ package org.apache.hadoop.mapreduce.v2.app.webapp;
 import static org.apache.hadoop.mapreduce.v2.app.webapp.AMParams.APP_ID;
 import static org.junit.Assert.assertEquals;
 
+import java.io.ByteArrayOutputStream;
+import java.io.InputStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Map.Entry;
 
+import javax.net.ssl.SSLException;
+
+import junit.framework.Assert;
+
+import org.apache.commons.httpclient.HttpStatus;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.http.HttpConfig.Policy;
+import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.mapreduce.v2.api.records.JobState;
 import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
 import org.apache.hadoop.mapreduce.v2.app.AppContext;
+import org.apache.hadoop.mapreduce.v2.app.MRApp;
 import org.apache.hadoop.mapreduce.v2.app.MockAppContext;
 import org.apache.hadoop.mapreduce.v2.app.MockJobs;
+import org.apache.hadoop.mapreduce.v2.app.client.ClientService;
+import org.apache.hadoop.mapreduce.v2.app.client.MRClientService;
 import org.apache.hadoop.mapreduce.v2.app.job.Job;
 import org.apache.hadoop.mapreduce.v2.app.job.Task;
 import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
 import org.apache.hadoop.mapreduce.v2.util.MRApps;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
+import org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer;
 import org.apache.hadoop.yarn.webapp.WebApps;
 import org.apache.hadoop.yarn.webapp.test.WebAppTests;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 import org.junit.Test;
 
+import com.google.common.net.HttpHeaders;
 import com.google.inject.Injector;
 
 public class TestAMWebApp {
@@ -147,7 +169,96 @@ public class TestAMWebApp {
     WebAppTests.testPage(SingleCounterPage.class, AppContext.class,
                          appContext, params);
   }
-  
+
+  @Test
+  public void testMRWebAppSSLDisabled() throws Exception {
+    MRApp app = new MRApp(2, 2, true, this.getClass().getName(), true) {
+      @Override
+      protected ClientService createClientService(AppContext context) {
+        return new MRClientService(context);
+      }
+    };
+    Configuration conf = new Configuration();
+    // MR is explicitly disabling SSL, even though setting as HTTPS_ONLY
+    conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY, Policy.HTTPS_ONLY.name());
+    Job job = app.submit(conf);
+
+    String hostPort =
+        NetUtils.getHostPortString(((MRClientService) app.getClientService())
+          .getWebApp().getListenerAddress());
+    // http:// should be accessible
+    URL httpUrl = new URL("http://" + hostPort);
+    HttpURLConnection conn = (HttpURLConnection) httpUrl.openConnection();
+    InputStream in = conn.getInputStream();
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+    IOUtils.copyBytes(in, out, 1024);
+    Assert.assertTrue(out.toString().contains("MapReduce Application"));
+
+    // https:// is not accessible.
+    URL httpsUrl = new URL("https://" + hostPort);
+    try {
+      HttpURLConnection httpsConn =
+          (HttpURLConnection) httpsUrl.openConnection();
+      httpsConn.getInputStream();
+      Assert.fail("https:// is not accessible, expected to fail");
+    } catch (Exception e) {
+      Assert.assertTrue(e instanceof SSLException);
+    }
+
+    app.waitForState(job, JobState.SUCCEEDED);
+    app.verifyCompleted();
+  }
+
+  static String webProxyBase = null;
+  public static class TestAMFilterInitializer extends AmFilterInitializer {
+
+    @Override
+    protected String getApplicationWebProxyBase() {
+      return webProxyBase;
+    }
+  }
+
+  @Test
+  public void testMRWebAppRedirection() throws Exception {
+
+    String[] schemePrefix =
+        { WebAppUtils.HTTP_PREFIX, WebAppUtils.HTTPS_PREFIX };
+    for (String scheme : schemePrefix) {
+      MRApp app = new MRApp(2, 2, true, this.getClass().getName(), true) {
+        @Override
+        protected ClientService createClientService(AppContext context) {
+          return new MRClientService(context);
+        }
+      };
+      Configuration conf = new Configuration();
+      conf.set(YarnConfiguration.PROXY_ADDRESS, "9.9.9.9");
+      conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY, scheme
+        .equals(WebAppUtils.HTTPS_PREFIX) ? Policy.HTTPS_ONLY.name()
+          : Policy.HTTP_ONLY.name());
+      webProxyBase = "/proxy/" + app.getAppID();
+      conf.set("hadoop.http.filter.initializers",
+        TestAMFilterInitializer.class.getName());
+      Job job = app.submit(conf);
+      String hostPort =
+          NetUtils.getHostPortString(((MRClientService) app.getClientService())
+            .getWebApp().getListenerAddress());
+      URL httpUrl = new URL("http://" + hostPort + "/mapreduce");
+
+      HttpURLConnection conn = (HttpURLConnection) httpUrl.openConnection();
+      conn.setInstanceFollowRedirects(false);
+      conn.connect();
+      String expectedURL =
+          scheme + conf.get(YarnConfiguration.PROXY_ADDRESS)
+              + ProxyUriUtils.getPath(app.getAppID(), "/mapreduce");
+      Assert.assertEquals(expectedURL,
+        conn.getHeaderField(HttpHeaders.LOCATION));
+      Assert.assertEquals(HttpStatus.SC_MOVED_TEMPORARILY,
+        conn.getResponseCode());
+      app.waitForState(job, JobState.SUCCEEDED);
+      app.verifyCompleted();
+    }
+  }
+
   public static void main(String[] args) {
     WebApps.$for("yarn", AppContext.class, new MockAppContext(0, 8, 88, 4)).
         at(58888).inDevMode().start(new AMWebApp()).joinThread();

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java

@@ -35,6 +35,8 @@ import javax.servlet.http.HttpServlet;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.http.HttpConfig.Policy;
+import org.apache.hadoop.http.HttpConfig;
 import org.apache.hadoop.http.HttpServer2;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -86,6 +88,7 @@ public class WebApps {
     int port = 0;
     boolean findPort = false;
     Configuration conf;
+    Policy httpPolicy = null;
     boolean devMode = false;
     private String spnegoPrincipalKey;
     private String spnegoKeytabKey;
@@ -142,7 +145,13 @@ public class WebApps {
       this.conf = conf;
       return this;
     }
-    
+
+    public Builder<T> withHttpPolicy(Configuration conf, Policy httpPolicy) {
+      this.conf = conf;
+      this.httpPolicy = httpPolicy;
+      return this;
+    }
+
     public Builder<T> withHttpSpnegoPrincipalKey(String spnegoPrincipalKey) {
       this.spnegoPrincipalKey = spnegoPrincipalKey;
       return this;
@@ -218,10 +227,18 @@ public class WebApps {
             System.exit(1);
           }
         }
+        String httpScheme;
+        if (this.httpPolicy == null) {
+          httpScheme = WebAppUtils.getHttpSchemePrefix(conf);
+        } else {
+          httpScheme =
+              (httpPolicy == Policy.HTTPS_ONLY) ? WebAppUtils.HTTPS_PREFIX
+                  : WebAppUtils.HTTP_PREFIX;
+        }
         HttpServer2.Builder builder = new HttpServer2.Builder()
             .setName(name)
             .addEndpoint(
-                URI.create(WebAppUtils.getHttpSchemePrefix(conf) + bindAddress
+                URI.create(httpScheme + bindAddress
                     + ":" + port)).setConf(conf).setFindPort(findPort)
             .setACL(new AdminACLsManager(conf).getAdminAcl())
             .setPathSpec(pathList.toArray(new String[0]));
@@ -236,7 +253,7 @@ public class WebApps {
               .setSecurityEnabled(UserGroupInformation.isSecurityEnabled());
         }
 
-        if (YarnConfiguration.useHttps(conf)) {
+        if (httpScheme.equals(WebAppUtils.HTTPS_PREFIX)) {
           WebAppUtils.loadSslConfiguration(builder);
         }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java

@@ -36,6 +36,9 @@ import org.apache.hadoop.yarn.util.ConverterUtils;
 @Private
 @Evolving
 public class WebAppUtils {
+  public static final String HTTPS_PREFIX = "https://";
+  public static final String HTTP_PREFIX = "http://";
+
   public static void setRMWebAppPort(Configuration conf, int port) {
     String hostname = getRMWebAppURLWithoutScheme(conf);
     hostname =
@@ -180,7 +183,7 @@ public class WebAppUtils {
    * @return the schmeme (HTTP / HTTPS)
    */
   public static String getHttpSchemePrefix(Configuration conf) {
-    return YarnConfiguration.useHttps(conf) ? "https://" : "http://";
+    return YarnConfiguration.useHttps(conf) ? HTTPS_PREFIX : HTTP_PREFIX;
   }
 
   /**

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java

@@ -27,6 +27,8 @@ import org.apache.hadoop.http.FilterInitializer;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 
+import com.google.common.annotations.VisibleForTesting;
+
 public class AmFilterInitializer extends FilterInitializer {
   private static final String FILTER_NAME = "AM_PROXY_FILTER";
   private static final String FILTER_CLASS = AmIpFilter.class.getCanonicalName();
@@ -37,10 +39,13 @@ public class AmFilterInitializer extends FilterInitializer {
     String proxy = WebAppUtils.getProxyHostAndPort(conf);
     String[] parts = proxy.split(":");
     params.put(AmIpFilter.PROXY_HOST, parts[0]);
-    params.put(AmIpFilter.PROXY_URI_BASE,
-        WebAppUtils.getHttpSchemePrefix(conf) + proxy +
-        System.getenv(ApplicationConstants.APPLICATION_WEB_PROXY_BASE_ENV));
+    params.put(AmIpFilter.PROXY_URI_BASE, WebAppUtils.getHttpSchemePrefix(conf)
+        + proxy + getApplicationWebProxyBase());
     container.addFilter(FILTER_NAME, FILTER_CLASS, params);
   }
 
+  @VisibleForTesting
+  protected String getApplicationWebProxyBase() {
+    return System.getenv(ApplicationConstants.APPLICATION_WEB_PROXY_BASE_ENV);
+  }
 }