BUG-17186. Allow wildcard bind for additional YARN/MR endpoints requested by customer.

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java

@@ -38,6 +38,8 @@ public class JHAdminConfig {
   public static final int DEFAULT_MR_HISTORY_PORT = 10020;
   public static final String DEFAULT_MR_HISTORY_ADDRESS = "0.0.0.0:" +
       DEFAULT_MR_HISTORY_PORT;
+  public static final String MR_HISTORY_BIND_HOST = MR_HISTORY_PREFIX
+      + "bind-host";
   
   /** The address of the History server admin interface. */
   public static final String JHS_ADMIN_ADDRESS = MR_HISTORY_PREFIX
@@ -45,6 +47,8 @@ public class JHAdminConfig {
   public static final int DEFAULT_JHS_ADMIN_PORT = 10033;
   public static final String DEFAULT_JHS_ADMIN_ADDRESS = "0.0.0.0:"
       + DEFAULT_JHS_ADMIN_PORT;
+  public static final String JHS_ADMIN_BIND_HOST = MR_HISTORY_PREFIX
+      + "admin.bind-host";
 
   /** ACL of who can be admin of Job history server. */
   public static final String JHS_ADMIN_ACL = MR_HISTORY_PREFIX + "admin.acl";
@@ -144,6 +148,8 @@ public class JHAdminConfig {
   public static final int DEFAULT_MR_HISTORY_WEBAPP_PORT = 19888;
   public static final String DEFAULT_MR_HISTORY_WEBAPP_ADDRESS =
     "0.0.0.0:" + DEFAULT_MR_HISTORY_WEBAPP_PORT;
+  public static final String MR_HISTORY_WEBAPP_BIND_HOST =
+      MR_HISTORY_PREFIX + "webapp.bind-host";
   
   /**The https address the history server webapp is on.*/
   public static final String MR_HISTORY_WEBAPP_HTTPS_ADDRESS =
@@ -151,6 +157,8 @@ public class JHAdminConfig {
   public static final int DEFAULT_MR_HISTORY_WEBAPP_HTTPS_PORT = 19890;
   public static final String DEFAULT_MR_HISTORY_WEBAPP_HTTPS_ADDRESS =
       "0.0.0.0:" + DEFAULT_MR_HISTORY_WEBAPP_HTTPS_PORT;
+  public static final String MR_HISTORY_WEBAPP_HTTPS_BIND_HOST =
+      MR_HISTORY_PREFIX + "webapp.https.bind-host";
   
   /**The kerberos principal to be used for spnego filter for history server*/
   public static final String MR_WEBAPP_SPNEGO_USER_NAME_KEY =

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRWebAppUtil.java

@@ -29,6 +29,7 @@ import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.ipc.RPCUtil;
 
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
@@ -105,11 +106,15 @@ public class MRWebAppUtil {
   
   public static InetSocketAddress getJHSWebBindAddress(Configuration conf) {
     if (httpPolicyInJHS == Policy.HTTPS_ONLY) {
-      return conf.getSocketAddr(JHAdminConfig.MR_HISTORY_WEBAPP_HTTPS_ADDRESS,
+      return RPCUtil.getSocketAddr(conf,
+          JHAdminConfig.MR_HISTORY_WEBAPP_HTTPS_BIND_HOST,
+          JHAdminConfig.MR_HISTORY_WEBAPP_HTTPS_ADDRESS,
           JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_HTTPS_ADDRESS,
           JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_HTTPS_PORT);
     } else {
-      return conf.getSocketAddr(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS,
+      return RPCUtil.getSocketAddr(conf,
+          JHAdminConfig.MR_HISTORY_WEBAPP_BIND_HOST,
+          JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS,
           JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_ADDRESS,
           JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_PORT);
     }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java

@@ -83,6 +83,7 @@ import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.service.AbstractService;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.ipc.RPCUtil;
 import org.apache.hadoop.yarn.ipc.YarnRPC;
 import org.apache.hadoop.yarn.util.Records;
 import org.apache.hadoop.yarn.webapp.WebApp;
@@ -118,7 +119,9 @@ public class HistoryClientService extends AbstractService {
     Configuration conf = getConfig();
     YarnRPC rpc = YarnRPC.create(conf);
     initializeWebApp(conf);
-    InetSocketAddress address = conf.getSocketAddr(
+    InetSocketAddress address = RPCUtil.getSocketAddr(
+        conf,
+        JHAdminConfig.MR_HISTORY_BIND_HOST,
         JHAdminConfig.MR_HISTORY_ADDRESS,
         JHAdminConfig.DEFAULT_MR_HISTORY_ADDRESS,
         JHAdminConfig.DEFAULT_MR_HISTORY_PORT);

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java

@@ -34,6 +34,7 @@ import org.apache.hadoop.security.Groups;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authorize.AccessControlList;
 import org.apache.hadoop.security.authorize.ProxyUsers;
+import org.apache.hadoop.yarn.ipc.RPCUtil;
 import org.apache.hadoop.yarn.logaggregation.AggregatedLogDeletionService;
 import org.apache.hadoop.security.proto.RefreshUserMappingsProtocolProtos.RefreshUserMappingsProtocolService;
 import org.apache.hadoop.security.protocolPB.RefreshUserMappingsProtocolPB;
@@ -94,7 +95,9 @@ public class HSAdminServer extends AbstractService implements HSAdminProtocol {
 
     WritableRpcEngine.ensureInitialized();
 
-    clientRpcAddress = conf.getSocketAddr(JHAdminConfig.JHS_ADMIN_ADDRESS,
+    clientRpcAddress = RPCUtil.getSocketAddr(conf,
+        JHAdminConfig.JHS_ADMIN_BIND_HOST,
+        JHAdminConfig.JHS_ADMIN_ADDRESS,
         JHAdminConfig.DEFAULT_JHS_ADMIN_ADDRESS,
         JHAdminConfig.DEFAULT_JHS_ADMIN_PORT);
     clientRpcServer = new RPC.Builder(conf)

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java

@@ -249,6 +249,8 @@ public class YarnConfiguration extends Configuration {
   public static final int DEFAULT_RM_ADMIN_PORT = 8033;
   public static final String DEFAULT_RM_ADMIN_ADDRESS = "0.0.0.0:" +
       DEFAULT_RM_ADMIN_PORT;
+  public static final String RM_ADMIN_BIND_HOST =
+    RM_PREFIX + "admin.bind-host";
   
   /**Number of threads used to handle RM admin interface.*/
   public static final String RM_ADMIN_CLIENT_THREAD_COUNT =
@@ -712,6 +714,7 @@ public class YarnConfiguration extends Configuration {
   
   /** NM Webapp address.**/
   public static final String NM_WEBAPP_ADDRESS = NM_PREFIX + "webapp.address";
+  public static final String NM_WEBAPP_BIND_HOST = NM_PREFIX + "webapp.bind-host";
   public static final int DEFAULT_NM_WEBAPP_PORT = 8042;
   public static final String DEFAULT_NM_WEBAPP_ADDRESS = "0.0.0.0:" +
     DEFAULT_NM_WEBAPP_PORT;
@@ -719,6 +722,8 @@ public class YarnConfiguration extends Configuration {
   /** NM Webapp https address.**/
   public static final String NM_WEBAPP_HTTPS_ADDRESS = NM_PREFIX
       + "webapp.https.address";
+  public static final String NM_WEBAPP_HTTPS_BIND_HOST = NM_PREFIX
+      + "webapp.https.bind-host";
   public static final int DEFAULT_NM_WEBAPP_HTTPS_PORT = 8044;
   public static final String DEFAULT_NM_WEBAPP_HTTPS_ADDRESS = "0.0.0.0:"
       + DEFAULT_NM_WEBAPP_HTTPS_PORT; 
@@ -1133,6 +1138,10 @@ public class YarnConfiguration extends Configuration {
   public static final String TIMELINE_SERVICE_WEBAPP_ADDRESS =
       TIMELINE_SERVICE_PREFIX  + "webapp.address";
 
+  /** The listening endpoint for the timeline service web application.*/
+  public static final String TIMELINE_SERVICE_WEBAPP_BIND_HOST =
+      TIMELINE_SERVICE_PREFIX + "webapp.bind-host";
+
   public static final int DEFAULT_TIMELINE_SERVICE_WEBAPP_PORT = 8188;
   public static final String DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS =
       "0.0.0.0:" + DEFAULT_TIMELINE_SERVICE_WEBAPP_PORT;
@@ -1141,6 +1150,10 @@ public class YarnConfiguration extends Configuration {
   public static final String TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS =
       TIMELINE_SERVICE_PREFIX + "webapp.https.address";
 
+  /** The listening endpoint for the timeline service web application.*/
+  public static final String TIMELINE_SERVICE_WEBAPP_HTTPS_BIND_HOST =
+      TIMELINE_SERVICE_PREFIX + "webapp.https.bind-host";
+  
   public static final int DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT = 8190;
   public static final String DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS =
       "0.0.0.0:" + DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java

@@ -170,14 +170,29 @@ public class WebAppUtils {
     return sb.toString();
   }
   
-  public static String getNMWebAppURLWithoutScheme(Configuration conf) {
+  public static String getNMWebAppBindURLWithoutScheme(Configuration conf) {
+    String address;
+    String bindHost;
+
     if (YarnConfiguration.useHttps(conf)) {
-      return conf.get(YarnConfiguration.NM_WEBAPP_HTTPS_ADDRESS,
+      address = conf.get(YarnConfiguration.NM_WEBAPP_HTTPS_ADDRESS,
         YarnConfiguration.DEFAULT_NM_WEBAPP_HTTPS_ADDRESS);
+      bindHost = conf.getTrimmed(
+          YarnConfiguration.NM_WEBAPP_HTTPS_BIND_HOST);
     } else {
-      return conf.get(YarnConfiguration.NM_WEBAPP_ADDRESS,
-        YarnConfiguration.DEFAULT_NM_WEBAPP_ADDRESS);
+      address = conf.get(YarnConfiguration.NM_WEBAPP_ADDRESS,
+                         YarnConfiguration.DEFAULT_NM_WEBAPP_ADDRESS);
+      bindHost = conf.getTrimmed(
+          YarnConfiguration.NM_WEBAPP_BIND_HOST);
+    }
+
+    // If the bind-host setting exists then it overrides the hostname
+    // portion of the corresponding address.
+    if (bindHost != null && !bindHost.isEmpty()) {
+      address = bindHost + ":" + address.split(":")[1];
     }
+          
+    return address;
   }
 
   public static String getAHSWebAppURLWithoutScheme(Configuration conf) {
@@ -189,7 +204,32 @@ public class WebAppUtils {
         YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS);
     }
   }
-  
+
+  public static String getAHSWebAppBindURLWithoutScheme(Configuration conf) {
+    String address;
+    String bindHost;
+
+    if (YarnConfiguration.useHttps(conf)) {
+      address = conf.get(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS,
+        YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS);
+      bindHost = conf.getTrimmed(
+        YarnConfiguration.TIMELINE_SERVICE_WEBAPP_HTTPS_BIND_HOST);
+    } else {
+      address = conf.get(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
+        YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS);
+      bindHost = conf.getTrimmed(
+        YarnConfiguration.TIMELINE_SERVICE_WEBAPP_BIND_HOST); 
+    }
+    
+    // If bindHost is specified it overrides the hostname portion of the
+    // address.
+    if (bindHost != null && !bindHost.isEmpty()) {
+      address = bindHost + ":" + address.split(":")[1];
+    }
+    
+    return address;
+  }
+
   /**
    * if url has scheme then it will be returned as it is else it will return
    * url with scheme.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java

@@ -149,7 +149,7 @@ public class ApplicationHistoryServer extends CompositeService {
   }
 
   protected void startWebApp() {
-    String bindAddress = WebAppUtils.getAHSWebAppURLWithoutScheme(getConfig());
+    String bindAddress = WebAppUtils.getAHSWebAppBindURLWithoutScheme(getConfig());
     LOG.info("Instantiating AHSWebApp at " + bindAddress);
     try {
       webApp =

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java

@@ -22,7 +22,6 @@ import static org.apache.hadoop.yarn.util.StringHelper.pajoin;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.http.HttpConfig;
 import org.apache.hadoop.service.AbstractService;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
@@ -55,7 +54,7 @@ public class WebServer extends AbstractService {
 
   @Override
   protected void serviceStart() throws Exception {
-    String bindAddress = WebAppUtils.getNMWebAppURLWithoutScheme(getConfig());
+    String bindAddress = WebAppUtils.getNMWebAppBindURLWithoutScheme(getConfig());
     
     LOG.info("Instantiating NMWebApp at " + bindAddress);
     try {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java

@@ -88,7 +88,12 @@ public class AdminService extends CompositeService implements
   private boolean autoFailoverEnabled;
 
   private Server server;
+
+  // Address to use for connection.
   private InetSocketAddress masterServiceAddress;
+
+  // Address to use for binding. May be a wildcard address.
+  private InetSocketAddress masterServiceBindAddress;
   private AccessControlList adminAcl;
 
   private final RecordFactory recordFactory = 
@@ -115,6 +120,14 @@ public class AdminService extends CompositeService implements
         YarnConfiguration.RM_ADMIN_ADDRESS,
         YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS,
         YarnConfiguration.DEFAULT_RM_ADMIN_PORT);
+
+    masterServiceBindAddress = RPCUtil.getSocketAddr(
+        conf,
+        YarnConfiguration.RM_ADMIN_BIND_HOST,
+        YarnConfiguration.RM_ADMIN_ADDRESS,
+        YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS,
+        YarnConfiguration.DEFAULT_RM_ADMIN_PORT);
+
     adminAcl = new AccessControlList(conf.get(
         YarnConfiguration.YARN_ADMIN_ACL,
         YarnConfiguration.DEFAULT_YARN_ADMIN_ACL));
@@ -138,7 +151,7 @@ public class AdminService extends CompositeService implements
     Configuration conf = getConfig();
     YarnRPC rpc = YarnRPC.create(conf);
     this.server = (Server) rpc.getServer(
-        ResourceManagerAdministrationProtocol.class, this, masterServiceAddress,
+        ResourceManagerAdministrationProtocol.class, this, masterServiceBindAddress,
         conf, null,
         conf.getInt(YarnConfiguration.RM_ADMIN_CLIENT_THREAD_COUNT,
             YarnConfiguration.DEFAULT_RM_ADMIN_CLIENT_THREAD_COUNT));
@@ -168,7 +181,7 @@ public class AdminService extends CompositeService implements
 
     this.server.start();
     conf.updateConnectAddr(YarnConfiguration.RM_ADMIN_ADDRESS,
-        server.getListenerAddress());
+      masterServiceAddress);
   }
 
   protected void stopServer() throws Exception {