Browse Source

YARN-4283 Avoid unsafe split and append on fields that might be IPv6 literals

Elliott Clark 10 years ago
parent
commit
009e67021f
11 changed files with 116 additions and 74 deletions
  1. 11 9
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
  2. 10 10
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
  3. 51 31
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java
  4. 11 3
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java
  5. 11 6
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/lib/TestZKClient.java
  6. 2 1
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
  7. 3 3
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
  8. 5 2
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
  9. 4 3
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java
  10. 4 2
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
  11. 4 4
      hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java

+ 11 - 9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java

@@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.api.records.URL;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 
 
+import com.google.common.net.HostAndPort;
 
 
 /**
 /**
  * This class contains a set of utilities which help converting data structures
  * This class contains a set of utilities which help converting data structures
@@ -152,26 +153,27 @@ public class ConverterUtils {
   public static String toString(ContainerId cId) {
   public static String toString(ContainerId cId) {
     return cId == null ? null : cId.toString();
     return cId == null ? null : cId.toString();
   }
   }
-  
+
   public static NodeId toNodeIdWithDefaultPort(String nodeIdStr) {
   public static NodeId toNodeIdWithDefaultPort(String nodeIdStr) {
-    if (nodeIdStr.indexOf(":") < 0) {
-      return toNodeId(nodeIdStr + ":0");
-    }
-    return toNodeId(nodeIdStr);
+    HostAndPort hp = HostAndPort.fromString(nodeIdStr);
+    hp = hp.withDefaultPort(0);
+    return toNodeId(hp.toString());
   }
   }
 
 
   public static NodeId toNodeId(String nodeIdStr) {
   public static NodeId toNodeId(String nodeIdStr) {
-    String[] parts = nodeIdStr.split(":");
-    if (parts.length != 2) {
+    HostAndPort hp = HostAndPort.fromString(nodeIdStr);
+    if (!hp.hasPort()) {
       throw new IllegalArgumentException("Invalid NodeId [" + nodeIdStr
       throw new IllegalArgumentException("Invalid NodeId [" + nodeIdStr
           + "]. Expected host:port");
           + "]. Expected host:port");
     }
     }
     try {
     try {
+      String hostPortStr = hp.toString();
+      String host = hostPortStr.substring(0, hostPortStr.lastIndexOf(":"));
       NodeId nodeId =
       NodeId nodeId =
-          NodeId.newInstance(parts[0].trim(), Integer.parseInt(parts[1]));
+          NodeId.newInstance(host, hp.getPort());
       return nodeId;
       return nodeId;
     } catch (NumberFormatException e) {
     } catch (NumberFormatException e) {
-      throw new IllegalArgumentException("Invalid port: " + parts[1], e);
+      throw new IllegalArgumentException("Invalid port: " + hp.getPort(), e);
     }
     }
   }
   }
 
 

+ 10 - 10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java

@@ -37,6 +37,8 @@ import org.apache.hadoop.yarn.conf.HAUtil;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.util.RMHAUtils;
 import org.apache.hadoop.yarn.util.RMHAUtils;
 
 
+import com.google.common.net.HostAndPort;
+
 @Private
 @Private
 @Evolving
 @Evolving
 public class WebAppUtils {
 public class WebAppUtils {
@@ -51,15 +53,13 @@ public class WebAppUtils {
 
 
   public static void setRMWebAppPort(Configuration conf, int port) {
   public static void setRMWebAppPort(Configuration conf, int port) {
     String hostname = getRMWebAppURLWithoutScheme(conf);
     String hostname = getRMWebAppURLWithoutScheme(conf);
-    hostname =
-        (hostname.contains(":")) ? hostname.substring(0, hostname.indexOf(":"))
-            : hostname;
-    setRMWebAppHostnameAndPort(conf, hostname, port);
+    HostAndPort hp = HostAndPort.fromString(hostname);
+    setRMWebAppHostnameAndPort(conf, hp.getHostText(), port);
   }
   }
 
 
   public static void setRMWebAppHostnameAndPort(Configuration conf,
   public static void setRMWebAppHostnameAndPort(Configuration conf,
       String hostname, int port) {
       String hostname, int port) {
-    String resolvedAddress = hostname + ":" + port;
+    String resolvedAddress = HostAndPort.fromParts(hostname, port).toString();
     if (YarnConfiguration.useHttps(conf)) {
     if (YarnConfiguration.useHttps(conf)) {
       conf.set(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, resolvedAddress);
       conf.set(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, resolvedAddress);
     } else {
     } else {
@@ -69,12 +69,11 @@ public class WebAppUtils {
   
   
   public static void setNMWebAppHostNameAndPort(Configuration conf,
   public static void setNMWebAppHostNameAndPort(Configuration conf,
       String hostName, int port) {
       String hostName, int port) {
+    String hostPortString = HostAndPort.fromParts(hostName, port).toString();
     if (YarnConfiguration.useHttps(conf)) {
     if (YarnConfiguration.useHttps(conf)) {
-      conf.set(YarnConfiguration.NM_WEBAPP_HTTPS_ADDRESS,
-          hostName + ":" + port);
+      conf.set(YarnConfiguration.NM_WEBAPP_HTTPS_ADDRESS, hostPortString);
     } else {
     } else {
-      conf.set(YarnConfiguration.NM_WEBAPP_ADDRESS,
-          hostName + ":" + port);
+      conf.set(YarnConfiguration.NM_WEBAPP_ADDRESS, hostPortString);
     }
     }
   }
   }
   
   
@@ -241,7 +240,8 @@ public class WebAppUtils {
     String host = conf.getTrimmed(hostProperty);
     String host = conf.getTrimmed(hostProperty);
     if (host != null && !host.isEmpty()) {
     if (host != null && !host.isEmpty()) {
       if (webAppURLWithoutScheme.contains(":")) {
       if (webAppURLWithoutScheme.contains(":")) {
-        webAppURLWithoutScheme = host + ":" + webAppURLWithoutScheme.split(":")[1];
+        String[] splits = webAppURLWithoutScheme.split(":");
+        webAppURLWithoutScheme = host + ":" + splits[splits.length - 1];
       }
       }
       else {
       else {
         throw new YarnRuntimeException("webAppURLWithoutScheme must include port specification but doesn't: " +
         throw new YarnRuntimeException("webAppURLWithoutScheme must include port specification but doesn't: " +

+ 51 - 31
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java

@@ -18,6 +18,8 @@
 
 
 package org.apache.hadoop.yarn.conf;
 package org.apache.hadoop.yarn.conf;
 
 
+import com.google.common.net.HostAndPort;
+import org.apache.hadoop.net.NetUtils;
 import org.junit.Assert;
 import org.junit.Assert;
 
 
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
@@ -53,7 +55,7 @@ public class TestYarnConfiguration {
     conf.set(YarnConfiguration.RM_ADDRESS, "rmtesting:9999");
     conf.set(YarnConfiguration.RM_ADDRESS, "rmtesting:9999");
     String rmWebUrl = WebAppUtils.getRMWebAppURLWithScheme(conf);
     String rmWebUrl = WebAppUtils.getRMWebAppURLWithScheme(conf);
     String[] parts = rmWebUrl.split(":");
     String[] parts = rmWebUrl.split(":");
-    Assert.assertEquals("RM Web URL Port is incrrect", 24543,
+    Assert.assertEquals("RM Web URL Port is incorrect", 24543,
         Integer.valueOf(parts[parts.length - 1]).intValue());
         Integer.valueOf(parts[parts.length - 1]).intValue());
     Assert.assertNotSame(
     Assert.assertNotSame(
         "RM Web Url not resolved correctly. Should not be rmtesting",
         "RM Web Url not resolved correctly. Should not be rmtesting",
@@ -92,9 +94,9 @@ public class TestYarnConfiguration {
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
     assertEquals(
     assertEquals(
-      new InetSocketAddress(
-        YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[0],
-        YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
+        new InetSocketAddress(
+            YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[0],
+            YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
         resourceTrackerAddress);
         resourceTrackerAddress);
 
 
     //with address
     //with address
@@ -105,9 +107,9 @@ public class TestYarnConfiguration {
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
     assertEquals(
     assertEquals(
-      new InetSocketAddress(
-        "10.0.0.1",
-        YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
+        new InetSocketAddress(
+            "10.0.0.1",
+            YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
         resourceTrackerAddress);
         resourceTrackerAddress);
 
 
     //address and socket
     //address and socket
@@ -118,9 +120,23 @@ public class TestYarnConfiguration {
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
     assertEquals(
     assertEquals(
-      new InetSocketAddress(
-        "10.0.0.2",
-        5001),
+        new InetSocketAddress(
+            "10.0.0.2",
+            5001),
+        resourceTrackerAddress);
+
+    // IPv6 address and socket
+    conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
+        "[2401:db00:20:a01e:face:0:5:0]:5001");
+    resourceTrackerAddress = conf.getSocketAddr(
+        YarnConfiguration.RM_BIND_HOST,
+        YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
+        YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
+        YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
+    assertEquals(
+        new InetSocketAddress(
+            "2401:db00:20:a01e:face:0:5:0",
+            5001),
         resourceTrackerAddress);
         resourceTrackerAddress);
 
 
     //bind host only
     //bind host only
@@ -132,9 +148,9 @@ public class TestYarnConfiguration {
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
     assertEquals(
     assertEquals(
-      new InetSocketAddress(
-        "10.0.0.3",
-        YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
+        new InetSocketAddress(
+            "10.0.0.3",
+            YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
         resourceTrackerAddress);
         resourceTrackerAddress);
 
 
     //bind host and address no port
     //bind host and address no port
@@ -146,9 +162,9 @@ public class TestYarnConfiguration {
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
     assertEquals(
     assertEquals(
-      new InetSocketAddress(
-        "0.0.0.0",
-        YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
+        new InetSocketAddress(
+            "0.0.0.0",
+            YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
         resourceTrackerAddress);
         resourceTrackerAddress);
 
 
     //bind host and address with port
     //bind host and address with port
@@ -160,9 +176,9 @@ public class TestYarnConfiguration {
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
     assertEquals(
     assertEquals(
-      new InetSocketAddress(
-        "0.0.0.0",
-        5003),
+        new InetSocketAddress(
+            "0.0.0.0",
+            5003),
         resourceTrackerAddress);
         resourceTrackerAddress);
 
 
   }
   }
@@ -176,9 +192,8 @@ public class TestYarnConfiguration {
     //no override, old behavior.  Won't work on a host named "yo.yo.yo"
     //no override, old behavior.  Won't work on a host named "yo.yo.yo"
     conf = new YarnConfiguration();
     conf = new YarnConfiguration();
     conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, "yo.yo.yo");
     conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, "yo.yo.yo");
-    serverAddress = new InetSocketAddress(
-        YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[0],
-        Integer.valueOf(YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[1]));
+    serverAddress = newInetSocketAddressFromHostPort(
+        YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS);
 
 
     resourceTrackerConnectAddress = conf.updateConnectAddr(
     resourceTrackerConnectAddress = conf.updateConnectAddr(
         YarnConfiguration.RM_BIND_HOST,
         YarnConfiguration.RM_BIND_HOST,
@@ -186,15 +201,14 @@ public class TestYarnConfiguration {
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
         serverAddress);
         serverAddress);
 
 
-    assertFalse(resourceTrackerConnectAddress.toString().startsWith("yo.yo.yo"));
+    assertFalse(NetUtils.getSocketAddressString(resourceTrackerConnectAddress)
+        .startsWith("yo.yo.yo"));
 
 
     //cause override with address
     //cause override with address
     conf = new YarnConfiguration();
     conf = new YarnConfiguration();
     conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, "yo.yo.yo");
     conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, "yo.yo.yo");
     conf.set(YarnConfiguration.RM_BIND_HOST, "0.0.0.0");
     conf.set(YarnConfiguration.RM_BIND_HOST, "0.0.0.0");
-    serverAddress = new InetSocketAddress(
-        YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[0],
-        Integer.valueOf(YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[1]));
+    serverAddress = newInetSocketAddressFromHostPort(YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS);
 
 
     resourceTrackerConnectAddress = conf.updateConnectAddr(
     resourceTrackerConnectAddress = conf.updateConnectAddr(
         YarnConfiguration.RM_BIND_HOST,
         YarnConfiguration.RM_BIND_HOST,
@@ -202,7 +216,8 @@ public class TestYarnConfiguration {
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
         serverAddress);
         serverAddress);
 
 
-    assertTrue(resourceTrackerConnectAddress.toString().startsWith("yo.yo.yo"));
+    assertTrue(NetUtils.getSocketAddressString(resourceTrackerConnectAddress)
+        .startsWith("yo.yo.yo"));
 
 
     //tests updateConnectAddr won't add suffix to NM service address configurations
     //tests updateConnectAddr won't add suffix to NM service address configurations
     conf = new YarnConfiguration();
     conf = new YarnConfiguration();
@@ -211,9 +226,7 @@ public class TestYarnConfiguration {
     conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
     conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
     conf.set(YarnConfiguration.RM_HA_ID, "rm1");
     conf.set(YarnConfiguration.RM_HA_ID, "rm1");
 
 
-    serverAddress = new InetSocketAddress(
-        YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS.split(":")[0],
-        Integer.valueOf(YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS.split(":")[1]));
+    serverAddress = newInetSocketAddressFromHostPort(YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS);
 
 
     InetSocketAddress localizerAddress = conf.updateConnectAddr(
     InetSocketAddress localizerAddress = conf.updateConnectAddr(
         YarnConfiguration.NM_BIND_HOST,
         YarnConfiguration.NM_BIND_HOST,
@@ -221,8 +234,15 @@ public class TestYarnConfiguration {
         YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS,
         YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS,
         serverAddress);
         serverAddress);
 
 
-    assertTrue(localizerAddress.toString().startsWith("yo.yo.yo"));
+    assertTrue(NetUtils.getSocketAddressString(localizerAddress)
+        .startsWith("yo.yo.yo"));
     assertNull(conf.get(
     assertNull(conf.get(
         HAUtil.addSuffix(YarnConfiguration.NM_LOCALIZER_ADDRESS, "rm1")));
         HAUtil.addSuffix(YarnConfiguration.NM_LOCALIZER_ADDRESS, "rm1")));
   }
   }
+
+  private InetSocketAddress newInetSocketAddressFromHostPort(
+      String hostPort) {
+    HostAndPort hp = HostAndPort.fromString(hostPort);
+    return new InetSocketAddress(hp.getHostText(), hp.getPort());
+  }
 }
 }

+ 11 - 3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java

@@ -30,7 +30,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
 import org.junit.Test;
 import org.junit.Test;
 
 
 public class TestConverterUtils {
 public class TestConverterUtils {
-  
+
   @Test
   @Test
   public void testConvertUrlWithNoPort() throws URISyntaxException {
   public void testConvertUrlWithNoPort() throws URISyntaxException {
     Path expectedPath = new Path("hdfs://foo.com");
     Path expectedPath = new Path("hdfs://foo.com");
@@ -90,14 +90,22 @@ public class TestConverterUtils {
   @Test
   @Test
   public void testNodeIdWithDefaultPort() throws URISyntaxException {
   public void testNodeIdWithDefaultPort() throws URISyntaxException {
     NodeId nid;
     NodeId nid;
-    
+
     nid = ConverterUtils.toNodeIdWithDefaultPort("node:10");
     nid = ConverterUtils.toNodeIdWithDefaultPort("node:10");
     assertEquals(nid.getPort(), 10);
     assertEquals(nid.getPort(), 10);
     assertEquals(nid.getHost(), "node");
     assertEquals(nid.getHost(), "node");
-    
+
     nid = ConverterUtils.toNodeIdWithDefaultPort("node");
     nid = ConverterUtils.toNodeIdWithDefaultPort("node");
     assertEquals(nid.getPort(), 0);
     assertEquals(nid.getPort(), 0);
     assertEquals(nid.getHost(), "node");
     assertEquals(nid.getHost(), "node");
+
+    nid = ConverterUtils.toNodeIdWithDefaultPort("[2401:db00:20:a01e:face:0:5:0]:10");
+    assertEquals(nid.getPort(), 10);
+    assertEquals(nid.getHost(), "[2401:db00:20:a01e:face:0:5:0]");
+
+    nid = ConverterUtils.toNodeIdWithDefaultPort("[2401:db00:20:a01e:face:0:5:0]");
+    assertEquals(nid.getPort(), 0);
+    assertEquals(nid.getHost(), "[2401:db00:20:a01e:face:0:5:0]");
   }
   }
 
 
   @Test(expected = IllegalArgumentException.class)
   @Test(expected = IllegalArgumentException.class)

+ 11 - 6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/lib/TestZKClient.java

@@ -26,6 +26,7 @@ import java.io.OutputStream;
 import java.net.InetSocketAddress;
 import java.net.InetSocketAddress;
 import java.net.Socket;
 import java.net.Socket;
 
 
+import com.google.common.net.HostAndPort;
 import org.junit.Assert;
 import org.junit.Assert;
 
 
 import org.apache.hadoop.yarn.lib.ZKClient;
 import org.apache.hadoop.yarn.lib.ZKClient;
@@ -82,8 +83,9 @@ public class TestZKClient  {
     long start = System.currentTimeMillis();
     long start = System.currentTimeMillis();
     while (true) {
     while (true) {
       try {
       try {
-        String host = hp.split(":")[0];
-        int port = Integer.parseInt(hp.split(":")[1]);
+        HostAndPort hap = HostAndPort.fromString(hp);
+        String host = hap.getHostText();
+        int port = hap.getPort();
         send4LetterWord(host, port, "stat");
         send4LetterWord(host, port, "stat");
       } catch (IOException e) {
       } catch (IOException e) {
         return true;
         return true;
@@ -106,8 +108,9 @@ public class TestZKClient  {
     long start = System.currentTimeMillis();
     long start = System.currentTimeMillis();
     while (true) {
     while (true) {
       try {
       try {
-        String host = hp.split(":")[0];
-        int port = Integer.parseInt(hp.split(":")[1]);
+        HostAndPort hap = HostAndPort.fromString(hp);
+        String host = hap.getHostText();
+        int port = hap.getPort();
         // if there are multiple hostports, just take the first one
         // if there are multiple hostports, just take the first one
         String result = send4LetterWord(host, port, "stat");
         String result = send4LetterWord(host, port, "stat");
         if (result.startsWith("Zookeeper version:")) {
         if (result.startsWith("Zookeeper version:")) {
@@ -146,7 +149,8 @@ public class TestZKClient  {
     }
     }
     File dataDir = createTmpDir(BASETEST);
     File dataDir = createTmpDir(BASETEST);
     zks = new ZooKeeperServer(dataDir, dataDir, 3000);
     zks = new ZooKeeperServer(dataDir, dataDir, 3000);
-    final int PORT = Integer.parseInt(hostPort.split(":")[1]);
+    HostAndPort hp = HostAndPort.fromString(hostPort);
+    final int PORT = hp.getPort();
     if (factory == null) {
     if (factory == null) {
       factory = new NIOServerCnxnFactory();
       factory = new NIOServerCnxnFactory();
       factory.configure(new InetSocketAddress(PORT), maxCnxns);
       factory.configure(new InetSocketAddress(PORT), maxCnxns);
@@ -167,7 +171,8 @@ public class TestZKClient  {
         zkDb.close();
         zkDb.close();
       } catch (IOException ie) {
       } catch (IOException ie) {
       }
       }
-      final int PORT = Integer.parseInt(hostPort.split(":")[1]);
+      HostAndPort hp = HostAndPort.fromString(hostPort);
+      final int PORT = hp.getPort();
 
 
       Assert.assertTrue("waiting for server down",
       Assert.assertTrue("waiting for server down",
           waitForServerDown("127.0.0.1:" + PORT,
           waitForServerDown("127.0.0.1:" + PORT,

+ 2 - 1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java

@@ -37,6 +37,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
 
 
+import com.google.common.net.HostAndPort;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -474,7 +475,7 @@ public class ContainerManagerImpl extends CompositeService implements
       //hostname found when querying for our hostname with the specified
       //hostname found when querying for our hostname with the specified
       //address, combine the specified address with the actual port listened
       //address, combine the specified address with the actual port listened
       //on by the server
       //on by the server
-      hostOverride = nmAddress.split(":")[0];
+      hostOverride = HostAndPort.fromString(nmAddress).getHostText();
     }
     }
 
 
     // setup node ID
     // setup node ID

+ 3 - 3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java

@@ -19,6 +19,7 @@
 package org.apache.hadoop.yarn.server.resourcemanager;
 package org.apache.hadoop.yarn.server.resourcemanager;
 
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.net.HostAndPort;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -962,9 +963,8 @@ public class ResourceManager extends CompositeService implements Recoverable {
       builder.withServlet(ProxyUriUtils.PROXY_SERVLET_NAME,
       builder.withServlet(ProxyUriUtils.PROXY_SERVLET_NAME,
           ProxyUriUtils.PROXY_PATH_SPEC, WebAppProxyServlet.class);
           ProxyUriUtils.PROXY_PATH_SPEC, WebAppProxyServlet.class);
       builder.withAttribute(WebAppProxy.FETCHER_ATTRIBUTE, fetcher);
       builder.withAttribute(WebAppProxy.FETCHER_ATTRIBUTE, fetcher);
-      String[] proxyParts = proxyHostAndPort.split(":");
-      builder.withAttribute(WebAppProxy.PROXY_HOST_ATTRIBUTE, proxyParts[0]);
-
+      builder.withAttribute(WebAppProxy.PROXY_HOST_ATTRIBUTE,
+          HostAndPort.fromString(proxyHostAndPort).getHostText());
     }
     }
     webApp = builder.start(new RMWebApp(this));
     webApp = builder.start(new RMWebApp(this));
   }
   }

+ 5 - 2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java

@@ -24,6 +24,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.List;
 import java.util.Map;
 import java.util.Map;
 
 
+import com.google.common.net.HostAndPort;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerState;
 import org.apache.hadoop.yarn.api.records.ContainerState;
@@ -75,8 +76,10 @@ public class MockNM {
     this.vCores = vcores;
     this.vCores = vcores;
     this.resourceTracker = resourceTracker;
     this.resourceTracker = resourceTracker;
     this.version = version;
     this.version = version;
-    String[] splits = nodeIdStr.split(":");
-    nodeId = BuilderUtils.newNodeId(splits[0], Integer.parseInt(splits[1]));
+    HostAndPort hostAndPort = HostAndPort.fromString(nodeIdStr);
+    String hostPortStr = hostAndPort.toString();
+    String host = hostPortStr.substring(0, hostPortStr.lastIndexOf(":"));
+    nodeId = BuilderUtils.newNodeId(host, hostAndPort.getPort());
   }
   }
 
 
   public NodeId getNodeId() {
   public NodeId getNodeId() {

+ 4 - 3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java

@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.webproxy;
 import java.io.IOException;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URI;
 
 
+import com.google.common.net.HostAndPort;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.http.HttpServer2;
 import org.apache.hadoop.http.HttpServer2;
 import org.apache.hadoop.security.authorize.AccessControlList;
 import org.apache.hadoop.security.authorize.AccessControlList;
@@ -56,7 +57,8 @@ public class WebAppProxy extends AbstractService {
   
   
   @Override
   @Override
   protected void serviceInit(Configuration conf) throws Exception {
   protected void serviceInit(Configuration conf) throws Exception {
-    String auth =  conf.get(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION);
+    String auth =  conf.get(
+        CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION);
     if (auth == null || "simple".equals(auth)) {
     if (auth == null || "simple".equals(auth)) {
       isSecurityEnabled = false;
       isSecurityEnabled = false;
     } else if ("kerberos".equals(auth)) {
     } else if ("kerberos".equals(auth)) {
@@ -67,8 +69,7 @@ public class WebAppProxy extends AbstractService {
           " of " + auth);
           " of " + auth);
     }
     }
     String proxy = WebAppUtils.getProxyHostAndPort(conf);
     String proxy = WebAppUtils.getProxyHostAndPort(conf);
-    String[] proxyParts = proxy.split(":");
-    proxyHost = proxyParts[0];
+    proxyHost = HostAndPort.fromString(proxy).getHostText();
 
 
     fetcher = new AppReportFetcher(conf);
     fetcher = new AppReportFetcher(conf);
     bindAddress = conf.get(YarnConfiguration.PROXY_ADDRESS);
     bindAddress = conf.get(YarnConfiguration.PROXY_ADDRESS);

+ 4 - 2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java

@@ -29,18 +29,20 @@ import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.net.HostAndPort;
 
 
 public class AmFilterInitializer extends FilterInitializer {
 public class AmFilterInitializer extends FilterInitializer {
   private static final String FILTER_NAME = "AM_PROXY_FILTER";
   private static final String FILTER_NAME = "AM_PROXY_FILTER";
   private static final String FILTER_CLASS = AmIpFilter.class.getCanonicalName();
   private static final String FILTER_CLASS = AmIpFilter.class.getCanonicalName();
-  
+
   @Override
   @Override
   public void initFilter(FilterContainer container, Configuration conf) {
   public void initFilter(FilterContainer container, Configuration conf) {
     Map<String, String> params = new HashMap<>();
     Map<String, String> params = new HashMap<>();
     List<String> proxies = WebAppUtils.getProxyHostsAndPortsForAmFilter(conf);
     List<String> proxies = WebAppUtils.getProxyHostsAndPortsForAmFilter(conf);
     StringBuilder sb = new StringBuilder();
     StringBuilder sb = new StringBuilder();
     for (String proxy : proxies) {
     for (String proxy : proxies) {
-      sb.append(proxy.split(":")[0]).append(AmIpFilter.PROXY_HOSTS_DELIMITER);
+      sb.append(HostAndPort.fromString(proxy).getHostText())
+          .append(AmIpFilter.PROXY_HOSTS_DELIMITER);
     }
     }
     sb.setLength(sb.length() - 1);
     sb.setLength(sb.length() - 1);
     params.put(AmIpFilter.PROXY_HOSTS, sb.toString());
     params.put(AmIpFilter.PROXY_HOSTS, sb.toString());

+ 4 - 4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java

@@ -41,6 +41,7 @@ import javax.servlet.http.HttpServlet;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 import javax.servlet.http.HttpServletResponse;
 
 
+import com.google.common.net.HostAndPort;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.http.HttpServer2;
 import org.apache.hadoop.http.HttpServer2;
@@ -442,11 +443,10 @@ public class TestWebAppProxyServlet {
       proxyServer.setAttribute(FETCHER_ATTRIBUTE,
       proxyServer.setAttribute(FETCHER_ATTRIBUTE,
           appReportFetcher );
           appReportFetcher );
       proxyServer.setAttribute(IS_SECURITY_ENABLED_ATTRIBUTE, Boolean.TRUE);
       proxyServer.setAttribute(IS_SECURITY_ENABLED_ATTRIBUTE, Boolean.TRUE);
-      
+
       String proxy = WebAppUtils.getProxyHostAndPort(conf);
       String proxy = WebAppUtils.getProxyHostAndPort(conf);
-      String[] proxyParts = proxy.split(":");
-      String proxyHost = proxyParts[0];
-      
+      String proxyHost = HostAndPort.fromString(proxy).getHostText();
+
       proxyServer.setAttribute(PROXY_HOST_ATTRIBUTE, proxyHost);
       proxyServer.setAttribute(PROXY_HOST_ATTRIBUTE, proxyHost);
       proxyServer.start();
       proxyServer.start();
       LOG.info("Proxy server is started at port {}",
       LOG.info("Proxy server is started at port {}",