YARN-3528. Tests with 12345 as hard-coded port break jenkins. Contributed by Brahma Reddy Battula.

Conflicts:
	hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/ServerSocketUtil.java

@@ -48,8 +48,8 @@ public class ServerSocketUtil {
       if (tryPort == 0) {
         continue;
       }
-      LOG.info("Using port " + tryPort);
       try (ServerSocket s = new ServerSocket(tryPort)) {
+        LOG.info("Using port " + tryPort);
         return tryPort;
       } catch (IOException e) {
         tries++;

hadoop-yarn-project/CHANGES.txt

@@ -947,6 +947,9 @@ Release 2.8.0 - UNRELEASED
 
     YARN-4246. NPE while listing app attempt. (Nijel S F via rohithsharmaks)
 
+    YARN-3528. Tests with 12345 as hard-coded port break jenkins.
+    (Brahma Reddy Battula via ozawa)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java

@@ -39,6 +39,7 @@ import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.fs.UnsupportedFileSystemException;
+import org.apache.hadoop.net.ServerSocketUtil;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
 import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
@@ -215,7 +216,7 @@ public class TestNodeManagerReboot {
     
   }
 
-  private void restartNM(int maxTries) {
+  private void restartNM(int maxTries) throws IOException {
     nm.stop();
     nm = new MyNodeManager();
     nm.start();
@@ -296,7 +297,7 @@ public class TestNodeManagerReboot {
 
   private class MyNodeManager extends NodeManager {
 
-    public MyNodeManager() {
+    public MyNodeManager() throws IOException {
       super();
       this.init(createNMConfig());
     }
@@ -315,11 +316,13 @@ public class TestNodeManagerReboot {
       return delService;
     }
 
-    private YarnConfiguration createNMConfig() {
+    private YarnConfiguration createNMConfig() throws IOException {
       YarnConfiguration conf = new YarnConfiguration();
       conf.setInt(YarnConfiguration.NM_PMEM_MB, 5 * 1024); // 5GB
-      conf.set(YarnConfiguration.NM_ADDRESS, "127.0.0.1:12345");
-      conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, "127.0.0.1:12346");
+      conf.set(YarnConfiguration.NM_ADDRESS,
+          "127.0.0.1:" + ServerSocketUtil.getPort(49152, 10));
+      conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, "127.0.0.1:"
+          + ServerSocketUtil.getPort(49153, 10));
       conf.set(YarnConfiguration.NM_LOG_DIRS, logsDir.getAbsolutePath());
       conf.set(YarnConfiguration.NM_LOCAL_DIRS, nmLocalDir.getAbsolutePath());
       conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java

@@ -41,6 +41,7 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.UnsupportedFileSystemException;
+import org.apache.hadoop.net.ServerSocketUtil;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.SecretManager;
 import org.apache.hadoop.util.Shell;
@@ -153,7 +154,8 @@ public class TestNodeManagerResync {
   protected void testContainerPreservationOnResyncImpl(TestNodeManager1 nm,
       boolean isWorkPreservingRestartEnabled)
       throws IOException, YarnException, InterruptedException {
-    YarnConfiguration conf = createNMConfig();
+    int port = ServerSocketUtil.getPort(49153, 10);
+    YarnConfiguration conf = createNMConfig(port);
     conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED,
         isWorkPreservingRestartEnabled);
 
@@ -162,7 +164,7 @@ public class TestNodeManagerResync {
       nm.start();
       ContainerId cId = TestNodeManagerShutdown.createContainerId();
       TestNodeManagerShutdown.startContainer(nm, cId, localFS, tmpDir,
-          processStartFile);
+          processStartFile, port);
 
       nm.setExistingContainerId(cId);
       Assert.assertEquals(1, ((TestNodeManager1) nm).getNMRegistrationCount());
@@ -191,7 +193,8 @@ public class TestNodeManagerResync {
   public void testBlockNewContainerRequestsOnStartAndResync()
       throws IOException, InterruptedException, YarnException {
     NodeManager nm = new TestNodeManager2();
-    YarnConfiguration conf = createNMConfig();
+    int port = ServerSocketUtil.getPort(49154, 10);
+    YarnConfiguration conf = createNMConfig(port);
     conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, false);
     nm.init(conf);
     nm.start();
@@ -199,7 +202,7 @@ public class TestNodeManagerResync {
     // Start the container in running state
     ContainerId cId = TestNodeManagerShutdown.createContainerId();
     TestNodeManagerShutdown.startContainer(nm, cId, localFS, tmpDir,
-      processStartFile);
+        processStartFile, port);
 
     nm.getNMDispatcher().getEventHandler()
       .handle(new NodeManagerEvent(NodeManagerEventType.RESYNC));
@@ -380,11 +383,12 @@ public class TestNodeManagerResync {
     }
   }
 
-  private YarnConfiguration createNMConfig() {
+  private YarnConfiguration createNMConfig(int port) throws IOException {
     YarnConfiguration conf = new YarnConfiguration();
     conf.setInt(YarnConfiguration.NM_PMEM_MB, 5*1024); // 5GB
-    conf.set(YarnConfiguration.NM_ADDRESS, "127.0.0.1:12345");
-    conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, "127.0.0.1:12346");
+    conf.set(YarnConfiguration.NM_ADDRESS, "127.0.0.1:" + port);
+    conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, "127.0.0.1:"
+        + ServerSocketUtil.getPort(49155, 10));
     conf.set(YarnConfiguration.NM_LOG_DIRS, logsDir.getAbsolutePath());
     conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
       remoteLogsDir.getAbsolutePath());
@@ -393,6 +397,10 @@ public class TestNodeManagerResync {
     return conf;
   }
 
+  private YarnConfiguration createNMConfig() throws IOException {
+    return createNMConfig(ServerSocketUtil.getPort(49156, 10));
+  }
+
   class TestNodeManager1 extends NodeManager {
 
     private int registrationCount = 0;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java

@@ -33,12 +33,12 @@ import java.util.List;
 import java.util.Map;
 
 import org.junit.Assert;
-
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.UnsupportedFileSystemException;
 import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.net.ServerSocketUtil;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
@@ -142,9 +142,10 @@ public class TestNodeManagerShutdown {
   public void testKillContainersOnShutdown() throws IOException,
       YarnException {
     nm = new TestNodeManager();
-    nm.init(createNMConfig());
+    int port = ServerSocketUtil.getPort(49157, 10);
+    nm.init(createNMConfig(port));
     nm.start();
-    startContainer(nm, cId, localFS, tmpDir, processStartFile);
+    startContainer(nm, cId, localFS, tmpDir, processStartFile, port);
     
     final int MAX_TRIES=20;
     int numTries = 0;
@@ -186,7 +187,8 @@ public class TestNodeManagerShutdown {
   }
 
   public static void startContainer(NodeManager nm, ContainerId cId,
-      FileContext localFS, File scriptFileDir, File processStartFile) 
+      FileContext localFS, File scriptFileDir, File processStartFile,
+      final int port)
           throws IOException, YarnException {
     File scriptFile =
         createUnhaltingScriptFile(cId, scriptFileDir, processStartFile);
@@ -195,8 +197,8 @@ public class TestNodeManagerShutdown {
         recordFactory.newRecordInstance(ContainerLaunchContext.class);
 
     NodeId nodeId = BuilderUtils.newNodeId(InetAddress.getByName("localhost")
-        .getCanonicalHostName(), 12345);
- 
+        .getCanonicalHostName(), port);
+    
     URL localResourceUri =
         ConverterUtils.getYarnUrlFromPath(localFS
             .makeQualified(new Path(scriptFile.getAbsolutePath())));
@@ -215,7 +217,7 @@ public class TestNodeManagerShutdown {
     List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
     containerLaunchContext.setCommands(commands);
     final InetSocketAddress containerManagerBindAddress =
-        NetUtils.createSocketAddrForHost("127.0.0.1", 12345);
+        NetUtils.createSocketAddrForHost("127.0.0.1", port);
     UserGroupInformation currentUser = UserGroupInformation
         .createRemoteUser(cId.toString());
     org.apache.hadoop.security.token.Token<NMTokenIdentifier> nmToken =
@@ -232,7 +234,7 @@ public class TestNodeManagerShutdown {
             Configuration conf = new Configuration();
             YarnRPC rpc = YarnRPC.create(conf);
             InetSocketAddress containerManagerBindAddress =
-                NetUtils.createSocketAddrForHost("127.0.0.1", 12345);
+                NetUtils.createSocketAddrForHost("127.0.0.1", port);
             return (ContainerManagementProtocol) rpc.getProxy(ContainerManagementProtocol.class,
               containerManagerBindAddress, conf);
           }
@@ -264,11 +266,12 @@ public class TestNodeManagerShutdown {
     return containerId;
   }
   
-  private YarnConfiguration createNMConfig() {
+  private YarnConfiguration createNMConfig(int port) throws IOException {
     YarnConfiguration conf = new YarnConfiguration();
     conf.setInt(YarnConfiguration.NM_PMEM_MB, 5*1024); // 5GB
-    conf.set(YarnConfiguration.NM_ADDRESS, "127.0.0.1:12345");
-    conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, "127.0.0.1:12346");
+    conf.set(YarnConfiguration.NM_ADDRESS, "127.0.0.1:" + port);
+    conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, "127.0.0.1:"
+        + ServerSocketUtil.getPort(49158, 10));
     conf.set(YarnConfiguration.NM_LOG_DIRS, logsDir.getAbsolutePath());
     conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogsDir.getAbsolutePath());
     conf.set(YarnConfiguration.NM_LOCAL_DIRS, nmLocalDir.getAbsolutePath());
@@ -276,6 +279,10 @@ public class TestNodeManagerShutdown {
     return conf;
   }
   
+  private YarnConfiguration createNMConfig() throws IOException {
+    return createNMConfig(ServerSocketUtil.getPort(49157, 10));
+  }
+
   /**
    * Creates a script to run a container that will run forever unless
    * stopped by external means.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java

@@ -57,6 +57,7 @@ import org.apache.hadoop.io.retry.RetryPolicy;
 import org.apache.hadoop.io.retry.RetryProxy;
 import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.net.ServerSocketUtil;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier;
 import org.apache.hadoop.service.Service.STATE;
@@ -140,7 +141,7 @@ public class TestNodeStatusUpdater {
   private AtomicBoolean assertionFailedInThread = new AtomicBoolean(false);
 
   @Before
-  public void setUp() {
+  public void setUp() throws IOException {
     nmLocalDir.mkdirs();
     tmpDir.mkdirs();
     logsDir.mkdirs();
@@ -1122,7 +1123,7 @@ public class TestNodeStatusUpdater {
   }
 
   @Test
-  public void testNMRegistration() throws InterruptedException {
+  public void testNMRegistration() throws InterruptedException, IOException {
     nm = new NodeManager() {
       @Override
       protected NodeStatusUpdater createNodeStatusUpdater(Context context,
@@ -1512,7 +1513,8 @@ public class TestNodeStatusUpdater {
       throws Exception {
     final long connectionWaitSecs = 1000;
     final long connectionRetryIntervalMs = 1000;
-    YarnConfiguration conf = createNMConfig();
+    int port = ServerSocketUtil.getPort(49156, 10);
+    YarnConfiguration conf = createNMConfig(port);
     conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS,
         connectionWaitSecs);
     conf.setLong(YarnConfiguration
@@ -1528,7 +1530,7 @@ public class TestNodeStatusUpdater {
     ContainerId cId = TestNodeManagerShutdown.createContainerId();
     FileContext localFS = FileContext.getLocalFSFileContext();
     TestNodeManagerShutdown.startContainer(nm, cId, localFS, nmLocalDir,
-      new File("start_file.txt"));
+        new File("start_file.txt"), port);
 
     try {
       syncBarrier.await(10000, TimeUnit.MILLISECONDS);
@@ -1542,7 +1544,8 @@ public class TestNodeStatusUpdater {
   }
 
   @Test
-  public void testRMVersionLessThanMinimum() throws InterruptedException {
+  public void testRMVersionLessThanMinimum() throws InterruptedException,
+      IOException {
     final AtomicInteger numCleanups = new AtomicInteger(0);
     YarnConfiguration conf = createNMConfig();
     conf.set(YarnConfiguration.NM_RESOURCEMANAGER_MINIMUM_VERSION, "3.0.0");
@@ -1810,17 +1813,19 @@ public class TestNodeStatusUpdater {
         this.registeredNodes.size());
   }
 
-  private YarnConfiguration createNMConfig() {
+  private YarnConfiguration createNMConfig(int port) throws IOException {
     YarnConfiguration conf = new YarnConfiguration();
     String localhostAddress = null;
     try {
-      localhostAddress = InetAddress.getByName("localhost").getCanonicalHostName();
+      localhostAddress = InetAddress.getByName("localhost")
+          .getCanonicalHostName();
     } catch (UnknownHostException e) {
       Assert.fail("Unable to get localhost address: " + e.getMessage());
     }
     conf.setInt(YarnConfiguration.NM_PMEM_MB, 5 * 1024); // 5GB
-    conf.set(YarnConfiguration.NM_ADDRESS, localhostAddress + ":12345");
-    conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, localhostAddress + ":12346");
+    conf.set(YarnConfiguration.NM_ADDRESS, localhostAddress + ":" + port);
+    conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, localhostAddress + ":"
+        + ServerSocketUtil.getPort(49160, 10));
     conf.set(YarnConfiguration.NM_LOG_DIRS, logsDir.getAbsolutePath());
     conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
       remoteLogsDir.getAbsolutePath());
@@ -1829,6 +1834,10 @@ public class TestNodeStatusUpdater {
     return conf;
   }
 
+  private YarnConfiguration createNMConfig() throws IOException {
+    return createNMConfig(ServerSocketUtil.getPort(49170, 10));
+  }
+
   private NodeManager getNodeManager(final NodeAction nodeHeartBeatAction) {
     return new NodeManager() {
       @Override

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java

@@ -29,13 +29,13 @@ import java.util.Map;
 
 import org.apache.hadoop.yarn.server.nodemanager.executor.DeletionAsUserContext;
 import org.junit.Assert;
-
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.UnsupportedFileSystemException;
+import org.apache.hadoop.net.ServerSocketUtil;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.SecretManager.InvalidToken;
 import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
@@ -167,7 +167,7 @@ public abstract class BaseContainerManagerTest {
     LOG.info("Created localDir in " + localDir.getAbsolutePath());
     LOG.info("Created tmpDir in " + tmpDir.getAbsolutePath());
 
-    String bindAddress = "0.0.0.0:12345";
+    String bindAddress = "0.0.0.0:" + ServerSocketUtil.getPort(49162, 10);
     conf.set(YarnConfiguration.NM_ADDRESS, bindAddress);
     conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir.getAbsolutePath());
     conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java

@@ -45,6 +45,7 @@ import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.UnsupportedFileSystemException;
 import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.net.ServerSocketUtil;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Shell;
@@ -125,7 +126,7 @@ public class TestContainerManagerRecovery extends BaseContainerManagerTest {
     LOG.info("Created localDir in " + localDir.getAbsolutePath());
     LOG.info("Created tmpDir in " + tmpDir.getAbsolutePath());
 
-    String bindAddress = "0.0.0.0:12345";
+    String bindAddress = "0.0.0.0:"+ServerSocketUtil.getPort(49160, 10);
     conf.set(YarnConfiguration.NM_ADDRESS, bindAddress);
     conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir.getAbsolutePath());
     conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java

@@ -208,7 +208,6 @@ public class TestContainersMonitor extends BaseContainerManagerTest {
     ApplicationId appId = ApplicationId.newInstance(0, 0);
     ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
     ContainerId cId = ContainerId.newContainerId(appAttemptId, 0);
-    int port = 12345;
 
     URL resource_alpha =
         ConverterUtils.getYarnUrlFromPath(localFS