YARN-3374. Collector's web server should randomly bind an available port. Contributed by Zhijie Shen

(cherry picked from commit 3aa898e734a1e4368ddf1d0bbd31f9b4de53ceba)

hadoop-yarn-project/CHANGES.txt

@@ -44,6 +44,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
     YARN-3040. Make putEntities operation be aware of the app's context. (Zhijie Shen 
     via junping_du)
 
+    YARN-3374. Collector's web server should randomly bind an available port. (
+    Zhijie Shen via junping_du)
+
   IMPROVEMENTS
 
   OPTIMIZATIONS

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java

@@ -1787,6 +1787,7 @@ public class YarnConfiguration extends Configuration {
   /** The listening endpoint for the timeline service application.*/
   public static final String TIMELINE_SERVICE_BIND_HOST =
       TIMELINE_SERVICE_PREFIX + "bind-host";
+  public static final String DEFAULT_TIMELINE_SERVICE_BIND_HOST = "0.0.0.0";
 
   /** The number of threads to handle client RPC API requests. */
   public static final String TIMELINE_SERVICE_HANDLER_THREAD_COUNT =

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java

@@ -210,22 +210,17 @@ public class TimelineCollectorManager extends CompositeService {
    */
   private void startWebApp() {
     Configuration conf = getConfig();
-    // use the same ports as the old ATS for now; we could create new properties
-    // for the new timeline service if needed
-    String bindAddress = WebAppUtils.getWebAppBindURL(conf,
-        YarnConfiguration.TIMELINE_SERVICE_BIND_HOST,
-        WebAppUtils.getAHSWebAppURLWithoutScheme(conf));
-    this.timelineRestServerBindAddress = WebAppUtils.getResolvedAddress(
-        NetUtils.createSocketAddr(bindAddress));
-    LOG.info("Instantiating the per-node collector webapp at " +
-        timelineRestServerBindAddress);
+    String bindAddress = conf.get(YarnConfiguration.TIMELINE_SERVICE_BIND_HOST,
+        YarnConfiguration.DEFAULT_TIMELINE_SERVICE_BIND_HOST) + ":0";
     try {
       Configuration confForInfoServer = new Configuration(conf);
       confForInfoServer.setInt(HttpServer2.HTTP_MAX_THREADS, 10);
       HttpServer2.Builder builder = new HttpServer2.Builder()
           .setName("timeline")
           .setConf(conf)
-          .addEndpoint(URI.create("http://" + bindAddress));
+          .addEndpoint(URI.create(
+              (YarnConfiguration.useHttps(conf) ? "https://" : "http://") +
+                  bindAddress));
       timelineRestServer = builder.build();
       // TODO: replace this by an authentication filter in future.
       HashMap<String, String> options = new HashMap<>();
@@ -249,6 +244,11 @@ public class TimelineCollectorManager extends CompositeService {
       LOG.error(msg, e);
       throw new YarnRuntimeException(msg, e);
     }
+    //TODO: We need to think of the case of multiple interfaces
+    this.timelineRestServerBindAddress = WebAppUtils.getResolvedAddress(
+        timelineRestServer.getConnectorAddress(0));
+    LOG.info("Instantiated the per-node collector webapp at " +
+        timelineRestServerBindAddress);
   }
 
   private void reportNewCollectorToNM(ApplicationId appId)

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java

@@ -18,7 +18,9 @@
 
 package org.apache.hadoop.yarn.server.timelineservice.collector;
 
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 import static org.mockito.Mockito.any;
@@ -63,6 +65,16 @@ public class TestTimelineCollectorManager {
     }
   }
 
+  @Test
+  public void testStartWebApp() throws Exception {
+    assertNotNull(collectorManager.getRestServerBindAddress());
+    String address = collectorManager.getRestServerBindAddress();
+    String[] parts = address.split(":");
+    assertEquals(2, parts.length);
+    assertNotNull(parts[0]);
+    assertTrue(Integer.valueOf(parts[1]) > 0);
+  }
+
   @Test(timeout=60000)
   public void testMultithreadedAdd() throws Exception {
     final int NUM_APPS = 5;