YARN-500. Fixed YARN webapps to not roll-over ports when explicitly asked to use non-ephemeral ports. Contributed by Kenji Kikushima.

svn merge --ignore-ancestry -c 1468739 ../../trunk/


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1468740 13f79535-47bb-0310-9956-ffa450edef68

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/JobInfo.java

@@ -287,7 +287,7 @@ public class JobInfo {
             avgShuffleTime += (attempt.getShuffleFinishTime() - attempt
                 .getLaunchTime());
             avgMergeTime += attempt.getSortFinishTime()
-                - attempt.getShuffleFinishTime();
+                - attempt.getLaunchTime();
             avgReduceTime += (attempt.getFinishTime() - attempt
                 .getShuffleFinishTime());
           }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEntities.java

@@ -94,11 +94,11 @@ public class TestJobHistoryEntities {
     assertEquals(1, completedJob.getAMInfos().size());
     assertEquals(10, completedJob.getCompletedMaps());
     assertEquals(1, completedJob.getCompletedReduces());
-    assertEquals(12, completedJob.getTasks().size());
+    assertEquals(11, completedJob.getTasks().size());
     //Verify tasks loaded at this point.
     assertEquals(true, completedJob.tasksLoaded.get());
     assertEquals(10, completedJob.getTasks(TaskType.MAP).size());
-    assertEquals(2, completedJob.getTasks(TaskType.REDUCE).size());
+    assertEquals(1, completedJob.getTasks(TaskType.REDUCE).size());
     assertEquals("user", completedJob.getUserName());
     assertEquals(JobState.SUCCEEDED, completedJob.getState());
     JobReport jobReport = completedJob.getReport();
@@ -119,7 +119,7 @@ public class TestJobHistoryEntities {
     Map<TaskId, Task> mapTasks = completedJob.getTasks(TaskType.MAP);
     Map<TaskId, Task> reduceTasks = completedJob.getTasks(TaskType.REDUCE);
     assertEquals(10, mapTasks.size());
-    assertEquals(2, reduceTasks.size());
+    assertEquals(1, reduceTasks.size());
     
     Task mt1 = mapTasks.get(mt1Id);
     assertEquals(1, mt1.getAttempts().size());
@@ -134,7 +134,7 @@ public class TestJobHistoryEntities {
     assertEquals(TaskState.SUCCEEDED, rt1Report.getTaskState());
     assertEquals(rt1Id, rt1Report.getTaskId());
   }
-
+  
   @Test (timeout=10000)
   public void testCompletedTaskAttempt() throws Exception {
     HistoryFileInfo info = mock(HistoryFileInfo.class);
@@ -196,12 +196,12 @@ public class TestJobHistoryEntities {
     assertEquals("default",completedJob.getQueueName());
     // progress
     assertEquals(1.0, completedJob.getProgress(),0.001);
-    // 12 rows in answer
-    assertEquals(12,completedJob.getTaskAttemptCompletionEvents(0,1000).length);
+    // 11 rows in answer
+    assertEquals(11,completedJob.getTaskAttemptCompletionEvents(0,1000).length);
     // select first 10 rows
     assertEquals(10,completedJob.getTaskAttemptCompletionEvents(0,10).length);
     // select 5-10 rows include 5th
-    assertEquals(7,completedJob.getTaskAttemptCompletionEvents(5,10).length);
+    assertEquals(6,completedJob.getTaskAttemptCompletionEvents(5,10).length);
 
     // without errors
     assertEquals(1,completedJob.getDiagnostics().size());

hadoop-yarn-project/CHANGES.txt

@@ -176,7 +176,10 @@ Release 2.0.5-beta - UNRELEASED
     YARN-412. Fixed FifoScheduler to check hostname of a NodeManager rather
     than its host:port during scheduling which caused incorrect locality for
     containers. (Roger Hoover via acmurthy)
-    
+
+    YARN-500. Fixed YARN webapps to not roll-over ports when explicitly asked
+    to use non-ephemeral ports. (Kenji Kikushima via vinodkv)
+
 Release 2.0.4-alpha - UNRELEASED
 
   INCOMPATIBLE CHANGES

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java

@@ -97,13 +97,14 @@ public class WebApps {
     public Builder<T> at(String bindAddress) {
       String[] parts = StringUtils.split(bindAddress, ':');
       if (parts.length == 2) {
-        return at(parts[0], Integer.parseInt(parts[1]), true);
+        int port = Integer.parseInt(parts[1]);
+        return at(parts[0], port, port == 0);
       }
       return at(bindAddress, 0, true);
     }
 
     public Builder<T> at(int port) {
-      return at("0.0.0.0", port, false);
+      return at("0.0.0.0", port, port == 0);
     }
 
     public Builder<T> at(String address, int port, boolean findPort) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java

@@ -161,6 +161,30 @@ public class TestWebApp {
     app.stop();
   }
 
+  @Test(expected=org.apache.hadoop.yarn.webapp.WebAppException.class)
+  public void testCreateWithBindAddressNonZeroPort() {
+    WebApp app = WebApps.$for(this).at("0.0.0.0:50000").start();
+    int port = app.getListenerAddress().getPort();
+    assertEquals(50000, port);
+    // start another WebApp with same NonZero port
+    WebApp app2 = WebApps.$for(this).at("0.0.0.0:50000").start();
+    // An exception occurs (findPort disabled)
+    app.stop();
+    app2.stop();
+  }
+
+  @Test(expected=org.apache.hadoop.yarn.webapp.WebAppException.class)
+  public void testCreateWithNonZeroPort() {
+    WebApp app = WebApps.$for(this).at(50000).start();
+    int port = app.getListenerAddress().getPort();
+    assertEquals(50000, port);
+    // start another WebApp with same NonZero port
+    WebApp app2 = WebApps.$for(this).at(50000).start();
+    // An exception occurs (findPort disabled)
+    app.stop();
+    app2.stop();
+  }
+
   @Test public void testServePaths() {
     WebApp app = WebApps.$for("test", this).start();
     assertEquals("/test", app.getRedirectPath());