HADOOP-1101. In web ui, split shuffle statisitics from reduce statistics, and add some averages. Contributed by Devaraj.

git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk@521900 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -9,6 +9,9 @@ Trunk (unreleased changes)
  2. HADOOP-1063.  Fix a race condition in MiniDFSCluster test code.
     (Hairong Kuang via cutting)
 
+ 3. HADOOP-1101.  In web ui, split shuffle statistics from reduce
+    statistics, and add some task averages.  (Devaraj Das via cutting)
+
 
 Release 0.12.2 - 2007-23-17
 

src/webapps/job/analysejobhistory.jsp

@@ -48,33 +48,67 @@
 	JobHistory.Task [] mapTasks = new JobHistory.Task[finishedMaps]; 
 	JobHistory.Task [] reduceTasks = new JobHistory.Task[finishedReduces]; 
 	int mapIndex = 0 , reduceIndex=0; 
+	long avgMapTime = 0;
+	long avgReduceTime = 0;
+	long avgShuffleTime = 0;
 	
 	for( JobHistory.Task task : tasks.values() ) {
+	  long avgFinishTime = (task.getLong(Keys.FINISH_TIME) - 
+                      task.getLong(Keys.START_TIME));
 	  if( Values.MAP.name().equals(task.get(Keys.TASK_TYPE)) ){
 		  mapTasks[mapIndex++] = task ; 
-	  }else{
-	    reduceTasks[reduceIndex++] = task; 
+		  avgMapTime += avgFinishTime;
+	  }else{ 
+	    Map<String, TaskAttempt> attempts = task.getTaskAttempts();
+	    for (JobHistory.TaskAttempt attempt : attempts.values()) {
+	      if (attempt.get(Keys.TASK_STATUS).equals(Values.SUCCESS.name())) {
+	        reduceTasks[reduceIndex++] = attempt;
+   	        avgShuffleTime += (attempt.getLong(Keys.SHUFFLE_FINISHED) - 
+                               attempt.getLong(Keys.START_TIME));
+   	        avgReduceTime += (attempt.getLong(Keys.FINISH_TIME) - 
+                              attempt.getLong(Keys.SHUFFLE_FINISHED));
+   	        break;
+	      }
+	    }
 	  }
 	}
 	 
-	Comparator<JobHistory.Task> c = new Comparator<JobHistory.Task>(){
+	if (finishedMaps > 0) {
+	  avgMapTime /= finishedMaps;
+	}
+	if (finishedReduces > 0) {
+	  avgReduceTime /= finishedReduces;
+	  avgShuffleTime /= finishedReduces;
+	}
+	Comparator<JobHistory.Task> cMap = new Comparator<JobHistory.Task>(){
 	  public int compare(JobHistory.Task t1, JobHistory.Task t2){
 	    Long l1 = new Long(t1.getLong(Keys.FINISH_TIME) - t1.getLong(Keys.START_TIME)); 
 	    Long l2 = new Long(t2.getLong(Keys.FINISH_TIME) - t2.getLong(Keys.START_TIME)) ;
 	    return l2.compareTo(l1); 
 	  }
 	}; 
-	Arrays.sort(mapTasks, c);
-	Arrays.sort(reduceTasks, c); 
+	Comparator<JobHistory.Task> cShuffle = new Comparator<JobHistory.Task>(){
+	  public int compare(JobHistory.Task t1, JobHistory.Task t2){
+	    Long l1 = new Long(t1.getLong(Keys.SHUFFLE_FINISHED) - 
+	                       t1.getLong(Keys.START_TIME)); 
+	    Long l2 = new Long(t2.getLong(Keys.SHUFFLE_FINISHED) - 
+	                       t2.getLong(Keys.START_TIME)) ;
+	    return l2.compareTo(l1); 
+	  }
+	}; 
+	Arrays.sort(mapTasks, cMap);
+	Arrays.sort(reduceTasks, cShuffle); 
 	
 	JobHistory.Task minMap = mapTasks[mapTasks.length-1] ;
-	JobHistory.Task minReduce = reduceTasks[reduceTasks.length-1] ;
+	JobHistory.Task minShuffle = reduceTasks[reduceTasks.length-1] ;
 	
 %>
 
 <h3>Time taken by best performing Map task 
 <a href="taskdetailshistory.jsp?jobid=<%=jobid%>&jobTrackerId=<%=jobTrackerId%>&taskid=<%=minMap.get(Keys.TASKID)%>">
 <%=minMap.get(Keys.TASKID) %></a> : <%=StringUtils.formatTimeDiff(minMap.getLong(Keys.FINISH_TIME), minMap.getLong(Keys.START_TIME) ) %></h3>
+<h3>Average time taken by Map tasks: 
+<%=StringUtils.formatTimeDiff(avgMapTime, 0) %></h3>
 <h3>Worse performing map tasks</h3>
 <table border="2" cellpadding="5" cellspacing="2">
 <tr><td>Task Id</td><td>Time taken</td></tr>
@@ -90,10 +124,96 @@
 	}
 %>
 </table>
+<%  
+    Comparator<JobHistory.Task> cFinishMapRed = 
+      new Comparator<JobHistory.Task>() {
+      public int compare(JobHistory.Task t1, JobHistory.Task t2){
+        Long l1 = new Long(t1.getLong(Keys.FINISH_TIME)); 
+        Long l2 = new Long(t2.getLong(Keys.FINISH_TIME));
+        return l2.compareTo(l1); 
+      }
+    };
+    Arrays.sort(mapTasks, cFinishMapRed);
+    JobHistory.Task lastMap = mapTasks[0] ;
+%>
+<h3>The last Map task 
+<a href="taskdetailshistory.jsp?jobid=<%=jobid%>&jobTrackerId=<%=jobTrackerId%>
+&taskid=<%=lastMap.get(Keys.TASKID)%>"><%=lastMap.get(Keys.TASKID) %></a> 
+finished at (relative to the Job launch time): 
+<%=StringUtils.getFormattedTimeWithDiff(dateFormat, 
+                              lastMap.getLong(Keys.FINISH_TIME), 
+                              job.getLong(Keys.LAUNCH_TIME) ) %></h3>
+<hr/>
+<h3>Time taken by best performing shuffle
+<a href="taskdetailshistory.jsp?jobid=<%=jobid%>&jobTrackerId=<%=jobTrackerId%>
+&taskid=<%=minShuffle.get(Keys.TASKID)%>"><%=minShuffle.get(Keys.TASKID)%></a> : 
+<%=StringUtils.formatTimeDiff(minShuffle.getLong(Keys.SHUFFLE_FINISHED), 
+                              minShuffle.getLong(Keys.START_TIME) ) %></h3>
+<h3>Average time taken by Shuffle: 
+<%=StringUtils.formatTimeDiff(avgShuffleTime, 0) %></h3>
+<h3>Worse performing Shuffle(s)</h3>
+<table border="2" cellpadding="5" cellspacing="2">
+<tr><td>Task Id</td><td>Time taken</td></tr>
+<%
+	for( int i=0;i<showTasks && i<reduceTasks.length; i++){
+%>
+      <tr>
+	    <td><a href="taskdetailshistory.jsp?jobid=<%=jobid%>&jobTrackerId=
+	    <%=jobTrackerId%>&taskid=<%=reduceTasks[i].get(Keys.TASKID)%>">
+			<%=reduceTasks[i].get(Keys.TASKID) %></a></td>
+	    <td><%=
+	      StringUtils.formatTimeDiff(
+	                       reduceTasks[i].getLong(Keys.SHUFFLE_FINISHED),
+	                       reduceTasks[i].getLong(Keys.START_TIME)) %>
+	    </td>
+	  </tr>
+<%
+	}
+%>
+</table>
+<%  
+    Comparator<JobHistory.Task> cFinishShuffle = 
+      new Comparator<JobHistory.Task>() {
+      public int compare(JobHistory.Task t1, JobHistory.Task t2){
+        Long l1 = new Long(t1.getLong(Keys.SHUFFLE_FINISHED)); 
+        Long l2 = new Long(t2.getLong(Keys.SHUFFLE_FINISHED));
+        return l2.compareTo(l1); 
+      }
+    };
+    Arrays.sort(reduceTasks, cFinishShuffle);
+    JobHistory.Task lastShuffle = reduceTasks[0] ;
+%>
+
+<h3>The last Shuffle  
+<a href="taskdetailshistory.jsp?jobid=<%=jobid%>&jobTrackerId=<%=jobTrackerId%>
+&taskid=<%=lastShuffle.get(Keys.TASKID)%>"><%=lastShuffle.get(Keys.TASKID)%>
+</a> finished at (relative to the Job launch time): 
+<%=StringUtils.getFormattedTimeWithDiff(dateFormat,
+                              lastShuffle.getLong(Keys.SHUFFLE_FINISHED), 
+                              job.getLong(Keys.LAUNCH_TIME) ) %></h3>
+
+<%
+	Comparator<JobHistory.Task> cReduce = new Comparator<JobHistory.Task>(){
+	  public int compare(JobHistory.Task t1, JobHistory.Task t2){
+	    Long l1 = new Long(t1.getLong(Keys.FINISH_TIME) - 
+	                       t1.getLong(Keys.SHUFFLE_FINISHED)); 
+	    Long l2 = new Long(t2.getLong(Keys.FINISH_TIME) - 
+	                       t2.getLong(Keys.SHUFFLE_FINISHED));
+	    return l2.compareTo(l1); 
+	  }
+	}; 
+	Arrays.sort(reduceTasks, cReduce); 
+	JobHistory.Task minReduce = reduceTasks[reduceTasks.length-1] ;
+%>
+<hr/>
 <h3>Time taken by best performing Reduce task : 
 <a href="taskdetailshistory.jsp?jobid=<%=jobid%>&jobTrackerId=<%=jobTrackerId%>&taskid=<%=minReduce.get(Keys.TASKID)%>">
-<%=minReduce.get(Keys.TASKID) %></a> : <%=StringUtils.formatTimeDiff(minReduce.getLong(Keys.FINISH_TIME), minReduce.getLong(Keys.START_TIME) ) %></h3>
+<%=minReduce.get(Keys.TASKID) %></a> : 
+<%=StringUtils.formatTimeDiff(minReduce.getLong(Keys.FINISH_TIME),
+    minReduce.getLong(Keys.SHUFFLE_FINISHED) ) %></h3>
 
+<h3>Average time taken by Reduce tasks: 
+<%=StringUtils.formatTimeDiff(avgReduceTime, 0) %></h3>
 <h3>Worse performing reduce tasks</h3>
 <table border="2" cellpadding="5" cellspacing="2">
 <tr><td>Task Id</td><td>Time taken</td></tr>
@@ -103,11 +223,25 @@
 		<tr>
 			<td><a href="taskdetailshistory.jsp?jobid=<%=jobid%>&jobTrackerId=<%=jobTrackerId%>&taskid=<%=reduceTasks[i].get(Keys.TASKID)%>">
 			<%=reduceTasks[i].get(Keys.TASKID) %></a></td>
-			<td><%=StringUtils.formatTimeDiff(reduceTasks[i].getLong(Keys.FINISH_TIME), reduceTasks[i].getLong(Keys.START_TIME)) %></td>
+			<td><%=StringUtils.formatTimeDiff(
+			    reduceTasks[i].getLong(Keys.FINISH_TIME), 
+			    reduceTasks[i].getLong(Keys.SHUFFLE_FINISHED)) %></td>
 		</tr>
 <%
 	}
 %>
 </table>
+<%  
+    Arrays.sort(reduceTasks, cFinishMapRed);
+    JobHistory.Task lastReduce = reduceTasks[0] ;
+%>
+
+<h3>The last Reduce task 
+<a href="taskdetailshistory.jsp?jobid=<%=jobid%>&jobTrackerId=<%=jobTrackerId%>
+&taskid=<%=lastReduce.get(Keys.TASKID)%>"><%=lastReduce.get(Keys.TASKID)%>
+</a> finished at (relative to the Job launch time): 
+<%=StringUtils.getFormattedTimeWithDiff(dateFormat,
+                              lastReduce.getLong(Keys.FINISH_TIME), 
+                              job.getLong(Keys.LAUNCH_TIME) ) %></h3>
  </center>
  </body></html>