commit 0d4d9249af5dd6d20a2bf9dc7fc104c421dfb7a9

Author: Hemanth Yamijala <yhemanth@apache.org>
Date:   Thu Oct 15 11:37:22 2009 +0530

    MAPREDUCE:277 from https://issues.apache.org/jira/secure/attachment/12421419/patch-277-0.20.txt
    
    +++ b/YAHOO-CHANGES.txt
    +65. MAPREDUCE-277. Makes job history counters available on the job history
    +    viewers. (Jothi Padmanabhan via ddas)
    +


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.20-security-patches@1077017 13f79535-47bb-0310-9956-ffa450edef68

src/mapred/org/apache/hadoop/mapred/HistoryViewer.java

@@ -19,10 +19,14 @@
 package org.apache.hadoop.mapred;
 
 import java.io.IOException;
+import java.text.DecimalFormat;
+import java.text.Format;
+import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.Arrays;
 import java.util.Comparator;
 import java.util.Date;
+import java.util.Iterator;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeMap;
@@ -32,6 +36,8 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.mapred.Counters.Counter;
+import org.apache.hadoop.mapred.Counters.Group;
 import org.apache.hadoop.mapred.DefaultJobHistoryParser.*;
 import org.apache.hadoop.mapred.JobHistory.*;
 import org.apache.hadoop.util.StringUtils;
@@ -89,7 +95,7 @@ class HistoryViewer {
     }
   }
   
-  public void print() throws IOException{
+  public void print() throws IOException {
     printJobDetails();
     printTaskSummary();
     printJobAnalysis();
@@ -117,7 +123,7 @@ class HistoryViewer {
     printFailedAttempts(filter);
   }
 
-  private void printJobDetails() {
+  private void printJobDetails() throws IOException {
     StringBuffer jobDetails = new StringBuffer();
     jobDetails.append("\nHadoop job: " ).append(jobId);
     jobDetails.append("\n=====================================");
@@ -140,10 +146,58 @@ class HistoryViewer {
                         job.getLong(Keys.LAUNCH_TIME)));
     jobDetails.append("\nStatus: ").append(((job.get(Keys.JOB_STATUS) == "") ? 
                       "Incomplete" :job.get(Keys.JOB_STATUS)));
+    try {
+      printCounters(jobDetails, job);
+    } catch (ParseException p) {
+      throw new IOException(p);
+    }
     jobDetails.append("\n=====================================");
     System.out.println(jobDetails.toString());
   }
   
+  private void printCounters(StringBuffer buff, JobInfo job) 
+      throws ParseException {
+    Counters mapCounters = 
+      Counters.fromEscapedCompactString(job.get(Keys.MAP_COUNTERS));
+    Counters reduceCounters = 
+      Counters.fromEscapedCompactString(job.get(Keys.REDUCE_COUNTERS));
+    Counters totalCounters = 
+      Counters.fromEscapedCompactString(job.get(Keys.COUNTERS));
+    
+    // Killed jobs might not have counters
+    if (totalCounters == null) {
+      return;
+    }
+    buff.append("\nCounters: \n\n");
+    buff.append(String.format("|%1$-30s|%2$-30s|%3$-10s|%4$-10s|%5$-10s|", 
+      "Group Name",
+      "Counter name",
+      "Map Value",
+      "Reduce Value",
+      "Total Value"));
+    buff.append("\n------------------------------------------"+
+      "---------------------------------------------");
+    for (String groupName : totalCounters.getGroupNames()) {
+      Group totalGroup = totalCounters.getGroup(groupName);
+      Group mapGroup = mapCounters.getGroup(groupName);
+      Group reduceGroup = reduceCounters.getGroup(groupName);
+      Format decimal = new DecimalFormat();
+      Iterator<Counter> ctrItr = totalGroup.iterator();
+      while (ctrItr.hasNext()) {
+        Counter counter = ctrItr.next();
+        String name = counter.getDisplayName();
+        String mapValue = decimal.format(mapGroup.getCounter(name));
+        String reduceValue = decimal.format(reduceGroup.getCounter(name));
+        String totalValue = decimal.format(counter.getValue());
+        buff.append(
+          String.format("\n|%1$-30s|%2$-30s|%3$-10s|%4$-10s|%5$-10s", 
+          totalGroup.getDisplayName(),
+          counter.getDisplayName(),
+          mapValue, reduceValue, totalValue));
+      }
+    }
+  }
+  
   private void printTasks(String taskType, String taskStatus) {
     Map<String, JobHistory.Task> tasks = job.getAllTasks();
     StringBuffer taskList = new StringBuffer();

src/mapred/org/apache/hadoop/mapred/JobHistory.java

@@ -266,7 +266,7 @@ public class JobHistory {
     FINISHED_MAPS, FINISHED_REDUCES, JOB_STATUS, TASKID, HOSTNAME, TASK_TYPE, 
     ERROR, TASK_ATTEMPT_ID, TASK_STATUS, COPY_PHASE, SORT_PHASE, REDUCE_PHASE, 
     SHUFFLE_FINISHED, SORT_FINISHED, COUNTERS, SPLITS, JOB_PRIORITY, HTTP_PORT, 
-    TRACKER_NAME, STATE_STRING, VERSION
+    TRACKER_NAME, STATE_STRING, VERSION, MAP_COUNTERS, REDUCE_COUNTERS
   }
 
   /**
@@ -1334,7 +1334,9 @@ public class JobHistory {
     public static void logFinished(JobID jobId, long finishTime, 
                                    int finishedMaps, int finishedReduces,
                                    int failedMaps, int failedReduces,
-                                   Counters counters){
+                                   Counters mapCounters,
+                                   Counters reduceCounters,
+                                   Counters counters) {
       if (!disableHistory){
         // close job file for this job
         ArrayList<PrintWriter> writer = fileManager.getWriters(jobId); 
@@ -1345,6 +1347,7 @@ public class JobHistory {
                                      Keys.JOB_STATUS, Keys.FINISHED_MAPS, 
                                      Keys.FINISHED_REDUCES,
                                      Keys.FAILED_MAPS, Keys.FAILED_REDUCES,
+                                     Keys.MAP_COUNTERS, Keys.REDUCE_COUNTERS,
                                      Keys.COUNTERS},
                          new String[] {jobId.toString(),  Long.toString(finishTime), 
                                        Values.SUCCESS.name(), 
@@ -1352,6 +1355,8 @@ public class JobHistory {
                                        String.valueOf(finishedReduces),
                                        String.valueOf(failedMaps), 
                                        String.valueOf(failedReduces),
+                                       mapCounters.makeEscapedCompactString(),
+                                       reduceCounters.makeEscapedCompactString(),
                                        counters.makeEscapedCompactString()});
           for (PrintWriter out : writer) {
             out.close();

src/mapred/org/apache/hadoop/mapred/JobInProgress.java

@@ -2338,7 +2338,8 @@ class JobInProgress {
       JobHistory.JobInfo.logFinished(this.status.getJobID(), finishTime, 
                                      this.finishedMapTasks, 
                                      this.finishedReduceTasks, failedMapTasks, 
-                                     failedReduceTasks, getCounters());
+                                     failedReduceTasks, getMapCounters(),
+                                     getReduceCounters(), getCounters());
       // Note that finalize will close the job history handles which garbage collect
       // might try to finalize
       garbageCollect();

src/test/org/apache/hadoop/mapred/TestJobHistory.java

@@ -555,6 +555,14 @@ public class TestJobHistory extends TestCase {
     assertTrue("Counters of job obtained from history file did not " +
                "match the expected value",
                c.makeEscapedCompactString().equals(values.get(Keys.COUNTERS)));
+    Counters m = jip.getMapCounters();
+    assertTrue("Map Counters of job obtained from history file did not " +
+               "match the expected value", m.makeEscapedCompactString().
+               equals(values.get(Keys.MAP_COUNTERS)));
+    Counters r = jip.getReduceCounters();
+    assertTrue("Reduce Counters of job obtained from history file did not " +
+               "match the expected value", r.makeEscapedCompactString().
+               equals(values.get(Keys.REDUCE_COUNTERS)));
 
     // Validate number of total maps, total reduces, finished maps,
     // finished reduces, failed maps, failed recudes

src/webapps/job/jobdetailshistory.jsp

@@ -6,7 +6,7 @@
   import="org.apache.hadoop.fs.*"
   import="org.apache.hadoop.mapred.*"
   import="org.apache.hadoop.util.*"
-  import="java.text.SimpleDateFormat"
+  import="java.text.*"
   import="org.apache.hadoop.mapred.JobHistory.*"
 %>
 
@@ -23,7 +23,14 @@
     FileSystem fs = (FileSystem) application.getAttribute("fileSys");
     JobInfo job = JSPUtil.getJobInfo(request, fs);
 %>
-<html><body>
+
+<html>
+<head>
+<title>Hadoop Job <%=jobid%> on History Viewer</title>
+<link rel="stylesheet" type="text/css" href="/static/hadoop.css">
+</head>
+<body>
+
 <h2>Hadoop Job <%=jobid %> on <a href="jobhistory.jsp">History Viewer</a></h2>
 
 <b>User: </b> <%=job.get(Keys.USER) %><br/> 
@@ -188,6 +195,68 @@
 </tr>
 </table>
 
+<br>
+<br>
+
+<table border=2 cellpadding="5" cellspacing="2">
+  <tr>
+  <th><br/></th>
+  <th>Counter</th>
+  <th>Map</th>
+  <th>Reduce</th>
+  <th>Total</th>
+</tr>
+
+<%  
+
+ Counters totalCounters = 
+   Counters.fromEscapedCompactString(job.get(Keys.COUNTERS));
+ Counters mapCounters = 
+   Counters.fromEscapedCompactString(job.get(Keys.MAP_COUNTERS));
+ Counters reduceCounters = 
+   Counters.fromEscapedCompactString(job.get(Keys.REDUCE_COUNTERS));
+
+ if (totalCounters != null) {
+   for (String groupName : totalCounters.getGroupNames()) {
+     Counters.Group totalGroup = totalCounters.getGroup(groupName);
+     Counters.Group mapGroup = mapCounters.getGroup(groupName);
+     Counters.Group reduceGroup = reduceCounters.getGroup(groupName);
+  
+     Format decimal = new DecimalFormat();
+  
+     boolean isFirst = true;
+     Iterator<Counters.Counter> ctrItr = totalGroup.iterator();
+     while(ctrItr.hasNext()) {
+       Counters.Counter counter = ctrItr.next();
+       String name = counter.getDisplayName();
+       String mapValue = 
+         decimal.format(mapGroup.getCounter(name));
+       String reduceValue = 
+         decimal.format(reduceGroup.getCounter(name));
+       String totalValue = decimal.format(counter.getCounter());
+%>
+       <tr>
+<%
+       if (isFirst) {
+         isFirst = false;
+%>
+         <td rowspan="<%=totalGroup.size()%>"><%=totalGroup.getDisplayName()%></td>
+<%
+       }
+%>
+       <td><%=counter.getDisplayName()%></td>
+       <td align="right"><%=mapValue%></td>
+       <td align="right"><%=reduceValue%></td>
+       <td align="right"><%=totalValue%></td>
+     </tr>
+<%
+      }
+    }
+  }
+%>
+</table>
+<br>
+
 <br/>
  <%
     DefaultJobHistoryParser.FailedOnNodesFilter filter = 
@@ -222,6 +291,7 @@
  %>
 </table>
 <br/>
+
  <%
     DefaultJobHistoryParser.KilledOnNodesFilter killedFilter =
                  new DefaultJobHistoryParser.KilledOnNodesFilter();

src/webapps/job/taskdetailshistory.jsp

@@ -35,10 +35,11 @@
 <%
   }
 %>
-<td>Finish Time</td><td>Host</td><td>Error</td><td>Task Logs</td></tr>
+<td>Finish Time</td><td>Host</td><td>Error</td><td>Task Logs</td>
+<td>Counters</td></tr>
 <%
   for (JobHistory.TaskAttempt attempt : task.getTaskAttempts().values()) {
-    printTaskAttempt(attempt, type, out);
+    printTaskAttempt(attempt, type, out, encodedLogFileName);
   }
 %>
 </table>
@@ -60,8 +61,9 @@
 %>
 <%!
   private void printTaskAttempt(JobHistory.TaskAttempt taskAttempt,
-                                String type, JspWriter out) 
-  throws IOException {
+                                String type, JspWriter out,
+                                String logFile) 
+  throws Exception {
     out.print("<tr>"); 
     out.print("<td>" + taskAttempt.get(Keys.TASK_ATTEMPT_ID) + "</td>");
     out.print("<td>" + StringUtils.getFormattedTimeWithDiff(dateFormat,
@@ -97,6 +99,22 @@
         out.print("n/a");
     }
     out.print("</td>");
+    Counters counters = 
+      Counters.fromEscapedCompactString(taskAttempt.get(Keys.COUNTERS));
+    if (counters != null) {
+      TaskAttemptID attemptId = 
+        TaskAttemptID.forName(taskAttempt.get(Keys.TASK_ATTEMPT_ID));
+      TaskID taskId = attemptId.getTaskID();
+      org.apache.hadoop.mapreduce.JobID jobId = taskId.getJobID();
+      out.print("<td>" 
+       + "<a href=\"/taskstatshistory.jsp?jobid=" + jobId
+           + "&taskid=" + taskId
+           + "&attemptid=" + attemptId
+           + "&logFile=" + logFile + "\">"
+           + counters.size() + "</a></td>");
+    } else {
+      out.print("<td></td>");
+    }
     out.print("</tr>"); 
   }
 %>

src/webapps/job/taskstatshistory.jsp

@@ -0,0 +1,106 @@
+<%
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file 
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+%>
+<%@ page
+  contentType="text/html; charset=UTF-8"
+  import="javax.servlet.http.*"
+  import="java.io.*"
+  import="java.util.*"
+  import="org.apache.hadoop.mapred.*"
+  import="org.apache.hadoop.fs.*"
+  import="org.apache.hadoop.util.*"
+  import="java.text.*"
+  import="org.apache.hadoop.mapred.JobHistory.*" 
+%>
+<%! private static SimpleDateFormat dateFormat = new SimpleDateFormat("d/MM HH:mm:ss") ;
+    private static final long serialVersionUID = 1L;
+%>
+
+<%
+  String jobid = request.getParameter("jobid");
+  String attemptid = request.getParameter("attemptid");
+  String taskid = request.getParameter("taskid");
+  String logFile = request.getParameter("logFile");
+  String encodedLogFileName = 
+    JobHistory.JobInfo.encodeJobHistoryFilePath(logFile);
+
+  Format decimal = new DecimalFormat();
+
+  FileSystem fs = (FileSystem) application.getAttribute("fileSys");
+  JobHistory.JobInfo job = JSPUtil.getJobInfo(request, fs);
+
+  JobHistory.Task task = job.getAllTasks().get(taskid);
+  JobHistory.TaskAttempt attempt = task.getTaskAttempts().get(attemptid);
+
+  Counters counters = 
+    Counters.fromEscapedCompactString(attempt.get(Keys.COUNTERS));
+%>
+
+<html>
+  <head>
+    <title>Counters for <%=attemptid%></title>
+  </head>
+<body>
+<h1>Counters for <%=attemptid%></h1>
+
+<hr>
+
+<%
+  if (counters == null) {
+%>
+    <h3>No counter information found for this attempt</h3>
+<%
+  } else {    
+%>
+    <table>
+<%
+      for (String groupName : counters.getGroupNames()) {
+        Counters.Group group = counters.getGroup(groupName);
+        String displayGroupName = group.getDisplayName();
+%>
+        <tr>
+          <td colspan="3"><br/><b><%=displayGroupName%></b></td>
+        </tr>
+<%
+        Iterator<Counters.Counter> ctrItr = group.iterator();
+        while(ctrItr.hasNext()) {
+          Counters.Counter counter = ctrItr.next();
+          String displayCounterName = counter.getDisplayName();
+          long value = counter.getCounter();
+%>
+          <tr>
+            <td width="50"></td>
+            <td><%=displayCounterName%></td>
+            <td align="right"><%=decimal.format(value)%></td>
+          </tr>
+<%
+        }
+      }
+%>
+    </table>
+<%
+  }
+%>
+
+<hr>
+<a href="jobdetailshistory.jsp?jobid=<%=jobid%>&logFile=<%=encodedLogFileName%>">Go back to the job</a><br>
+<a href="jobtracker.jsp">Go back to JobTracker</a><br>
+<%
+out.println(ServletUtil.htmlFooter());
+%>