HADOOP-271. Add links from jobtracker's web ui to tasktracker's web ui. Also attempt to log a thread dump of child processes before they're killed.

git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk@415765 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -16,6 +16,10 @@ Trunk (unreleased changes)
     tasktracker, greatly increasing code coverage.
     (Milind Bhandarkar via cutting)
 
+ 5. HADOOP-271.  Add links from jobtracker's web ui to tasktracker's
+    web ui.  Also attempt to log a thread dump of child processes
+    before they're killed.  (omalley via cutting)
+
 
 Release 0.3.2 - 2006-06-09
 

src/java/org/apache/hadoop/mapred/JobInProgress.java

@@ -235,6 +235,14 @@ class JobInProgress {
       return reduces;
     }
     
+    /**
+     * Get the job configuration
+     * @return the job's configuration
+     */
+    JobConf getJobConf() {
+      return conf;
+    }
+    
     /**
      * Return a treeset of completed TaskInProgress objects
      */
@@ -604,7 +612,7 @@ class JobInProgress {
                                           TaskStatus.FAILED,
                                           reason,
                                           reason,
-                                          hostname);
+                                          trackerName);
        failedTask(tip, taskid, status, trackerName);
     }
        

src/java/org/apache/hadoop/mapred/JobTracker.java

@@ -128,8 +128,11 @@ public class JobTracker implements MRConstants, InterTrackerProtocol, JobSubmiss
                   }
                   if (tip != null) {
                      JobInProgress job = tip.getJob();
+                     String trackerName = getAssignedTracker(taskId);
+                     TaskTrackerStatus trackerStatus = 
+                       getTaskTracker(trackerName);
                      job.failedTask(tip, taskId, "Error launching task", 
-                                    "n/a", "n/a");
+                                    trackerStatus.getHost(), trackerName);
                   }
                   itr.remove();
                 } else {
@@ -1029,6 +1032,15 @@ public class JobTracker implements MRConstants, InterTrackerProtocol, JobSubmiss
 	return tip.getTaskStatuses();
     }
 
+    /**
+     * Get tracker name for a given task id.
+     * @param taskId the name of the task
+     * @return The name of the task tracker
+     */
+    public synchronized String getAssignedTracker(String taskId) {
+      return (String) taskidToTrackerMap.get(taskId);
+    }
+    
     ///////////////////////////////////////////////////////////////
     // JobTracker methods
     ///////////////////////////////////////////////////////////////
@@ -1055,7 +1067,7 @@ public class JobTracker implements MRConstants, InterTrackerProtocol, JobSubmiss
     void updateTaskStatuses(TaskTrackerStatus status) {
         for (Iterator it = status.taskReports(); it.hasNext(); ) {
             TaskStatus report = (TaskStatus) it.next();
-            report.setHostname(status.getHost());
+            report.setTaskTracker(status.getTrackerName());
             String taskId = report.getTaskId();
             TaskInProgress tip = (TaskInProgress) taskidToTIPMap.get(taskId);
             if (tip == null) {

src/java/org/apache/hadoop/mapred/TaskStatus.java

@@ -37,20 +37,20 @@ class TaskStatus implements Writable {
     private int runState;
     private String diagnosticInfo;
     private String stateString;
-    private String hostname;
+    private String taskTracker;
 
     public TaskStatus() {}
 
     public TaskStatus(String taskid, boolean isMap, float progress,
                       int runState, String diagnosticInfo,
-                      String stateString, String hostname) {
+                      String stateString, String taskTracker) {
         this.taskid = taskid;
         this.isMap = isMap;
         this.progress = progress;
         this.runState = runState;
         this.diagnosticInfo = diagnosticInfo;
         this.stateString = stateString;
-	this.hostname = hostname;
+        this.taskTracker = taskTracker;
     }
     
     public String getTaskId() { return taskid; }
@@ -58,8 +58,8 @@ class TaskStatus implements Writable {
     public float getProgress() { return progress; }
     public void setProgress(float progress) { this.progress = progress; } 
     public int getRunState() { return runState; }
-    public String getHostname() {return hostname;}
-    public void setHostname(String host) { this.hostname = host;}
+    public String getTaskTracker() {return taskTracker;}
+    public void setTaskTracker(String tracker) { this.taskTracker = tracker;}
     public void setRunState(int runState) { this.runState = runState; }
     public String getDiagnosticInfo() { return diagnosticInfo; }
     public void setDiagnosticInfo(String info) { this.diagnosticInfo = info; }

src/java/org/apache/hadoop/mapred/TaskTracker.java

@@ -249,6 +249,28 @@ public class TaskTracker
       return jobClient;
     }
 
+    /**
+     * Are we running under killall-less operating system.
+     */
+    private static boolean isWindows = 
+      System.getProperty("os.name").startsWith("Windows");
+    
+    /**
+     * Get the call stacks for all java processes on this system.
+     * Obviously, this is only useful for debugging.
+     */
+    private static void getCallStacks() {
+      if (LOG.isDebugEnabled() && !isWindows) {
+         try {
+          Process proc = 
+            Runtime.getRuntime().exec("killall -QUIT java");
+          proc.waitFor();
+        } catch (IOException ie) {
+          LOG.warn(StringUtils.stringifyException(ie));
+        } catch (InterruptedException ie) {}
+      }
+    }
+    
     /**
      * Main service loop.  Will stay in this loop forever.
      */
@@ -341,6 +363,7 @@ public class TaskTracker
                                      (timeSinceLastReport / 1000) + 
                                      " seconds. Killing.";
                         LOG.info(tip.getTask().getTaskId() + ": " + msg);
+                        getCallStacks();
                         tip.reportDiagnosticInfo(msg);
                         try {
                           tip.killAndCleanup(true);
@@ -582,7 +605,13 @@ public class TaskTracker
         /**
          */
         public synchronized TaskStatus createStatus() {
-            TaskStatus status = new TaskStatus(task.getTaskId(), task.isMapTask(), progress, runstate, diagnosticInfo.toString(), (stateString == null) ? "" : stateString, "");
+            TaskStatus status = 
+              new TaskStatus(task.getTaskId(), 
+                             task.isMapTask(),
+                             progress, runstate, 
+                             diagnosticInfo.toString(), 
+                             (stateString == null) ? "" : stateString, 
+                              getName());
             if (diagnosticInfo.length() > 0) {
                 diagnosticInfo = new StringBuffer();
             }
@@ -902,6 +931,7 @@ public class TaskTracker
                     LOG.info("Ping exception: " + msg);
                     remainingRetries -=1;
                     if (remainingRetries == 0) {
+                      getCallStacks();
                       LOG.warn("Last retry, killing "+taskid);
                       System.exit(65);
                     }

src/webapps/job/jobfailures.jsp

@@ -20,12 +20,20 @@
     String tipId = tip.getTIPId();
     for(int i=0; i < statuses.length; ++i) {
       if (statuses[i].getRunState() == TaskStatus.FAILED) {
+        String taskTrackerName = statuses[i].getTaskTracker();
+        TaskTrackerStatus taskTracker = tracker.getTaskTracker(taskTrackerName);
         out.print("<tr><td>" + statuses[i].getTaskId() +
                   "</td><td><a href=\"/taskdetails.jsp?jobid="+ jobId + 
                   "&taskid=" + tipId + "\">" + tipId +
-                  "</a></td><td>" + statuses[i].getHostname() +
-                  "</td><td>" + statuses[i].getDiagnosticInfo() +
-                  "</td></tr>\n");
+                  "</a></td>");
+        if (taskTracker == null) {
+          out.print("<td>" + taskTrackerName + "</td>");
+        } else {
+          out.print("<td><a href=\"http://" + taskTracker.getHost() + ":" +
+                    taskTracker.getHttpPort() + "\">" +  taskTracker.getHost() + 
+                    "</a></td>");
+        }
+        out.print("<td>" + statuses[i].getDiagnosticInfo() + "</td></tr>\n");
       }
     }
   }

src/webapps/job/taskdetails.jsp

@@ -52,8 +52,16 @@
   <%
     for (int i = 0; i < ts.length; i++) {
       TaskStatus status = ts[i];
+      String taskTrackerName = status.getTaskTracker();
+      TaskTrackerStatus taskTracker = tracker.getTaskTracker(taskTrackerName);
       out.print("<tr><td>" + status.getTaskId() + "</td>");
-      out.print("<td>" + status.getHostname() + "</td>");
+      if (taskTracker == null) {
+        out.print("<td>" + taskTrackerName + "</td>");
+      } else {
+        out.print("<td><a href=\"http://" + taskTracker.getHost() + ":" +
+                  taskTracker.getHttpPort() + "\">" +  taskTracker.getHost() + 
+                  "</a></td>");
+      }
       out.print("<td>");
       writeString(out, status.getRunState()); 
       out.print("</td>");