MAPREDUCE-2490. Add logging to graylist and blacklist activity to aid

diagnosis of related issues. Contributed by Jonathan Eagles


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.20-security@1125589 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -32,6 +32,9 @@ Release 0.20.205.0 - unreleased
     MAPREDUCE-2514. Fix typo in TaskTracker ReinitTrackerAction log message.
     (Jonathan Eagles via cdouglas)
 
+    MAPREDUCE-2490. Add logging to graylist and blacklist activity to aid
+    diagnosis of related issues. (Jonathan Eagles via cdouglas)
+
 Release 0.20.204.0 - unreleased
 
   BUG FIXES

src/mapred/org/apache/hadoop/mapred/JobTracker.java

@@ -887,21 +887,25 @@ public class JobTracker implements MRConstants, InterTrackerProtocol,
     }
 
     private void incrBlacklistedTrackers(int count) {
+      LOG.info("Incrementing blacklisted trackers by " + count);
       numBlacklistedTrackers += count;
       getInstrumentation().addBlackListedTrackers(count);
     }
 
     private void decrBlacklistedTrackers(int count) {
+      LOG.info("Decrementing blacklisted trackers by " + count);
       numBlacklistedTrackers -= count;
       getInstrumentation().decBlackListedTrackers(count);
     }
 
     private void incrGraylistedTrackers(int count) {
+      LOG.info("Incrementing graylisted trackers by " + count);
       numGraylistedTrackers += count;
       getInstrumentation().addGrayListedTrackers(count);
     }
 
     private void decrGraylistedTrackers(int count) {
+      LOG.info("Decrementing graylisted trackers by " + count);
       numGraylistedTrackers -= count;
       getInstrumentation().decGrayListedTrackers(count);
     }
@@ -988,13 +992,13 @@ public class JobTracker implements MRConstants, InterTrackerProtocol,
       if (listed && rfbSet.contains(rfb)) {
         if (fi.removeBlacklistedReason(rfb, gray)) {
           if (fi.getReasonForBlacklisting(gray).isEmpty()) {
+            LOG.info("Un" + (gray? "gray" : "black") + "listing tracker : " +
+                     hostName);
             if (gray) {
               decrGraylistedTrackers(getNumTaskTrackersOnHost(hostName));
             } else {
               addHostCapacity(hostName);
             }
-            LOG.info("Un" + (gray? "gray" : "black") + "listing tracker : " +
-                     hostName);
             fi.unBlacklist(gray);
             // We have unblack/graylisted tracker, so tracker should definitely
             // be healthy. Check fault count; if zero, don't keep it in memory.
@@ -1034,11 +1038,11 @@ public class JobTracker implements MRConstants, InterTrackerProtocol,
         if (fi != null) {
           // a tracker can be both blacklisted and graylisted, so check both
           if (fi.isGraylisted()) {
-            LOG.info("Removing " + hostName + " from graylist");
+            LOG.info("Marking " + hostName + " healthy from graylist");
             decrGraylistedTrackers(getNumTaskTrackersOnHost(hostName));
           }
           if (fi.isBlacklisted()) {
-            LOG.info("Removing " + hostName + " from blacklist");
+            LOG.info("Marking " + hostName + " healthy from blacklist");
             addHostCapacity(hostName);
           }
           // no need for fi.unBlacklist() for either one:  fi is already gone
@@ -4889,15 +4893,18 @@ public class JobTracker implements MRConstants, InterTrackerProtocol,
   // Remove a tracker from the system
   private void removeTracker(TaskTracker tracker) {
     String trackerName = tracker.getTrackerName();
+    String hostName = JobInProgress.convertTrackerNameToHostName(trackerName);
     // Remove completely after marking the tasks as 'KILLED'
     lostTaskTracker(tracker);
     // tracker is lost; if it is blacklisted and/or graylisted, remove
     // it from the relevant count(s) of trackers in the cluster
     if (isBlacklisted(trackerName)) {
-     faultyTrackers.decrBlacklistedTrackers(1);
+      LOG.info("Removing " + hostName + " from blacklist");
+      faultyTrackers.decrBlacklistedTrackers(1);
     }
     if (isGraylisted(trackerName)) {
-     faultyTrackers.decrGraylistedTrackers(1);
+      LOG.info("Removing " + hostName + " from graylist");
+      faultyTrackers.decrGraylistedTrackers(1);
     }
     updateTaskTrackerStatus(trackerName, null);
     statistics.taskTrackerRemoved(trackerName);