MAPREDUCE-2490. Add logging to graylist and blacklist activity to aid

diagnosis of related issues. (Jonathan Eagles via cdouglas)


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.20-security-204@1136713 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -36,6 +36,9 @@ Release 0.20.204.0 - unreleased
 
     HADOOP-7356. Fix bin/hadoop scripts (eyang via omalley)
 
+    MAPREDUCE-2490. Add logging to graylist and blacklist activity to aid
+    diagnosis of related issues. (Jonathan Eagles via cdouglas)
+
     MAPREDUCE-2447. Fix Child.java to set Task.jvmContext sooner to avoid
     corner cases in error handling. (Siddharth Seth via acmurthy) 
 

src/mapred/org/apache/hadoop/mapred/JobTracker.java

@@ -887,21 +887,25 @@ public class JobTracker implements MRConstants, InterTrackerProtocol,
     }
 
     private void incrBlacklistedTrackers(int count) {
+      LOG.info("Incrementing blacklisted trackers by " + count);
       numBlacklistedTrackers += count;
       getInstrumentation().addBlackListedTrackers(count);
     }
 
     private void decrBlacklistedTrackers(int count) {
+      LOG.info("Decrementing blacklisted trackers by " + count);
       numBlacklistedTrackers -= count;
       getInstrumentation().decBlackListedTrackers(count);
     }
 
     private void incrGraylistedTrackers(int count) {
+      LOG.info("Incrementing graylisted trackers by " + count);
       numGraylistedTrackers += count;
       getInstrumentation().addGrayListedTrackers(count);
     }
 
     private void decrGraylistedTrackers(int count) {
+      LOG.info("Decrementing graylisted trackers by " + count);
       numGraylistedTrackers -= count;
       getInstrumentation().decGrayListedTrackers(count);
     }
@@ -986,13 +990,13 @@ public class JobTracker implements MRConstants, InterTrackerProtocol,
       if (listed && rfbSet.contains(rfb)) {
         if (fi.removeBlacklistedReason(rfb, gray)) {
           if (fi.getReasonForBlacklisting(gray).isEmpty()) {
+            LOG.info("Un" + (gray? "gray" : "black") + "listing tracker : " +
+                     hostName);
             if (gray) {
               decrGraylistedTrackers(getNumTaskTrackersOnHost(hostName));
             } else {
               addHostCapacity(hostName);
             }
-            LOG.info("Un" + (gray? "gray" : "black") + "listing tracker : " +
-                     hostName);
             fi.unBlacklist(gray);
             // We have unblack/graylisted tracker, so tracker should definitely
             // be healthy. Check fault count; if zero, don't keep it in memory.
@@ -1032,11 +1036,11 @@ public class JobTracker implements MRConstants, InterTrackerProtocol,
         if (fi != null) {
           // a tracker can be both blacklisted and graylisted, so check both
           if (fi.isGraylisted()) {
-            LOG.info("Removing " + hostName + " from graylist");
+            LOG.info("Marking " + hostName + " healthy from graylist");
             decrGraylistedTrackers(getNumTaskTrackersOnHost(hostName));
           }
           if (fi.isBlacklisted()) {
-            LOG.info("Removing " + hostName + " from blacklist");
+            LOG.info("Marking " + hostName + " healthy from blacklist");
             addHostCapacity(hostName);
           }
           // no need for fi.unBlacklist() for either one:  fi is already gone
@@ -4880,15 +4884,18 @@ public class JobTracker implements MRConstants, InterTrackerProtocol,
   // Remove a tracker from the system
   private void removeTracker(TaskTracker tracker) {
     String trackerName = tracker.getTrackerName();
+    String hostName = JobInProgress.convertTrackerNameToHostName(trackerName);
     // Remove completely after marking the tasks as 'KILLED'
     lostTaskTracker(tracker);
     // tracker is lost; if it is blacklisted and/or graylisted, remove
     // it from the relevant count(s) of trackers in the cluster
     if (isBlacklisted(trackerName)) {
-     faultyTrackers.decrBlacklistedTrackers(1);
+      LOG.info("Removing " + hostName + " from blacklist");
+      faultyTrackers.decrBlacklistedTrackers(1);
     }
     if (isGraylisted(trackerName)) {
-     faultyTrackers.decrGraylistedTrackers(1);
+      LOG.info("Removing " + hostName + " from graylist");
+      faultyTrackers.decrGraylistedTrackers(1);
     }
     updateTaskTrackerStatus(trackerName, null);
     statistics.taskTrackerRemoved(trackerName);