HDFS-4832. Namenode doesn't change the number of missing blocks in safemode when DNs rejoin or leave. Contributed by Ravi Prakash.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1490806 13f79535-47bb-0310-9956-ffa450edef68

hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt

@@ -14,6 +14,10 @@ Release 0.23.9 - UNRELEASED
 
     HDFS-4862. SafeModeInfo.isManual() returns true when resources are low even
     if it wasn't entered into manually (Ravi Prakash via kihwal)
+
+    HDFS-4832. Namenode doesn't change the number of missing blocks in
+    safemode when DNs rejoin or leave (Ravi Prakash via kihwal)
+
 Release 0.23.8 - 2013-06-05
   
   INCOMPATIBLE CHANGES

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java

@@ -1806,7 +1806,7 @@ public class BlockManager {
       return storedBlock;
     }
 
-    // do not try to handle over/under-replicated blocks during safe mode
+    // do not try to handle over/under-replicated blocks during first safe mode
     if (!namesystem.isPopulatingReplQueues()) {
       return storedBlock;
     }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java

@@ -915,7 +915,13 @@ public class DatanodeManager {
 
         heartbeatManager.updateHeartbeat(nodeinfo, capacity, dfsUsed,
             remaining, blockPoolUsed, xceiverCount, failedVolumes);
-        
+
+        // If we are in safemode, do not send back any recovery / replication
+        // requests. Don't even drain the existing queue of work.
+        if(namesystem.isInSafeMode()) {
+          return new DatanodeCommand[0];
+        }
+
         //check lease recovery
         BlockInfoUnderConstruction[] blocks = nodeinfo
             .getLeaseRecoveryCommand(Integer.MAX_VALUE);

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java

@@ -202,7 +202,7 @@ class HeartbeatManager implements DatanodeStatistics {
     final DatanodeManager dm = blockManager.getDatanodeManager();
     // It's OK to check safe mode w/o taking the lock here, we re-check
     // for safe mode after taking the lock before removing a datanode.
-    if (namesystem.isInSafeMode()) {
+    if (namesystem.isInStartupSafeMode()) {
       return;
     }
     boolean allAlive = false;
@@ -224,7 +224,7 @@ class HeartbeatManager implements DatanodeStatistics {
         // acquire the fsnamesystem lock, and then remove the dead node.
         namesystem.writeLock();
         try {
-          if (namesystem.isInSafeMode()) {
+          if (namesystem.isInStartupSafeMode()) {
             return;
           }
           synchronized(this) {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java

@@ -2946,7 +2946,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
      * 
      * @see SafeModeInfo
      */
-    private SafeModeInfo(boolean resourcesLow) {
+    private SafeModeInfo(boolean resourcesLow, boolean isReplQueuesInited) {
       this.threshold = 1.5f;  // this threshold can never be reached
       this.datanodeThreshold = Integer.MAX_VALUE;
       this.extension = Integer.MAX_VALUE;
@@ -2955,6 +2955,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
       this.blockTotal = -1;
       this.blockSafe = -1;
       this.resourcesLow = resourcesLow;
+      this.initializedReplQueues = isReplQueuesInited;
       enter();
       reportStatus("STATE* Safe mode is ON.", true);
     }
@@ -3005,7 +3006,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
         }
         if(needUpgrade) {
           // switch to manual safe mode
-          safeMode = new SafeModeInfo(false);
+          safeMode = new SafeModeInfo(false, isPopulatingReplQueues());
           return;
         }
       }
@@ -3356,6 +3357,10 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
       && safeMode.isOn();
   }
 
+  /**
+   * Check if replication queues are to be populated
+   * @return true when node is HAState.Active and not in the very first safemode
+   */
   @Override
   public boolean isPopulatingReplQueues() {
     // safeMode is volatile, and may be set to null at any time
@@ -3452,7 +3457,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
       // is entirely stable on disk as soon as we're in safe mode.
       getEditLog().logSyncAll();
       if (!isInSafeMode()) {
-        safeMode = new SafeModeInfo(resourcesLow);
+        safeMode = new SafeModeInfo(resourcesLow, isPopulatingReplQueues());
         return;
       }
       if (resourcesLow) {

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java

@@ -51,4 +51,25 @@ public class TestFSNamesystem {
     assertTrue("After entering safemode due to low resources FSNamesystem."
       + "isInSafeMode still returned false", fsn.isInSafeMode());
   }
+
+  @Test
+  public void testReplQueuesActiveAfterStartupSafemode() throws IOException, InterruptedException{
+    Configuration conf = new Configuration();
+
+    FSEditLog fsEditLog = Mockito.mock(FSEditLog.class);
+    FSImage fsImage = Mockito.mock(FSImage.class);
+    Mockito.when(fsImage.getEditLog()).thenReturn(fsEditLog);
+
+    FSNamesystem fsNamesystem = new FSNamesystem(fsImage, conf);
+    FSNamesystem fsn = Mockito.spy(fsNamesystem);
+
+    fsn.leaveSafeMode(false);
+    assertTrue("FSNamesystem didn't leave safemode", !fsn.isInSafeMode());
+    assertTrue("Replication queues weren't being populated even after leaving "
+      + "safemode", fsn.isPopulatingReplQueues());
+    fsn.enterSafeMode(false);
+    assertTrue("FSNamesystem didn't enter safemode", fsn.isInSafeMode());
+    assertTrue("Replication queues weren't being populated after entering "
+      + "safemode 2nd time", fsn.isPopulatingReplQueues());
+  }
 }