HDFS-9787. SNNs stop uploading FSImage to ANN once isPrimaryCheckPointer changed to false. (Contributed by Guocui Mi)

hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt

@@ -935,6 +935,9 @@ Trunk (Unreleased)
       HDFS-9755. Erasure Coding: allow to use multiple EC policies in striping
       related tests [Part 2]. (Rui Li via zhz)
 
+      HDFS-9787. SNNs stop uploading FSImage to ANN once isPrimaryCheckPointer
+      changed to false. (Guocui Mi via vinayakumarb)
+
 Release 2.9.0 - UNRELEASED
 
   INCOMPATIBLE CHANGES

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyCheckpointer.java

@@ -65,6 +65,7 @@ public class StandbyCheckpointer {
   private final Configuration conf;
   private final FSNamesystem namesystem;
   private long lastCheckpointTime;
+  private long lastUploadTime;
   private final CheckpointerThread thread;
   private final ThreadFactory uploadThreadFactory;
   private List<URL> activeNNAddresses;
@@ -252,6 +253,7 @@ public class StandbyCheckpointer {
         break;
       }
     }
+    lastUploadTime = monotonicNow();
 
     // we are primary if we successfully updated the ANN
     this.isPrimaryCheckPointer = success;
@@ -362,6 +364,7 @@ public class StandbyCheckpointer {
       // Reset checkpoint time so that we don't always checkpoint
       // on startup.
       lastCheckpointTime = monotonicNow();
+      lastUploadTime = monotonicNow();
       while (shouldRun) {
         boolean needRollbackCheckpoint = namesystem.isNeedRollbackFsImage();
         if (!needRollbackCheckpoint) {
@@ -414,7 +417,9 @@ public class StandbyCheckpointer {
 
             // on all nodes, we build the checkpoint. However, we only ship the checkpoint if have a
             // rollback request, are the checkpointer, are outside the quiet period.
-            boolean sendRequest = isPrimaryCheckPointer || secsSinceLast >= checkpointConf.getQuietPeriod();
+            final long secsSinceLastUpload = (now - lastUploadTime) / 1000;
+            boolean sendRequest = isPrimaryCheckPointer
+                || secsSinceLastUpload >= checkpointConf.getQuietPeriod();
             doCheckpoint(sendRequest);
 
             // reset needRollbackCheckpoint to false only when we finish a ckpt

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java

@@ -458,6 +458,48 @@ public class TestStandbyCheckpoints {
     t.join();
   }
 
+  /**
+   * Test for the case standby NNs can upload FSImage to ANN after
+   * become non-primary standby NN. HDFS-9787
+   */
+  @Test(timeout=300000)
+  public void testNonPrimarySBNUploadFSImage() throws Exception {
+    // Shutdown all standby NNs.
+    for (int i = 1; i < NUM_NNS; i++) {
+      cluster.shutdownNameNode(i);
+
+      // Checkpoint as fast as we can, in a tight loop.
+      cluster.getConfiguration(i).setInt(
+        DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_KEY, 1);
+    }
+
+    doEdits(0, 10);
+    cluster.transitionToStandby(0);
+
+    // Standby NNs do checkpoint without active NN available.
+    for (int i = 1; i < NUM_NNS; i++) {
+      cluster.restartNameNode(i, false);
+    }
+    cluster.waitClusterUp();
+
+    for (int i = 0; i < NUM_NNS; i++) {
+      // Once the standby catches up, it should do a checkpoint
+      // and save to local directories.
+      HATestUtil.waitForCheckpoint(cluster, 1, ImmutableList.of(12));
+    }
+
+    cluster.transitionToActive(0);
+
+    // Wait for 2 seconds to expire last upload time.
+    Thread.sleep(2000);
+
+    doEdits(11, 20);
+    nns[0].getRpcServer().rollEditLog();
+
+    // One of standby NNs should also upload it back to the active.
+    HATestUtil.waitForCheckpoint(cluster, 0, ImmutableList.of(23));
+  }
+
   private void doEdits(int start, int stop) throws IOException {
     for (int i = start; i < stop; i++) {
       Path p = new Path("/test" + i);