HDFS-11338: [SPS]: Fix timeout issue in unit tests caused by longger NN down time. Contributed by Wei Zhou and Rakesh R

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java

@@ -726,7 +726,7 @@ public class BlockManager implements BlockStatsMXBean {
 
   public void close() {
     if (sps != null) {
-      sps.stop(false);
+      sps.deactivate(false);
     }
     bmSafeMode.close();
     try {
@@ -741,6 +741,7 @@ public class BlockManager implements BlockStatsMXBean {
     datanodeManager.close();
     pendingReconstruction.stop();
     blocksMap.close();
+    stopSPSGracefully();
   }
 
   /** @return the datanodeManager */
@@ -5067,9 +5068,17 @@ public class BlockManager implements BlockStatsMXBean {
       LOG.info("Storage policy satisfier is already stopped.");
       return;
     }
-    sps.stop(true);
+    sps.deactivate(true);
   }
 
+  /**
+   * Timed wait to stop storage policy satisfier daemon threads.
+   */
+  public void stopSPSGracefully() {
+    if (sps != null) {
+      sps.stopGracefully();
+    }
+  }
   /**
    * @return True if storage policy satisfier running.
    */

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BlockStorageMovementAttemptedItems.java

@@ -130,20 +130,33 @@ public class BlockStorageMovementAttemptedItems {
   }
 
   /**
-   * Stops the monitor thread.
+   * Sets running flag to false. Also, this will interrupt monitor thread and
+   * clear all the queued up tasks.
    */
-  public synchronized void stop() {
+  public synchronized void deactivate() {
     monitorRunning = false;
     if (timerThread != null) {
       timerThread.interrupt();
-      try {
-        timerThread.join(3000);
-      } catch (InterruptedException ie) {
-      }
     }
     this.clearQueues();
   }
 
+  /**
+   * Timed wait to stop monitor thread.
+   */
+  synchronized void stopGracefully() {
+    if (timerThread == null) {
+      return;
+    }
+    if (monitorRunning) {
+      deactivate();
+    }
+    try {
+      timerThread.join(3000);
+    } catch (InterruptedException ie) {
+    }
+  }
+
   /**
    * This class contains information of an attempted trackID. Information such
    * as, (a)last attempted time stamp, (b)whether all the blocks in the trackID

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java

@@ -1324,7 +1324,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
       if (blockManager != null) {
         blockManager.deactivateSPS();
       }
-
       stopSecretManager();
       leaseManager.stopMonitor();
       if (nnrmthread != null) {
@@ -1363,6 +1362,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
         // Don't want to keep replication queues when not in Active.
         blockManager.clearQueues();
         blockManager.setInitializedReplQueues(false);
+        blockManager.stopSPSGracefully();
       }
     } finally {
       writeUnlock("stopActiveServices");

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java

@@ -115,22 +115,21 @@ public class StoragePolicySatisfier implements Runnable {
   }
 
   /**
-   * Stop storage policy satisfier demon thread.
+   * Deactivates storage policy satisfier by stopping its services.
    *
-   * @param reconfigStop
+   * @param reconfig
+   *          true represents deactivating SPS service as requested by admin,
+   *          false otherwise
    */
-  public synchronized void stop(boolean reconfigStop) {
+  public synchronized void deactivate(boolean reconfig) {
     isRunning = false;
     if (storagePolicySatisfierThread == null) {
       return;
     }
+
     storagePolicySatisfierThread.interrupt();
-    try {
-      storagePolicySatisfierThread.join(3000);
-    } catch (InterruptedException ie) {
-    }
-    this.storageMovementsMonitor.stop();
-    if (reconfigStop) {
+    this.storageMovementsMonitor.deactivate();
+    if (reconfig) {
       LOG.info("Stopping StoragePolicySatisfier, as admin requested to "
           + "deactivate it.");
       this.clearQueuesWithNotification();
@@ -140,6 +139,23 @@ public class StoragePolicySatisfier implements Runnable {
     }
   }
 
+  /**
+   * Timed wait to stop storage policy satisfier daemon threads.
+   */
+  public synchronized void stopGracefully() {
+    if (isRunning) {
+      deactivate(true);
+    }
+    this.storageMovementsMonitor.stopGracefully();
+    if (storagePolicySatisfierThread == null) {
+      return;
+    }
+    try {
+      storagePolicySatisfierThread.join(3000);
+    } catch (InterruptedException ie) {
+    }
+  }
+
   /**
    * Check whether StoragePolicySatisfier is running.
    * @return true if running
@@ -162,7 +178,7 @@ public class StoragePolicySatisfier implements Runnable {
       if (!isRunning) {
         // Stopping monitor thread and clearing queues as well
         this.clearQueues();
-        this.storageMovementsMonitor.stop();
+        this.storageMovementsMonitor.stopGracefully();
         LOG.error(
             "Stopping StoragePolicySatisfier thread " + "as Mover ID file "
                 + HdfsServerConstants.MOVER_ID_PATH.toString()
@@ -194,7 +210,7 @@ public class StoragePolicySatisfier implements Runnable {
           isRunning = false;
           // Stopping monitor thread and clearing queues as well
           this.clearQueues();
-          this.storageMovementsMonitor.stop();
+          this.storageMovementsMonitor.stopGracefully();
         }
         if (!namesystem.isRunning()) {
           LOG.info("Stopping StoragePolicySatisfier.");

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockStorageMovementAttemptedItems.java

@@ -47,7 +47,8 @@ public class TestBlockStorageMovementAttemptedItems {
   @After
   public void teardown() {
     if (bsmAttemptedItems != null) {
-      bsmAttemptedItems.stop();
+      bsmAttemptedItems.deactivate();
+      bsmAttemptedItems.stopGracefully();
     }
   }