HDFS-7922. ShortCircuitCache#close is not releasing ScheduledThreadPoolExecutors (Rakesh R via Colin P. McCabe)

(cherry picked from commit 3c7adaaf3571c91fee80585472d2a81402a53e2b)

hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt

@@ -94,6 +94,9 @@ Release 2.8.0 - UNRELEASED
     HDFS-6945. BlockManager should remove a block from excessReplicateMap and
     decrement ExcessBlocks metric when the block is removed. (aajisaka)
 
+    HDFS-7922. ShortCircuitCache#close is not releasing
+    ScheduledThreadPoolExecutors (Rakesh R via Colin P. McCabe)
+
 Release 2.7.0 - UNRELEASED
 
   INCOMPATIBLE CHANGES

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java

@@ -916,6 +916,34 @@ public class ShortCircuitCache implements Closeable {
     } finally {
       lock.unlock();
     }
+
+    releaserExecutor.shutdown();
+    cleanerExecutor.shutdown();
+    // wait for existing tasks to terminate
+    try {
+      if (!releaserExecutor.awaitTermination(30, TimeUnit.SECONDS)) {
+        LOG.error("Forcing SlotReleaserThreadPool to shutdown!");
+        releaserExecutor.shutdownNow();
+      }
+    } catch (InterruptedException e) {
+      releaserExecutor.shutdownNow();
+      Thread.currentThread().interrupt();
+      LOG.error("Interrupted while waiting for SlotReleaserThreadPool "
+          + "to terminate", e);
+    }
+
+    // wait for existing tasks to terminate
+    try {
+      if (!cleanerExecutor.awaitTermination(30, TimeUnit.SECONDS)) {
+        LOG.error("Forcing CleanerThreadPool to shutdown!");
+        cleanerExecutor.shutdownNow();
+      }
+    } catch (InterruptedException e) {
+      cleanerExecutor.shutdownNow();
+      Thread.currentThread().interrupt();
+      LOG.error("Interrupted while waiting for CleanerThreadPool "
+          + "to terminate", e);
+    }
     IOUtils.cleanup(LOG, shmManager);
   }
 

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java

@@ -203,7 +203,7 @@ public class TestShortCircuitCache {
     cache.close();
   }
 
-  @Test(timeout=60000)
+  @Test(timeout=100000)
   public void testExpiry() throws Exception {
     final ShortCircuitCache cache =
         new ShortCircuitCache(2, 1, 1, 10000000, 1, 10000000, 0);