HDFS-12837. Intermittent failure in TestReencryptionWithKMS.

(cherry picked from commit b37074be5ab35c238e18bb9c3b89db6d7f8d0986)

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionHandler.java

@@ -617,7 +617,9 @@ public class ReencryptionHandler implements Runnable {
       while (shouldPauseForTesting) {
         LOG.info("Sleeping in the re-encrypt handler for unit test.");
         synchronized (reencryptionHandler) {
-          reencryptionHandler.wait(30000);
+          if (shouldPauseForTesting) {
+            reencryptionHandler.wait(30000);
+          }
         }
         LOG.info("Continuing re-encrypt handler after pausing.");
       }

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryption.java

@@ -67,6 +67,7 @@ import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
@@ -206,8 +207,7 @@ public class TestReencryption {
     ZoneReencryptionStatus zs = it.next();
     assertEquals(zone.toString(), zs.getZoneName());
     assertEquals(ZoneReencryptionStatus.State.Completed, zs.getState());
-    assertTrue(zs.getCompletionTime() > 0);
-    assertTrue(zs.getCompletionTime() > zs.getSubmissionTime());
+    verifyZoneCompletionTime(zs);
     assertNotEquals(fei0.getEzKeyVersionName(), zs.getEzKeyVersionName());
     assertEquals(fei1.getEzKeyVersionName(), zs.getEzKeyVersionName());
     assertEquals(10, zs.getFilesReencrypted());
@@ -599,14 +599,27 @@ public class TestReencryption {
     final ZoneReencryptionStatus zs = it.next();
     assertEquals(zone.toString(), zs.getZoneName());
     assertEquals(ZoneReencryptionStatus.State.Completed, zs.getState());
-    assertTrue(zs.getCompletionTime() > 0);
-    assertTrue(zs.getCompletionTime() > zs.getSubmissionTime());
+    verifyZoneCompletionTime(zs);
     if (fei != null) {
       assertNotEquals(fei.getEzKeyVersionName(), zs.getEzKeyVersionName());
     }
     assertEquals(expectedFiles, zs.getFilesReencrypted());
   }
 
+  /**
+   * Verify the zone status' completion time is larger than 0, and is no less
+   * than submission time.
+   */
+  private void verifyZoneCompletionTime(final ZoneReencryptionStatus zs) {
+    assertNotNull(zs);
+    assertTrue("Completion time should be positive. " + zs.getCompletionTime(),
+        zs.getCompletionTime() > 0);
+    assertTrue("Completion time " + zs.getCompletionTime()
+            + " should be no less than submission time "
+            + zs.getSubmissionTime(),
+        zs.getCompletionTime() >= zs.getSubmissionTime());
+  }
+
   @Test
   public void testReencryptLoadedFromFsimage() throws Exception {
     /*
@@ -1475,7 +1488,7 @@ public class TestReencryption {
       }
 
       @Override
-      public void reencryptEncryptedKeys() throws IOException {
+      public synchronized void reencryptEncryptedKeys() throws IOException {
         if (exceptionCount > 0) {
           exceptionCount--;
           try {
@@ -1536,8 +1549,7 @@ public class TestReencryption {
     assertEquals(zone.toString(), zs.getZoneName());
     assertEquals(ZoneReencryptionStatus.State.Completed, zs.getState());
     assertTrue(zs.isCanceled());
-    assertTrue(zs.getCompletionTime() > 0);
-    assertTrue(zs.getCompletionTime() > zs.getSubmissionTime());
+    verifyZoneCompletionTime(zs);
     assertEquals(0, zs.getFilesReencrypted());
 
     assertTrue(getUpdater().isRunning());
@@ -1559,8 +1571,7 @@ public class TestReencryption {
     assertEquals(zone.toString(), zs.getZoneName());
     assertEquals(ZoneReencryptionStatus.State.Completed, zs.getState());
     assertFalse(zs.isCanceled());
-    assertTrue(zs.getCompletionTime() > 0);
-    assertTrue(zs.getCompletionTime() > zs.getSubmissionTime());
+    verifyZoneCompletionTime(zs);
     assertEquals(10, zs.getFilesReencrypted());
   }
 
@@ -1578,8 +1589,7 @@ public class TestReencryption {
     assertEquals(zone.toString(), zs.getZoneName());
     assertEquals(ZoneReencryptionStatus.State.Completed, zs.getState());
     assertTrue(zs.isCanceled());
-    assertTrue(zs.getCompletionTime() > 0);
-    assertTrue(zs.getCompletionTime() > zs.getSubmissionTime());
+    verifyZoneCompletionTime(zs);
     assertEquals(0, zs.getFilesReencrypted());
 
     // verify re-encryption works after restart.
@@ -1591,8 +1601,7 @@ public class TestReencryption {
     assertEquals(zone.toString(), zs.getZoneName());
     assertEquals(ZoneReencryptionStatus.State.Completed, zs.getState());
     assertFalse(zs.isCanceled());
-    assertTrue(zs.getCompletionTime() > 0);
-    assertTrue(zs.getCompletionTime() > zs.getSubmissionTime());
+    verifyZoneCompletionTime(zs);
     assertEquals(10, zs.getFilesReencrypted());
   }
 
@@ -1678,8 +1687,7 @@ public class TestReencryption {
     ZoneReencryptionStatus zs = it.next();
     assertEquals(zone.toString(), zs.getZoneName());
     assertEquals(ZoneReencryptionStatus.State.Completed, zs.getState());
-    assertTrue(zs.getCompletionTime() > 0);
-    assertTrue(zs.getCompletionTime() > zs.getSubmissionTime());
+    verifyZoneCompletionTime(zs);
     assertEquals(10, zs.getFilesReencrypted());
   }
 
@@ -1735,7 +1743,7 @@ public class TestReencryption {
       }
 
       @Override
-      public void reencryptEncryptedKeys() throws IOException {
+      public synchronized void reencryptEncryptedKeys() throws IOException {
         if (exceptionCount > 0) {
           --exceptionCount;
           throw new IOException("Injected KMS failure");
@@ -1771,8 +1779,7 @@ public class TestReencryption {
     ZoneReencryptionStatus zs = it.next();
     assertEquals(zone.toString(), zs.getZoneName());
     assertEquals(ZoneReencryptionStatus.State.Completed, zs.getState());
-    assertTrue(zs.getCompletionTime() > 0);
-    assertTrue(zs.getCompletionTime() > zs.getSubmissionTime());
+    verifyZoneCompletionTime(zs);
     assertEquals(5, zs.getFilesReencrypted());
     assertEquals(5, zs.getNumReencryptionFailures());
   }
@@ -1787,7 +1794,8 @@ public class TestReencryption {
       }
 
       @Override
-      public void reencryptUpdaterProcessOneTask() throws IOException {
+      public synchronized void reencryptUpdaterProcessOneTask()
+          throws IOException {
         if (exceptionCount > 0) {
           --exceptionCount;
           throw new IOException("Injected process task failure");
@@ -1823,8 +1831,7 @@ public class TestReencryption {
     ZoneReencryptionStatus zs = it.next();
     assertEquals(zone.toString(), zs.getZoneName());
     assertEquals(ZoneReencryptionStatus.State.Completed, zs.getState());
-    assertTrue(zs.getCompletionTime() > 0);
-    assertTrue(zs.getCompletionTime() > zs.getSubmissionTime());
+    verifyZoneCompletionTime(zs);
     assertEquals(5, zs.getFilesReencrypted());
     assertEquals(1, zs.getNumReencryptionFailures());
   }
@@ -1840,7 +1847,8 @@ public class TestReencryption {
       }
 
       @Override
-      public void reencryptUpdaterProcessCheckpoint() throws IOException {
+      public synchronized void reencryptUpdaterProcessCheckpoint()
+          throws IOException {
         if (exceptionCount > 0) {
           --exceptionCount;
           throw new IOException("Injected process checkpoint failure");
@@ -1876,8 +1884,7 @@ public class TestReencryption {
     ZoneReencryptionStatus zs = it.next();
     assertEquals(zone.toString(), zs.getZoneName());
     assertEquals(ZoneReencryptionStatus.State.Completed, zs.getState());
-    assertTrue(zs.getCompletionTime() > 0);
-    assertTrue(zs.getCompletionTime() > zs.getSubmissionTime());
+    verifyZoneCompletionTime(zs);
     assertEquals(10, zs.getFilesReencrypted());
     assertEquals(1, zs.getNumReencryptionFailures());
   }
@@ -1892,7 +1899,8 @@ public class TestReencryption {
       }
 
       @Override
-      public void reencryptUpdaterProcessOneTask() throws IOException {
+      public synchronized void reencryptUpdaterProcessOneTask()
+          throws IOException {
         if (exceptionCount > 0) {
           --exceptionCount;
           throw new RetriableException("Injected process task failure");
@@ -1929,8 +1937,7 @@ public class TestReencryption {
     ZoneReencryptionStatus zs = it.next();
     assertEquals(zone.toString(), zs.getZoneName());
     assertEquals(ZoneReencryptionStatus.State.Completed, zs.getState());
-    assertTrue(zs.getCompletionTime() > 0);
-    assertTrue(zs.getCompletionTime() > zs.getSubmissionTime());
+    verifyZoneCompletionTime(zs);
     assertEquals(10, zs.getFilesReencrypted());
     assertEquals(0, zs.getNumReencryptionFailures());
   }