HADOOP-16789. In TestZKFailoverController, restore changes from HADOOP-11149 that were dropped by HDFS-6440. Contributed by Jim Brennan.

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestZKFailoverController.java

@@ -575,42 +575,41 @@ public class TestZKFailoverController extends ClientBaseWithFixes {
     cluster.getZkfc(1).gracefulFailoverToYou();
     cluster.getZkfc(0).gracefulFailoverToYou();
   }
-  
-  private int runFC(DummyHAService target, String ... args) throws Exception {
-    DummyZKFC zkfc = new DummyZKFC(conf, target);
-    return zkfc.run(args);
-  }
 
-  @Test(timeout = 25000)
+  @Test
   public void testGracefulFailoverMultipleZKfcs() throws Exception {
-    try {
-      cluster.start(3);
+    cluster.start(3);
 
-      cluster.waitForActiveLockHolder(0);
+    cluster.waitForActiveLockHolder(0);
 
-      // failover to first
-      cluster.getService(1).getZKFCProxy(conf, 5000).gracefulFailover();
-      cluster.waitForActiveLockHolder(1);
+    // failover to first
+    cluster.getService(1).getZKFCProxy(conf, 5000).gracefulFailover();
+    cluster.waitForActiveLockHolder(1);
 
-      // failover to second
-      cluster.getService(2).getZKFCProxy(conf, 5000).gracefulFailover();
-      cluster.waitForActiveLockHolder(2);
+    // failover to second
+    cluster.getService(2).getZKFCProxy(conf, 5000).gracefulFailover();
+    cluster.waitForActiveLockHolder(2);
 
-      // failover back to original
-      cluster.getService(0).getZKFCProxy(conf, 5000).gracefulFailover();
-      cluster.waitForActiveLockHolder(0);
+    // failover back to original
+    cluster.getService(0).getZKFCProxy(conf, 5000).gracefulFailover();
+    cluster.waitForActiveLockHolder(0);
 
-      Thread.sleep(10000); // allow to quiesce
+    GenericTestUtils.waitFor(new Supplier<Boolean>() {
+      @Override
+      public Boolean get() {
+        return cluster.getService(0).fenceCount == 0 &&
+            cluster.getService(1).fenceCount == 0 &&
+            cluster.getService(2).fenceCount == 0 &&
+            cluster.getService(0).activeTransitionCount == 2 &&
+            cluster.getService(1).activeTransitionCount == 1 &&
+            cluster.getService(2).activeTransitionCount == 1;
+      }
+    }, 100, 60 * 1000);
+  }
 
-      assertEquals(0, cluster.getService(0).fenceCount);
-      assertEquals(0, cluster.getService(1).fenceCount);
-      assertEquals(0, cluster.getService(2).fenceCount);
-      assertEquals(2, cluster.getService(0).activeTransitionCount);
-      assertEquals(1, cluster.getService(1).activeTransitionCount);
-      assertEquals(1, cluster.getService(2).activeTransitionCount);
-    } finally {
-      cluster.stop();
-    }
+  private int runFC(DummyHAService target, String ... args) throws Exception {
+    DummyZKFC zkfc = new DummyZKFC(conf, target);
+    return zkfc.run(args);
   }
 
 }