HADOOP-12706. TestLocalFsFCStatistics#testStatisticsThreadLocalDataCleanUp times out occasionally. Contributed by Sangjin Lee and Colin Patrick McCabe

(cherry picked from commit cdf88952599a43b1ef5adda792bfb195c7529fad)

Conflicts:

	hadoop-common-project/hadoop-common/CHANGES.txt

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -21,6 +21,9 @@ Release 2.6.4 - UNRELEASED
     HADOOP-12107. long running apps may have a huge number of StatisticsData
     instances under FileSystem (Sangjin Lee via Ming Ma)
 
+    HADOOP-12706. TestLocalFsFCStatistics#testStatisticsThreadLocalDataCleanUp
+    times out occasionally (Sangjin Lee and Colin Patrick McCabe via jlowe)
+
 Release 2.6.3 - 2015-12-17
 
   INCOMPATIBLE CHANGES

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java

@@ -32,6 +32,8 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.FileSystem.Statistics;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.Assert;
@@ -46,6 +48,8 @@ import com.google.common.util.concurrent.Uninterruptibles;
  * </p>
  */
 public abstract class FCStatisticsBaseTest {
+  private static final Log LOG = LogFactory.getLog(FCStatisticsBaseTest.class);
+
   static protected int blockSize = 512;
   static protected int numBlocks = 1;
   
@@ -110,7 +114,7 @@ public abstract class FCStatisticsBaseTest {
     fc.delete(filePath, true);
   }
 
-  @Test(timeout=60000)
+  @Test(timeout=70000)
   public void testStatisticsThreadLocalDataCleanUp() throws Exception {
     final Statistics stats = new Statistics("test");
     // create a small thread pool to test the statistics
@@ -137,17 +141,24 @@ public abstract class FCStatisticsBaseTest {
     es.shutdownNow();
     es.awaitTermination(1, TimeUnit.MINUTES);
     es = null;
-    System.gc();
+    System.gc(); // force GC to garbage collect threads
 
-    // wait for up to 10 seconds
+    // wait for up to 60 seconds
     GenericTestUtils.waitFor(new Supplier<Boolean>() {
           @Override
           public Boolean get() {
             int size = stats.getAllThreadLocalDataSize();
             allDataSize.set(size);
-            return size == 0;
+            if (size == 0) {
+              return true;
+            }
+            LOG.warn("not all references have been cleaned up; still " +
+                allDataSize.get() + " references left");
+            LOG.warn("triggering another GC");
+            System.gc();
+            return false;
           }
-        }, 1000, 10*1000);
+        }, 500, 60*1000);
     Assert.assertEquals(0, allDataSize.get());
     Assert.assertEquals(size, stats.getReadOps());
   }