HDFS-6614. Merging change r1609536 from branch-2 to branch-2.5.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2.5@1609540 13f79535-47bb-0310-9956-ffa450edef68

hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt

@@ -518,6 +518,9 @@ Release 2.5.0 - UNRELEASED
 
     HDFS-5411. Update Bookkeeper dependency to 4.2.3. (Rakesh R via umamahesh)
 
+    HDFS-6631. TestPread#testHedgedReadLoopTooManyTimes fails intermittently.
+    (Liang Xie via cnauroth)
+
   BREAKDOWN OF HDFS-2006 SUBTASKS AND RELATED JIRAS
 
     HDFS-6299. Protobuf for XAttr and client-side implementation. (Yi Liu via umamahesh)

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClientFaultInjector.java

@@ -52,4 +52,6 @@ public class DFSClientFaultInjector {
   public void startFetchFromDatanode() {}
 
   public void fetchFromDatanodeException() {}
+
+  public void readFromDatanodeDelay() {}
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java

@@ -1044,6 +1044,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
           throw new IOException("truncated return from reader.read(): " +
                                 "excpected " + len + ", got " + nread);
         }
+        DFSClientFaultInjector.get().readFromDatanodeDelay();
         return;
       } catch (ChecksumException e) {
         String msg = "fetchBlockByteRange(). Got a checksum exception for "

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java

@@ -292,11 +292,12 @@ public class TestPread {
     DFSClientFaultInjector.instance = Mockito
         .mock(DFSClientFaultInjector.class);
     DFSClientFaultInjector injector = DFSClientFaultInjector.instance;
+    final int sleepMs = 100;
     Mockito.doAnswer(new Answer<Void>() {
       @Override
       public Void answer(InvocationOnMock invocation) throws Throwable {
         if (true) {
-          Thread.sleep(hedgedReadTimeoutMillis + 1);
+          Thread.sleep(hedgedReadTimeoutMillis + sleepMs);
           if (DFSClientFaultInjector.exceptionNum.compareAndSet(0, 1)) {
             System.out.println("-------------- throw Checksum Exception");
             throw new ChecksumException("ChecksumException test", 100);
@@ -305,6 +306,15 @@ public class TestPread {
         return null;
       }
     }).when(injector).fetchFromDatanodeException();
+    Mockito.doAnswer(new Answer<Void>() {
+      @Override
+      public Void answer(InvocationOnMock invocation) throws Throwable {
+        if (true) {
+          Thread.sleep(sleepMs * 2);
+        }
+        return null;
+      }
+    }).when(injector).readFromDatanodeDelay();
 
     MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2)
         .format(true).build();