MAPREDUCE-5601. ShuffleHandler fadvises file regions as DONTNEED even when fetch fails (Sandy Ryza)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1537855 13f79535-47bb-0310-9956-ffa450edef68

hadoop-mapreduce-project/CHANGES.txt

@@ -171,6 +171,9 @@ Release 2.3.0 - UNRELEASED
     MAPREDUCE-5487. In task processes, JobConf is unnecessarily loaded again
     in Limits (Sandy Ryza)
 
+    MAPREDUCE-5601. ShuffleHandler fadvises file regions as DONTNEED even when
+    fetch fails (Sandy Ryza)
+
   BUG FIXES
 
     MAPREDUCE-5316. job -list-attempt-ids command does not handle illegal

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java

@@ -69,6 +69,14 @@ public class FadvisedFileRegion extends DefaultFileRegion {
     if (readaheadRequest != null) {
       readaheadRequest.cancel();
     }
+    super.releaseExternalResources();
+  }
+  
+  /**
+   * Call when the transfer completes successfully so we can advise the OS that
+   * we don't need the region to be cached anymore.
+   */
+  public void transferSuccessful() {
     if (manageOsCache && getCount() > 0) {
       try {
         NativeIO.POSIX.posixFadviseIfPossible(identifier,
@@ -78,6 +86,5 @@ public class FadvisedFileRegion extends DefaultFileRegion {
         LOG.warn("Failed to manage OS cache for " + identifier, t);
       }
     }
-    super.releaseExternalResources();
   }
 }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java

@@ -625,6 +625,9 @@ public class ShuffleHandler extends AuxiliaryService {
             //      attribute to appropriate spill output
           @Override
           public void operationComplete(ChannelFuture future) {
+            if (future.isSuccess()) {
+              partition.transferSuccessful();
+            }
             partition.releaseExternalResources();
           }
         });