Don't always query jobtracker for all needed map outputs, instead just

for a random sample.  When the total number of splits was large, the
jobtracker was spending most of its time servicing these requests.
Also reduce the frequency of these requests.  Long-term we may need a
different algorithm here to ensure that reduces are more promptly and
efficiently notified of map completions.


git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk@383622 13f79535-47bb-0310-9956-ffa450edef68

src/java/org/apache/hadoop/mapred/ReduceTaskRunner.java

@@ -55,9 +55,13 @@ class ReduceTaskRunner extends TaskRunner {
     while (needed.size() > 0) {
       getTask().reportProgress(getTracker());
 
-      // get list of available map output locations from job tracker
-      String[][] neededStrings = new String[needed.size()][];
-      for (int i = 0; i < needed.size(); i++) {
+      // query for a just a random subset of needed segments so that we don't
+      // overwhelm jobtracker.  ideally perhaps we could send a more compact
+      // representation of all needed, i.e., a bit-vector
+      Collections.shuffle(needed);
+      int checkSize = Math.min(10, needed.size());
+      String[][] neededStrings = new String[checkSize][];
+      for (int i = 0; i < checkSize; i++) {
           neededStrings[i] = (String[]) needed.elementAt(i);
       }
       MapOutputLocation[] locs =
@@ -68,7 +72,7 @@ class ReduceTaskRunner extends TaskRunner {
           if (killed) {
             return false;
           }
-          Thread.sleep(1000);
+          Thread.sleep(10000);
         } catch (InterruptedException e) {
         }
         continue;