MAPREDUCE-2489. Jobsplits with random hostnames can make the queue unusable (Jeffrey Naisbit via mahadev)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.20-security@1156814 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -61,7 +61,10 @@ Release 0.20.205.0 - unreleased
 
     HADOOP-7472. RPC client should deal with IP address change.
     (Kihwal Lee via suresh)
-
+  
+    MAPREDUCE-2489. Jobsplits with random hostnames can make the queue unusable
+    (Jeffrey Naisbit via mahadev)
+    
 Release 0.20.204.0 - unreleased
 
   NEW FEATURES

src/core/org/apache/hadoop/net/CachedDNSToSwitchMapping.java

@@ -17,9 +17,6 @@
  */
 package org.apache.hadoop.net;
 
-import java.net.InetAddress;
-import java.net.SocketException;
-import java.net.UnknownHostException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -85,33 +82,7 @@ public class CachedDNSToSwitchMapping implements DNSToSwitchMapping {
     }
     return result;
   }
-  
-  /**
-   * Resolves host names and adds them to the cache.
-   * Unlike the 'resolve" method, this won't hide UnknownHostExceptions
-   * 
-   * @param names to resolve
-   * @return List of resolved names
-   * @throws UnknownHostException if any hosts cannot be resolved
-   */  
-  public List<String> resolveValidHosts(List<String> names) 
-    throws UnknownHostException {
-    if (names.isEmpty()) {
-      return new ArrayList<String>();
-    }
-    List<String> addresses = new ArrayList<String>(names.size());
-    for (String name : names) {
-      addresses.add(InetAddress.getByName(name).getHostAddress());
-    }
-    
-    List<String> uncachedHosts = this.getUncachedHosts(names);
 
-    // Resolve the uncached hosts
-    List<String> resolvedHosts = rawMapping.resolveValidHosts(uncachedHosts);
-    this.cacheResolvedHosts(uncachedHosts, resolvedHosts);
-    return this.getCachedHosts(addresses);
-  }
-  
   public List<String> resolve(List<String> names) {
     // normalize all input names to be in the form of IP addresses
     names = NetUtils.normalizeHostNames(names);

src/core/org/apache/hadoop/net/DNSToSwitchMapping.java

@@ -20,11 +20,16 @@ package org.apache.hadoop.net;
 import java.util.List;
 import java.net.UnknownHostException;
 
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
 /**
  * An interface that should be implemented to allow pluggable 
  * DNS-name/IP-address to RackID resolvers.
  *
  */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
 public interface DNSToSwitchMapping {
   /**
    * Resolves a list of DNS-names/IP-addresses and returns back a list of
@@ -41,22 +46,4 @@ public interface DNSToSwitchMapping {
    */
   public List<String> resolve(List<String> names);
 
-  /**
-   * Resolves a list of DNS-names/IP-addresses and returns back a list of
-   * switch information (network paths). One-to-one correspondence must be 
-   * maintained between the elements in the lists. 
-   * Consider an element in the argument list - x.y.com. The switch information
-   * that is returned must be a network path of the form /foo/rack, 
-   * where / is the root, and 'foo' is the switch where 'rack' is connected.
-   * Note the hostname/ip-address is not part of the returned path.
-   * The network topology of the cluster would determine the number of
-   * components in the network path.  Unlike 'resolve', names must be 
-   * resolvable
-   * @param names
-   * @return list of resolved network paths
-   * @throws UnknownHostException if any hosts are not resolvable
-   */
-  public List<String> resolveValidHosts(List<String> names)
-    throws UnknownHostException;
-
 }

src/core/org/apache/hadoop/net/ScriptBasedMapping.java

@@ -20,7 +20,6 @@ package org.apache.hadoop.net;
 
 import java.util.*;
 import java.io.*;
-import java.net.UnknownHostException;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -115,18 +114,7 @@ implements Configurable
     
     return m;
   }
-  
-  public List<String> resolveValidHosts(List<String> names) 
-    throws UnknownHostException {
-    List<String> result = this.resolve(names);
-    if (result != null) {
-      return result;
-    } else {
-      throw new UnknownHostException(
-          "Unknown host(s) returned from ScriptBasedMapping");
-    }
-  }
-  
+
   private String runResolveCommand(List<String> args) {
     int loopCount = 0;
     if (args.size() == 0) {

src/mapred/org/apache/hadoop/mapred/JobTracker.java

@@ -3201,7 +3201,7 @@ public class JobTracker implements MRConstants, InterTrackerProtocol,
   public Node resolveAndAddToTopology(String name) throws UnknownHostException {
     List <String> tmpList = new ArrayList<String>(1);
     tmpList.add(name);
-    List <String> rNameList = dnsToSwitchMapping.resolveValidHosts(tmpList);
+    List <String> rNameList = dnsToSwitchMapping.resolve(tmpList);
     String rName = rNameList.get(0);
     String networkLoc = NodeBase.normalize(rName);
     return addHostToNodeMapping(name, networkLoc);

src/test/org/apache/hadoop/net/StaticMapping.java

@@ -18,7 +18,6 @@
 package org.apache.hadoop.net;
 
 import java.util.*;
-import java.net.UnknownHostException;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
@@ -60,8 +59,4 @@ public class StaticMapping extends Configured implements DNSToSwitchMapping {
       return m;
     }
   }
-  public List<String> resolveValidHosts(List<String> names)
-    throws UnknownHostException {
-    return this.resolve(names);
-  }
 }

src/test/org/apache/hadoop/net/TestScriptBasedMapping.java

@@ -19,7 +19,6 @@ package org.apache.hadoop.net;
 
 import java.util.ArrayList;
 import java.util.List;
-import java.net.UnknownHostException;
 
 import org.apache.hadoop.conf.Configuration;
 
@@ -49,37 +48,5 @@ public class TestScriptBasedMapping extends TestCase {
     List<String> result = mapping.resolve(names);
     assertNull(result);
   }
-  
-  public void testResolveValidInvalidHostException() {
-    names = new ArrayList<String>();
-    names.add("1.com"); // Add invalid hostname that doesn't resolve
-    boolean exceptionThrown = false;
-    try {
-      mapping.resolveValidHosts(names);
-    } catch (UnknownHostException e) {
-      exceptionThrown = true;
-    }
-    assertTrue(
-        "resolveValidHosts did not throw UnknownHostException for invalid host",
-        exceptionThrown);
-  }
-
-  public void testResolveValidHostNoException() {
-    conf.setInt(ScriptBasedMapping.SCRIPT_ARG_COUNT_KEY,
-        ScriptBasedMapping.MIN_ALLOWABLE_ARGS);
-    conf.set(ScriptBasedMapping.SCRIPT_FILENAME_KEY, "echo");
-    mapping.setConf(conf);    
 
-    names = new ArrayList<String>();
-    names.add("some.machine.name");
-    names.add("other.machine.name");
-    
-    boolean exceptionThrown = false;
-    try {
-      mapping.resolveValidHosts(names);
-    } catch (UnknownHostException e) {
-      exceptionThrown = true;
-    }
-    assertFalse("resolveValidHosts threw Exception for valid host", exceptionThrown);
-  }
 }