HDFS-14327. Using FQDN instead of IP to access servers with DNS resolving. Contributed by Fengnan Li.

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSDomainNameResolver.java

@@ -22,8 +22,9 @@ import java.net.InetAddress;
 import java.net.UnknownHostException;
 
 /**
- * DNSDomainNameResolver takes one domain name and returns all of the IP
- * addresses from the underlying DNS service.
+ * DNSDomainNameResolver wraps up the default DNS service for forward/reverse
+ * DNS lookup. It also provides a function to resolve a host name to all of
+ * fully qualified domain names belonging to the IPs from this host name
  */
 public class DNSDomainNameResolver implements DomainNameResolver {
   @Override
@@ -31,4 +32,32 @@ public class DNSDomainNameResolver implements DomainNameResolver {
       throws UnknownHostException {
     return InetAddress.getAllByName(domainName);
   }
+
+  @Override
+  public String getHostnameByIP(InetAddress address) {
+    String host = address.getCanonicalHostName();
+    if (host != null && host.length() != 0
+        && host.charAt(host.length()-1) == '.') {
+      host = host.substring(0, host.length()-1);
+    }
+    return host;
+  }
+
+  @Override
+  public String[] getAllResolvedHostnameByDomainName(
+      String domainName, boolean useFQDN) throws UnknownHostException {
+    InetAddress[] addresses = getAllByDomainName(domainName);
+    String[] hosts = new String[addresses.length];
+    if (useFQDN) {
+      for (int i = 0; i < addresses.length; i++) {
+        hosts[i] = getHostnameByIP(addresses[i]);
+      }
+    } else {
+      for (int i = 0; i < addresses.length; i++) {
+        hosts[i] = addresses[i].getHostAddress();
+      }
+    }
+
+    return hosts;
+  }
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DomainNameResolver.java

@@ -36,4 +36,27 @@ public interface DomainNameResolver {
    */
   InetAddress[] getAllByDomainName(String domainName)
       throws UnknownHostException;
+
+  /**
+   * Reverse lookup an IP address and get the fully qualified domain name(fqdn).
+   *
+   * @param address
+   * @return fully qualified domain name
+   */
+  String getHostnameByIP(InetAddress address);
+
+  /**
+   * This function combines getAllByDomainName and getHostnameByIP, for a single
+   * domain name, it will first do a forward lookup to get all of IP addresses,
+   * then for each IP address, it will do a reverse lookup to get the fqdn.
+   * This function is necessary in secure environment since Kerberos uses fqdn
+   * in the service principal instead of IP.
+   *
+   * @param domainName
+   * @return all fully qualified domain names belonging to the IPs resolved from
+   * the input domainName
+   * @throws UnknownHostException
+   */
+   String[] getAllResolvedHostnameByDomainName(
+       String domainName, boolean useFQDN) throws UnknownHostException;
 }

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/MockDomainNameResolver.java

@@ -19,14 +19,16 @@ package org.apache.hadoop.net;
 
 import java.net.InetAddress;
 import java.net.UnknownHostException;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.TreeMap;
 
 import com.google.common.annotations.VisibleForTesting;
 
 /**
- * This mock resolver class returns the predefined resolving results.
- * By default it uses a default "test.foo.bar" domain with two IP addresses.
+ * This mock resolver class returns the predefined resolving/reverse lookup
+ * results. By default it uses a default "test.foo.bar" domain with two
+ * IP addresses.
  */
 public class MockDomainNameResolver implements DomainNameResolver {
 
@@ -37,16 +39,21 @@ public class MockDomainNameResolver implements DomainNameResolver {
   public static final byte[] BYTE_ADDR_2 = new byte[]{10, 1, 1, 2};
   public static final String ADDR_1 = "10.1.1.1";
   public static final String ADDR_2 = "10.1.1.2";
+  public static final String FQDN_1 = "host01.com";
+  public static final String FQDN_2 = "host02.com";
 
   /** Internal mapping of domain names and IP addresses. */
   private Map<String, InetAddress[]> addrs = new TreeMap<>();
-
+  /** Internal mapping from IP addresses to fqdns. */
+  private Map<InetAddress, String> ptrMap = new HashMap<>();
 
   public MockDomainNameResolver() {
     try {
       InetAddress nn1Address = InetAddress.getByAddress(BYTE_ADDR_1);
       InetAddress nn2Address = InetAddress.getByAddress(BYTE_ADDR_2);
       addrs.put(DOMAIN, new InetAddress[]{nn1Address, nn2Address});
+      ptrMap.put(nn1Address, FQDN_1);
+      ptrMap.put(nn2Address, FQDN_2);
     } catch (UnknownHostException e) {
       throw new RuntimeException(e);
     }
@@ -61,6 +68,29 @@ public class MockDomainNameResolver implements DomainNameResolver {
     return addrs.get(domainName);
   }
 
+  @Override
+  public String getHostnameByIP(InetAddress address) {
+    return ptrMap.containsKey(address) ? ptrMap.get(address) : null;
+  }
+
+  @Override
+  public String[] getAllResolvedHostnameByDomainName(
+      String domainName, boolean useFQDN) throws UnknownHostException {
+    InetAddress[] addresses = getAllByDomainName(domainName);
+    String[] hosts = new String[addresses.length];
+    if (useFQDN) {
+      for (int i = 0; i < hosts.length; i++) {
+        hosts[i] = this.ptrMap.get(addresses[i]);
+      }
+    } else {
+      for (int i = 0; i < hosts.length; i++) {
+        hosts[i] = addresses[i].getHostAddress();
+      }
+    }
+
+    return hosts;
+  }
+
   @VisibleForTesting
   public void setAddressMap(Map<String, InetAddress[]> addresses) {
     this.addrs = addresses;

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java

@@ -291,6 +291,8 @@ public interface HdfsClientConfigKeys {
     String  RESOLVE_ADDRESS_NEEDED_KEY = PREFIX + "resolve-needed";
     boolean RESOLVE_ADDRESS_NEEDED_DEFAULT = false;
     String RESOLVE_SERVICE_KEY = PREFIX + "resolver.impl";
+    String  RESOLVE_ADDRESS_TO_FQDN = PREFIX + "useFQDN";
+    boolean RESOLVE_ADDRESS_TO_FQDN_DEFAULT = true;
   }
 
   /** dfs.client.write configuration properties */

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/AbstractNNFailoverProxyProvider.java

@@ -19,7 +19,6 @@
 package org.apache.hadoop.hdfs.server.namenode.ha;
 
 import java.io.IOException;
-import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.net.URI;
 import java.util.ArrayList;
@@ -180,7 +179,7 @@ public abstract class AbstractNNFailoverProxyProvider<T> implements
 
     Collection<InetSocketAddress> addressesOfNns = addressesInNN.values();
     try {
-      addressesOfNns = getResolvedAddressesIfNecessary(addressesOfNns, uri);
+      addressesOfNns = getResolvedHostsIfNecessary(addressesOfNns, uri);
     } catch (IOException e) {
       throw new RuntimeException(e);
     }
@@ -209,7 +208,7 @@ public abstract class AbstractNNFailoverProxyProvider<T> implements
    * @return The collection of resolved IP addresses.
    * @throws IOException If there are issues resolving the addresses.
    */
-  Collection<InetSocketAddress> getResolvedAddressesIfNecessary(
+  Collection<InetSocketAddress> getResolvedHostsIfNecessary(
       Collection<InetSocketAddress> addressesOfNns, URI nameNodeUri)
           throws IOException {
     // 'host' here is usually the ID of the nameservice when address
@@ -223,6 +222,11 @@ public abstract class AbstractNNFailoverProxyProvider<T> implements
       // Early return is no resolve is necessary
       return addressesOfNns;
     }
+    // decide whether to access server by IP or by host name
+    String useFQDNKeyWithHost =
+        HdfsClientConfigKeys.Failover.RESOLVE_ADDRESS_TO_FQDN + "." + host;
+    boolean requireFQDN = conf.getBoolean(useFQDNKeyWithHost,
+        HdfsClientConfigKeys.Failover.RESOLVE_ADDRESS_TO_FQDN_DEFAULT);
 
     Collection<InetSocketAddress> addressOfResolvedNns = new ArrayList<>();
     DomainNameResolver dnr = DomainNameResolverFactory.newInstance(
@@ -232,12 +236,12 @@ public abstract class AbstractNNFailoverProxyProvider<T> implements
     LOG.info("Namenode domain name will be resolved with {}",
         dnr.getClass().getName());
     for (InetSocketAddress address : addressesOfNns) {
-      InetAddress[] resolvedAddresses = dnr.getAllByDomainName(
-          address.getHostName());
+      String[] resolvedHostNames = dnr.getAllResolvedHostnameByDomainName(
+          address.getHostName(), requireFQDN);
       int port = address.getPort();
-      for (InetAddress raddress : resolvedAddresses) {
+      for (String hostname : resolvedHostNames) {
         InetSocketAddress resolvedAddress = new InetSocketAddress(
-            raddress, port);
+            hostname, port);
         addressOfResolvedNns.add(resolvedAddress);
       }
     }

hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConfiguredFailoverProxyProvider.java

@@ -142,7 +142,8 @@ public class TestConfiguredFailoverProxyProvider {
    * Add more DNS related settings to the passed in configuration.
    * @param config Configuration file to add settings to.
    */
-  private void addDNSSettings(Configuration config, boolean hostResolvable) {
+  private void addDNSSettings(Configuration config,
+                              boolean hostResolvable, boolean useFQDN) {
     config.set(
         HdfsClientConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX + "." + ns3, "nn");
     String domain = hostResolvable
@@ -163,6 +164,10 @@ public class TestConfiguredFailoverProxyProvider {
     config.setBoolean(
         HdfsClientConfigKeys.Failover.RANDOM_ORDER + "." + ns3,
         true);
+    config.setBoolean(
+        HdfsClientConfigKeys.Failover.RESOLVE_ADDRESS_TO_FQDN + "." + ns3,
+        useFQDN
+    );
   }
 
   /**
@@ -250,17 +255,18 @@ public class TestConfiguredFailoverProxyProvider {
         nn1Count.get() + nn2Count.get() + nn3Count.get());
   }
 
-  @Test
-  public void testResolveDomainNameUsingDNS() throws Exception {
+  private void testResolveDomainNameUsingDNS(boolean useFQDN) throws Exception {
     Configuration dnsConf = new Configuration(conf);
-    addDNSSettings(dnsConf, true);
+    addDNSSettings(dnsConf, true, useFQDN);
 
     // Mock ClientProtocol
     Map<InetSocketAddress, ClientProtocol> proxyMap = new HashMap<>();
     final AtomicInteger nn1Count = addClientMock(
-        MockDomainNameResolver.BYTE_ADDR_1, proxyMap);
+        useFQDN ? MockDomainNameResolver.FQDN_1 : MockDomainNameResolver.ADDR_1,
+        proxyMap);
     final AtomicInteger nn2Count = addClientMock(
-        MockDomainNameResolver.BYTE_ADDR_2, proxyMap);
+        useFQDN ? MockDomainNameResolver.FQDN_2 : MockDomainNameResolver.ADDR_2,
+        proxyMap);
 
     // Get a client multiple times
     final Map<String, AtomicInteger> proxyResults = new HashMap<>();
@@ -280,16 +286,18 @@ public class TestConfiguredFailoverProxyProvider {
       proxy.getStats();
     }
 
+    String resolvedHost1 = useFQDN ?
+        MockDomainNameResolver.FQDN_1 : "/" + MockDomainNameResolver.ADDR_1;
+    String resolvedHost2 = useFQDN ?
+        MockDomainNameResolver.FQDN_2 : "/" + MockDomainNameResolver.ADDR_2;
     // Check we got the proper addresses
     assertEquals(2, proxyResults.size());
     assertTrue(
         "nn1 wasn't returned: " + proxyResults,
-        proxyResults.containsKey(
-            "/" + MockDomainNameResolver.ADDR_1 + ":8020"));
+        proxyResults.containsKey(resolvedHost1 + ":8020"));
     assertTrue(
         "nn2 wasn't returned: " + proxyResults,
-        proxyResults.containsKey(
-            "/" + MockDomainNameResolver.ADDR_2 + ":8020"));
+        proxyResults.containsKey(resolvedHost2 + ":8020"));
 
     // Check that the Namenodes were invoked
     assertEquals(NUM_ITERATIONS, nn1Count.get() + nn2Count.get());
@@ -304,10 +312,18 @@ public class TestConfiguredFailoverProxyProvider {
         nn2Count.get() > 0);
   }
 
+  @Test
+  public void testResolveDomainNameUsingDNS() throws Exception {
+    // test resolving to IP
+    testResolveDomainNameUsingDNS(false);
+    // test resolving to FQDN
+    testResolveDomainNameUsingDNS(true);
+  }
+
   @Test
   public void testResolveDomainNameUsingDNSUnknownHost() throws Exception {
     Configuration dnsConf = new Configuration(conf);
-    addDNSSettings(dnsConf, false);
+    addDNSSettings(dnsConf, false, false);
 
     Map<InetSocketAddress, ClientProtocol> proxyMap = new HashMap<>();
     exception.expect(RuntimeException.class);
@@ -321,19 +337,18 @@ public class TestConfiguredFailoverProxyProvider {
 
   /**
    * Add a ClientProtocol mock for the proxy.
-   * @param addr IP address for the destination.
+   * @param host host name for the destination.
    * @param proxyMap Map containing the client for each target address.
    * @return The counter for the number of calls to this target.
    * @throws Exception If the client cannot be created.
    */
   private AtomicInteger addClientMock(
-      byte[] addr, Map<InetSocketAddress, ClientProtocol> proxyMap)
-          throws Exception {
+      String host, Map<InetSocketAddress, ClientProtocol> proxyMap)
+      throws Exception {
 
     final AtomicInteger counter = new AtomicInteger(0);
-    InetAddress inetAddr = InetAddress.getByAddress(addr);
     InetSocketAddress inetSockerAddr =
-        new InetSocketAddress(inetAddr, rpcPort);
+        new InetSocketAddress(host, rpcPort);
 
     final ClientProtocol cpMock = mock(ClientProtocol.class);
     when(cpMock.getStats()).thenAnswer(createAnswer(counter, 1));

hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml

@@ -3798,6 +3798,20 @@
   </description>
 </property>
 
+<property>
+  <name>dfs.client.failover.resolver.useFQDN</name>
+  <value>true</value>
+  <description>
+    Determines whether the resolved result is fully qualified domain name instead
+    of pure IP address(es). The config name can be extended with an optional
+    nameservice ID (of form dfs.client.failover.resolver.impl[.nameservice]) to
+    configure specific nameservices when multiple nameservices exist.
+    In secure environment, this has to be enabled since Kerberos is using fqdn
+    in machine's principal therefore accessing servers by IP won't be recognized
+    by the KDC.
+  </description>
+</property>
+
 <property>
   <name>dfs.client.key.provider.cache.expiry</name>
   <value>864000000</value>