HADOOP-7472. RPC client should deal with IP address change. (Kihwal Lee via suresh)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.20-security@1156353 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -59,6 +59,9 @@ Release 0.20.205.0 - unreleased
     MAPREDUCE-2780. Use a utility method to set service in token. 
     (Daryn Sharp via jitendra)
 
+    HADOOP-7472. RPC client should deal with IP address change.
+    (Kihwal Lee via suresh)
+
 Release 0.20.204.0 - unreleased
 
   NEW FEATURES

src/core/org/apache/hadoop/ipc/Client.java

@@ -383,6 +383,27 @@ public class Client {
       saslRpcClient = new SaslRpcClient(authMethod, token, serverPrincipal);
       return saslRpcClient.saslConnect(in2, out2);
     }
+
+    /**
+     * Update the server address if the address corresponding to the host
+     * name has changed.
+     *
+     * @return true if an addr change was detected.
+     * @throws IOException when the hostname cannot be resolved.
+     */
+    private synchronized boolean updateAddress() throws IOException {
+      // Do a fresh lookup with the old host name.
+      InetSocketAddress currentAddr =  new InetSocketAddress(
+                               server.getHostName(), server.getPort());
+
+      if (!server.equals(currentAddr)) {
+        LOG.warn("Address change detected. Old: " + server.toString() +
+                                 " New: " + currentAddr.toString());
+        server = currentAddr;
+        return true;
+      }
+      return false;
+    }
     
     private synchronized void setupConnection() throws IOException {
       short ioFailures = 0;
@@ -413,7 +434,7 @@ public class Client {
           }
           
           // connection time out is 20s
-          NetUtils.connect(this.socket, remoteId.getAddress(), 20000);
+          NetUtils.connect(this.socket, server, 20000);
           if (rpcTimeout > 0) {
             pingInterval = rpcTimeout;  // rpcTimeout overwrites pingInterval
           }
@@ -421,11 +442,20 @@ public class Client {
           this.socket.setSoTimeout(pingInterval);
           return;
         } catch (SocketTimeoutException toe) {
+          /* Check for an address change and update the local reference.
+           * Reset the failure counter if the address was changed
+           */
+          if (updateAddress()) {
+            timeoutFailures = ioFailures = 0;
+          }
           /* The max number of retries is 45,
            * which amounts to 20s*45 = 15 minutes retries.
            */
           handleConnectionFailure(timeoutFailures++, 45, toe);
         } catch (IOException ie) {
+          if (updateAddress()) {
+            timeoutFailures = ioFailures = 0;
+          }
           handleConnectionFailure(ioFailures++, maxRetries, ie);
         }
       }