HDFS-6648. Order of namenodes in ConfiguredFailoverProxyProvider is undefined. Contributed by Inigo Goiri

(cherry picked from commit b104f3a282ab8e15314446e71189eb3a3b5dd9a6)

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java

@@ -351,7 +351,7 @@ public class DFSUtilClient {
   static Map<String, InetSocketAddress> getAddressesForNameserviceId(
       Configuration conf, String nsId, String defaultValue, String... keys) {
     Collection<String> nnIds = getNameNodeIds(conf, nsId);
-    Map<String, InetSocketAddress> ret = Maps.newHashMap();
+    Map<String, InetSocketAddress> ret = Maps.newLinkedHashMap();
     for (String nnId : emptyAsSingletonNull(nnIds)) {
       String suffix = concatSuffixes(nsId, nnId);
       String address = getConfValue(defaultValue, suffix, conf, keys);

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java

@@ -276,6 +276,8 @@ public interface HdfsClientConfigKeys {
     String  CONNECTION_RETRIES_ON_SOCKET_TIMEOUTS_KEY =
         PREFIX + "connection.retries.on.timeouts";
     int     CONNECTION_RETRIES_ON_SOCKET_TIMEOUTS_DEFAULT = 0;
+    String  RANDOM_ORDER = PREFIX + "random.order";
+    boolean RANDOM_ORDER_DEFAULT = false;
   }
 
   /** dfs.client.write configuration properties */

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java

@@ -23,6 +23,7 @@ import java.net.InetSocketAddress;
 import java.net.URI;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -43,9 +44,10 @@ import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 
 /**
- * A FailoverProxyProvider implementation which allows one to configure two URIs
- * to connect to during fail-over. The first configured address is tried first,
- * and on a fail-over event the other address is tried.
+ * A FailoverProxyProvider implementation which allows one to configure
+ * multiple URIs to connect to during fail-over. A random configured address is
+ * tried first, and on a fail-over event the other addresses are tried
+ * sequentially in a random order.
  */
 public class ConfiguredFailoverProxyProvider<T> extends
     AbstractNNFailoverProxyProvider<T> {
@@ -124,6 +126,13 @@ public class ConfiguredFailoverProxyProvider<T> extends
       for (InetSocketAddress address : addressesOfNns) {
         proxies.add(new AddressRpcProxyPair<T>(address));
       }
+      // Randomize the list to prevent all clients pointing to the same one
+      boolean randomized = conf.getBoolean(
+          HdfsClientConfigKeys.Failover.RANDOM_ORDER,
+          HdfsClientConfigKeys.Failover.RANDOM_ORDER_DEFAULT);
+      if (randomized) {
+        Collections.shuffle(proxies);
+      }
 
       // The client may have a delegation token set for the logical
       // URI of the cluster. Clone this token to apply to each of the