HDFS-17743: Added support for random datanode ordering in getBlockLocations() (#7447)

Added support random datanode ordering in getBlockLocations() and put it behind a configuration parameter with the default behavior remaining unchanged.

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java

@@ -1131,4 +1131,22 @@ public class NetworkTopology {
     numOfEmptyRacks = count;
     LOG.debug("Current numOfEmptyRacks is {}", numOfEmptyRacks);
   }
+
+  /**
+   * Randomly permute the active nodes of the node array.
+   *
+   * @param nodes     Available replicas with the requested data
+   * @param activeLen Number of active nodes at the front of the array
+   */
+  public void shuffle(Node[] nodes, int activeLen) {
+    List<Node> list = new ArrayList<>(activeLen);
+    for (int i = 0; i < activeLen; i++) {
+      list.add(nodes[i]);
+    }
+    Collections.shuffle(list, getRandom());
+
+    for (int i = 0; i < activeLen; i++) {
+      nodes[i] = list.get(i);
+    }
+  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java

@@ -352,6 +352,12 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final boolean
       DFS_NAMENODE_CORRUPT_BLOCK_DELETE_IMMEDIATELY_ENABLED_DEFAULT = true;
 
+  public static final String
+      DFS_NAMENODE_RANDOM_NODE_ORDER_ENABLED =
+      "dfs.namenode.random.node.order.enabled";
+  public static final boolean
+      DFS_NAMENODE_RANDOM_NODE_ORDER_ENABLED_DEFAULT = false;
+
   @Deprecated
   public static final String  DFS_WEBHDFS_USER_PATTERN_KEY =
       HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY;

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java

@@ -228,6 +228,8 @@ public class DatanodeManager {
    */
   private final long timeBetweenResendingCachingDirectivesMs;
 
+  private final boolean randomNodeOrderEnabled;
+
   DatanodeManager(final BlockManager blockManager, final Namesystem namesystem,
       final Configuration conf) throws IOException {
     this.namesystem = namesystem;
@@ -359,6 +361,9 @@ public class DatanodeManager {
     this.blocksPerPostponedMisreplicatedBlocksRescan = conf.getLong(
         DFSConfigKeys.DFS_NAMENODE_BLOCKS_PER_POSTPONEDBLOCKS_RESCAN_KEY,
         DFSConfigKeys.DFS_NAMENODE_BLOCKS_PER_POSTPONEDBLOCKS_RESCAN_KEY_DEFAULT);
+    this.randomNodeOrderEnabled = conf.getBoolean(
+        DFSConfigKeys.DFS_NAMENODE_RANDOM_NODE_ORDER_ENABLED,
+        DFSConfigKeys.DFS_NAMENODE_RANDOM_NODE_ORDER_ENABLED_DEFAULT);
   }
 
   /**
@@ -662,12 +667,16 @@ public class DatanodeManager {
       --lastActiveIndex;
     }
     int activeLen = lastActiveIndex + 1;
-    if(nonDatanodeReader) {
-      networktopology.sortByDistanceUsingNetworkLocation(client,
-          lb.getLocations(), activeLen, createSecondaryNodeSorter());
+    if (!randomNodeOrderEnabled) {
+      if(nonDatanodeReader) {
+        networktopology.sortByDistanceUsingNetworkLocation(client,
+            lb.getLocations(), activeLen, createSecondaryNodeSorter());
+      } else {
+        networktopology.sortByDistance(client, lb.getLocations(), activeLen,
+            createSecondaryNodeSorter());
+      }
     } else {
-      networktopology.sortByDistance(client, lb.getLocations(), activeLen,
-          createSecondaryNodeSorter());
+      networktopology.shuffle(lb.getLocations(), activeLen);
     }
     // move PROVIDED storage to the end to prefer local replicas.
     lb.moveProvidedToEnd(activeLen);

hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml

@@ -6445,6 +6445,15 @@
     </description>
   </property>
 
+  <property>
+    <name>dfs.namenode.random.node.order.enabled</name>
+    <value>false</value>
+    <description>
+      Whether to randomized the order of data nodes of a block or sort by network
+      distance (default behavior) before returning to clients.
+    </description>
+  </property>
+
   <property>
     <name>dfs.journalnode.edits.dir.perm</name>
     <value>700</value>

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopology.java

@@ -23,9 +23,12 @@ import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
+import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Random;
 import java.util.Set;
@@ -649,4 +652,37 @@ public class TestNetworkTopology {
     cluster.recommissionNode(n1);
     assertEquals(6, cluster.getNumOfNonEmptyRacks());
   }
+
+  @Test
+  public void testShuffle() {
+    testShuffleInternal(0);
+    testShuffleInternal(1);
+    testShuffleInternal(2);
+    testShuffleInternal(3);
+  }
+
+  private void testShuffleInternal(int activeLen) {
+    // Produce the sequence used for later validation
+    List<Integer> idxList = new ArrayList<>();
+    for (int i = 0; i < activeLen; ++i) {
+      idxList.add(i);
+    }
+    cluster.setRandomSeed(0xDEADBEEF);
+    Collections.shuffle(idxList, cluster.getRandom());
+    for (int i = activeLen; i < 3; ++i) {
+      idxList.add(i);
+    }
+
+    // array contains both active and other nodes
+    DatanodeDescriptor[] testNodes = new DatanodeDescriptor[3];
+    testNodes[0] = dataNodes[0];
+    testNodes[1] = dataNodes[1];
+    testNodes[2] = dataNodes[2];
+    cluster.setRandomSeed(0xDEADBEEF);
+    cluster.shuffle(testNodes, activeLen);
+
+    for (int i = 0; i < testNodes.length; ++i) {
+      Assert.assertEquals(testNodes[i], dataNodes[idxList.get(i)]);
+    }
+  }
 }