HDFS-10715. NPE when applying AvailableSpaceBlockPlacementPolicy. Contributed by Guangbin Zhu.

(cherry picked from commit ef432579a7763cc0e482fe049027c6e5325eb034)
(cherry picked from commit 68638661271e6e69c220b19fec7cbc7ae5e6208e)

# Conflicts:
#	hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java

hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt

@@ -303,6 +303,9 @@ Release 2.7.4 - UNRELEASED
     HDFS-8498. Blocks can be committed with wrong size. (Jing Zhao)
     Backport HDFS-11732 by Zhe Zhang.
 
+    HDFS-10715. NPE when applying AvailableSpaceBlockPlacementPolicy.
+    (Guangbin Zhu)
+
 Release 2.7.3 - 2016-08-25
 
   INCOMPATIBLE CHANGES

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java

@@ -71,13 +71,17 @@ public class AvailableSpaceBlockPlacementPolicy extends
   protected DatanodeDescriptor chooseDataNode(String scope) {
     DatanodeDescriptor a = (DatanodeDescriptor) clusterMap.chooseRandom(scope);
     DatanodeDescriptor b = (DatanodeDescriptor) clusterMap.chooseRandom(scope);
-    int ret = compareDataNode(a, b);
-    if (ret == 0) {
-      return a;
-    } else if (ret < 0) {
-      return (RAND.nextInt(100) < balancedPreference) ? a : b;
+    if (a != null && b != null){
+      int ret = compareDataNode(a, b);
+      if (ret == 0) {
+        return a;
+      } else if (ret < 0) {
+        return (RAND.nextInt(100) < balancedPreference) ? a : b;
+      } else {
+        return (RAND.nextInt(100) < balancedPreference) ? b : a;
+      }
     } else {
-      return (RAND.nextInt(100) < balancedPreference) ? b : a;
+      return a == null ? b : a;
     }
   }
 

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java

@@ -20,6 +20,8 @@ package org.apache.hadoop.hdfs.server.blockmanagement;
 
 import java.io.File;
 import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
@@ -30,6 +32,7 @@ import org.apache.hadoop.hdfs.TestBlockStoragePolicy;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.net.NetworkTopology;
+import org.apache.hadoop.net.Node;
 import org.apache.hadoop.test.PathUtils;
 import org.junit.AfterClass;
 import org.junit.Assert;
@@ -158,6 +161,21 @@ public class TestAvailableSpaceBlockPlacementPolicy {
     Assert.assertTrue(possibility < 0.55);
   }
 
+  @Test
+  public void testChooseDataNode() {
+    try {
+      Collection<Node> allNodes = new ArrayList<>(dataNodes.length);
+      Collections.addAll(allNodes, dataNodes);
+      if (placementPolicy instanceof AvailableSpaceBlockPlacementPolicy){
+        // exclude all datanodes when chooseDataNode, no NPE should be thrown
+        ((AvailableSpaceBlockPlacementPolicy)placementPolicy)
+                .chooseDataNode("~");
+      }
+    }catch (NullPointerException npe){
+      Assert.fail("NPE should not be thrown");
+    }
+  }
+
   @AfterClass
   public static void teardownCluster() {
     if (namenode != null) {