HADOOP-994. In HDFS, limit the number of blocks invalidated at once. Contributed by Dhruba.

git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk@513984 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -195,6 +195,10 @@ Trunk (unreleased changes)
 60. HADOOP-1056.  Check HDFS include/exclude node lists with both IP
     address and hostname.  (Wendy Chien via cutting)
 
+61. HADOOP-994.  In HDFS, limit the number of blocks invalidated at
+    once.  Large lists were causing datenodes to timeout.
+    (Dhruba Borthakur via cutting) 
+
 
 Release 0.11.2 - 2007-02-16
 

src/java/org/apache/hadoop/dfs/FSNamesystem.java

@@ -2047,10 +2047,20 @@ class FSNamesystem implements FSConstants {
         for (Iterator<Block> it = node.getBlockIterator(); it.hasNext(); ) {
             Block b = it.next();
 
-            if (! dir.isValidBlock(b) && ! pendingCreateBlocks.contains(b)) {
-                obsolete.add(b);
-                NameNode.stateChangeLog.debug("BLOCK* NameSystem.processReport: "
+            // 
+            // A block report can only send BLOCK_INVALIDATE_CHUNK number of
+            // blocks to be deleted. If there are more blocks to be deleted, 
+            // they are added to recentInvalidateSets and will be sent out
+            // thorugh succeeding heartbeat responses.
+            //
+            if (obsolete.size() > FSConstants.BLOCK_INVALIDATE_CHUNK) {
+                addToInvalidates(b, node);
+            } else {
+                if (! dir.isValidBlock(b) && ! pendingCreateBlocks.contains(b)) {
+                  obsolete.add(b);
+                  NameNode.stateChangeLog.debug("BLOCK* NameSystem.processReport: "
                         +"ask "+nodeID.getName()+" to delete "+b.getBlockName() );
+                }
             }
         }
         return (Block[]) obsolete.toArray(new Block[obsolete.size()]);