HDFS-7503. Namenode restart after large deletions can cause slow processReport. (Arpit Agarwal)

CHANGES.txt

@@ -256,6 +256,9 @@ Release 1.3.0 - unreleased
 
     MAPREDUCE-6170. TestUlimit failure on JDK8. (bc Wong via kasha)
 
+    HDFS-7503. Namenode restart after large deletions can cause slow
+    processReport (Arpit Agarwal)
+
 Release 1.2.2 - unreleased
 
   INCOMPATIBLE CHANGES

src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java

@@ -3934,9 +3934,9 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, FSClusterSt
    * The given node is reporting all its blocks.  Use this info to 
    * update the (machine-->blocklist) and (block-->machinelist) tables.
    */
-  public synchronized void processReport(DatanodeID nodeID, 
-                                         BlockListAsLongs newReport
-                                        ) throws IOException {
+  public synchronized Collection<Block> processReport(
+      DatanodeID nodeID,
+      BlockListAsLongs newReport) throws IOException {
     long startTime = now();
     if (NameNode.stateChangeLog.isDebugEnabled()) {
       NameNode.stateChangeLog.debug("BLOCK* processReport: "
@@ -3961,7 +3961,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, FSClusterSt
       NameNode.stateChangeLog.info("BLOCK* processReport: "
           + "discarded non-initial block report from " + nodeID.getName()
           + " because namenode still in startup phase");
-      return;
+      return null;
     }
 
     //
@@ -3980,9 +3980,6 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, FSClusterSt
       addStoredBlock(b, node, null);
     }
     for (Block b : toInvalidate) {
-      NameNode.stateChangeLog.info("BLOCK* processReport: " 
-          + b + " on " + node.getName() + " size " + b.getNumBytes()
-          + " does not belong to any file");
       addToInvalidates(b, node);
     }
     long endTime = now();
@@ -3991,6 +3988,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean, FSClusterSt
         + nodeID.getName() + ", blocks: " + newReport.getNumberOfBlocks()
         + ", processing time: " + (endTime - startTime) + " msecs");
     node.processedBlockReport();
+    return toInvalidate;
   }
 
   /**

src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java

@@ -1130,7 +1130,17 @@ public class NameNode implements ClientProtocol, DatanodeProtocol,
     stateChangeLog.debug("*BLOCK* NameNode.blockReport: "
            +"from "+nodeReg.getName()+" "+blist.getNumberOfBlocks() +" blocks");
 
-    namesystem.processReport(nodeReg, blist);
+    Collection<Block> blocksInvalidated =
+        namesystem.processReport(nodeReg, blist);
+
+    final String node = nodeReg.toString();
+    if (blocksInvalidated != null) {
+      for (Block b : blocksInvalidated) {
+        stateChangeLog.info("BLOCK* processReport: "
+                                + b + " on " + node + " size " + b.getNumBytes()
+                                + " does not belong to any file");
+      }
+    }
     if (getFSImage().isUpgradeFinalized())
       return DatanodeCommand.FINALIZE;
     return null;