HADOOP-1133. Add tool to analyze and debug namenode on a production cluster. Contributed by Dhruba Borthakur.

git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk@525596 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -106,6 +106,9 @@ Trunk (unreleased changes)
 33. HADOOP-1187.  Improve DFS Scalability: avoid scanning entire list of
     datanodes in getAdditionalBlocks.  (Dhruba Borthakur via tomwhite)
 
+34. HADOOP-1133.  Add tool to analyze and debug namenode on a production
+    cluster.  (Dhruba Borthakur via tomwhite)
+
 
 Release 0.12.3 (not yet released)
 

src/java/org/apache/hadoop/dfs/ClientProtocol.java

@@ -30,7 +30,7 @@ import org.apache.hadoop.ipc.VersionedProtocol;
 interface ClientProtocol extends VersionedProtocol {
 
     /*
-     * 10: finalizeUpgrade() added
+     * 11: metasave() added
      */
     public static final long versionID = 10L;  
   
@@ -355,4 +355,11 @@ interface ClientProtocol extends VersionedProtocol {
      * @throws IOException
      */
     public void finalizeUpgrade() throws IOException;
+
+   /**
+    * Dumps namenode data structures into specified file. If file
+    * already exists, then append.
+    * @throws IOException
+    */
+    public void metaSave(String filename) throws IOException;
 }

src/java/org/apache/hadoop/dfs/DFSAdmin.java

@@ -216,6 +216,23 @@ public class DFSAdmin extends FsShell {
       return exitCode;
     }
 
+    /**
+     * Dumps DFS data structures into specified file.
+     * Usage: java DFSAdmin -metasave filename
+     * @param argv List of of command line parameters.
+     * @param idx The index of the command that is being processed.
+     * @exception IOException if an error accoured wile accessing
+     *            the file or path.
+     */
+    public int metaSave(String[] argv, int idx) throws IOException {
+      String pathname = argv[idx];
+      DistributedFileSystem dfs = (DistributedFileSystem) fs;
+      dfs.metaSave(pathname);
+      System.out.println("Created file " + pathname + " on server " +
+                          dfs.getUri());
+      return 0;
+    }
+
     /**
      * Displays format of commands.
      * @param cmd The command that is being executed.
@@ -233,12 +250,16 @@ public class DFSAdmin extends FsShell {
           } else if ("-finalizeUpgrade".equals(cmd)) {
             System.err.println("Usage: java DFSAdmin"
                 + " [-finalizeUpgrade]");
+           } else if ("-metasave".equals(cmd)) {
+             System.err.println("Usage: java DFSAdmin"
+                 + " [-metasave filename]");
           } else {
             System.err.println("Usage: java DFSAdmin");
             System.err.println("           [-report]");
             System.err.println("           [-safemode enter | leave | get | wait]");
             System.err.println("           [-refreshNodes]");
             System.err.println("           [-finalizeUpgrade]");
+            System.err.println("           [-metasave filename]");
             System.err.println("           [-help [cmd]]");
           }
     }
@@ -282,6 +303,11 @@ public class DFSAdmin extends FsShell {
                   printUsage(cmd);
                   return exitCode;
                 }
+        } else if ("-metasave".equals(cmd)) {
+                if (argv.length != 2) {
+                  printUsage(cmd);
+                  return exitCode;
+                }
         }
 
 
@@ -307,6 +333,8 @@ public class DFSAdmin extends FsShell {
                 exitCode = refreshNodes();
             } else if ("-finalizeUpgrade".equals(cmd)) {
                 exitCode = finalizeUpgrade();
+            } else if ("-metasave".equals(cmd)) {
+                 exitCode = metaSave(argv, i);
             } else if ("-help".equals(cmd)) {
                 if (i < argv.length) {
                     printHelp(argv[i]);

src/java/org/apache/hadoop/dfs/DFSClient.java

@@ -375,6 +375,17 @@ class DFSClient implements FSConstants {
     public void refreshNodes() throws IOException {
       namenode.refreshNodes();
     }
+
+    /**
+     * Dumps DFS data structures into specified file.
+     * See {@link ClientProtocol#metaSave()} 
+     * for more details.
+     * 
+     * @see ClientProtocol#metaSave()
+     */
+    public void metaSave(String pathname) throws IOException {
+      namenode.metaSave(pathname);
+    }
     
     /**
      * @see ClientProtocol#finalizeUpgrade()

src/java/org/apache/hadoop/dfs/DatanodeInfo.java

@@ -166,6 +166,30 @@ public class DatanodeInfo extends DatanodeID implements Node {
     return buffer.toString();
   }
 
+  /** A formatted string for printing the status of the DataNode. */
+  String dumpDatanode() {
+    StringBuffer buffer = new StringBuffer();
+    long c = getCapacity();
+    long r = getRemaining();
+    long u = c - r;
+    buffer.append(name);
+    if(!NetworkTopology.DEFAULT_RACK.equals(location)) {
+        buffer.append(" "+location);
+    }
+    if (isDecommissioned()) {
+      buffer.append(" DD");
+    } else if (isDecommissionInProgress()) {
+      buffer.append(" DP");
+    } else {
+      buffer.append(" IN");
+    }
+    buffer.append(" " + c + "(" + FsShell.byteDesc(c)+")");
+    buffer.append(" " + u + "(" + FsShell.byteDesc(u)+")");
+    buffer.append(" " + FsShell.limitDecimal(((1.0*u)/c)*100,2)+"%");
+    buffer.append(" " + new Date(lastUpdate));
+    return buffer.toString();
+  }
+
   /**
    * Start decommissioning a node.
    * old state.

src/java/org/apache/hadoop/dfs/DistributedFileSystem.java

@@ -311,6 +311,14 @@ public class DistributedFileSystem extends ChecksumFileSystem {
       dfs.finalizeUpgrade();
     }
 
+    /*
+     * Requests the namenode to dump data strcutures into specified 
+     * file.
+     */
+    public void metaSave(String pathname) throws IOException {
+      dfs.metaSave(pathname);
+    }
+
     /**
      * We need to find the blocks that didn't match.  Likely only one 
      * is corrupt but we will report both to the namenode.  In the future,
@@ -416,6 +424,13 @@ public class DistributedFileSystem extends ChecksumFileSystem {
       ((RawDistributedFileSystem)fs).finalizeUpgrade();
     }
 
+    /*
+     * Dumps dfs data structures into specified file.
+     */
+     public void metaSave(String pathname) throws IOException {
+       ((RawDistributedFileSystem)fs).metaSave(pathname);
+     }
+
     /**
      * We need to find the blocks that didn't match.  Likely only one 
      * is corrupt but we will report both to the namenode.  In the future,

src/java/org/apache/hadoop/dfs/FSNamesystem.java

@@ -345,6 +345,59 @@ class FSNamesystem implements FSConstants {
           }
         }
     }
+
+    /**
+     * Dump all metadata into specified file
+     */
+    void metaSave(String filename) throws IOException {
+      File file = new File(System.getProperty("hadoop.log.dir"), 
+                           filename);
+      PrintWriter out = new PrintWriter(new BufferedWriter(
+                                        new FileWriter(file, true)));
+ 
+
+      //
+      // Dump contents of neededReplication
+      //
+      synchronized (neededReplications) {
+        out.println("Metasave: Blocks waiting for replication: " + 
+                    neededReplications.size());
+        if (neededReplications.size() > 0) {
+          for (Iterator<Block> it = neededReplications.iterator(); 
+               it.hasNext();) {
+            Block block = it.next();
+            Collection<DatanodeDescriptor> containingNodes = blocksMap.get(block);
+            out.print(block);
+            if (containingNodes != null) {
+              for (Iterator<DatanodeDescriptor> jt = containingNodes.iterator();
+                   jt.hasNext(); ) {
+                DatanodeDescriptor node = jt.next();
+                out.print(" " + node + " : " );
+              }
+            }
+            out.println("");
+          }
+        }
+      }
+
+      //
+      // Dump blocks from pendingReplication
+      //
+      pendingReplications.metaSave(out);
+
+      //
+      // Dump blocks that are waiting to be deleted
+      //
+      dumpRecentInvalidateSets(out);
+
+      //
+      // Dump all datanodes
+      //
+      datanodeDump(out);
+
+      out.flush();
+      out.close();
+    }
     
     /* get replication factor of a block */
     private int getReplication( Block block ) {
@@ -1052,6 +1105,34 @@ class FSNamesystem implements FSConstants {
       invalidateSet.add(b);
     }
 
+    /**
+     * dumps the contents of recentInvalidateSets
+     */
+    private synchronized void dumpRecentInvalidateSets(PrintWriter out) {
+      Collection<Collection<Block>> values = recentInvalidateSets.values();
+      Iterator it = recentInvalidateSets.entrySet().iterator();
+      if (values.size() == 0) {
+        out.println("Metasave: Blocks waiting deletion: 0");
+        return;
+      }
+      out.println("Metasave: Blocks waiting deletion from " +
+                   values.size() + " datanodes.");
+      while (it.hasNext()) {
+        Map.Entry entry = (Map.Entry) it.next();
+        String storageId = (String) entry.getKey();
+        DatanodeDescriptor node = datanodeMap.get(storageId);
+        Collection<Block> blklist = (Collection<Block>) entry.getValue();
+        if (blklist.size() > 0) {
+          out.print(node.getName());
+          for (Iterator jt = blklist.iterator(); jt.hasNext();) {
+            Block block = (Block) jt.next();
+            out.print(" " + block); 
+          }
+          out.println("");
+        }
+      }
+    }
+
     /**
      * Invalidates the given block on the given datanode.
      */
@@ -2380,6 +2461,18 @@ class FSNamesystem implements FSConstants {
       }
     }
 
+    /**
+     * Prints information about all datanodes.
+     */
+    private synchronized void datanodeDump(PrintWriter out) {
+      synchronized (datanodeMap) {
+        out.println("Metasave: Number of datanodes: " + datanodeMap.size());
+        for(Iterator<DatanodeDescriptor> it = datanodeMap.values().iterator(); it.hasNext(); ) {
+          DatanodeDescriptor node = it.next();
+          out.println(node.dumpDatanode());
+        }
+      }
+    }
 
     /**
      * Start decommissioning the specified datanode. 

src/java/org/apache/hadoop/dfs/NameNode.java

@@ -585,6 +585,13 @@ public class NameNode implements ClientProtocol, DatanodeProtocol, FSConstants {
       getFSImage().finalizeUpgrade();
     }
 
+    /**
+     * Dumps namenode state into specified file
+     */
+    public void metaSave(String filename) throws IOException {
+      namesystem.metaSave(filename);
+    }
+
     ////////////////////////////////////////////////////////////////
     // DatanodeProtocol
     ////////////////////////////////////////////////////////////////

src/java/org/apache/hadoop/dfs/PendingReplicationBlocks.java

@@ -19,7 +19,9 @@ package org.apache.hadoop.dfs;
 
 import org.apache.commons.logging.*;
 import org.apache.hadoop.util.*;
+import java.io.*;
 import java.util.*;
+import java.sql.Time;
 
 /***************************************************
  * PendingReplicationBlocks does the bookkeeping of all
@@ -228,4 +230,24 @@ class PendingReplicationBlocks {
     } catch (InterruptedException ie) {
     }
   }
+
+  /**
+   * Iterate through all items and print them.
+   */
+  void metaSave(PrintWriter out) {
+    synchronized (pendingReplications) {
+      out.println("Metasave: Blocks being replicated: " +
+                  pendingReplications.size());
+      Iterator iter = pendingReplications.entrySet().iterator();
+      while (iter.hasNext()) {
+        Map.Entry entry = (Map.Entry) iter.next();
+        PendingBlockInfo pendingBlock = (PendingBlockInfo) entry.getValue();
+        Block block = (Block) entry.getKey();
+        out.println(block + 
+                    " StartTime: " + new Time(pendingBlock.timeStamp) +
+                    " NumReplicaInProgress: " + 
+                    pendingBlock.numReplicasInProgress);
+      }
+    }
+  }
 }