ZOOKEEPER-3142: Extend SnapshotFormatter to dump data in json format

Author: Brian Nixon <nixon@fb.com>

Reviewers: Michael Han <hanm@apache.org>

Closes #619 from enixon/extend-sf

build.xml

@@ -59,7 +59,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
     <property name="kerby.version" value="1.1.0"/>
 
     <property name="clover.version" value="4.2.1" />
-
+    <property name="json.version" value="1.1.1"/>
 
     <!-- ====================================================== -->
     <!-- Project properties                                     -->

ivy.xml

@@ -63,6 +63,9 @@
       <artifact name="netty" type="jar" conf="default"/>
     </dependency>
 
+    <dependency org="com.googlecode.json-simple" name="json-simple" rev="${json.version}" >
+      <exclude org="junit" module="junit"/>
+    </dependency>
     <dependency org="junit" name="junit" rev="${junit.version}" conf="test->default"/>
 	<dependency org="org.mockito" name="mockito-all" rev="${mockito.version}"
                conf="test->default"/>

src/java/main/org/apache/zookeeper/server/SnapshotFormatter.java

@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -20,8 +20,10 @@ package org.apache.zookeeper.server;
 
 import java.io.BufferedInputStream;
 import java.io.FileInputStream;
+import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.Base64;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.Map;
@@ -34,71 +36,130 @@ import org.apache.jute.InputArchive;
 import org.apache.yetus.audience.InterfaceAudience;
 import org.apache.zookeeper.data.StatPersisted;
 import org.apache.zookeeper.server.persistence.FileSnap;
+import org.apache.zookeeper.server.persistence.Util;
+import org.json.simple.JSONValue;
+
+import static org.apache.zookeeper.server.persistence.FileSnap.SNAPSHOT_FILE_PREFIX;
 
 /**
  * Dump a snapshot file to stdout.
+ *
+ * For JSON format, followed https://dev.yorhel.nl/ncdu/jsonfmt
  */
 @InterfaceAudience.Public
 public class SnapshotFormatter {
 
+    // per-znode counter so ncdu treats each as a unique object
+    private static Integer INODE_IDX = 1000;
+
     /**
      * USAGE: SnapshotFormatter snapshot_file
      */
     public static void main(String[] args) throws Exception {
-        if (args.length != 1) {
-            System.err.println("USAGE: SnapshotFormatter snapshot_file");
+        String snapshotFile = null;
+        boolean dumpData = false;
+        boolean dumpJson = false;
+
+        int i;
+        for (i = 0; i < args.length; i++) {
+            if (args[i].equals("-d")) {
+                dumpData = true;
+            } else if (args[i].equals("-json")) {
+                dumpJson = true;
+            } else {
+                snapshotFile = args[i];
+                i++;
+                break;
+            }
+        }
+        if (args.length != i || snapshotFile == null) {
+            System.err.println("USAGE: SnapshotFormatter [-d|-json] snapshot_file");
+            System.err.println("       -d dump the data for each znode");
+            System.err.println("       -json dump znode info in json format");
             System.exit(ExitCode.INVALID_INVOCATION.getValue());
         }
 
-        new SnapshotFormatter().run(args[0]);
+        if (dumpData && dumpJson) {
+            System.err.println("Cannot specify both data dump (-d) and json mode (-json) in same call");
+            System.exit(ExitCode.INVALID_INVOCATION.getValue());
+        }
+
+        new SnapshotFormatter().run(snapshotFile, dumpData, dumpJson);
     }
-    
-    public void run(String snapshotFileName) throws IOException {
-        InputStream is = new CheckedInputStream(
+
+    public void run(String snapshotFileName, boolean dumpData, boolean dumpJson)
+        throws IOException {
+        File snapshotFile = new File(snapshotFileName);
+        try (InputStream is = new CheckedInputStream(
                 new BufferedInputStream(new FileInputStream(snapshotFileName)),
-                new Adler32());
-        InputArchive ia = BinaryInputArchive.getArchive(is);
-        
-        FileSnap fileSnap = new FileSnap(null);
+                new Adler32())) {
+            InputArchive ia = BinaryInputArchive.getArchive(is);
+
+            FileSnap fileSnap = new FileSnap(null);
+
+            DataTree dataTree = new DataTree();
+            Map<Long, Integer> sessions = new HashMap<Long, Integer>();
 
-        DataTree dataTree = new DataTree();
-        Map<Long, Integer> sessions = new HashMap<Long, Integer>();
-        
-        fileSnap.deserialize(dataTree, sessions, ia);
+            fileSnap.deserialize(dataTree, sessions, ia);
+            long fileNameZxid = Util.getZxidFromName(snapshotFile.getName(), SNAPSHOT_FILE_PREFIX);
 
-        printDetails(dataTree, sessions);
+            if (dumpJson) {
+                printSnapshotJson(dataTree);
+            } else {
+                printDetails(dataTree, sessions, dumpData, fileNameZxid);
+            }
+        }
     }
 
-    private void printDetails(DataTree dataTree, Map<Long, Integer> sessions) {
-        printZnodeDetails(dataTree);
+    private void printDetails(
+        DataTree dataTree, Map<Long, Integer> sessions,
+        boolean dumpData, long fileNameZxid
+    ) {
+        long dtZxid = printZnodeDetails(dataTree, dumpData);
         printSessionDetails(dataTree, sessions);
+        System.out.println(String.format("----%nLast zxid: 0x%s", Long.toHexString(Math.max(fileNameZxid, dtZxid))));
     }
 
-    private void printZnodeDetails(DataTree dataTree) {
-        System.out.println(String.format("ZNode Details (count=%d):",
-                dataTree.getNodeCount()));
-        
-        printZnode(dataTree, "/");
+    private long printZnodeDetails(DataTree dataTree, boolean dumpData) {
+        System.out.println(String.format(
+            "ZNode Details (count=%d):",
+            dataTree.getNodeCount()
+        ));
+
+        final long zxid = printZnode(dataTree, "/", dumpData);
         System.out.println("----");
+        return zxid;
     }
 
-    private void printZnode(DataTree dataTree, String name) {
+    private long printZnode(DataTree dataTree, String name, boolean dumpData) {
         System.out.println("----");
         DataNode n = dataTree.getNode(name);
         Set<String> children;
-        synchronized(n) { // keep findbugs happy
+        long zxid;
+        synchronized (n) { // keep findbugs happy
             System.out.println(name);
             printStat(n.stat);
-            if (n.data != null) {
-                System.out.println("  dataLength = " + n.data.length);
+            zxid = Math.max(n.stat.getMzxid(), n.stat.getPzxid());
+            if (dumpData) {
+                System.out.println("  data = " + (n.data == null ? "" :
+                                                      Base64.getEncoder().encodeToString(n.data)));
             } else {
-                System.out.println("  no data");
+                System.out.println("  dataLength = "
+                                       + (n.data == null ? 0 : n.data.length));
             }
             children = n.getChildren();
         }
-        for (String child : children) {
-            printZnode(dataTree, name + (name.equals("/") ? "" : "/") + child);
+        if (children != null) {
+            for (String child : children) {
+                long cxid = printZnode(
+                    dataTree,
+                    name + (name.equals("/") ? "" : "/") + child,
+                    dumpData
+                );
+                zxid = Math.max(zxid, cxid);
+            }
         }
+        return zxid;
     }
 
     private void printSessionDetails(DataTree dataTree, Map<Long, Integer> sessions) {
@@ -106,7 +167,8 @@ public class SnapshotFormatter {
         for (Map.Entry<Long, Integer> e : sessions.entrySet()) {
             long sid = e.getKey();
             System.out.println(String.format("%#016x, %d, %d",
-                    sid, e.getValue(), dataTree.getEphemerals(sid).size()));
+                                             sid, e.getValue(), dataTree.getEphemerals(sid).size()
+            ));
         }
     }
 
@@ -125,4 +187,55 @@ public class SnapshotFormatter {
     private void printHex(String prefix, long value) {
         System.out.println(String.format("  %s = %#016x", prefix, value));
     }
+
+    private void printSnapshotJson(final DataTree dataTree) {
+        System.out.printf("[1,0,{\"progname\":\"SnapshotFormatter.java\",\"progver\":\"0.01\",\"timestamp\":%d}", System.currentTimeMillis());
+        printZnodeJson(dataTree, "/");
+        System.out.print("]");
+    }
+
+    private void printZnodeJson(final DataTree dataTree, final String fullPath) {
+
+        final DataNode n = dataTree.getNode(fullPath);
+
+        if (null == n) {
+            System.err.println("DataTree Node for " + fullPath + " doesn't exist");
+            return;
+        }
+
+        final String name = fullPath.equals("/") ? fullPath : fullPath.substring(fullPath.lastIndexOf(
+            "/") + 1);
+
+        System.out.print(",");
+
+        int dataLen;
+        synchronized (n) { // keep findbugs happy
+            dataLen = (n.data == null) ? 0 : n.data.length;
+        }
+        StringBuilder nodeSB = new StringBuilder();
+        nodeSB.append("{");
+        nodeSB.append("\"name\":\"").append(JSONValue.escape(name)).append("\"").append(",");
+        nodeSB.append("\"asize\":").append(dataLen).append(",");
+        nodeSB.append("\"dsize\":").append(dataLen).append(",");
+        nodeSB.append("\"dev\":").append(0).append(",");
+        nodeSB.append("\"ino\":").append(++INODE_IDX);
+        nodeSB.append("}");
+
+        Set<String> children;
+        synchronized (n) { // keep findbugs happy
+            children = n.getChildren();
+        }
+        if (children != null && children.size() > 0) {
+            System.out.print("[" + nodeSB);
+            for (String child : children) {
+                printZnodeJson(
+                    dataTree,
+                    fullPath + (fullPath.equals("/") ? "" : "/") + child
+                );
+            }
+            System.out.print("]");
+        } else {
+            System.out.print(nodeSB);
+        }
+    }
 }