ZOOKEEPER-572. add ability for operator to examine state of watches currently registered with a server (phunt via mahadev)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/zookeeper/trunk@903025 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -263,6 +263,9 @@ NEW FEATURES:
 
   ZOOKEEPER-496. zookeeper-tree utility for export, import and incremental updates (anirban roy via breed)
 
+  ZOOKEEPER-572. add ability for operator to examine state of watches
+  currently registered with a server (phunt via mahadev)
+  
 Release 3.2.0 - 2009-06-30
 
 Non-backward compatible changes:

docs/zookeeperAdmin.html

@@ -1307,6 +1307,10 @@ server.3=zoo3:2888:3888</span>
 <p>Tests if server is running in a non-error state. The server
             will respond with imok if it is running. Otherwise it will not
             respond at all.</p>
+<p>A response of "imok" does not necessarily indicate that the
+            server has joined the quorum, just that the server process is active
+            and bound to the specified client port. Use "stat" for details on
+            state wrt quorum and client connection information.</p>
 </dd>
 
         
@@ -1333,6 +1337,36 @@ server.3=zoo3:2888:3888</span>
 <p>Lists brief details for the server and connected
             clients.</p>
 </dd>
+
+        
+<dt>
+<term>wchs</term>
+</dt>
+<dd>
+<p>Lists brief information on watches for the server.</p>
+</dd>
+
+        
+<dt>
+<term>wchc</term>
+</dt>
+<dd>
+<p>Lists detailed information on watches for the server, by session.
+            This outputs a list of sessions(connections) with associated watches
+            (paths). Note, depending on the number of watches this operation may be
+            expensive (ie impact server performance), use it carefully.</p>
+</dd>
+
+        
+<dt>
+<term>wchp</term>
+</dt>
+<dd>
+<p>Lists detailed information on watches for the server, by path.
+            This outputs a list of paths (znodes) with associated sessions. Note,
+            depending on the number of watches this operation may be expensive
+            (ie impact server performance), use it carefully.</p>
+</dd>
       
 </dl>
 <p>Here's an example of the <strong>ruok</strong>
@@ -1340,7 +1374,7 @@ server.3=zoo3:2888:3888</span>
 <pre class="code">$ echo ruok | nc 127.0.0.1 5111
 imok
 </pre>
-<a name="N10435"></a><a name="sc_dataFileManagement"></a>
+<a name="N1044C"></a><a name="sc_dataFileManagement"></a>
 <h3 class="h4">Data File Management</h3>
 <p>ZooKeeper stores its data in a data directory and its transaction
       log in a transaction log directory. By default these two directories are
@@ -1348,7 +1382,7 @@ imok
       transaction log files in a separate directory than the data files.
       Throughput increases and latency decreases when transaction logs reside
       on a dedicated log devices.</p>
-<a name="N1043E"></a><a name="The+Data+Directory"></a>
+<a name="N10455"></a><a name="The+Data+Directory"></a>
 <h4>The Data Directory</h4>
 <p>This directory has two files in it:</p>
 <ul>
@@ -1394,14 +1428,14 @@ imok
         idempotent nature of its updates. By replaying the transaction log
         against fuzzy snapshots ZooKeeper gets the state of the system at the
         end of the log.</p>
-<a name="N1047A"></a><a name="The+Log+Directory"></a>
+<a name="N10491"></a><a name="The+Log+Directory"></a>
 <h4>The Log Directory</h4>
 <p>The Log Directory contains the ZooKeeper transaction logs.
         Before any update takes place, ZooKeeper ensures that the transaction
         that represents the update is written to non-volatile storage. A new
         log file is started each time a snapshot is begun. The log file's
         suffix is the first zxid written to that log.</p>
-<a name="N10484"></a><a name="sc_filemanagement"></a>
+<a name="N1049B"></a><a name="sc_filemanagement"></a>
 <h4>File Management</h4>
 <p>The format of snapshot and log files does not change between
         standalone ZooKeeper servers and different configurations of
@@ -1421,7 +1455,7 @@ imok
         this document for more details on setting a retention policy
         and maintenance of ZooKeeper storage.
         </p>
-<a name="N10499"></a><a name="sc_commonProblems"></a>
+<a name="N104B0"></a><a name="sc_commonProblems"></a>
 <h3 class="h4">Things to Avoid</h3>
 <p>Here are some common problems you can avoid by configuring
       ZooKeeper correctly:</p>
@@ -1475,7 +1509,7 @@ imok
 </dd>
       
 </dl>
-<a name="N104BD"></a><a name="sc_bestPractices"></a>
+<a name="N104D4"></a><a name="sc_bestPractices"></a>
 <h3 class="h4">Best Practices</h3>
 <p>For best results, take note of the following list of good
       Zookeeper practices:</p>

src/docs/src/documentation/content/xdocs/zookeeperAdmin.xml

@@ -941,6 +941,7 @@ server.3=zoo3:2888:3888</computeroutput></para>
             of packets received/sent, session id, operation latencies,
             last operation performed, etc...</para>
           </listitem>
+
         </varlistentry>
 
         <varlistentry>
@@ -975,6 +976,11 @@ server.3=zoo3:2888:3888</computeroutput></para>
             <para>Tests if server is running in a non-error state. The server
             will respond with imok if it is running. Otherwise it will not
             respond at all.</para>
+
+            <para>A response of "imok" does not necessarily indicate that the
+            server has joined the quorum, just that the server process is active
+            and bound to the specified client port. Use "stat" for details on
+            state wrt quorum and client connection information.</para>
           </listitem>
         </varlistentry>
 
@@ -1002,6 +1008,36 @@ server.3=zoo3:2888:3888</computeroutput></para>
             clients.</para>
           </listitem>
         </varlistentry>
+
+        <varlistentry>
+          <term>wchs</term>
+
+          <listitem>
+            <para>Lists brief information on watches for the server.</para>
+          </listitem>
+        </varlistentry>
+
+        <varlistentry>
+          <term>wchc</term>
+
+          <listitem>
+            <para>Lists detailed information on watches for the server, by session.
+            This outputs a list of sessions(connections) with associated watches
+            (paths). Note, depending on the number of watches this operation may be
+            expensive (ie impact server performance), use it carefully.</para>
+          </listitem>
+        </varlistentry>
+
+        <varlistentry>
+          <term>wchp</term>
+
+          <listitem>
+            <para>Lists detailed information on watches for the server, by path.
+            This outputs a list of paths (znodes) with associated sessions. Note,
+            depending on the number of watches this operation may be expensive
+            (ie impact server performance), use it carefully.</para>
+          </listitem>
+        </varlistentry>
       </variablelist>
 
       <para>Here's an example of the <emphasis role="bold">ruok</emphasis>

src/java/main/org/apache/zookeeper/server/DataTree.java

@@ -19,6 +19,7 @@
 package org.apache.zookeeper.server;
 
 import java.io.IOException;
+import java.io.PrintWriter;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
@@ -1074,21 +1075,41 @@ public class DataTree {
         setupQuota();
     }
 
-    public String dumpEphemerals() {
+    /**
+     * Summary of the watches on the datatree.
+     * @param pwriter the output to write to
+     */
+    public synchronized void dumpWatchesSummary(PrintWriter pwriter) {
+        pwriter.print(dataWatches.toString());
+    }
+
+    /**
+     * Write a text dump of all the watches on the datatree.
+     * Warning, this is expensive, use sparingly!
+     * @param pwriter the output to write to
+     */
+    public synchronized void dumpWatches(PrintWriter pwriter, boolean byPath) {
+        dataWatches.dumpWatches(pwriter, byPath);
+    }
+
+    /**
+     * Write a text dump of all the ephemerals in the datatree.
+     * @param pwriter the output to write to
+     */
+    public void dumpEphemerals(PrintWriter pwriter) {
         Set<Long> keys = ephemerals.keySet();
-        StringBuilder sb = new StringBuilder("Sessions with Ephemerals ("
-                + keys.size() + "):\n");
+        pwriter.println("Sessions with Ephemerals ("
+                + keys.size() + "):");
         for (long k : keys) {
-            sb.append("0x" + Long.toHexString(k));
-            sb.append(":\n");
+            pwriter.print("0x" + Long.toHexString(k));
+            pwriter.println(":");
             HashSet<String> tmp = ephemerals.get(k);
             synchronized (tmp) {
                 for (String path : tmp) {
-                    sb.append("\t" + path + "\n");
+                    pwriter.println("\t" + path);
                 }
             }
         }
-        return sb.toString();
     }
 
     public void removeCnxn(Watcher watcher) {

src/java/main/org/apache/zookeeper/server/FinalRequestProcessor.java

@@ -200,7 +200,7 @@ public class FinalRequestProcessor implements RequestProcessor {
                 break;
             }
             case OpCode.setACL: {
-                lastOp = "SETD";
+                lastOp = "SETA";
                 rsp = new SetACLResponse(rc.stat);
                 err = Code.get(rc.err);
                 break;

src/java/main/org/apache/zookeeper/server/NIOServerCnxn.java

@@ -18,9 +18,13 @@
 
 package org.apache.zookeeper.server;
 
+import java.io.BufferedWriter;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.io.Writer;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.nio.ByteBuffer;
@@ -188,10 +192,17 @@ public class NIOServerCnxn implements Watcher, ServerCnxn {
                     InetAddress addr = cnxn.sock.socket().getInetAddress();
                     Set<NIOServerCnxn> s = ipMap.get(addr);
                     if (s == null) {
-                        s = new HashSet<NIOServerCnxn>();
+                        // in general we will see 1 connection from each
+                        // host, setting the initial cap to 2 allows us
+                        // to minimize mem usage in the common case
+                        // of 1 entry --  we need to set the initial cap
+                        // to 2 to avoid rehash when the first entry is added
+                        s = new HashSet<NIOServerCnxn>(2);
+                        s.add(cnxn);
+                        ipMap.put(addr,s);
+                    } else {
+                        s.add(cnxn);
                     }
-                    s.add(cnxn);
-                    ipMap.put(addr,s);
                 }
             }
         }
@@ -399,7 +410,7 @@ public class NIOServerCnxn implements Watcher, ServerCnxn {
             LOG.error("Unexpected Exception: ", e);
         }
     }
-    
+
     private static class CloseRequestException extends IOException {
         private static final long serialVersionUID = -7854505709816442681L;
 
@@ -466,6 +477,7 @@ public class NIOServerCnxn implements Watcher, ServerCnxn {
                     if (incomingBuffer == lenBuffer) { // start of next request
                         incomingBuffer.flip();
                         isPayload = readLength(k);
+                        incomingBuffer.clear();
                     } else {
                         // continuation
                         isPayload = true;
@@ -740,162 +752,384 @@ public class NIOServerCnxn implements Watcher, ServerCnxn {
         }
     }
 
-    /** Return if four letter word found and responded to, otw false **/
-    private boolean checkFourLetterWord(SelectionKey k, int len)
-        throws IOException
-    {
-        // We take advantage of the limited size of the length to look
-        // for cmds. They are all 4-bytes which fits inside of an int
-        if (len == ruokCmd) {
-            LOG.info("Processing ruok command from "
-                    + sock.socket().getRemoteSocketAddress());
-            packetReceived();
+    /*
+     * See <a href="{@docRoot}/../../../docs/zookeeperAdmin.html#sc_zkCommands">
+     * Zk Admin</a>. this link is for all the commands.
+     */
+    private final static int consCmd =
+        ByteBuffer.wrap("cons".getBytes()).getInt();
 
-            sendBuffer(imok.duplicate());
-            k.interestOps(SelectionKey.OP_WRITE);
-            return true;
-        } else if (len == getTraceMaskCmd) {
-            LOG.info("Processing getracemask command from "
-                    + sock.socket().getRemoteSocketAddress());
-            packetReceived();
+    /*
+     * See <a href="{@docRoot}/../../../docs/zookeeperAdmin.html#sc_zkCommands">
+     * Zk Admin</a>. this link is for all the commands.
+     */
+    private final static int crstCmd =
+        ByteBuffer.wrap("crst".getBytes()).getInt();
 
-            long traceMask = ZooTrace.getTextTraceLevel();
-            ByteBuffer resp = ByteBuffer.allocate(8);
-            resp.putLong(traceMask);
-            resp.flip();
-            sendBuffer(resp);
-            k.interestOps(SelectionKey.OP_WRITE);
-            return true;
-        } else if (len == setTraceMaskCmd) {
-            LOG.info("Processing settracemask command from "
-                    + sock.socket().getRemoteSocketAddress());
-            incomingBuffer = ByteBuffer.allocate(8);
+    /*
+     * See <a href="{@docRoot}/../../../docs/zookeeperAdmin.html#sc_zkCommands">
+     * Zk Admin</a>. this link is for all the commands.
+     */
+    private final static int dumpCmd =
+        ByteBuffer.wrap("dump".getBytes()).getInt();
 
-            int rc = sock.read(incomingBuffer);
-            if (rc < 0) {
-                throw new IOException("Read error");
-            }
-            packetReceived();
+    /*
+     * See <a href="{@docRoot}/../../../docs/zookeeperAdmin.html#sc_zkCommands">
+     * Zk Admin</a>. this link is for all the commands.
+     */
+    private final static int enviCmd =
+        ByteBuffer.wrap("envi".getBytes()).getInt();
 
-            System.out.println("rc=" + rc);
-            incomingBuffer.flip();
-            long traceMask = incomingBuffer.getLong();
-            ZooTrace.setTextTraceLevel(traceMask);
-            ByteBuffer resp = ByteBuffer.allocate(8);
-            resp.putLong(traceMask);
-            resp.flip();
-            sendBuffer(resp);
-            k.interestOps(SelectionKey.OP_WRITE);
-            return true;
-        } else if (len == dumpCmd) {
-            LOG.info("Processing dump command from "
-                    + sock.socket().getRemoteSocketAddress());
-            packetReceived();
+    /*
+     * See <a href="{@docRoot}/../../../docs/zookeeperAdmin.html#sc_zkCommands">
+     * Zk Admin</a>. this link is for all the commands.
+     */
+    private final static int getTraceMaskCmd =
+        ByteBuffer.wrap("gtmk".getBytes()).getInt();
 
-            if (zk == null) {
-                sendBuffer(ByteBuffer.wrap("ZooKeeper not active \n"
-                        .getBytes()));
-            } else {
-                StringBuilder sb = new StringBuilder();
-                sb.append("SessionTracker dump: \n");
-                sb.append(zk.sessionTracker.toString()).append("\n");
-                sb.append("ephemeral nodes dump:\n");
-                sb.append(zk.getZKDatabase().dumpEphemerals()).append("\n");
+    /*
+     * See <a href="{@docRoot}/../../../docs/zookeeperAdmin.html#sc_zkCommands">
+     * Zk Admin</a>. this link is for all the commands.
+     */
+    private final static int ruokCmd =
+        ByteBuffer.wrap("ruok".getBytes()).getInt();
+    /*
+     * See <a href="{@docRoot}/../../../docs/zookeeperAdmin.html#sc_zkCommands">
+     * Zk Admin</a>. this link is for all the commands.
+     */
+    private final static int setTraceMaskCmd =
+        ByteBuffer.wrap("stmk".getBytes()).getInt();
+
+    /*
+     * See <a href="{@docRoot}/../../../docs/zookeeperAdmin.html#sc_zkCommands">
+     * Zk Admin</a>. this link is for all the commands.
+     */
+    private final static int srvrCmd =
+        ByteBuffer.wrap("srvr".getBytes()).getInt();
+
+    /*
+     * See <a href="{@docRoot}/../../../docs/zookeeperAdmin.html#sc_zkCommands">
+     * Zk Admin</a>. this link is for all the commands.
+     */
+    private final static int srstCmd =
+        ByteBuffer.wrap("srst".getBytes()).getInt();
+
+    /*
+     * See <a href="{@docRoot}/../../../docs/zookeeperAdmin.html#sc_zkCommands">
+     * Zk Admin</a>. this link is for all the commands.
+     */
+    private final static int statCmd =
+        ByteBuffer.wrap("stat".getBytes()).getInt();
+
+    /*
+     * See <a href="{@docRoot}/../../../docs/zookeeperAdmin.html#sc_zkCommands">
+     * Zk Admin</a>. this link is for all the commands.
+     */
+    private final static int wchcCmd =
+        ByteBuffer.wrap("wchc".getBytes()).getInt();
+
+    /*
+     * See <a href="{@docRoot}/../../../docs/zookeeperAdmin.html#sc_zkCommands">
+     * Zk Admin</a>. this link is for all the commands.
+     */
+    private final static int wchpCmd =
+        ByteBuffer.wrap("wchp".getBytes()).getInt();
+
+    /*
+     * See <a href="{@docRoot}/../../../docs/zookeeperAdmin.html#sc_zkCommands">
+     * Zk Admin</a>. this link is for all the commands.
+     */
+    private final static int wchsCmd =
+        ByteBuffer.wrap("wchs".getBytes()).getInt();
+
+    private final static HashMap<Integer, String> cmd2String =
+        new HashMap<Integer, String>();
+
+    // specify all of the commands that are available
+    static {
+        cmd2String.put(consCmd, "cons");
+        cmd2String.put(crstCmd, "crst");
+        cmd2String.put(dumpCmd, "dump");
+        cmd2String.put(enviCmd, "envi");
+        cmd2String.put(getTraceMaskCmd, "gtmk");
+        cmd2String.put(ruokCmd, "ruok");
+        cmd2String.put(setTraceMaskCmd, "stmk");
+        cmd2String.put(srstCmd, "srst");
+        cmd2String.put(srvrCmd, "srvr");
+        cmd2String.put(statCmd, "stat");
+        cmd2String.put(wchcCmd, "wchc");
+        cmd2String.put(wchpCmd, "wchp");
+        cmd2String.put(wchsCmd, "wchs");
+    }
+
+    /**
+     * This class wraps the sendBuffer method of NIOServerCnxn. It is
+     * responsible for chunking up the response to a client. Rather
+     * than cons'ing up a response fully in memory, which may be large
+     * for some commands, this class chunks up the result.
+     */
+    private class SendBufferWriter extends Writer {
+        private StringBuffer sb = new StringBuffer();
+
+        /* FYI: clearing the READ interestOps on the key results in
+         * the cnxn being closed in doIO.
+         */
+
+        /**
+         * Check if we are ready to send another chunk.
+         * @param force force sending, even if not a full chunk
+         */
+        private void checkFlush(boolean force) {
+            if ((force && sb.length() > 0) || sb.length() > 2048) {
                 sendBuffer(ByteBuffer.wrap(sb.toString().getBytes()));
+                // including op_read keeps doio from closing the conn
+                wakeup(SelectionKey.OP_READ
+                    | SelectionKey.OP_WRITE);
+
+                // clear our internal buffer
+                sb.setLength(0);
             }
-            k.interestOps(SelectionKey.OP_WRITE);
-            return true;
-        } else if (len == statCmd || len == srvrCmd) {
-            LOG.info("Processing " + (len == statCmd ? "stat" : "srvr") 
-                    + " command from "
-                    + sock.socket().getRemoteSocketAddress());
-            packetReceived();
+        }
 
-            StringBuilder sb = new StringBuilder();
-            if (zk != null){
-                sb.append("Zookeeper version: ").append(Version.getFullVersion())
-                    .append("\n");
-                if (len == statCmd) {
-                    sb.append("Clients:\n");
-                    synchronized(factory.cnxns){
-                        for(NIOServerCnxn c : factory.cnxns){
-                            sb.append(((CnxnStats)c.getStats()).toString(true));
-                        }
-                    }
-                    sb.append("\n");
-                }
-                sb.append(zk.serverStats().toString());
-                sb.append("Node count: ").append(zk.getZKDatabase().getNodeCount()).
-                    append("\n");
-            } else {
-                sb.append("ZooKeeperServer not running\n");
+        /**
+         * Wakeup the selector. This is necessary as the cnxn is
+         * waiting for interestOps to be satisfied. If we want the
+         * selector to wakeup immediately (rather than the last
+         * select(timeout) period) we need to force a wakeup.
+         * @param sel the new interest ops
+         */
+        private void wakeup(int sel) {
+            synchronized(factory) {
+                sk.selector().wakeup();
+                sk.interestOps(sel);
             }
+        }
 
-            sendBuffer(ByteBuffer.wrap(sb.toString().getBytes()));
-            k.interestOps(SelectionKey.OP_WRITE);
-            return true;
-        } else if (len == consCmd) {
-            LOG.info("Processing cons command from "
-                    + sock.socket().getRemoteSocketAddress());
-            packetReceived();
+        @Override
+        public void close() throws IOException {
+            if (sb == null) return;
 
-            StringBuilder sb = new StringBuilder();
-            if (zk != null){
-                synchronized(factory.cnxns){
-                    for(NIOServerCnxn c : factory.cnxns){
-                        sb.append(((CnxnStats)c.getStats()).toString(false));
-                    }
-                }
-                sb.append("\n");
-            } else {
-                sb.append("ZooKeeperServer not running\n");
-            }
+            checkFlush(true);
 
-            sendBuffer(ByteBuffer.wrap(sb.toString().getBytes()));
-            k.interestOps(SelectionKey.OP_WRITE);
-            return true;
-        } else if (len == enviCmd) {
-            LOG.info("Processing envi command from "
-                    + sock.socket().getRemoteSocketAddress());
-            packetReceived();
+            // nothing left, please close
+            wakeup(SelectionKey.OP_WRITE);
 
-            StringBuilder sb = new StringBuilder();
+            sb = null; // clear out the ref to ensure no reuse
+        }
 
-            List<Environment.Entry> env = Environment.list();
+        @Override
+        public void flush() throws IOException {
+            checkFlush(true);
+        }
 
-            sb.append("Environment:\n");
-            for(Environment.Entry e : env) {
-                sb.append(e.getKey()).append("=").append(e.getValue())
-                    .append("\n");
-            }
+        @Override
+        public void write(char[] cbuf, int off, int len) throws IOException {
+            sb.append(cbuf, off, len);
+            checkFlush(false);
+        }
+    }
 
-            sendBuffer(ByteBuffer.wrap(sb.toString().getBytes()));
-            k.interestOps(SelectionKey.OP_WRITE);
-            return true;
-        } else if (len == srstCmd) {
-            LOG.info("Processing srst command from "
-                    + sock.socket().getRemoteSocketAddress());
-            packetReceived();
+    private static final String ZK_NOT_SERVING =
+        "This ZooKeeper instance is not currently serving requests";
 
-            zk.serverStats().reset();
+    /** Return if four letter word found and responded to, otw false **/
+    private boolean checkFourLetterWord(final SelectionKey k, final int len)
+        throws IOException
+    {
+        // We take advantage of the limited size of the length to look
+        // for cmds. They are all 4-bytes which fits inside of an int
+        String cmd = cmd2String.get(len);
+        if (cmd == null) {
+            return false;
+        }
+        LOG.info("Processing " + cmd + " command from "
+                + sock.socket().getRemoteSocketAddress());
+        packetReceived();
 
-            sendBuffer(ByteBuffer.wrap("Server stats reset.\n".getBytes()));
-            k.interestOps(SelectionKey.OP_WRITE);
-            return true;
-        } else if (len == crstCmd) {
-            LOG.info("Processing crst command from "
-                    + sock.socket().getRemoteSocketAddress());
-            packetReceived();
+        final PrintWriter pwriter = new PrintWriter(
+                new BufferedWriter(new SendBufferWriter()));
+        boolean threadWillClosePWriter = false;
+        try {
+            if (len == ruokCmd) {
+                pwriter.print("imok");
+                return true;
+            } else if (len == getTraceMaskCmd) {
+                long traceMask = ZooTrace.getTextTraceLevel();
+                pwriter.print(traceMask);
+                return true;
+            } else if (len == setTraceMaskCmd) {
+                int rc = sock.read(incomingBuffer);
+                if (rc < 0) {
+                    throw new IOException("Read error");
+                }
 
-            synchronized(factory.cnxns){
-                for(NIOServerCnxn c : factory.cnxns){
-                    c.getStats().reset();
+                incomingBuffer.flip();
+                long traceMask = incomingBuffer.getLong();
+                ZooTrace.setTextTraceLevel(traceMask);
+                pwriter.print(traceMask);
+                return true;
+            } else if (len == enviCmd) {
+                List<Environment.Entry> env = Environment.list();
+
+                pwriter.println("Environment:");
+                for(Environment.Entry e : env) {
+                    pwriter.print(e.getKey());
+                    pwriter.print("=");
+                    pwriter.println(e.getValue());
                 }
-            }
+                return true;
+            } else if (len == srstCmd) {
+                if (zk == null) {
+                    pwriter.println(ZK_NOT_SERVING);
+                    return true;
+                }
+                zk.serverStats().reset();
+                pwriter.println("Server stats reset.");
+                return true;
+            } else if (len == crstCmd) {
+                if (zk == null) {
+                    pwriter.println(ZK_NOT_SERVING);
+                    return true;
+                }
+                synchronized(factory.cnxns){
+                    for(NIOServerCnxn c : factory.cnxns){
+                        c.getStats().reset();
+                    }
+                }
+                pwriter.println("Connection stats reset.");
+                return true;
+            } else if (len == dumpCmd) {
+                if (zk == null) {
+                    pwriter.println(ZK_NOT_SERVING);
+                    return true;
+                }
+                // this could be a long running task, spawn a thread so
+                // that we don't block the processing of other requests
+                threadWillClosePWriter = true;
+                new Thread() {
+                    @Override
+                    public void run() {
+                        try {
+                            pwriter.println("SessionTracker dump:");
+                            zk.sessionTracker.dumpSessions(pwriter);
+                            pwriter.println("ephemeral nodes dump:");
+                            zk.dumpEphemerals(pwriter);
+                        } finally {
+                            pwriter.flush();
+                            pwriter.close();
+                        }
+                    }
+                }.start();
 
-            sendBuffer(ByteBuffer.wrap("Connection stats reset.\n".getBytes()));
-            k.interestOps(SelectionKey.OP_WRITE);
-            return true;
+                return true;
+            } else if (len == statCmd || len == srvrCmd) {
+                if (zk == null) {
+                    pwriter.println(ZK_NOT_SERVING);
+                    return true;
+                }
+                // this could be a long running task, spawn a thread so
+                // that we don't block the processing of other requests
+                threadWillClosePWriter = true;
+                new Thread() {
+                    @SuppressWarnings("unchecked")
+                    @Override
+                    public void run() {
+                        try {
+                            pwriter.print("Zookeeper version: ");
+                            pwriter.println(Version.getFullVersion());
+                            if (len == statCmd) {
+                                pwriter.println("Clients:");
+                                // clone should be faster than iteration
+                                // ie give up the cnxns lock faster
+                                HashSet<NIOServerCnxn> cnxns;
+                                synchronized(factory.cnxns){
+                                    cnxns = (HashSet<NIOServerCnxn>)factory
+                                        .cnxns.clone();
+                                }
+                                for(NIOServerCnxn c : cnxns){
+                                    ((CnxnStats)c.getStats())
+                                        .dumpConnectionInfo(pwriter, true);
+                                }
+                                pwriter.println();
+                            }
+                            pwriter.print(zk.serverStats().toString());
+                            pwriter.print("Node count: ");
+                            pwriter.println(zk.getZKDatabase().getNodeCount());
+                        } finally {
+                            pwriter.flush();
+                            pwriter.close();
+                        }
+                    }
+                }.start();
+                return true;
+            } else if (len == consCmd) {
+                if (zk == null) {
+                    pwriter.println(ZK_NOT_SERVING);
+                    return true;
+                }
+                // this could be a long running task, spawn a thread so
+                // that we don't block the processing of other requests
+                threadWillClosePWriter = true;
+                new Thread() {
+                    @SuppressWarnings("unchecked")
+                    @Override
+                    public void run() {
+                        try {
+                            // clone should be faster than iteration
+                            // ie give up the cnxns lock faster
+                            HashSet<NIOServerCnxn> cnxns;
+                            synchronized(factory.cnxns){
+                                cnxns = (HashSet<NIOServerCnxn>)factory
+                                    .cnxns.clone();
+                            }
+                            for(NIOServerCnxn c : cnxns){
+                                ((CnxnStats)c.getStats())
+                                    .dumpConnectionInfo(pwriter, false);
+                            }
+                            pwriter.println();
+                        } finally {
+                            pwriter.flush();
+                            pwriter.close();
+                        }
+                    }
+                }.start();
+                return true;
+            } else if (len == wchpCmd || len == wchcCmd || len == wchsCmd) {
+                if (zk == null) {
+                    pwriter.println(ZK_NOT_SERVING);
+                    return true;
+                }
+                // this could be a long running task, spawn a thread so
+                // that we don't block the processing of other requests
+                threadWillClosePWriter = true;
+                new Thread() {
+                    @Override
+                    public void run() {
+                        try {
+                            DataTree dt = zk.getZKDatabase().getDataTree();
+                            if (len == wchsCmd) {
+                                dt.dumpWatchesSummary(pwriter);
+                            } else if (len == wchpCmd) {
+                                dt.dumpWatches(pwriter, true);
+                            } else {
+                                dt.dumpWatches(pwriter, false);
+                            }
+                            pwriter.println();
+                        } finally {
+                            pwriter.flush();
+                            pwriter.close();
+                        }
+                    }
+                }.start();
+                return true;
+            }
+        } finally {
+            // if we spawned a thread it is responsible for eventually
+            // flushing and closeing the writer
+            if (!threadWillClosePWriter) {
+                pwriter.flush();
+                pwriter.close();
+            }
         }
         return false;
     }
@@ -1303,13 +1537,19 @@ public class NIOServerCnxn implements Watcher, ServerCnxn {
             return lastLatency;
         }
 
-        /** Prints brief stats information for the connection.
-         * 
-         * @see toString(boolean) for detailed stats
+        /**
+         * Prints detailed stats information for the connection.
+         *
+         * @see dumpConnectionInfo(PrintWriter, boolean) for brief stats
          */
         @Override
         public String toString() {
-            return toString(false);
+            StringWriter sw = new StringWriter();
+            PrintWriter pwriter = new PrintWriter(sw);
+            dumpConnectionInfo(pwriter, false);
+            pwriter.flush();
+            pwriter.close();
+            return sw.toString();
         }
 
         /**
@@ -1317,41 +1557,55 @@ public class NIOServerCnxn implements Watcher, ServerCnxn {
          * @param brief iff true prints brief details, otw full detail
          * @return information about this connection
          */
-        public String toString(boolean brief) {
-            StringBuilder sb = new StringBuilder();
+        public synchronized void
+        dumpConnectionInfo(PrintWriter pwriter, boolean brief)
+        {
             Channel channel = sk.channel();
             if (channel instanceof SocketChannel) {
-                sb.append(" ").append(((SocketChannel)channel).socket()
-                                .getRemoteSocketAddress())
-                  .append("[").append(Integer.toHexString(sk.interestOps()))
-                  .append("](queued=").append(getOutstandingRequests())
-                  .append(",recved=").append(getPacketsReceived())
-                  .append(",sent=").append(getPacketsSent());
+                pwriter.print(" ");
+                pwriter.print(((SocketChannel)channel).socket()
+                        .getRemoteSocketAddress());
+                pwriter.print("[");
+                pwriter.print(Integer.toHexString(sk.interestOps()));
+                pwriter.print("](queued=");
+                pwriter.print(getOutstandingRequests());
+                pwriter.print(",recved=");
+                pwriter.print(getPacketsReceived());
+                pwriter.print(",sent=");
+                pwriter.print(getPacketsSent());
 
                 if (!brief) {
                     long sessionId = getSessionId();
                     if (sessionId != 0) {
-                        sb.append(",sid=0x").append(Long.toHexString(sessionId))
-                            .append(",lop=").append(getLastOperation())
-                            .append(",est=").append(getEstablished().getTime())
-                            .append(",to=").append(getSessionTimeout());
+                        pwriter.print(",sid=0x");
+                        pwriter.print(Long.toHexString(sessionId));
+                        pwriter.print(",lop=");
+                        pwriter.print(getLastOperation());
+                        pwriter.print(",est=");
+                        pwriter.print(getEstablished().getTime());
+                        pwriter.print(",to=");
+                        pwriter.print(getSessionTimeout());
                         long lastCxid = getLastCxid();
                         if (lastCxid >= 0) {
-                            sb.append(",lcxid=0x")
-                                .append(Long.toHexString(lastCxid));
+                            pwriter.print(",lcxid=0x");
+                            pwriter.print(Long.toHexString(lastCxid));
                         }
-                        sb.append(",lzxid=0x")
-                                .append(Long.toHexString(getLastZxid()))
-                            .append(",lresp=").append(getLastResponseTime())
-                            .append(",llat=").append(getLastLatency())
-                            .append(",minlat=").append(getMinLatency())
-                            .append(",avglat=").append(getAvgLatency())
-                            .append(",maxlat=").append(getMaxLatency());
+                        pwriter.print(",lzxid=0x");
+                        pwriter.print(Long.toHexString(getLastZxid()));
+                        pwriter.print(",lresp=");
+                        pwriter.print(getLastResponseTime());
+                        pwriter.print(",llat=");
+                        pwriter.print(getLastLatency());
+                        pwriter.print(",minlat=");
+                        pwriter.print(getMinLatency());
+                        pwriter.print(",avglat=");
+                        pwriter.print(getAvgLatency());
+                        pwriter.print(",maxlat=");
+                        pwriter.print(getMaxLatency());
                     }
                 }
-                sb.append(")\n");
+                pwriter.println(")");
             }
-            return sb.toString();
         }
     }
 

src/java/main/org/apache/zookeeper/server/PrepRequestProcessor.java

@@ -140,8 +140,8 @@ public class PrepRequestProcessor extends Thread implements RequestProcessor {
                         acl = n.acl;
                         children = n.getChildren();
                     }
-                    lastChange = new ChangeRecord(-1, path, n.stat, 
-                        children != null ? children.size() : 0, 
+                    lastChange = new ChangeRecord(-1, path, n.stat,
+                        children != null ? children.size() : 0,
                             zks.getZKDatabase().convertLong(acl));
                 }
             }
@@ -216,7 +216,7 @@ public class PrepRequestProcessor extends Thread implements RequestProcessor {
                 String path = createRequest.getPath();
                 int lastSlash = path.lastIndexOf('/');
                 if (lastSlash == -1 || path.indexOf('\0') != -1 || failCreate) {
-                    LOG.info("Invalid path " + path + " with session " +
+                    LOG.info("Invalid path " + path + " with session 0x" +
                             Long.toHexString(request.sessionId));
                     throw new KeeperException.BadArgumentsException(path);
                 }
@@ -237,7 +237,7 @@ public class PrepRequestProcessor extends Thread implements RequestProcessor {
                 try {
                     PathUtils.validatePath(path);
                 } catch(IllegalArgumentException ie) {
-                    LOG.info("Invalid path " + path + " with session " +
+                    LOG.info("Invalid path " + path + " with session 0x" +
                             Long.toHexString(request.sessionId));
                     throw new KeeperException.BadArgumentsException(path);
                 }

src/java/main/org/apache/zookeeper/server/ServerCnxn.java

@@ -20,7 +20,6 @@ package org.apache.zookeeper.server;
 
 import java.io.IOException;
 import java.net.InetSocketAddress;
-import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Date;
 
@@ -35,34 +34,6 @@ import org.apache.zookeeper.proto.ReplyHeader;
  * to the server.
  */
 public interface ServerCnxn extends Watcher {
-    /**
-     * See <a href="{@docRoot}/../../../docs/zookeeperAdmin.html#sc_zkCommands">
-     * Zk Admin</a>. this link is for all the commands.
-     */
-    final static int ruokCmd = ByteBuffer.wrap("ruok".getBytes()).getInt();
-
-    final static int dumpCmd = ByteBuffer.wrap("dump".getBytes()).getInt();
-
-    final static int statCmd = ByteBuffer.wrap("stat".getBytes()).getInt();
-
-    final static int srvrCmd = ByteBuffer.wrap("srvr".getBytes()).getInt();
-
-    final static int consCmd = ByteBuffer.wrap("cons".getBytes()).getInt();
-
-    final static int setTraceMaskCmd = ByteBuffer.wrap("stmk".getBytes())
-            .getInt();
-
-    final static int getTraceMaskCmd = ByteBuffer.wrap("gtmk".getBytes())
-            .getInt();
-
-    final static int enviCmd = ByteBuffer.wrap("envi".getBytes()).getInt();
-
-    final static int srstCmd = ByteBuffer.wrap("srst".getBytes()).getInt();
-
-    final static int crstCmd = ByteBuffer.wrap("crst".getBytes()).getInt();
-
-    final static ByteBuffer imok = ByteBuffer.wrap("imok".getBytes());
-
     // This is just an arbitrary object to represent requests issued by
     // (aka owned by) this class
     final public static Object me = new Object();

src/java/main/org/apache/zookeeper/server/SessionTracker.java

@@ -18,6 +18,8 @@
 
 package org.apache.zookeeper.server;
 
+import java.io.PrintWriter;
+
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.KeeperException.SessionExpiredException;
 import org.apache.zookeeper.KeeperException.SessionMovedException;
@@ -63,4 +65,10 @@ public interface SessionTracker {
     void checkSession(long sessionId, Object owner) throws KeeperException.SessionExpiredException, SessionMovedException;
 
     void setOwner(long id, Object owner) throws SessionExpiredException;
+
+    /**
+     * Text dump of session information, suitable for debugging.
+     * @param pwriter the output writer
+     */
+    void dumpSessions(PrintWriter pwriter);
 }

src/java/main/org/apache/zookeeper/server/SessionTrackerImpl.java

@@ -18,6 +18,8 @@
 
 package org.apache.zookeeper.server;
 
+import java.io.PrintWriter;
+import java.io.StringWriter;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Date;
@@ -103,21 +105,32 @@ public class SessionTrackerImpl extends Thread implements SessionTracker {
 
     volatile long currentTime;
 
-    @Override
-    synchronized public String toString() {
-        StringBuilder sb = new StringBuilder("Session Sets (")
-                .append(sessionSets.size()).append("):\n");
+    synchronized public void dumpSessions(PrintWriter pwriter) {
+        pwriter.print("Session Sets (");
+        pwriter.print(sessionSets.size());
+        pwriter.println("):");
         ArrayList<Long> keys = new ArrayList<Long>(sessionSets.keySet());
         Collections.sort(keys);
         for (long time : keys) {
-            sb.append(sessionSets.get(time).sessions.size())
-                .append(" expire at ").append(new Date(time)).append(":\n");
+            pwriter.print(sessionSets.get(time).sessions.size());
+            pwriter.print(" expire at ");
+            pwriter.print(new Date(time));
+            pwriter.println(":");
             for (SessionImpl s : sessionSets.get(time).sessions) {
-                sb.append("\t0x").append(Long.toHexString(s.sessionId))
-                    .append("\n");
+                pwriter.print("\t0x");
+                pwriter.println(Long.toHexString(s.sessionId));
             }
         }
-        return sb.toString();
+    }
+
+    @Override
+    synchronized public String toString() {
+        StringWriter sw = new StringWriter();
+        PrintWriter pwriter = new PrintWriter(sw);
+        dumpSessions(pwriter);
+        pwriter.flush();
+        pwriter.close();
+        return sw.toString();
     }
 
     @Override
@@ -196,7 +209,7 @@ public class SessionTrackerImpl extends Thread implements SessionTracker {
         }
     }
 
-   
+
     synchronized public long createSession(int sessionTimeout) {
         addSession(nextSessionId, sessionTimeout);
         return nextSessionId++;
@@ -209,13 +222,13 @@ public class SessionTrackerImpl extends Thread implements SessionTracker {
             sessionsById.put(id, s);
             if (LOG.isTraceEnabled()) {
                 ZooTrace.logTraceMessage(LOG, ZooTrace.SESSION_TRACE_MASK,
-                        "SessionTrackerImpl --- Adding session 0x" 
+                        "SessionTrackerImpl --- Adding session 0x"
                         + Long.toHexString(id) + " " + sessionTimeout);
             }
         } else {
             if (LOG.isTraceEnabled()) {
                 ZooTrace.logTraceMessage(LOG, ZooTrace.SESSION_TRACE_MASK,
-                        "SessionTrackerImpl --- Existing session 0x" 
+                        "SessionTrackerImpl --- Existing session 0x"
                         + Long.toHexString(id) + " " + sessionTimeout);
             }
         }
@@ -224,21 +237,21 @@ public class SessionTrackerImpl extends Thread implements SessionTracker {
 
     synchronized public void checkSession(long sessionId, Object owner) throws KeeperException.SessionExpiredException, KeeperException.SessionMovedException {
         SessionImpl session = sessionsById.get(sessionId);
-		if (session == null) {
+        if (session == null) {
             throw new KeeperException.SessionExpiredException();
         }
-		if (session.owner == null) {
-			session.owner = owner;
-		} else if (session.owner != owner) {
-			throw new KeeperException.SessionMovedException();
-		}
+        if (session.owner == null) {
+            session.owner = owner;
+        } else if (session.owner != owner) {
+            throw new KeeperException.SessionMovedException();
+        }
     }
 
-	synchronized public void setOwner(long id, Object owner) throws SessionExpiredException {
-		SessionImpl session = sessionsById.get(id);
-		if (session == null) {
+    synchronized public void setOwner(long id, Object owner) throws SessionExpiredException {
+        SessionImpl session = sessionsById.get(id);
+        if (session == null) {
             throw new KeeperException.SessionExpiredException();
         }
-		session.owner = owner;
-	}
+        session.owner = owner;
+    }
 }

src/java/main/org/apache/zookeeper/server/WatchManager.java

@@ -18,9 +18,11 @@
 
 package org.apache.zookeeper.server;
 
+import java.io.PrintWriter;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Set;
+import java.util.Map.Entry;
 
 import org.apache.log4j.Logger;
 import org.apache.zookeeper.WatchedEvent;
@@ -35,10 +37,10 @@ import org.apache.zookeeper.Watcher.Event.KeeperState;
 public class WatchManager {
     private static final Logger LOG = Logger.getLogger(WatchManager.class);
 
-    private HashMap<String, HashSet<Watcher>> watchTable = 
+    private final HashMap<String, HashSet<Watcher>> watchTable =
         new HashMap<String, HashSet<Watcher>>();
 
-    private HashMap<Watcher, HashSet<String>> watch2Paths = 
+    private final HashMap<Watcher, HashSet<String>> watch2Paths =
         new HashMap<Watcher, HashSet<String>>();
 
     public synchronized int size(){
@@ -48,13 +50,17 @@ public class WatchManager {
     public synchronized void addWatch(String path, Watcher watcher) {
         HashSet<Watcher> list = watchTable.get(path);
         if (list == null) {
-            list = new HashSet<Watcher>();
+            // don't waste memory if there are few watches on a node
+            // rehash when the 4th entry is added, doubling size thereafter
+            // seems like a good compromise
+            list = new HashSet<Watcher>(4);
             watchTable.put(path, list);
         }
         list.add(watcher);
 
         HashSet<String> paths = watch2Paths.get(watcher);
         if (paths == null) {
+            // cnxns typically have many watches, so use default cap here
             paths = new HashSet<String>();
             watch2Paths.put(watcher, paths);
         }
@@ -80,7 +86,7 @@ public class WatchManager {
     public Set<Watcher> triggerWatch(String path, EventType type) {
         return triggerWatch(path, type, null);
     }
-    
+
     public Set<Watcher> triggerWatch(String path, EventType type, Set<Watcher> supress) {
         WatchedEvent e = new WatchedEvent(type,
                 KeeperState.SyncConnected, path);
@@ -110,4 +116,51 @@ public class WatchManager {
         }
         return watchers;
     }
+
+    /**
+     * Brief description of this object.
+     */
+    @Override
+    public synchronized String toString() {
+        StringBuffer sb = new StringBuffer();
+
+        sb.append(watch2Paths.size()).append(" connections watching ")
+            .append(watchTable.size()).append(" paths\n");
+
+        int total = 0;
+        for (HashSet<String> paths : watch2Paths.values()) {
+            total += paths.size();
+        }
+        sb.append("Total watches:").append(total);
+
+        return sb.toString();
+    }
+
+    /**
+     * String representation of watches. Warning, may be large!
+     * @param byPath iff true output watches by paths, otw output
+     * watches by connection
+     * @return string representation of watches
+     */
+    public synchronized void dumpWatches(PrintWriter pwriter, boolean byPath) {
+        if (byPath) {
+            for (Entry<String, HashSet<Watcher>> e : watchTable.entrySet()) {
+                pwriter.println(e.getKey());
+                for (Watcher w : e.getValue()) {
+                    pwriter.print("\t0x");
+                    pwriter.print(Long.toHexString(((ServerCnxn)w).getSessionId()));
+                    pwriter.print("\n");
+                }
+            }
+        } else {
+            for (Entry<Watcher, HashSet<String>> e : watch2Paths.entrySet()) {
+                pwriter.print("0x");
+                pwriter.println(Long.toHexString(((ServerCnxn)e.getKey()).getSessionId()));
+                for (String path : e.getValue()) {
+                    pwriter.print("\t");
+                    pwriter.println(path);
+                }
+            }
+        }
+    }
 }

src/java/main/org/apache/zookeeper/server/ZKDatabase.java

@@ -20,6 +20,7 @@ package org.apache.zookeeper.server;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.io.PrintWriter;
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.LinkedList;
@@ -251,12 +252,11 @@ public class ZKDatabase {
     }
 
     /**
-     * get a string dump of all the ephemerals in
-     * the datatree
-     * @return the string dump of ephemerals
+     * write a text dump of all the ephemerals in the datatree
+     * @param pwriter the output to write to
      */
-    public String dumpEphemerals() {
-        return dataTree.dumpEphemerals();
+    public void dumpEphemerals(PrintWriter pwriter) {
+        dataTree.dumpEphemerals(pwriter);
     }
 
     /**

src/java/main/org/apache/zookeeper/server/ZooKeeperServer.java

@@ -18,13 +18,12 @@
 
 package org.apache.zookeeper.server;
 
-import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.IOException;
+import java.io.PrintWriter;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
@@ -32,9 +31,6 @@ import java.util.Random;
 import java.util.concurrent.ConcurrentHashMap;
 
 import org.apache.jute.BinaryInputArchive;
-import org.apache.jute.BinaryOutputArchive;
-import org.apache.jute.InputArchive;
-import org.apache.jute.OutputArchive;
 import org.apache.jute.Record;
 import org.apache.log4j.Logger;
 import org.apache.zookeeper.Environment;
@@ -48,12 +44,6 @@ import org.apache.zookeeper.proto.RequestHeader;
 import org.apache.zookeeper.server.SessionTracker.Session;
 import org.apache.zookeeper.server.SessionTracker.SessionExpirer;
 import org.apache.zookeeper.server.persistence.FileTxnSnapLog;
-import org.apache.zookeeper.server.persistence.FileTxnSnapLog.PlayBackListener;
-import org.apache.zookeeper.server.quorum.Leader;
-import org.apache.zookeeper.server.quorum.QuorumPacket;
-import org.apache.zookeeper.server.quorum.Leader.Proposal;
-import org.apache.zookeeper.server.util.SerializeUtils;
-import org.apache.zookeeper.txn.TxnHeader;
 
 /**
  * This class implements a simple standalone ZooKeeperServer. It sets up the
@@ -651,5 +641,8 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider {
         return "standalone";
     }
 
+    public void dumpEphemerals(PrintWriter pwriter) {
+    	zkDb.dumpEphemerals(pwriter);
+    }
     
 }

src/java/main/org/apache/zookeeper/server/quorum/LearnerSessionTracker.java

@@ -18,6 +18,7 @@
 
 package org.apache.zookeeper.server.quorum;
 
+import java.io.PrintWriter;
 import java.util.HashMap;
 import java.util.concurrent.ConcurrentHashMap;
 
@@ -37,10 +38,6 @@ public class LearnerSessionTracker implements SessionTracker {
     
     private ConcurrentHashMap<Long, Integer> sessionsWithTimeouts;
 
-    
-    /**
-     * 
-     */
     public LearnerSessionTracker(SessionExpirer expirer,
             ConcurrentHashMap<Long, Integer> sessionsWithTimeouts, long id) {
         this.expirer = expirer;
@@ -86,4 +83,10 @@ public class LearnerSessionTracker implements SessionTracker {
     public void setOwner(long sessionId, Object owner) {
         // Nothing to do here. Sessions are checked at the Leader
     }
+
+    public void dumpSessions(PrintWriter pwriter) {
+    	// the original class didn't have tostring impl, so just
+    	// dup what we had before
+    	pwriter.println(toString());
+    }
 }

src/java/test/org/apache/zookeeper/test/FourLetterWordsQuorumTest.java

@@ -40,6 +40,9 @@ public class FourLetterWordsQuorumTest extends QuorumBase {
             verify(hp, "srvr", "Outstanding");
             verify(hp, "cons", "queued");
             verify(hp, "dump", "Session");
+            verify(hp, "wchs", "watches");
+            verify(hp, "wchp", "");
+            verify(hp, "wchc", "");
 
             verify(hp, "srst", "reset");
             verify(hp, "crst", "reset");
@@ -56,12 +59,15 @@ public class FourLetterWordsQuorumTest extends QuorumBase {
             verify(hp, "cons", sid);
             verify(hp, "dump", sid);
 
-            zk.getData("/", false, null);
+            zk.getData("/", true, null);
 
             verify(hp, "stat", "queued");
             verify(hp, "srvr", "Outstanding");
             verify(hp, "cons", sid);
             verify(hp, "dump", sid);
+            verify(hp, "wchs", "watching 1");
+            verify(hp, "wchp", sid);
+            verify(hp, "wchc", sid);
 
             zk.close();
 
@@ -71,6 +77,9 @@ public class FourLetterWordsQuorumTest extends QuorumBase {
             verify(hp, "srvr", "Outstanding");
             verify(hp, "cons", "queued");
             verify(hp, "dump", "Session");
+            verify(hp, "wchs", "watch");
+            verify(hp, "wchp", "");
+            verify(hp, "wchc", "");
 
             verify(hp, "srst", "reset");
             verify(hp, "crst", "reset");

src/java/test/org/apache/zookeeper/test/FourLetterWordsTest.java

@@ -38,6 +38,9 @@ public class FourLetterWordsTest extends ClientBase {
         verify("srvr", "Outstanding");
         verify("cons", "queued");
         verify("dump", "Session");
+        verify("wchs", "watches");
+        verify("wchp", "");
+        verify("wchc", "");
 
         verify("srst", "reset");
         verify("crst", "reset");
@@ -54,13 +57,16 @@ public class FourLetterWordsTest extends ClientBase {
         verify("cons", sid);
         verify("dump", sid);
 
-        zk.getData("/", false, null);
+        zk.getData("/", true, null);
 
         verify("stat", "queued");
         verify("srvr", "Outstanding");
         verify("cons", sid);
         verify("dump", sid);
 
+        verify("wchs", "watching 1");
+        verify("wchp", sid);
+        verify("wchc", sid);
         zk.close();
 
         verify("ruok", "imok");
@@ -69,6 +75,9 @@ public class FourLetterWordsTest extends ClientBase {
         verify("srvr", "Outstanding");
         verify("cons", "queued");
         verify("dump", "Session");
+        verify("wchs", "watch");
+        verify("wchp", "");
+        verify("wchc", "");
 
         verify("srst", "reset");
         verify("crst", "reset");