ZOOKEEPER-744. Add monitoring four-letter word

git-svn-id: https://svn.apache.org/repos/asf/hadoop/zookeeper/trunk@960655 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -85,6 +85,8 @@ NEW FEATURES:
 
   ZOOKEEPER-773. Log visualisation (Ivan Kelly via phunt)
 
+  ZOOKEEPER-744. Add monitoring four-letter word (Savu Andrei via phunt)
+
 Release 3.3.0 - 2010-03-24
 
 Non-backward compatible changes:

src/docs/src/documentation/content/xdocs/zookeeperAdmin.xml

@@ -1155,6 +1155,45 @@ server.3=zoo3:2888:3888</programlisting>
             use it carefully.</para>
           </listitem>
         </varlistentry>
+
+
+        <varlistentry>
+          <term>mntr</term>
+
+          <listitem>
+            <para><emphasis role="bold">New in 3.4.0:</emphasis> Outputs a list 
+            of variables that could be used for monitoring the health of the cluster.</para>
+
+            <programlisting>$ echo mntr | nc localhost 2185
+
+zk_version  3.4.0
+zk_avg_latency  0
+zk_max_latency  0
+zk_min_latency  0
+zk_packets_received 70
+zk_packets_sent 69
+zk_outstanding_requests 0
+zk_server_state leader
+zk_znode_count   4
+zk_watch_count  0
+zk_ephemerals_count 0
+zk_approximate_data_size    27
+zk_followers    4                   - only exposed by the Leader
+zk_synced_followers 4               - only exposed by the Leader
+zk_pending_syncs    0               - only exposed by the Leader
+zk_open_file_descriptor_count 23    - only available on Unix platforms
+zk_max_file_descriptor_count 1024   - only available on Unix platforms
+</programlisting>
+
+          <para>The output is compatible with java properties format and the content 
+        may change over time (new keys added). Your scripts should expect changes.</para>
+
+          <para>ATTENTION: Some of the keys are platform specific and some of the keys are only exported by the Leader. </para>
+
+          <para>The output contains multiple lines with the following format:</para>
+          <programlisting>key \t value</programlisting>
+          </listitem>
+        </varlistentry>
       </variablelist>
 
       <para>Here's an example of the <emphasis role="bold">ruok</emphasis>

src/java/main/org/apache/zookeeper/server/DataTree.java

@@ -243,6 +243,15 @@ public class DataTree {
         return dataWatches.size() + childWatches.size();
     }
 
+    public int getEphemeralsCount() {
+        Map<Long, HashSet<String>> map = this.getEphemeralsMap();
+        int result = 0;
+        for (HashSet<String> set : map.values()) {
+            result += set.size();
+        }
+        return result;
+    }
+
     /**
      * Get the size of the nodes based on path and data length.
      *

src/java/main/org/apache/zookeeper/server/DataTreeBean.java

@@ -18,11 +18,7 @@
 
 package org.apache.zookeeper.server;
 
-import java.util.HashSet;
-import java.util.Map;
-
 import org.apache.zookeeper.jmx.ZKMBeanInfo;
-import org.apache.zookeeper.server.DataTree;
 
 /**
  * This class implements the data tree MBean.
@@ -43,12 +39,7 @@ public class DataTreeBean implements DataTreeMXBean, ZKMBeanInfo {
     }
 
     public int countEphemerals() {
-        Map<Long, HashSet<String>> map = dataTree.getEphemeralsMap();
-        int result = 0;
-        for (HashSet<String> set : map.values()) {
-            result += set.size();
-        }
-        return result;
+        return dataTree.getEphemeralsCount();
     }
 
     public int getWatchCount() {

src/java/main/org/apache/zookeeper/server/NIOServerCnxn.java

@@ -25,6 +25,8 @@ import java.io.InputStream;
 import java.io.PrintWriter;
 import java.io.StringWriter;
 import java.io.Writer;
+import java.lang.management.ManagementFactory;
+import java.lang.management.OperatingSystemMXBean;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.nio.ByteBuffer;
@@ -64,6 +66,10 @@ import org.apache.zookeeper.proto.RequestHeader;
 import org.apache.zookeeper.proto.WatcherEvent;
 import org.apache.zookeeper.server.auth.AuthenticationProvider;
 import org.apache.zookeeper.server.auth.ProviderRegistry;
+import org.apache.zookeeper.server.quorum.Leader;
+import org.apache.zookeeper.server.quorum.LeaderZooKeeperServer;
+
+import com.sun.management.UnixOperatingSystemMXBean;
 
 /**
  * This class handles communication with clients using NIO. There is one per
@@ -891,6 +897,14 @@ public class NIOServerCnxn implements Watcher, ServerCnxn {
     private final static int wchsCmd =
         ByteBuffer.wrap("wchs".getBytes()).getInt();
 
+    /*
+     * See <a href="{@docRoot}/../../../docs/zookeeperAdmin.html#sc_zkCommands">
+     * Zk Admin</a>. this link is for all the commands.
+     */
+    private final static int mntrCmd = ByteBuffer.wrap("mntr".getBytes())
+            .getInt();
+
+
     private final static HashMap<Integer, String> cmd2String =
         new HashMap<Integer, String>();
 
@@ -910,6 +924,7 @@ public class NIOServerCnxn implements Watcher, ServerCnxn {
         cmd2String.put(wchcCmd, "wchc");
         cmd2String.put(wchpCmd, "wchp");
         cmd2String.put(wchsCmd, "wchs");
+        cmd2String.put(mntrCmd, "mntr");
     }
 
     /**
@@ -1221,8 +1236,71 @@ public class NIOServerCnxn implements Watcher, ServerCnxn {
             }
         }
     }
-    
-    
+
+    private class MonitorCommand extends CommandThread {
+
+        MonitorCommand(PrintWriter pw) {
+            super(pw);
+        }
+
+        @Override
+        public void commandRun() {
+            if(zk == null) {
+                pw.println(ZK_NOT_SERVING);
+                return;
+            }
+            ZKDatabase zkdb = zk.getZKDatabase();
+            ServerStats stats = zk.serverStats();
+
+            print("version", Version.getFullVersion());
+
+            print("avg_latency", stats.getAvgLatency());
+            print("max_latency", stats.getMaxLatency());
+            print("min_latency", stats.getMinLatency());
+
+            print("packets_received", stats.getPacketsReceived());
+            print("packets_sent", stats.getPacketsSent());
+
+            print("outstanding_requests", stats.getOutstandingRequests());
+
+            print("server_state", stats.getServerState());
+            print("znode_count", zkdb.getNodeCount());
+
+            print("watch_count", zkdb.getDataTree().getWatchCount());
+            print("ephemerals_count", zkdb.getDataTree().getEphemeralsCount());
+            print("approximate_data_size", zkdb.getDataTree().approximateDataSize());
+
+            OperatingSystemMXBean osMbean = ManagementFactory.getOperatingSystemMXBean();
+            if(osMbean != null && osMbean instanceof UnixOperatingSystemMXBean) {
+                UnixOperatingSystemMXBean unixos = (UnixOperatingSystemMXBean)osMbean;
+
+                print("open_file_descriptor_count", unixos.getOpenFileDescriptorCount());
+                print("max_file_descriptor_count", unixos.getMaxFileDescriptorCount());
+            }
+
+            if(stats.getServerState() == "leader") {
+                Leader leader = ((LeaderZooKeeperServer)zk).getLeader();
+
+                print("followers", leader.learners.size());
+                print("synced_followers", leader.forwardingFollowers.size());
+                print("pending_syncs", leader.pendingSyncs.size());
+            }
+        }
+
+        private void print(String key, long number) {
+            print(key, "" + number);
+        }
+
+        private void print(String key, String value) {
+            pw.print("zk_");
+            pw.print(key);
+            pw.print("\t");
+            pw.println(value);
+        }
+
+    }
+
+
     /** Return if four letter word found and responded to, otw false **/
     private boolean checkFourLetterWord(final SelectionKey k, final int len)
     throws IOException
@@ -1308,6 +1386,10 @@ public class NIOServerCnxn implements Watcher, ServerCnxn {
             WatchCommand wcmd = new WatchCommand(pwriter, len);
             wcmd.start();
             return true;
+        } else if (len == mntrCmd) {
+            MonitorCommand mntr = new MonitorCommand(pwriter);
+            mntr.start();
+            return true;
         }
         return false;
     }

src/java/main/org/apache/zookeeper/server/WatchManager.java

@@ -44,7 +44,11 @@ public class WatchManager {
         new HashMap<Watcher, HashSet<String>>();
 
     public synchronized int size(){
-        return watchTable.size();
+        int result = 0;
+        for(Set<Watcher> watches : watchTable.values()) {
+            result += watches.size();
+        }
+        return result;
     }
 
     public synchronized void addWatch(String path, Watcher watcher) {

src/java/test/org/apache/zookeeper/test/ClientBase.java

@@ -427,6 +427,10 @@ public abstract class ClientBase extends ZKTestCase {
         JMXEnv.ensureOnly();
     }
 
+    protected ZooKeeperServer getServer() {
+        return serverFactory.getZooKeeperServer();
+    }
+
     protected void tearDownAll() throws Exception {
         synchronized (this) {
             for (ZooKeeper zk : allClients) {

src/java/test/org/apache/zookeeper/test/FourLetterWordsQuorumTest.java

@@ -89,6 +89,8 @@ public class FourLetterWordsQuorumTest extends QuorumBase {
             verify(hp, "stat", "Outstanding");
             verify(hp, "srvr", "Outstanding");
             verify(hp, "cons", "queued");
+
+            verify(hp, "mntr", "zk_version\t");
         }
     }
 

src/java/test/org/apache/zookeeper/test/FourLetterWordsTest.java

@@ -87,6 +87,7 @@ public class FourLetterWordsTest extends ClientBase {
         verify("stat", "Outstanding");
         verify("srvr", "Outstanding");
         verify("cons", "queued");
+        verify("mntr", "zk_server_state\tstandalone");
     }
 
     private void verify(String cmd, String expected) throws IOException {

src/java/test/org/apache/zookeeper/test/WatcherTest.java

@@ -29,6 +29,7 @@ import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.TestableZooKeeper;
 import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
 import org.apache.zookeeper.ZooKeeper;
 import org.apache.zookeeper.AsyncCallback.StatCallback;
 import org.apache.zookeeper.AsyncCallback.VoidCallback;
@@ -128,6 +129,38 @@ public class WatcherTest extends ClientBase {
         }
     }
 
+    @Test
+    public void testWatcherCount() 
+    throws IOException, InterruptedException, KeeperException {
+        ZooKeeper zk1 = null, zk2 = null;
+        try {
+            MyWatcher w1 = new MyWatcher();
+            zk1 = createClient(w1, hostPort);
+
+            MyWatcher w2 = new MyWatcher();
+            zk2 = createClient(w2, hostPort);
+
+            Stat stat = new Stat();
+            zk1.create("/watch-count-test", "value".getBytes(), Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL);
+            zk1.create("/watch-count-test-2", "value".getBytes(), Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL);
+
+            zk1.getData("/watch-count-test", w1, stat);
+            zk1.getData("/watch-count-test-2", w1, stat);
+            zk2.getData("/watch-count-test", w2, stat);
+
+            Assert.assertEquals(getServer().getZKDatabase().getDataTree().getWatchCount(), 3);
+
+        } finally {
+            if(zk1 != null) {
+                zk1.close();
+            }
+            if(zk2 != null) {
+                zk2.close();
+            }
+        }
+
+    }
+
     final static int COUNT = 100;
     boolean hasSeenDelete = true;
     /**