ZOOKEEPER-516 add support for 10 minute test ie "pre-commit" test

git-svn-id: https://svn.apache.org/repos/asf/hadoop/zookeeper/trunk@815574 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -78,6 +78,8 @@ IMPROVEMENTS:
   ZOOKEEPER-488. Fix zkServer.sh to add clover.jar in classpath
   (Giridharan Kesavan via gkesavan)  
 
+  ZOOKEEPER-516. add support for 10 minute test ie "pre-commit" test (phunt)
+
 NEW FEATURES:
 
 

build.xml

@@ -66,6 +66,7 @@
     <property name="test.junit.haltonfailure" value="no" />
     <property name="config.dir" value="${src.dir}/java/test/config" />
     <property name="test.junit.maxmem" value="512m" />
+    <property name="test.quick" value="no" />
     <property name="conf.dir" value="${basedir}/conf"/>
     <property name="docs.dir" value="${basedir}/docs"/>
     <property name="docs.src" value="${basedir}/src/docs"/>
@@ -615,6 +616,23 @@
        
     </target>
 
+    <condition property="quicktest">
+      <and>
+        <equals arg1="${test.quick}" arg2="yes"/>
+        <not>
+          <isset property="testcase"/>
+        </not>
+      </and>
+    </condition>
+    <condition property="fulltest">
+      <and>
+        <equals arg1="${test.quick}" arg2="no"/>
+        <not>
+          <isset property="testcase"/>
+        </not>
+      </and>
+    </condition>
+
     <target name="junit.run">
         <junit showoutput="${test.output}"
                printsummary="${test.junit.printsummary}"
@@ -628,14 +646,27 @@
           <sysproperty key="test.data.dir" value="${test.data.dir}" />
           <sysproperty key="log4j.configuration"
                        value="file:${basedir}/conf/log4j.properties" />
+          <!-- superDigest is used by the tests/main code. If this is not set
+               as part of starting the jvm there is no guarantee that the static
+               initializers in the java code will see this (esp when running
+               with junit fork mode set to "once")-->
+          <sysproperty key="zookeeper.DigestAuthenticationProvider.superDigest"
+                       value="super:D/InIHSb7yEEbrWz8b9l71RjZJU=" />
           <classpath refid="test.classpath"/>
           <classpath>
             <pathelement path="${build.testclasses}" />
           </classpath>
           <formatter type="${test.junit.output.format}" />
-          <batchtest todir="${test.log.dir}" unless="testcase">
-            <fileset dir="${test.src.dir}"
-                     includes="**/*${test.category}Test.java"/>
+          <batchtest todir="${test.log.dir}" if="quicktest">
+            <fileset dir="${test.src.dir}">
+              <include name="**/*${test.category}Test.java"/>
+              <exclude name="**/*HammerTest.java"/>
+            </fileset>
+          </batchtest>
+          <batchtest todir="${test.log.dir}" if="fulltest">
+            <fileset dir="${test.src.dir}">
+              <include name="**/*${test.category}Test.java"/>
+            </fileset>
           </batchtest>
           <batchtest todir="${test.log.dir}" if="testcase">
             <fileset dir="${test.src.dir}" includes="**/${testcase}.java"/>
@@ -700,7 +731,7 @@
             if="cppunit"
 		description="to run cppunit test cases">
         <property name="lib.cppunit" value="/usr/local/lib"/>
-	<exec dir="${test.cppunit.dir}" executable="make" failonerror="true">
+        <exec dir="${test.cppunit.dir}" executable="make" failonerror="true">
 		<env key="LD_LIBRARY_PATH" value="${lib.cppunit}"/>
 		<env key="PATH" path="${env.PATH};${c.src.dir};"/>
                 <env key="CALLER" value="ANT"/>

src/c/src/zk_log.c

@@ -26,7 +26,7 @@
 #include <time.h>
 
 #define TIME_NOW_BUF_SIZE 1024
-#define FORMAT_LOG_BUF_SIZE 2048
+#define FORMAT_LOG_BUF_SIZE 4096
 
 #ifdef THREADED
 #include <pthread.h>

src/c/src/zookeeper.c

@@ -602,7 +602,7 @@ zhandle_t *zookeeper_init(const char *host, watcher_fn watcher,
     log_env();
 
     LOG_INFO(("Initiating client connection, host=%s sessionTimeout=%d watcher=%p"
-          " sessionId=0x%llx sessionPasswd=%s context=%p flags=%d",
+          " sessionId=%#llx sessionPasswd=%s context=%p flags=%d",
               host,
               recv_timeout,
               watcher,
@@ -1473,14 +1473,14 @@ static int check_events(zhandle_t *zh, int events)
                     zh->state = ZOO_EXPIRED_SESSION_STATE;
                     errno = ESTALE;
                     return handle_socket_error_msg(zh,__LINE__,ZSESSIONEXPIRED,
-                            "session %llx has expired.",oldid);
+                            "sessionId=%#llx has expired.",oldid);
                 } else {
                     zh->recv_timeout = zh->primer_storage.timeOut;
                     zh->client_id.client_id = newid;
                  
                     memcpy(zh->client_id.passwd, &zh->primer_storage.passwd, sizeof(zh->client_id.passwd));
                     zh->state = ZOO_CONNECTED_STATE;
-                    LOG_INFO(("connected to server [%s] with session id=%llx",
+                    LOG_INFO(("connected to server [%s] with sessionId=%#llx",
                             format_endpoint_info(&zh->addrs[zh->connect_index]),newid));
                     /* we want the auth to be sent for, but since both call push to front
                        we need to call send_watch_set first */
@@ -1621,7 +1621,7 @@ void process_completions(zhandle_t *zh)
             int rc = hdr.err;
             switch (cptr->completion_type) {
             case COMPLETION_DATA:
-                LOG_DEBUG(("Calling COMPLETION_DATA for xid=%x rc=%d",cptr->xid,rc));
+                LOG_DEBUG(("Calling COMPLETION_DATA for xid=%#x rc=%d",cptr->xid,rc));
                 if (rc) {
                     cptr->c.data_result(rc, 0, 0, 0, cptr->data);
                 } else {
@@ -1633,7 +1633,7 @@ void process_completions(zhandle_t *zh)
                 }
                 break;
             case COMPLETION_STAT:
-                LOG_DEBUG(("Calling COMPLETION_STAT for xid=%x rc=%d",cptr->xid,rc));
+                LOG_DEBUG(("Calling COMPLETION_STAT for xid=%#x rc=%d",cptr->xid,rc));
                 if (rc) {
                     cptr->c.stat_result(rc, 0, cptr->data);
                 } else {
@@ -1644,7 +1644,7 @@ void process_completions(zhandle_t *zh)
                 }
                 break;
             case COMPLETION_STRINGLIST:
-                LOG_DEBUG(("Calling COMPLETION_STRINGLIST for xid=%x rc=%d",cptr->xid,rc));
+                LOG_DEBUG(("Calling COMPLETION_STRINGLIST for xid=%#x rc=%d",cptr->xid,rc));
                 if (rc) {
                     cptr->c.strings_result(rc, 0, cptr->data);
                 } else {
@@ -1655,7 +1655,7 @@ void process_completions(zhandle_t *zh)
                 }
                 break;
             case COMPLETION_STRING:
-                LOG_DEBUG(("Calling COMPLETION_STRING for xid=%x rc=%d",cptr->xid,rc));
+                LOG_DEBUG(("Calling COMPLETION_STRING for xid=%#x rc=%d",cptr->xid,rc));
                 if (rc) {
                     cptr->c.string_result(rc, 0, cptr->data);
                 } else {
@@ -1666,7 +1666,7 @@ void process_completions(zhandle_t *zh)
                 }
                 break;
             case COMPLETION_ACLLIST:
-                LOG_DEBUG(("Calling COMPLETION_ACLLIST for xid=%x rc=%d",cptr->xid,rc));
+                LOG_DEBUG(("Calling COMPLETION_ACLLIST for xid=%#x rc=%d",cptr->xid,rc));
                 if (rc) {
                     cptr->c.acl_result(rc, 0, 0, cptr->data);
                 } else {
@@ -1677,7 +1677,7 @@ void process_completions(zhandle_t *zh)
                 }
                 break;
             case COMPLETION_VOID:
-                LOG_DEBUG(("Calling COMPLETION_VOID for xid=%x rc=%d",cptr->xid,rc));
+                LOG_DEBUG(("Calling COMPLETION_VOID for xid=%#x rc=%d",cptr->xid,rc));
                 if (hdr.xid == PING_XID) {
                     // We want to skip the ping
                 } else {
@@ -1745,7 +1745,7 @@ int zookeeper_process(zhandle_t *zh, int events)
         if (hdr.zxid > 0) {
             zh->last_zxid = hdr.zxid;
         } else {
-            // fprintf(stderr, "Got %llx for %x\n", hdr.zxid, hdr.xid);
+            // fprintf(stderr, "Got %#x for %#x\n", hdr.zxid, hdr.xid);
         }
         
         if (hdr.xid == WATCHER_EVENT_XID) {
@@ -1799,7 +1799,7 @@ int zookeeper_process(zhandle_t *zh, int events)
                 // signaled and deallocated) and disconnect from the server
                 queue_completion(&zh->sent_requests,cptr,1);
                 return handle_socket_error_msg(zh, __LINE__,ZRUNTIMEINCONSISTENCY,
-                        "unexpected server response: expected %x, but received %x",
+                        "unexpected server response: expected %#x, but received %#x",
                         hdr.xid,cptr->xid);
             }
 
@@ -1827,7 +1827,7 @@ int zookeeper_process(zhandle_t *zh, int events)
                 sc->rc = rc;
                 switch(cptr->completion_type) {
                 case COMPLETION_DATA:
-                    LOG_DEBUG(("Calling COMPLETION_DATA for xid=%x rc=%d",
+                    LOG_DEBUG(("Calling COMPLETION_DATA for xid=%#x rc=%d",
                                cptr->xid, rc));
                     if (rc==0) {
                         struct GetDataResponse res;
@@ -1851,7 +1851,7 @@ int zookeeper_process(zhandle_t *zh, int events)
                     }
                     break;
                 case COMPLETION_STAT:
-                    LOG_DEBUG(("Calling COMPLETION_STAT for xid=%x rc=%d",
+                    LOG_DEBUG(("Calling COMPLETION_STAT for xid=%#x rc=%d",
                                cptr->xid, rc));
                     if (rc == 0) {
                         struct SetDataResponse res;
@@ -1861,7 +1861,7 @@ int zookeeper_process(zhandle_t *zh, int events)
                     }
                     break;
                 case COMPLETION_STRINGLIST:
-                    LOG_DEBUG(("Calling COMPLETION_STRINGLIST for xid=%x rc=%d",
+                    LOG_DEBUG(("Calling COMPLETION_STRINGLIST for xid=%#x rc=%d",
                                cptr->xid, rc));
                     if (rc == 0) {
                         struct GetChildrenResponse res;
@@ -1872,7 +1872,7 @@ int zookeeper_process(zhandle_t *zh, int events)
                     }
                     break;
                 case COMPLETION_STRING:
-                    LOG_DEBUG(("Calling COMPLETION_STRING for xid=%x rc=%d",
+                    LOG_DEBUG(("Calling COMPLETION_STRING for xid=%#x rc=%d",
                                cptr->xid, rc));
                     if (rc == 0) {
                         struct CreateResponse res;
@@ -1890,7 +1890,7 @@ int zookeeper_process(zhandle_t *zh, int events)
                     }
                     break;
                 case COMPLETION_ACLLIST:
-                    LOG_DEBUG(("Calling COMPLETION_ACLLIST for xid=%x rc=%d",
+                    LOG_DEBUG(("Calling COMPLETION_ACLLIST for xid=%#x rc=%d",
                                cptr->xid, rc));
                     if (rc == 0) {
                         struct GetACLResponse res;
@@ -1902,11 +1902,11 @@ int zookeeper_process(zhandle_t *zh, int events)
                     }
                     break;
                 case COMPLETION_VOID:
-                    LOG_DEBUG(("Calling COMPLETION_VOID for xid=%x rc=%d",
+                    LOG_DEBUG(("Calling COMPLETION_VOID for xid=%#x rc=%d",
                                cptr->xid, rc));
                     break;
                 default:
-                    LOG_DEBUG(("UNKNOWN response type xid=%x rc=%d",
+                    LOG_DEBUG(("UNKNOWN response type xid=%#x rc=%d",
                                cptr->xid, rc));
                     break;
                 }
@@ -2087,7 +2087,7 @@ int zookeeper_close(zhandle_t *zh)
     if(zh->state==ZOO_CONNECTED_STATE){
         struct oarchive *oa;
         struct RequestHeader h = { .xid = get_xid(), .type = CLOSE_OP};
-        LOG_INFO(("Closing zookeeper session %llx to [%s]\n",
+        LOG_INFO(("Closing zookeeper sessionId=%#llx to [%s]\n",
                 zh->client_id.client_id,format_current_endpoint_info(zh)));
         oa = create_buffer_oarchive();
         rc = serialize_RequestHeader(oa, "header", &h);
@@ -2104,7 +2104,7 @@ int zookeeper_close(zhandle_t *zh)
          * (but reasonable) number of milliseconds since we want the call to block*/
         rc=adaptor_send_queue(zh, 3000);
     }else{
-        LOG_INFO(("Freeing zookeeper resources for session %llx\n",
+        LOG_INFO(("Freeing zookeeper resources for sessionId=%#llx\n",
                 zh->client_id.client_id));
         rc = ZOK;
     } 
@@ -2194,7 +2194,7 @@ int zoo_awget(zhandle_t *zh, const char *path,
     /* We queued the buffer, so don't free it */
     close_buffer_oarchive(&oa, 0);
     
-    LOG_DEBUG(("Sending request xid=%x for path [%s] to %s",h.xid,path,
+    LOG_DEBUG(("Sending request xid=%#x for path [%s] to %s",h.xid,path,
             format_current_endpoint_info(zh)));
     /* make a best (non-blocking) effort to send the requests asap */
     adaptor_send_queue(zh, 0);
@@ -2235,7 +2235,7 @@ int zoo_aset(zhandle_t *zh, const char *path, const char *buffer, int buflen,
     /* We queued the buffer, so don't free it */
     close_buffer_oarchive(&oa, 0);
 
-    LOG_DEBUG(("Sending request xid=%x for path [%s] to %s",h.xid,path,
+    LOG_DEBUG(("Sending request xid=%#x for path [%s] to %s",h.xid,path,
             format_current_endpoint_info(zh)));
     /* make a best (non-blocking) effort to send the requests asap */
     adaptor_send_queue(zh, 0);
@@ -2283,7 +2283,7 @@ int zoo_acreate(zhandle_t *zh, const char *path, const char *value,
     /* We queued the buffer, so don't free it */
     close_buffer_oarchive(&oa, 0);
     
-    LOG_DEBUG(("Sending request xid=%x for path [%s] to %s",h.xid,path,
+    LOG_DEBUG(("Sending request xid=%#x for path [%s] to %s",h.xid,path,
             format_current_endpoint_info(zh)));
     /* make a best (non-blocking) effort to send the requests asap */
     adaptor_send_queue(zh, 0);
@@ -2322,7 +2322,7 @@ int zoo_adelete(zhandle_t *zh, const char *path, int version,
     /* We queued the buffer, so don't free it */
     close_buffer_oarchive(&oa, 0);
 
-    LOG_DEBUG(("Sending request xid=%x for path [%s] to %s",h.xid,path,
+    LOG_DEBUG(("Sending request xid=%#x for path [%s] to %s",h.xid,path,
             format_current_endpoint_info(zh)));
     /* make a best (non-blocking) effort to send the requests asap */
     adaptor_send_queue(zh, 0);
@@ -2367,7 +2367,7 @@ int zoo_awexists(zhandle_t *zh, const char *path,
     /* We queued the buffer, so don't free it */
     close_buffer_oarchive(&oa, 0);
 
-    LOG_DEBUG(("Sending request xid=%x for path [%s] to %s",h.xid,path,
+    LOG_DEBUG(("Sending request xid=%#x for path [%s] to %s",h.xid,path,
             format_current_endpoint_info(zh)));
     /* make a best (non-blocking) effort to send the requests asap */
     adaptor_send_queue(zh, 0);
@@ -2411,7 +2411,7 @@ int zoo_awget_children(zhandle_t *zh, const char *path,
     /* We queued the buffer, so don't free it */
     close_buffer_oarchive(&oa, 0);
 
-    LOG_DEBUG(("Sending request xid=%x for path [%s] to %s",h.xid,path,
+    LOG_DEBUG(("Sending request xid=%#x for path [%s] to %s",h.xid,path,
             format_current_endpoint_info(zh)));
     /* make a best (non-blocking) effort to send the requests asap */
     adaptor_send_queue(zh, 0);
@@ -2449,7 +2449,7 @@ int zoo_async(zhandle_t *zh, const char *path,
     /* We queued the buffer, so don't free it */
     close_buffer_oarchive(&oa, 0);
 
-    LOG_DEBUG(("Sending request xid=%x for path [%s] to %s",h.xid,path,
+    LOG_DEBUG(("Sending request xid=%#x for path [%s] to %s",h.xid,path,
             format_current_endpoint_info(zh)));
     /* make a best (non-blocking) effort to send the requests asap */
     adaptor_send_queue(zh, 0);
@@ -2488,7 +2488,7 @@ int zoo_aget_acl(zhandle_t *zh, const char *path, acl_completion_t completion,
     /* We queued the buffer, so don't free it */
     close_buffer_oarchive(&oa, 0);
 
-    LOG_DEBUG(("Sending request xid=%x for path [%s] to %s",h.xid,path,
+    LOG_DEBUG(("Sending request xid=%#x for path [%s] to %s",h.xid,path,
             format_current_endpoint_info(zh)));
     /* make a best (non-blocking) effort to send the requests asap */
     adaptor_send_queue(zh, 0);
@@ -2528,7 +2528,7 @@ int zoo_aset_acl(zhandle_t *zh, const char *path, int version,
     /* We queued the buffer, so don't free it */
     close_buffer_oarchive(&oa, 0);
 
-    LOG_DEBUG(("Sending request xid=%x for path [%s] to %s",h.xid,path,
+    LOG_DEBUG(("Sending request xid=%#x for path [%s] to %s",h.xid,path,
             format_current_endpoint_info(zh)));
     /* make a best (non-blocking) effort to send the requests asap */
     adaptor_send_queue(zh, 0);

src/c/tests/TestClient.cc

@@ -37,6 +37,8 @@ using namespace std;
 
 #include <zookeeper.h>
 
+#include "Util.h"
+
 #ifdef THREADED
     static void yield(zhandle_t *zh, int i)
     {
@@ -197,35 +199,37 @@ class Zookeeper_simpleSystem : public CPPUNIT_NS::TestFixture
     }
     
     zhandle_t *createClient(watchctx_t *ctx) {
-        zhandle_t *zk = zookeeper_init(hostPorts, watcher, 10000, 0,
-                                       ctx, 0);
+        zhandle_t *zk = zookeeper_init(hostPorts, watcher, 10000, 0, ctx, 0);
         ctx->zh = zk;
         sleep(1);
         return zk;
     }
     
-    zhandle_t *createchClient(watchctx_t *ctx) {
-        zhandle_t *zk = zookeeper_init(hp_chroot, watcher, 10000, 0,
-                                       ctx, 0);
+    zhandle_t *createchClient(watchctx_t *ctx, const char* chroot) {
+        zhandle_t *zk = zookeeper_init(chroot, watcher, 10000, 0, ctx, 0);
         ctx->zh = zk;
         sleep(1);
         return zk;
     }
         
+    FILE *logfile;
 public:
 
+    Zookeeper_simpleSystem() {
+      logfile = openlogfile("Zookeeper_simpleSystem");
+    }
+
+    ~Zookeeper_simpleSystem() {
+      if (logfile) {
+        fflush(logfile);
+        fclose(logfile);
+        logfile = 0;
+      }
+    }
 
     void setUp()
     {
-        char cmd[1024];
-        sprintf(cmd, "%s startClean %s", ZKSERVER_CMD, getHostPorts());
-        CPPUNIT_ASSERT(system(cmd) == 0);
-
-        struct sigaction act;
-        act.sa_handler = SIG_IGN;
-        sigemptyset(&act.sa_mask);
-        act.sa_flags = 0;
-        CPPUNIT_ASSERT(sigaction(SIGPIPE, &act, NULL) == 0);
+        zoo_set_log_stream(logfile);
     }
     
 
@@ -236,14 +240,13 @@ public:
     }
 
     void stopServer() {
-        tearDown();
+        char cmd[1024];
+        sprintf(cmd, "%s stop %s", ZKSERVER_CMD, getHostPorts());
+        CPPUNIT_ASSERT(system(cmd) == 0);
     }
 
     void tearDown()
     {
-        char cmd[1024];
-        sprintf(cmd, "%s stop %s", ZKSERVER_CMD, getHostPorts());
-        CPPUNIT_ASSERT(system(cmd) == 0);
     }
     
     void testPing()
@@ -252,31 +255,36 @@ public:
         watchctx_t ctxWC;
         zhandle_t *zkIdle = createClient(&ctxIdle);
         zhandle_t *zkWatchCreator = createClient(&ctxWC);
-        int rc;
-        char path[80];
+
         CPPUNIT_ASSERT(zkIdle);
         CPPUNIT_ASSERT(zkWatchCreator);
+
+        char path[80];
+        sprintf(path, "/testping");
+        int rc = zoo_create(zkWatchCreator, path, "", 0, &ZOO_OPEN_ACL_UNSAFE, 0, 0, 0);
+        CPPUNIT_ASSERT_EQUAL((int)ZOK, rc);
+
         for(int i = 0; i < 30; i++) {
-            sprintf(path, "/%i", i);
+            sprintf(path, "/testping/%i", i);
             rc = zoo_create(zkWatchCreator, path, "", 0, &ZOO_OPEN_ACL_UNSAFE, 0, 0, 0);
             CPPUNIT_ASSERT_EQUAL((int)ZOK, rc);
         }
 
         for(int i = 0; i < 30; i++) {
-            sprintf(path, "/%i", i);
+            sprintf(path, "/testping/%i", i);
             struct Stat stat;
             rc = zoo_exists(zkIdle, path, 1, &stat);
             CPPUNIT_ASSERT_EQUAL((int)ZOK, rc);
         }
 
         for(int i = 0; i < 30; i++) {
-            sprintf(path, "/%i", i);
+            sprintf(path, "/testping/%i", i);
             usleep(500000);
             rc = zoo_delete(zkWatchCreator, path, -1);
             CPPUNIT_ASSERT_EQUAL((int)ZOK, rc);
         }
         struct Stat stat;
-        CPPUNIT_ASSERT_EQUAL((int)ZNONODE, zoo_exists(zkIdle, "/0", 0, &stat));
+        CPPUNIT_ASSERT_EQUAL((int)ZNONODE, zoo_exists(zkIdle, "/testping/0", 0, &stat));
     }
 
     bool waitForEvent(zhandle_t *zh, watchctx_t *ctx, int seconds) {
@@ -291,7 +299,7 @@ public:
     
     static zhandle_t *async_zk;
     static volatile int count;
-    static char* hp_chroot;
+    static const char* hp_chroot;
     
     static void statCompletion(int rc, const struct Stat *stat, const void *data) {
         int tmp = (int) (long) data;
@@ -537,41 +545,41 @@ public:
 
     void testPath() {
         watchctx_t ctx;
-        char pathbuf[10];
+        char pathbuf[20];
         zhandle_t *zk = createClient(&ctx);
         CPPUNIT_ASSERT(zk);
         int rc = 0;
 
-        memset(pathbuf, 'X', 10);
-        rc = zoo_create(zk, "/path0", "", 0, 
+        memset(pathbuf, 'X', 20);
+        rc = zoo_create(zk, "/testpathpath0", "", 0, 
                         &ZOO_OPEN_ACL_UNSAFE, 0, pathbuf, 0);
         CPPUNIT_ASSERT_EQUAL((int) ZOK, rc);
         CPPUNIT_ASSERT_EQUAL('X', pathbuf[0]);
 
-        rc = zoo_create(zk, "/path1", "", 0, 
+        rc = zoo_create(zk, "/testpathpath1", "", 0, 
                         &ZOO_OPEN_ACL_UNSAFE, 0, pathbuf, 1);
         CPPUNIT_ASSERT_EQUAL((int) ZOK, rc);
         CPPUNIT_ASSERT(strlen(pathbuf) == 0);
 
-        rc = zoo_create(zk, "/path2", "", 0, 
+        rc = zoo_create(zk, "/testpathpath2", "", 0, 
                         &ZOO_OPEN_ACL_UNSAFE, 0, pathbuf, 2);
         CPPUNIT_ASSERT_EQUAL((int) ZOK, rc);
         CPPUNIT_ASSERT(strcmp(pathbuf, "/") == 0);
 
-        rc = zoo_create(zk, "/path3", "", 0, 
+        rc = zoo_create(zk, "/testpathpath3", "", 0, 
                         &ZOO_OPEN_ACL_UNSAFE, 0, pathbuf, 3);
         CPPUNIT_ASSERT_EQUAL((int) ZOK, rc);
-        CPPUNIT_ASSERT(strcmp(pathbuf, "/p") == 0);
+        CPPUNIT_ASSERT(strcmp(pathbuf, "/t") == 0);
 
-        rc = zoo_create(zk, "/path7", "", 0, 
-                        &ZOO_OPEN_ACL_UNSAFE, 0, pathbuf, 7);
+        rc = zoo_create(zk, "/testpathpath7", "", 0, 
+                        &ZOO_OPEN_ACL_UNSAFE, 0, pathbuf, 15);
         CPPUNIT_ASSERT_EQUAL((int) ZOK, rc);
-        CPPUNIT_ASSERT(strcmp(pathbuf, "/path7") == 0);
+        CPPUNIT_ASSERT(strcmp(pathbuf, "/testpathpath7") == 0);
 
-        rc = zoo_create(zk, "/path8", "", 0, 
-                        &ZOO_OPEN_ACL_UNSAFE, 0, pathbuf, 8);
+        rc = zoo_create(zk, "/testpathpath8", "", 0, 
+                        &ZOO_OPEN_ACL_UNSAFE, 0, pathbuf, 16);
         CPPUNIT_ASSERT_EQUAL((int) ZOK, rc);
-        CPPUNIT_ASSERT(strcmp(pathbuf, "/path8") == 0);
+        CPPUNIT_ASSERT(strcmp(pathbuf, "/testpathpath8") == 0);
     }
 
     void testPathValidation() {
@@ -638,13 +646,12 @@ public:
         const char* data = "garbage";
         const char* retStr = "/chroot"; 
         const char* root= "/";
-        hp_chroot = "127.0.0.1:22181/test/mahadev";
-        zk_ch = createchClient(&ctx_ch);
+        zk_ch = createchClient(&ctx_ch, "127.0.0.1:22181/testch1/mahadev");
         CPPUNIT_ASSERT(zk_ch != NULL);
         zk = createClient(&ctx);
-        rc = zoo_create(zk, "/test", "", 0, &ZOO_OPEN_ACL_UNSAFE, 0, 0, 0);
+        rc = zoo_create(zk, "/testch1", "", 0, &ZOO_OPEN_ACL_UNSAFE, 0, 0, 0);
         CPPUNIT_ASSERT_EQUAL((int) ZOK, rc);
-        rc = zoo_create(zk, "/test/mahadev", data, 7, &ZOO_OPEN_ACL_UNSAFE, 0, 0, 0);
+        rc = zoo_create(zk, "/testch1/mahadev", data, 7, &ZOO_OPEN_ACL_UNSAFE, 0, 0, 0);
         CPPUNIT_ASSERT_EQUAL((int) ZOK, rc);
         // try an exists with /
         len = 60;
@@ -662,12 +669,12 @@ public:
         CPPUNIT_ASSERT(count == 0);
         rc = zoo_create(zk_ch, "/chroot/child", "", 0, &ZOO_OPEN_ACL_UNSAFE, 0, 0, 0);
         CPPUNIT_ASSERT_EQUAL((int) ZOK, rc);
-        rc = zoo_exists(zk, "/test/mahadev/chroot/child", 0, &stat);
+        rc = zoo_exists(zk, "/testch1/mahadev/chroot/child", 0, &stat);
         CPPUNIT_ASSERT_EQUAL((int) ZOK, rc);
         
         rc = zoo_delete(zk_ch, "/chroot/child", -1);
         CPPUNIT_ASSERT_EQUAL((int) ZOK, rc);
-        rc = zoo_exists(zk, "/test/mahadev/chroot/child", 0, &stat);
+        rc = zoo_exists(zk, "/testch1/mahadev/chroot/child", 0, &stat);
         CPPUNIT_ASSERT_EQUAL((int) ZNONODE, rc);
         rc = zoo_wget(zk_ch, "/chroot", watcher_chroot_fn, (char*) retStr,
                       buf, &len, &stat);
@@ -723,7 +730,7 @@ public:
 
         async_zk = zk;
         for(i = 0; i < COUNT; i++) {
-            sprintf(path, "/%d", i);
+            sprintf(path, "/awar%d", i);
             rc = zoo_awexists(zk, path, watcher, &lctx[i], statCompletion, (void*)ZNONODE);
             CPPUNIT_ASSERT_EQUAL((int)ZOK, rc);
         }
@@ -731,14 +738,14 @@ public:
         yield(zk, 0);
 
         for(i = 0; i < COUNT/2; i++) {
-            sprintf(path, "/%d", i);
+            sprintf(path, "/awar%d", i);
             rc = zoo_acreate(zk, path, "", 0,  &ZOO_OPEN_ACL_UNSAFE, 0, stringCompletion, strdup(path));
             CPPUNIT_ASSERT_EQUAL((int)ZOK, rc);
         }
 
         yield(zk, 3);
         for(i = 0; i < COUNT/2; i++) {
-            sprintf(path, "/%d", i);
+            sprintf(path, "/awar%d", i);
             CPPUNIT_ASSERT_MESSAGE(path, waitForEvent(zk, &lctx[i], 5));
             evt = lctx[i].getEvent();
             CPPUNIT_ASSERT_EQUAL_MESSAGE(evt.path.c_str(), ZOO_CREATED_EVENT, evt.type);
@@ -746,7 +753,7 @@ public:
         }
 
         for(i = COUNT/2 + 1; i < COUNT*10; i++) {
-            sprintf(path, "/%d", i);
+            sprintf(path, "/awar%d", i);
             rc = zoo_acreate(zk, path, "", 0,  &ZOO_OPEN_ACL_UNSAFE, 0, stringCompletion, strdup(path));
             CPPUNIT_ASSERT_EQUAL((int)ZOK, rc);
         }
@@ -758,7 +765,7 @@ public:
         CPPUNIT_ASSERT(ctx.waitForConnected(zk));
         yield(zk, 3);
         for(i = COUNT/2+1; i < COUNT; i++) {
-            sprintf(path, "/%d", i);
+            sprintf(path, "/awar%d", i);
             CPPUNIT_ASSERT_MESSAGE(path, waitForEvent(zk, &lctx[i], 5));
             evt = lctx[i].getEvent();
             CPPUNIT_ASSERT_EQUAL_MESSAGE(evt.path, ZOO_CREATED_EVENT, evt.type);
@@ -918,22 +925,44 @@ public:
 
     void testWatcherAutoResetWithGlobal()
     {
+      {
         watchctx_t ctx;
         zhandle_t *zk = createClient(&ctx);
+        int rc = zoo_create(zk, "/testarwg", "", 0, &ZOO_OPEN_ACL_UNSAFE, 0, 0, 0);
+        CPPUNIT_ASSERT_EQUAL((int) ZOK, rc);
+        rc = zoo_create(zk, "/testarwg/arwg", "", 0, &ZOO_OPEN_ACL_UNSAFE, 0, 0, 0);
+        CPPUNIT_ASSERT_EQUAL((int) ZOK, rc);
+      }
+
+      {
+        watchctx_t ctx;
+        zhandle_t *zk = createchClient(&ctx, "127.0.0.1:22181/testarwg/arwg");
+
         testWatcherAutoReset(zk, &ctx, &ctx);
+      }
     }
 
     void testWatcherAutoResetWithLocal()
     {
+      {
         watchctx_t ctx;
-        watchctx_t lctx;
         zhandle_t *zk = createClient(&ctx);
+        int rc = zoo_create(zk, "/testarwl", "", 0, &ZOO_OPEN_ACL_UNSAFE, 0, 0, 0);
+        CPPUNIT_ASSERT_EQUAL((int) ZOK, rc);
+        rc = zoo_create(zk, "/testarwl/arwl", "", 0, &ZOO_OPEN_ACL_UNSAFE, 0, 0, 0);
+        CPPUNIT_ASSERT_EQUAL((int) ZOK, rc);
+      }
+
+      {
+        watchctx_t ctx;
+        watchctx_t lctx;
+        zhandle_t *zk = createchClient(&ctx, "127.0.0.1:22181/testarwl/arwl");
         testWatcherAutoReset(zk, &ctx, &lctx);
+      }
     }
 };
 
 volatile int Zookeeper_simpleSystem::count;
 zhandle_t *Zookeeper_simpleSystem::async_zk;
 const char Zookeeper_simpleSystem::hostPorts[] = "127.0.0.1:22181";
-char* Zookeeper_simpleSystem::hp_chroot;
 CPPUNIT_TEST_SUITE_REGISTRATION(Zookeeper_simpleSystem);

src/c/tests/TestClientRetry.cc

@@ -37,6 +37,8 @@ using namespace std;
 
 #include <zookeeper.h>
 
+#include "Util.h"
+
 #ifdef THREADED
     static void yield(zhandle_t *zh, int i)
     {
@@ -193,12 +195,30 @@ class Zookeeper_clientretry : public CPPUNIT_NS::TestFixture
         return zk;
     }
     
+    FILE *logfile;
 public:
 
+    Zookeeper_clientretry() {
+      logfile = openlogfile("Zookeeper_clientretry");
+    }
+
+    ~Zookeeper_clientretry() {
+      if (logfile) {
+        fflush(logfile);
+        fclose(logfile);
+        logfile = 0;
+      }
+    }
 
     void setUp()
     {
+        zoo_set_log_stream(logfile);
+
         char cmd[1024];
+        sprintf(cmd, "%s stop %s", ZKSERVER_CMD, getHostPorts());
+        CPPUNIT_ASSERT(system(cmd) == 0);
+
+        /* we are testing that if max cnxns is exceeded the server does the right thing */
         sprintf(cmd, "export ZKMAXCNXNS=1;%s startClean %s", ZKSERVER_CMD, getHostPorts());
         CPPUNIT_ASSERT(system(cmd) == 0);
 
@@ -214,6 +234,16 @@ public:
         char cmd[1024];
         sprintf(cmd, "%s stop %s", ZKSERVER_CMD, getHostPorts());
         CPPUNIT_ASSERT(system(cmd) == 0);
+
+        /* restart the server in "normal" mode */
+        sprintf(cmd, "%s startClean %s", ZKSERVER_CMD, getHostPorts());
+        CPPUNIT_ASSERT(system(cmd) == 0);
+
+        struct sigaction act;
+        act.sa_handler = SIG_IGN;
+        sigemptyset(&act.sa_mask);
+        act.sa_flags = 0;
+        CPPUNIT_ASSERT(sigaction(SIGPIPE, &act, NULL) == 0);
     }
 
     bool waitForEvent(zhandle_t *zh, watchctx_t *ctx, int seconds) {
@@ -229,7 +259,6 @@ public:
     void testRetry()
     {
       watchctx_t ctx1, ctx2;
-      zoo_set_debug_level((ZooLogLevel)0); // disable logging
       zhandle_t *zk1 = createClient(&ctx1);
       CPPUNIT_ASSERT_EQUAL(true, ctx1.waitForConnected(zk1));
       zhandle_t *zk2 = createClient(&ctx2);

src/c/tests/TestDriver.cc

@@ -24,13 +24,20 @@
 #include <cppunit/TextTestProgressListener.h>
 #include <cppunit/BriefTestProgressListener.h>
 #include <cppunit/extensions/TestFactoryRegistry.h>
+#include <signal.h>
+#include <stdlib.h>
 #include <stdexcept>
+#include <unistd.h>
+#include <sys/select.h>
 #include <cppunit/Exception.h>
 #include <cppunit/TestFailure.h>
 #include <cppunit/XmlOutputter.h>
+#include <cppunit/TestAssert.h>
 #include <fstream>
+#include <time.h> 
 
 #include "Util.h"
+#include "zookeeper_log.h"
 
 using namespace std;
 
@@ -65,31 +72,84 @@ public:
 
 CPPUNIT_NS_END
 
+class TimingListener : public CPPUNIT_NS::BriefTestProgressListener {
+ public:
+   void startTest( CPPUNIT_NS::Test *test )
+   {
+     gettimeofday(&_start_time, NULL);
+
+     CPPUNIT_NS::BriefTestProgressListener::startTest(test);
+   }
+  
+   void endTest( CPPUNIT_NS::Test *test )
+   {
+     struct timeval end;
+     gettimeofday(&end, NULL);
+
+     long seconds = end.tv_sec - _start_time.tv_sec;
+     long useconds = end.tv_usec - _start_time.tv_usec;
+
+     long mtime = seconds * 1000 + useconds/1000.0;
+     CPPUNIT_NS::stdCOut()  <<  " : elapsed " <<  mtime;
+     CPPUNIT_NS::BriefTestProgressListener::endTest(test);
+   }
+
+ private:
+   struct timeval _start_time;
+};
+
+class ZKServer {
+public:
+    ZKServer() {
+        char cmd[1024];
+        sprintf(cmd, "%s startClean %s", ZKSERVER_CMD, "127.0.0.1:22181");
+        CPPUNIT_ASSERT(system(cmd) == 0);
+
+        struct sigaction act;
+        act.sa_handler = SIG_IGN;
+        sigemptyset(&act.sa_mask);
+        act.sa_flags = 0;
+        CPPUNIT_ASSERT(sigaction(SIGPIPE, &act, NULL) == 0);
+    }
+    virtual ~ZKServer(){
+        char cmd[1024];
+        sprintf(cmd, "%s stop %s", ZKSERVER_CMD, "127.0.0.1:22181");
+        CPPUNIT_ASSERT(system(cmd) == 0);
+    }
+};
+
 int main( int argc, char* argv[] ) { 
    // if command line contains "-ide" then this is the post build check
    // => the output must be in the compiler error format.
    //bool selfTest = (argc > 1) && (std::string("-ide") == argv[1]);
    globalTestConfig.addConfigFromCmdLine(argc,argv);
 
+   ZKServer zkserver;
+
    // Create the event manager and test controller
    CPPUNIT_NS::TestResult controller;
    // Add a listener that colllects test result
    CPPUNIT_NS::TestResultCollector result;
    controller.addListener( &result );
    
-   // Add a listener that print dots as tests run.
-//   CPPUNIT_NS::TextTestProgressListener progress;
-   CPPUNIT_NS::BriefTestProgressListener progress;
+   // A listener that print dots as tests run.
+   // CPPUNIT_NS::TextTestProgressListener progress;
+   // CPPUNIT_NS::BriefTestProgressListener progress;
+
+   // brief + elapsed time
+   TimingListener progress;
    controller.addListener( &progress );
  
    CPPUNIT_NS::TestRunner runner;
    runner.addTest( CPPUNIT_NS::TestFactoryRegistry::getRegistry().makeTest() );
  
-   try
-   {
-     cout << "Running "  <<  globalTestConfig.getTestName();
+   try {
+     CPPUNIT_NS::stdCOut() << "Running " << endl;
+
+     zoo_set_debug_level(ZOO_LOG_LEVEL_INFO);
+     //zoo_set_debug_level(ZOO_LOG_LEVEL_DEBUG);
+
      runner.run( controller, globalTestConfig.getTestName());
-     cout<<endl;
 
      // Print test in a compiler compatible format.
      CPPUNIT_NS::EclipseOutputter outputter( &result,cout);
@@ -103,9 +163,8 @@ int main( int argc, char* argv[] ) {
      xml.write();
      file.close();
 #endif
-   }
-   catch ( std::invalid_argument &e )  // Test path not resolved
-   {
+   } catch ( std::invalid_argument &e ) {
+     // Test path not resolved
      cout<<"\nERROR: "<<e.what()<<endl;
      return 0;
    }

src/c/tests/TestOperations.cc

@@ -40,12 +40,26 @@ class Zookeeper_operations : public CPPUNIT_NS::TestFixture
     CPPUNIT_TEST(testConcurrentOperations1);
     CPPUNIT_TEST_SUITE_END();
     zhandle_t *zh;
+    FILE *logfile;
 
     static void watcher(zhandle_t *, int, int, const char *,void*){}
 public: 
+    Zookeeper_operations() {
+      logfile = openlogfile("Zookeeper_operations");
+    }
+
+    ~Zookeeper_operations() {
+      if (logfile) {
+        fflush(logfile);
+        fclose(logfile);
+        logfile = 0;
+      }
+    }
+
     void setUp()
     {
-        zoo_set_debug_level((ZooLogLevel)0); // disable logging
+        zoo_set_log_stream(logfile);
+
         zoo_deterministic_conn_order(0);
         zh=0;
     }
@@ -108,8 +122,8 @@ public:
         // process the send queue
         rc=zookeeper_interest(zh,&fd,&interest,&tv);
         CPPUNIT_ASSERT_EQUAL((int)ZOK,rc);
-        while((rc=zookeeper_process(zh,interest))==ZOK) printf("%d\n", rc);
-	printf("RC = %d", rc);
+        while((rc=zookeeper_process(zh,interest))==ZOK) millisleep(100); //printf("%d\n", rc);
+        //printf("RC = %d", rc);
         CPPUNIT_ASSERT_EQUAL((int)ZNOTHING,rc);
 
         CPPUNIT_ASSERT_EQUAL((int)ZOK,res1.rc_);

src/c/tests/TestWatchers.cc

@@ -36,12 +36,26 @@ class Zookeeper_watchers : public CPPUNIT_NS::TestFixture
 
     static void watcher(zhandle_t *, int, int, const char *,void*){}
     zhandle_t *zh;
+    FILE *logfile;
     
 public:
 
+    Zookeeper_watchers() {
+      logfile = openlogfile("Zookeeper_watchers");
+    }
+
+    ~Zookeeper_watchers() {
+      if (logfile) {
+        fflush(logfile);
+        fclose(logfile);
+        logfile = 0;
+      }
+    }
+
     void setUp()
     {
-        zoo_set_debug_level((ZooLogLevel)0); // disable logging
+        zoo_set_log_stream(logfile);
+
         zoo_deterministic_conn_order(0);
         zh=0;
     }

src/c/tests/TestZookeeperClose.cc

@@ -39,10 +39,25 @@ class Zookeeper_close : public CPPUNIT_NS::TestFixture
     CPPUNIT_TEST_SUITE_END();
     zhandle_t *zh;
     static void watcher(zhandle_t *, int, int, const char *,void*){}
+    FILE *logfile;
 public: 
+
+    Zookeeper_close() {
+      logfile = openlogfile("Zookeeper_close");
+    }
+
+    ~Zookeeper_close() {
+      if (logfile) {
+        fflush(logfile);
+        fclose(logfile);
+        logfile = 0;
+      }
+    }
+
     void setUp()
     {
-        zoo_set_debug_level((ZooLogLevel)0); // disable logging
+        zoo_set_log_stream(logfile);
+
         zoo_deterministic_conn_order(0);
         zh=0;
     }

src/c/tests/TestZookeeperInit.cc

@@ -54,12 +54,24 @@ class Zookeeper_init : public CPPUNIT_NS::TestFixture
     zhandle_t *zh;
     MockPthreadsNull* pthreadMock;   
     static void watcher(zhandle_t *, int , int , const char *,void*){}
+    FILE *logfile;
 public: 
-    Zookeeper_init():zh(0),pthreadMock(0){}
-    
+    Zookeeper_init():zh(0),pthreadMock(0){
+      logfile = openlogfile("Zookeeper_init");
+    }
+
+    ~Zookeeper_init() {
+      if (logfile) {
+        fflush(logfile);
+        fclose(logfile);
+        logfile = 0;
+      }
+    }
+
     void setUp()
     {
-    	zoo_set_debug_level((ZooLogLevel)0); // disable logging
+        zoo_set_log_stream(logfile);
+
         zoo_deterministic_conn_order(0);
 #ifdef THREADED
         // disable threading

src/c/tests/Util.cc

@@ -17,6 +17,7 @@
  */
 
 #include "Util.h"
+#include "string.h"
 
 const std::string EMPTY_STRING;
 
@@ -28,3 +29,23 @@ void millisleep(int ms){
     ts.tv_nsec=(ms%1000)*1000000; // to nanoseconds
     nanosleep(&ts,0);
 }
+
+FILE *openlogfile(const char* testname) {
+  char name[1024];
+  strcpy(name, "TEST-");
+  strncpy(name + 5, testname, sizeof(name) - 5);
+#ifdef THREADED
+  strcpy(name + strlen(name), "-mt.txt");
+#else
+  strcpy(name + strlen(name), "-st.txt");
+#endif
+
+  FILE *logfile = fopen(name, "a");
+
+  if (logfile == 0) {
+    fprintf(stderr, "Can't open log file %s!\n", name);
+    return 0;
+  }
+
+  return logfile;
+}

src/c/tests/Util.h

@@ -92,6 +92,7 @@ bool getValue(const std::map<K,V>& map,const K& k,V& v){
 
 // millisecond sleep
 void millisleep(int ms);
+FILE *openlogfile(const char* name);
 // evaluate given predicate until it returns true or the timeout 
 // (in millis) has expired
 template<class Predicate>

src/java/test/org/apache/zookeeper/server/DeserializationPerfTest.java

@@ -76,13 +76,13 @@ public class DeserializationPerfTest extends TestCase {
     @Test
     public void testWideDeserialize() throws
             InterruptedException, IOException, KeeperException.NodeExistsException, KeeperException.NoNodeException {
-        deserializeTree(2, 50000, 20);
+        deserializeTree(2, 10000, 20);
     }
 
     @Test
     public void testDeepDeserialize() throws
             InterruptedException, IOException, KeeperException.NodeExistsException, KeeperException.NoNodeException {
-        deserializeTree(1000, 1, 20);
+        deserializeTree(400, 1, 20);
     }
 
     @Test

src/java/test/org/apache/zookeeper/server/SerializationPerfTest.java

@@ -84,13 +84,13 @@ public class SerializationPerfTest extends TestCase {
     @Test
     public void testWideSerialize()
             throws InterruptedException, IOException, KeeperException.NodeExistsException, KeeperException.NoNodeException {
-        serializeTree(2, 50000, 20);
+        serializeTree(2, 10000, 20);
     }
 
     @Test
     public void testDeepSerialize()
             throws InterruptedException, IOException, KeeperException.NodeExistsException, KeeperException.NoNodeException {
-        serializeTree(1000, 1, 20);
+        serializeTree(400, 1, 20);
     }
 
     @Test

src/java/test/org/apache/zookeeper/test/AsyncHammerTest.java

@@ -0,0 +1,194 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zookeeper.test;
+
+import static org.apache.zookeeper.test.ClientBase.CONNECTION_TIMEOUT;
+import static org.apache.zookeeper.test.ClientBase.verifyThreadTerminated;
+
+import java.util.LinkedList;
+
+import junit.framework.TestCase;
+
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.AsyncCallback.DataCallback;
+import org.apache.zookeeper.AsyncCallback.StringCallback;
+import org.apache.zookeeper.AsyncCallback.VoidCallback;
+import org.apache.zookeeper.ZooDefs.Ids;
+import org.apache.zookeeper.data.Stat;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class AsyncHammerTest extends TestCase
+    implements StringCallback, VoidCallback, DataCallback
+{
+    private static final Logger LOG = Logger.getLogger(AsyncHammerTest.class);
+
+    private QuorumBase qb = new QuorumBase();
+
+    private volatile boolean bang;
+
+    @Before
+    @Override
+    protected void setUp() throws Exception {
+        LOG.info("STARTING " + getName());
+        qb.setUp();
+    }
+
+    protected void restart() throws Exception {
+        JMXEnv.setUp();
+        qb.startServers();
+    }
+
+    @After
+    @Override
+    protected void tearDown() throws Exception {
+        LOG.info("Test clients shutting down");
+        qb.tearDown();
+        LOG.info("FINISHED " + getName());
+    }
+
+    /**
+     * Create /test- sequence nodes asynchronously, max 30 outstanding
+     */
+    class HammerThread extends Thread
+        implements Watcher, StringCallback, VoidCallback
+    {
+        private static final int MAX_OUTSTANDING = 30;
+
+        private ZooKeeper zk;
+        private int outstanding;
+
+        public HammerThread(String name) {
+            super(name);
+        }
+
+        public void run() {
+            try {
+                zk = new ZooKeeper(qb.hostPort, CONNECTION_TIMEOUT, this);
+                while(bang) {
+                    incOutstanding(); // before create otw race
+                    zk.create("/test-", new byte[0], Ids.OPEN_ACL_UNSAFE,
+                            CreateMode.PERSISTENT_SEQUENTIAL, this, null);
+                }
+            } catch (InterruptedException e) {
+                if (bang) {
+                    LOG.error("sanity check failed!!!"); // sanity check
+                    return;
+                }
+            } catch (Exception e) {
+                LOG.error("Client create operation failed", e);
+                return;
+            } finally {
+                if (zk != null) {
+                    try {
+                        zk.close();
+                    } catch (InterruptedException e) {
+                        LOG.warn("Unexpected", e);
+                    }
+                }
+            }
+        }
+
+        private synchronized void incOutstanding() throws InterruptedException {
+            outstanding++;
+            while(outstanding > MAX_OUTSTANDING) {
+                wait();
+            }
+        }
+
+        private synchronized void decOutstanding() {
+            outstanding--;
+            assertTrue("outstanding >= 0", outstanding >= 0);
+            notifyAll();
+        }
+
+        public void process(WatchedEvent event) {
+            // ignore for purposes of this test
+        }
+
+        public void processResult(int rc, String path, Object ctx, String name) {
+            try {
+                decOutstanding();
+                zk.delete(path, -1, this, null);
+            } catch (Exception e) {
+                LOG.error("Client delete failed", e);
+            }
+        }
+
+        public void processResult(int rc, String path, Object ctx) {
+            // ignore for purposes of this test
+        }
+    }
+
+    @Test
+    public void testHammer() throws Exception {
+        bang = true;
+        Thread[] hammers = new Thread[100];
+        for (int i = 0; i < hammers.length; i++) {
+            hammers[i] = new HammerThread("HammerThread-" + i);
+            hammers[i].start();
+        }
+        Thread.sleep(5000); // allow the clients to run for max 5sec
+        bang = false;
+        for (int i = 0; i < hammers.length; i++) {
+            hammers[i].interrupt();
+            verifyThreadTerminated(hammers[i], 60000);
+        }
+        // before restart
+        QuorumBase qt = new QuorumBase();
+        qt.setUp();
+        qt.verifyRootOfAllServersMatch(qb.hostPort);
+        tearDown();
+
+        restart();
+
+        // after restart
+        qt.verifyRootOfAllServersMatch(qb.hostPort);
+    }
+
+    @SuppressWarnings("unchecked")
+    public void processResult(int rc, String path, Object ctx, String name) {
+        synchronized(ctx) {
+            ((LinkedList<Integer>)ctx).add(rc);
+            ctx.notifyAll();
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    public void processResult(int rc, String path, Object ctx) {
+        synchronized(ctx) {
+            ((LinkedList<Integer>)ctx).add(rc);
+            ctx.notifyAll();
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    public void processResult(int rc, String path, Object ctx, byte[] data,
+            Stat stat) {
+        synchronized(ctx) {
+            ((LinkedList<Integer>)ctx).add(rc);
+            ctx.notifyAll();
+        }
+    }
+}

src/java/test/org/apache/zookeeper/test/AsyncTest.java

@@ -19,7 +19,6 @@
 package org.apache.zookeeper.test;
 
 import static org.apache.zookeeper.test.ClientBase.CONNECTION_TIMEOUT;
-import static org.apache.zookeeper.test.ClientBase.verifyThreadTerminated;
 
 import java.io.IOException;
 import java.util.LinkedList;
@@ -52,8 +51,6 @@ public class AsyncTest extends TestCase
 
     private QuorumBase qb = new QuorumBase();
 
-    private volatile boolean bang;
-
     @Before
     @Override
     protected void setUp() throws Exception {
@@ -101,105 +98,6 @@ public class AsyncTest extends TestCase
         return zk;
     }
 
-    /**
-     * Create /test- sequence nodes asynchronously, max 30 outstanding
-     */
-    class HammerThread extends Thread
-        implements Watcher, StringCallback, VoidCallback
-    {
-        private static final int MAX_OUTSTANDING = 30;
-
-        private ZooKeeper zk;
-        private int outstanding;
-
-        public HammerThread(String name) {
-            super(name);
-        }
-
-        public void run() {
-            try {
-                zk = new ZooKeeper(qb.hostPort, CONNECTION_TIMEOUT, this);
-                while(bang) {
-                    incOutstanding(); // before create otw race
-                    zk.create("/test-", new byte[0], Ids.OPEN_ACL_UNSAFE,
-                            CreateMode.PERSISTENT_SEQUENTIAL, this, null);
-                }
-            } catch (InterruptedException e) {
-                if (bang) {
-                    LOG.error("sanity check failed!!!"); // sanity check
-                    return;
-                }
-            } catch (Exception e) {
-                LOG.error("Client create operation failed", e);
-                return;
-            } finally {
-                if (zk != null) {
-                    try {
-                        zk.close();
-                    } catch (InterruptedException e) {
-                        LOG.warn("Unexpected", e);
-                    }
-                }
-            }
-        }
-
-        private synchronized void incOutstanding() throws InterruptedException {
-            outstanding++;
-            while(outstanding > MAX_OUTSTANDING) {
-                wait();
-            }
-        }
-
-        private synchronized void decOutstanding() {
-            outstanding--;
-            assertTrue("outstanding >= 0", outstanding >= 0);
-            notifyAll();
-        }
-
-        public void process(WatchedEvent event) {
-            // ignore for purposes of this test
-        }
-
-        public void processResult(int rc, String path, Object ctx, String name) {
-            try {
-                decOutstanding();
-                zk.delete(path, -1, this, null);
-            } catch (Exception e) {
-                LOG.error("Client delete failed", e);
-            }
-        }
-
-        public void processResult(int rc, String path, Object ctx) {
-            // ignore for purposes of this test
-        }
-    }
-
-    @Test
-    public void testHammer() throws Exception {
-        bang = true;
-        Thread[] hammers = new Thread[100];
-        for (int i = 0; i < hammers.length; i++) {
-            hammers[i] = new HammerThread("HammerThread-" + i);
-            hammers[i].start();
-        }
-        Thread.sleep(5000); // allow the clients to run for max 5sec
-        bang = false;
-        for (int i = 0; i < hammers.length; i++) {
-            hammers[i].interrupt();
-            verifyThreadTerminated(hammers[i], 60000);
-        }
-        // before restart
-        QuorumBase qt = new QuorumBase();
-        qt.setUp();
-        qt.verifyRootOfAllServersMatch(qb.hostPort);
-        tearDown();
-
-        restart();
-
-        // after restart
-        qt.verifyRootOfAllServersMatch(qb.hostPort);
-    }
-
     LinkedList<Integer> results = new LinkedList<Integer>();
     @Test
     public void testAsync()
@@ -220,7 +118,7 @@ public class AsyncTest extends TestCase
                 }
             }
             assertEquals(0, (int) results.get(0));
-            assertEquals(Code.NOAUTH, Code.get((int) results.get(1)));
+            assertEquals(Code.NOAUTH, Code.get(results.get(1)));
             assertEquals(0, (int) results.get(2));
             assertEquals(0, (int) results.get(3));
             assertEquals(0, (int) results.get(4));

src/java/test/org/apache/zookeeper/test/ChrootClientTest.java

@@ -23,6 +23,7 @@ import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.ZooKeeper;
 import org.apache.zookeeper.ZooDefs.Ids;
 import org.junit.Before;
+import org.junit.Test;
 
 public class ChrootClientTest extends ClientTest {
     private static final Logger LOG = Logger.getLogger(ChrootClientTest.class);
@@ -46,4 +47,9 @@ public class ChrootClientTest extends ClientTest {
             zk.close();
         }
     }
+    
+    @Test
+    public void testPing() throws Exception {
+        // not necessary to repeat this, expensive and not chroot related
+    }
 }

src/java/test/org/apache/zookeeper/test/ClientBase.java

@@ -23,6 +23,8 @@ import java.io.File;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.OutputStream;
+import java.lang.management.ManagementFactory;
+import java.lang.management.OperatingSystemMXBean;
 import java.net.Socket;
 import java.util.Arrays;
 import java.util.LinkedList;
@@ -46,6 +48,8 @@ import org.apache.zookeeper.server.NIOServerCnxn;
 import org.apache.zookeeper.server.ZooKeeperServer;
 import org.apache.zookeeper.server.persistence.FileTxnLog;
 
+import com.sun.management.UnixOperatingSystemMXBean;
+
 public abstract class ClientBase extends TestCase {
     protected static final Logger LOG = Logger.getLogger(ClientBase.class);
 
@@ -140,12 +144,17 @@ public abstract class ClientBase extends TestCase {
 
     protected TestableZooKeeper createClient(CountdownWatcher watcher, String hp)
         throws IOException, InterruptedException
+    {
+        return createClient(watcher, hp, CONNECTION_TIMEOUT);
+    }
+
+    protected TestableZooKeeper createClient(CountdownWatcher watcher,
+            String hp, int timeout)
+        throws IOException, InterruptedException
     {
         watcher.reset();
-        TestableZooKeeper zk =
-            new TestableZooKeeper(hp, CONNECTION_TIMEOUT, watcher);
-        if (!watcher.clientConnected.await(CONNECTION_TIMEOUT,
-                TimeUnit.MILLISECONDS))
+        TestableZooKeeper zk = new TestableZooKeeper(hp, timeout, watcher);
+        if (!watcher.clientConnected.await(timeout, TimeUnit.MILLISECONDS))
         {
             fail("Unable to connect to server");
         }
@@ -261,7 +270,6 @@ public abstract class ClientBase extends TestCase {
         File tmpFile = File.createTempFile("test", ".junit", parentDir);
         // don't delete tmpFile - this ensures we don't attempt to create
         // a tmpDir with a duplicate name
-        tmpFile.delete();
         File tmpDir = new File(tmpFile + ".dir");
         assertFalse(tmpDir.exists()); // never true if tmpfile does it's job
         assertTrue(tmpDir.mkdirs());
@@ -338,6 +346,20 @@ public abstract class ClientBase extends TestCase {
 
         startServer();
 
+        /* some useful information - log the number of fds used before
+         * and after a test is run. Helps to verify we are freeing resources
+         * correctly. Unfortunately this only works on unix systems (the
+         * only place sun has implemented as part of the mgmt bean api.
+         */
+        OperatingSystemMXBean osMbean =
+            ManagementFactory.getOperatingSystemMXBean();
+        if (osMbean != null && osMbean instanceof UnixOperatingSystemMXBean) {
+            UnixOperatingSystemMXBean unixos =
+                (UnixOperatingSystemMXBean)osMbean;
+            LOG.info("Initial fdcount is: "
+                    + unixos.getOpenFileDescriptorCount());
+        }
+
         LOG.info("Client test setup finished");
     }
 
@@ -373,6 +395,21 @@ public abstract class ClientBase extends TestCase {
     @Override
     protected void tearDown() throws Exception {
         LOG.info("tearDown starting");
+
+        /* some useful information - log the number of fds used before
+         * and after a test is run. Helps to verify we are freeing resources
+         * correctly. Unfortunately this only works on unix systems (the
+         * only place sun has implemented as part of the mgmt bean api.
+         */
+        OperatingSystemMXBean osMbean =
+            ManagementFactory.getOperatingSystemMXBean();
+        if (osMbean != null && osMbean instanceof UnixOperatingSystemMXBean) {
+            UnixOperatingSystemMXBean unixos =
+                (UnixOperatingSystemMXBean)osMbean;
+            LOG.info("fdcount after test is: "
+                    + unixos.getOpenFileDescriptorCount());
+        }
+
         tearDownAll();
 
         stopServer();

src/java/test/org/apache/zookeeper/test/ClientHammerTest.java

@@ -0,0 +1,244 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zookeeper.test;
+
+import java.io.IOException;
+import java.util.Date;
+import java.util.List;
+
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.ZooDefs.Ids;
+import org.junit.Test;
+
+public class ClientHammerTest extends ClientBase {
+    protected static final Logger LOG = Logger.getLogger(ClientHammerTest.class);
+
+    @Override
+    protected void tearDown() throws Exception {
+        super.tearDown();
+        LOG.info("FINISHED " + getName());
+    }
+    private static final long HAMMERTHREAD_LATENCY = 5;
+
+    private static abstract class HammerThread extends Thread {
+        protected final int count;
+        protected volatile int current = 0;
+
+        HammerThread(String name, int count) {
+            super(name);
+            this.count = count;
+        }
+    }
+
+    private static class BasicHammerThread extends HammerThread {
+        private final ZooKeeper zk;
+        private final String prefix;
+
+        BasicHammerThread(String name, ZooKeeper zk, String prefix, int count) {
+            super(name, count);
+            this.zk = zk;
+            this.prefix = prefix;
+        }
+
+        public void run() {
+            byte b[] = new byte[256];
+            try {
+                for (; current < count; current++) {
+                    // Simulate a bit of network latency...
+                    Thread.sleep(HAMMERTHREAD_LATENCY);
+                    zk.create(prefix + current, b, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+                }
+            } catch (Throwable t) {
+                LOG.error("Client create operation failed", t);
+            } finally {
+                try {
+                    zk.close();
+                } catch (InterruptedException e) {
+                    LOG.warn("Unexpected", e);
+                }
+            }
+        }
+    }
+
+    private static class SuperHammerThread extends HammerThread {
+        private final ClientHammerTest parent;
+        private final String prefix;
+
+        SuperHammerThread(String name, ClientHammerTest parent, String prefix,
+                int count)
+        {
+            super(name, count);
+            this.parent = parent;
+            this.prefix = prefix;
+        }
+
+        public void run() {
+            byte b[] = new byte[256];
+            try {
+                for (; current < count; current++) {
+                    ZooKeeper zk = parent.createClient();
+                    try {
+                        zk.create(prefix + current, b, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+                    } finally {
+                        try {
+                            zk.close();
+                        } catch (InterruptedException e) {
+                            LOG.warn("Unexpected", e);
+                        }
+                    }
+                }
+            } catch (Throwable t) {
+                LOG.error("Client create operation failed", t);
+            }
+        }
+    }
+
+    /**
+     * Separate threads each creating a number of nodes. Each thread
+     * is using a non-shared (owned by thread) client for all node creations.
+     * @throws Throwable
+     */
+    @Test
+    public void testHammerBasic() throws Throwable {
+        runHammer(10, 1000);
+    }
+
+    public void runHammer(final int threadCount, final int childCount)
+        throws Throwable
+    {
+        try {
+            HammerThread[] threads = new HammerThread[threadCount];
+            long start = System.currentTimeMillis();
+            for (int i = 0; i < threads.length; i++) {
+                ZooKeeper zk = createClient();
+                String prefix = "/test-" + i;
+                zk.create(prefix, new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+                prefix += "/";
+                HammerThread thread =
+                    new BasicHammerThread("BasicHammerThread-" + i, zk, prefix,
+                            childCount);
+                thread.start();
+
+                threads[i] = thread;
+            }
+
+            verifyHammer(start, threads, childCount);
+        } catch (Throwable t) {
+            LOG.error("test failed", t);
+            throw t;
+        }
+    }
+
+    /**
+     * Separate threads each creating a number of nodes. Each thread
+     * is creating a new client for each node creation.
+     * @throws Throwable
+     */
+    @Test
+    public void testHammerSuper() throws Throwable {
+        try {
+            final int threadCount = 5;
+            final int childCount = 10;
+
+            HammerThread[] threads = new HammerThread[threadCount];
+            long start = System.currentTimeMillis();
+            for (int i = 0; i < threads.length; i++) {
+                String prefix = "/test-" + i;
+                {
+                    ZooKeeper zk = createClient();
+                    try {
+                        zk.create(prefix, new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+                    } finally {
+                        zk.close();
+                    }
+                }
+                prefix += "/";
+                HammerThread thread =
+                    new SuperHammerThread("SuperHammerThread-" + i, this,
+                            prefix, childCount);
+                thread.start();
+
+                threads[i] = thread;
+            }
+
+            verifyHammer(start, threads, childCount);
+        } catch (Throwable t) {
+            LOG.error("test failed", t);
+            throw t;
+        }
+    }
+
+    public void verifyHammer(long start, HammerThread[] threads, int childCount)
+        throws IOException, InterruptedException, KeeperException
+    {
+        // look for the clients to finish their create operations
+        LOG.info("Starting check for completed hammers");
+        int workingCount = threads.length;
+        for (int i = 0; i < 120; i++) {
+            Thread.sleep(10000);
+            for (HammerThread h : threads) {
+                if (!h.isAlive() || h.current == h.count) {
+                    workingCount--;
+                }
+            }
+            if (workingCount == 0) {
+                break;
+            }
+            workingCount = threads.length;
+        }
+        if (workingCount > 0) {
+            for (HammerThread h : threads) {
+                LOG.warn(h.getName() + " never finished creation, current:"
+                        + h.current);
+            }
+        } else {
+            LOG.info("Hammer threads completed creation operations");
+        }
+
+        for (HammerThread h : threads) {
+            final int safetyFactor = 3;
+            verifyThreadTerminated(h,
+                    threads.length * childCount
+                    * HAMMERTHREAD_LATENCY * safetyFactor);
+        }
+        LOG.info(new Date() + " Total time "
+                + (System.currentTimeMillis() - start));
+
+        ZooKeeper zk = createClient();
+        try {
+            LOG.info("******************* Connected to ZooKeeper" + new Date());
+            for (int i = 0; i < threads.length; i++) {
+                LOG.info("Doing thread: " + i + " " + new Date());
+                List<String> children =
+                    zk.getChildren("/test-" + i, false);
+                assertEquals(childCount, children.size());
+            }
+            for (int i = 0; i < threads.length; i++) {
+                List<String> children =
+                    zk.getChildren("/test-" + i, false);
+                assertEquals(childCount, children.size());
+            }
+        } finally {
+            zk.close();
+        }
+    }
+}

src/java/test/org/apache/zookeeper/test/ClientTest.java

@@ -19,9 +19,10 @@
 package org.apache.zookeeper.test;
 
 import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.lang.management.OperatingSystemMXBean;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.Date;
 import java.util.List;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeUnit;
@@ -43,6 +44,8 @@ import org.apache.zookeeper.data.Stat;
 import org.apache.zookeeper.server.PrepRequestProcessor;
 import org.junit.Test;
 
+import com.sun.management.UnixOperatingSystemMXBean;
+
 public class ClientTest extends ClientBase {
     protected static final Logger LOG = Logger.getLogger(ClientTest.class);
 
@@ -52,21 +55,25 @@ public class ClientTest extends ClientBase {
         LOG.info("FINISHED " + getName());
     }
 
+    /** Verify that pings are sent, keeping the "idle" client alive */
     @Test
     public void testPing() throws Exception {
         ZooKeeper zkIdle = null;
         ZooKeeper zkWatchCreator = null;
         try {
-            zkIdle = createClient();
+            CountdownWatcher watcher = new CountdownWatcher();
+            zkIdle = createClient(watcher, hostPort, 10000);
+
             zkWatchCreator = createClient();
 
-            for (int i = 0; i < 30; i++) {
-                zkWatchCreator.create("/" + i, new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+            for (int i = 0; i < 10; i++) {
+                zkWatchCreator.create("/" + i, new byte[0], Ids.OPEN_ACL_UNSAFE,
+                        CreateMode.PERSISTENT);
             }
-            for (int i = 0; i < 30; i++) {
+            for (int i = 0; i < 10; i++) {
                 zkIdle.exists("/" + i, true);
             }
-            for (int i = 0; i < 30; i++) {
+            for (int i = 0; i < 10; i++) {
                 Thread.sleep(1000);
                 zkWatchCreator.delete("/" + i, -1);
             }
@@ -593,209 +600,6 @@ public class ClientTest extends ClientBase {
         zk.close();
     }
 
-    private static final long HAMMERTHREAD_LATENCY = 5;
-
-    private static abstract class HammerThread extends Thread {
-        protected final int count;
-        protected volatile int current = 0;
-
-        HammerThread(String name, int count) {
-            super(name);
-            this.count = count;
-        }
-    }
-
-    private static class BasicHammerThread extends HammerThread {
-        private final ZooKeeper zk;
-        private final String prefix;
-
-        BasicHammerThread(String name, ZooKeeper zk, String prefix, int count) {
-            super(name, count);
-            this.zk = zk;
-            this.prefix = prefix;
-        }
-
-        public void run() {
-            byte b[] = new byte[256];
-            try {
-                for (; current < count; current++) {
-                    // Simulate a bit of network latency...
-                    Thread.sleep(HAMMERTHREAD_LATENCY);
-                    zk.create(prefix + current, b, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
-                }
-            } catch (Throwable t) {
-                LOG.error("Client create operation failed", t);
-            } finally {
-                try {
-                    zk.close();
-                } catch (InterruptedException e) {
-                    LOG.warn("Unexpected", e);
-                }
-            }
-        }
-    }
-
-    private static class SuperHammerThread extends HammerThread {
-        private final ClientTest parent;
-        private final String prefix;
-
-        SuperHammerThread(String name, ClientTest parent, String prefix,
-                int count)
-        {
-            super(name, count);
-            this.parent = parent;
-            this.prefix = prefix;
-        }
-
-        public void run() {
-            byte b[] = new byte[256];
-            try {
-                for (; current < count; current++) {
-                    ZooKeeper zk = parent.createClient();
-                    try {
-                        zk.create(prefix + current, b, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
-                    } finally {
-                        try {
-                            zk.close();
-                        } catch (InterruptedException e) {
-                            LOG.warn("Unexpected", e);
-                        }
-                    }
-                }
-            } catch (Throwable t) {
-                LOG.error("Client create operation failed", t);
-            }
-        }
-    }
-
-    /**
-     * Separate threads each creating a number of nodes. Each thread
-     * is using a non-shared (owned by thread) client for all node creations.
-     * @throws Throwable
-     */
-    @Test
-    public void testHammerBasic() throws Throwable {
-        try {
-            final int threadCount = 10;
-            final int childCount = 1000;
-
-            HammerThread[] threads = new HammerThread[threadCount];
-            long start = System.currentTimeMillis();
-            for (int i = 0; i < threads.length; i++) {
-                ZooKeeper zk = createClient();
-                String prefix = "/test-" + i;
-                zk.create(prefix, new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
-                prefix += "/";
-                HammerThread thread =
-                    new BasicHammerThread("BasicHammerThread-" + i, zk, prefix,
-                            childCount);
-                thread.start();
-
-                threads[i] = thread;
-            }
-
-            verifyHammer(start, threads, childCount);
-        } catch (Throwable t) {
-            LOG.error("test failed", t);
-            throw t;
-        }
-    }
-
-    /**
-     * Separate threads each creating a number of nodes. Each thread
-     * is creating a new client for each node creation.
-     * @throws Throwable
-     */
-    @Test
-    public void testHammerSuper() throws Throwable {
-        try {
-            final int threadCount = 5;
-            final int childCount = 10;
-
-            HammerThread[] threads = new HammerThread[threadCount];
-            long start = System.currentTimeMillis();
-            for (int i = 0; i < threads.length; i++) {
-                String prefix = "/test-" + i;
-                {
-                    ZooKeeper zk = createClient();
-                    try {
-                        zk.create(prefix, new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
-                    } finally {
-                        zk.close();
-                    }
-                }
-                prefix += "/";
-                HammerThread thread =
-                    new SuperHammerThread("SuperHammerThread-" + i, this,
-                            prefix, childCount);
-                thread.start();
-
-                threads[i] = thread;
-            }
-
-            verifyHammer(start, threads, childCount);
-        } catch (Throwable t) {
-            LOG.error("test failed", t);
-            throw t;
-        }
-    }
-
-    public void verifyHammer(long start, HammerThread[] threads, int childCount)
-        throws IOException, InterruptedException, KeeperException
-    {
-        // look for the clients to finish their create operations
-        LOG.info("Starting check for completed hammers");
-        int workingCount = threads.length;
-        for (int i = 0; i < 120; i++) {
-            Thread.sleep(10000);
-            for (HammerThread h : threads) {
-                if (!h.isAlive() || h.current == h.count) {
-                    workingCount--;
-                }
-            }
-            if (workingCount == 0) {
-                break;
-            }
-            workingCount = threads.length;
-        }
-        if (workingCount > 0) {
-            for (HammerThread h : threads) {
-                LOG.warn(h.getName() + " never finished creation, current:"
-                        + h.current);
-            }
-        } else {
-            LOG.info("Hammer threads completed creation operations");
-        }
-
-        for (HammerThread h : threads) {
-            final int safetyFactor = 3;
-            verifyThreadTerminated(h,
-                    threads.length * childCount
-                    * HAMMERTHREAD_LATENCY * safetyFactor);
-        }
-        LOG.info(new Date() + " Total time "
-                + (System.currentTimeMillis() - start));
-
-        ZooKeeper zk = createClient();
-        try {
-
-            LOG.info("******************* Connected to ZooKeeper" + new Date());
-            for (int i = 0; i < threads.length; i++) {
-                LOG.info("Doing thread: " + i + " " + new Date());
-                List<String> children =
-                    zk.getChildren("/test-" + i, false);
-                assertEquals(childCount, children.size());
-            }
-            for (int i = 0; i < threads.length; i++) {
-                List<String> children =
-                    zk.getChildren("/test-" + i, false);
-                assertEquals(childCount, children.size());
-            }
-        } finally {
-            zk.close();
-        }
-    }
-
     private class VerifyClientCleanup extends Thread {
         int count;
         int current = 0;
@@ -826,8 +630,26 @@ public class ClientTest extends ClientBase {
      */
     @Test
     public void testClientCleanup() throws Throwable {
-        final int threadCount = 20;
-        final int clientCount = 100;
+        OperatingSystemMXBean osMbean =
+            ManagementFactory.getOperatingSystemMXBean();
+        if (osMbean == null 
+                || !(osMbean instanceof UnixOperatingSystemMXBean))
+        {
+            LOG.warn("skipping testClientCleanup, only available on Unix");
+            return;
+        }
+
+        final int threadCount = 3;
+        final int clientCount = 10;
+
+        /* Log the number of fds used before and after a test is run. Verifies
+         * we are freeing resources correctly. Unfortunately this only works
+         * on unix systems (the only place sun has implemented as part of the
+         * mgmt bean api).
+         */
+        UnixOperatingSystemMXBean unixos =
+            (UnixOperatingSystemMXBean) osMbean;
+        long initialFdCount = unixos.getOpenFileDescriptorCount();
 
         VerifyClientCleanup threads[] = new VerifyClientCleanup[threadCount];
 
@@ -840,5 +662,10 @@ public class ClientTest extends ClientBase {
             threads[i].join(600000);
             assertTrue(threads[i].current == threads[i].count);
         }
+        
+        // if this fails it means we are not cleaning up after the closed
+        // sessions.
+        assertTrue("open fds after test are not significantly higher than before",
+                unixos.getOpenFileDescriptorCount() <= initialFdCount + 10);
     }
 }

src/java/test/org/apache/zookeeper/test/CnxManagerTest.java

@@ -19,24 +19,18 @@
 package org.apache.zookeeper.test;
 
 import java.io.File;
-import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.nio.ByteBuffer;
-import java.nio.channels.SocketChannel;
-import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
 
 import junit.framework.TestCase;
 
 import org.apache.log4j.Logger;
 import org.apache.zookeeper.PortAssignment;
-import org.apache.zookeeper.server.quorum.FastLeaderElection;
 import org.apache.zookeeper.server.quorum.QuorumCnxManager;
-import org.apache.zookeeper.server.quorum.QuorumCnxManager.Message;
 import org.apache.zookeeper.server.quorum.QuorumPeer;
-import org.apache.zookeeper.server.quorum.Vote;
+import org.apache.zookeeper.server.quorum.QuorumCnxManager.Message;
 import org.apache.zookeeper.server.quorum.QuorumPeer.QuorumServer;
 import org.apache.zookeeper.server.quorum.QuorumPeer.ServerState;
 import org.junit.Test;

src/java/test/org/apache/zookeeper/test/DBSizeTest.java

@@ -60,9 +60,10 @@ public class DBSizeTest extends ClientBase {
             zk.create(path, data, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
             long baseLatency = System.currentTimeMillis() - startTime;
             
-            for(int i = 0; i < 16; i++) {
+            for(int i = 0; i < 1 /* was 16 */; i++) {
                 startTime = System.currentTimeMillis();
-                zk.create(path + '/' + i, data, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+                zk.create(path + '/' + i, data, Ids.OPEN_ACL_UNSAFE,
+                        CreateMode.PERSISTENT);
                 long latency = System.currentTimeMillis() - startTime;
                 System.out.println("Latency = " + latency);
                 //assertTrue(latency < baseLatency + 10);
@@ -71,12 +72,14 @@ public class DBSizeTest extends ClientBase {
                             CreateMode.EPHEMERAL, new AsyncCallback.StringCallback() {
                         public void processResult(int rc, String path,
                                 Object ctx, String name) {
+                            // empty
                         }}, null);
                 }
             }
         } finally {
-            if(zk != null)
+            if (zk != null) {
                 zk.close();
+            }
         }
     }
 

src/java/test/org/apache/zookeeper/test/FLELostMessageTest.java

@@ -22,9 +22,7 @@ import java.io.File;
 import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.nio.ByteBuffer;
-import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.concurrent.Semaphore;
 
 import junit.framework.TestCase;
 

src/java/test/org/apache/zookeeper/test/HierarchicalQuorumTest.java

@@ -34,7 +34,6 @@ import org.apache.zookeeper.server.quorum.QuorumPeer;
 import org.apache.zookeeper.server.quorum.QuorumPeer.QuorumServer;
 import org.apache.zookeeper.server.quorum.flexible.QuorumHierarchical;
 import org.junit.After;
-import org.junit.Before;
 import org.junit.Test;
 
 /**
@@ -64,7 +63,7 @@ public class HierarchicalQuorumTest extends ClientBase {
     private int leport5;
 
     Properties qp;
-    private final ClientTest ct = new ClientTest();
+    private final ClientHammerTest cht = new ClientHammerTest();
     
     @Override
     protected void setUp() throws Exception {
@@ -113,8 +112,8 @@ public class HierarchicalQuorumTest extends ClientBase {
         qp.load(is);
         startServers();
 
-        ct.hostPort = hostPort;
-        ct.setUpAll();
+        cht.hostPort = hostPort;
+        cht.setUpAll();
         
         LOG.info("Setup finished");
     }
@@ -212,7 +211,7 @@ public class HierarchicalQuorumTest extends ClientBase {
     @Override
     protected void tearDown() throws Exception {
         LOG.info("TearDown started");
-        ct.tearDownAll();
+        cht.tearDownAll();
 
         LOG.info("Shutting down server 1");
         shutdown(s1);
@@ -267,6 +266,6 @@ public class HierarchicalQuorumTest extends ClientBase {
 
     @Test
     public void testHierarchicalQuorum() throws Throwable {
-        ct.testHammerBasic();
+        cht.runHammer(5, 10);
     }
 }

src/java/test/org/apache/zookeeper/test/QuorumBase.java

@@ -20,6 +20,8 @@ package org.apache.zookeeper.test;
 
 import java.io.File;
 import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.lang.management.OperatingSystemMXBean;
 import java.net.InetSocketAddress;
 import java.util.HashMap;
 import java.util.LinkedHashSet;
@@ -32,6 +34,8 @@ import org.apache.zookeeper.server.quorum.QuorumPeer;
 import org.apache.zookeeper.server.quorum.QuorumPeer.QuorumServer;
 import org.junit.After;
 
+import com.sun.management.UnixOperatingSystemMXBean;
+
 public class QuorumBase extends ClientBase {
     private static final Logger LOG = Logger.getLogger(QuorumBase.class);
 
@@ -72,6 +76,15 @@ public class QuorumBase extends ClientBase {
 
         startServers();
 
+        OperatingSystemMXBean osMbean =
+            ManagementFactory.getOperatingSystemMXBean();
+        if (osMbean != null && osMbean instanceof UnixOperatingSystemMXBean) {
+            UnixOperatingSystemMXBean unixos =
+                (UnixOperatingSystemMXBean)osMbean;
+            LOG.info("Initial fdcount is: "
+                    + unixos.getOpenFileDescriptorCount());
+        }
+
         LOG.info("Setup finished");
     }
     void startServers() throws Exception {
@@ -147,6 +160,16 @@ public class QuorumBase extends ClientBase {
     @Override
     protected void tearDown() throws Exception {
         LOG.info("TearDown started");
+        
+        OperatingSystemMXBean osMbean =
+            ManagementFactory.getOperatingSystemMXBean();
+        if (osMbean != null && osMbean instanceof UnixOperatingSystemMXBean) {
+            UnixOperatingSystemMXBean unixos =
+                (UnixOperatingSystemMXBean)osMbean;
+            LOG.info("fdcount after test is: "
+                    + unixos.getOpenFileDescriptorCount());
+        }
+
         shutdown(s1);
         shutdown(s2);
         shutdown(s3);

src/java/test/org/apache/zookeeper/test/QuorumHammerTest.java

@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zookeeper.test;
+import org.apache.log4j.Logger;
+import org.junit.Before;
+import org.junit.Test;
+
+public class QuorumHammerTest extends QuorumBase {
+    private static final Logger LOG = Logger.getLogger(QuorumHammerTest.class);
+    public static final long CONNECTION_TIMEOUT = ClientTest.CONNECTION_TIMEOUT;
+
+    private final QuorumBase qb = new QuorumBase();
+    private final ClientHammerTest cht = new ClientHammerTest();
+
+    @Before
+    @Override
+    protected void setUp() throws Exception {
+        qb.setUp();
+        cht.hostPort = qb.hostPort;
+        cht.setUpAll();
+    }
+
+    protected void tearDown() throws Exception {
+        cht.tearDownAll();
+        qb.tearDown();
+    }
+    
+    @Test
+    public void testHammerBasic() throws Throwable {
+        cht.testHammerBasic();
+    }
+}

src/java/test/org/apache/zookeeper/test/QuorumTest.java

@@ -17,8 +17,6 @@
  */
 
 package org.apache.zookeeper.test;
-import static org.apache.zookeeper.test.ClientBase.CONNECTION_TIMEOUT;
-
 import java.io.IOException;
 import java.util.ArrayList;
 
@@ -30,10 +28,10 @@ import org.apache.zookeeper.WatchedEvent;
 import org.apache.zookeeper.Watcher;
 import org.apache.zookeeper.ZooDefs;
 import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.ZooDefs.Ids;
 import org.apache.zookeeper.data.Stat;
 import org.apache.zookeeper.server.quorum.FollowerHandler;
 import org.apache.zookeeper.server.quorum.Leader;
-import org.apache.zookeeper.ZooDefs.Ids;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -62,11 +60,6 @@ public class QuorumTest extends QuorumBase {
         ct.testDeleteWithChildren();
     }
 
-    @Test
-    public void testHammerBasic() throws Throwable {
-        ct.testHammerBasic();
-    }
-
     @Test
     public void testPing() throws Exception {
         ct.testPing();

src/java/test/org/apache/zookeeper/test/SessionTest.java

@@ -100,6 +100,13 @@ public class SessionTest extends TestCase implements Watcher {
         return createClient(CONNECTION_TIMEOUT, watcher);
     }
 
+    private DisconnectableZooKeeper createClient(int timeout)
+        throws IOException, InterruptedException
+    {
+        CountdownWatcher watcher = new CountdownWatcher();
+        return createClient(timeout, watcher);
+    }
+
     private DisconnectableZooKeeper createClient(int timeout,
             CountdownWatcher watcher)
         throws IOException, InterruptedException
@@ -190,28 +197,27 @@ public class SessionTest extends TestCase implements Watcher {
      * Make sure ephemerals get cleaned up when a session times out.
      */
     public void testSessionTimeout() throws Exception {
-        int oldTimeout = CONNECTION_TIMEOUT;
-        CONNECTION_TIMEOUT = 5000;
-        try {
-            DisconnectableZooKeeper zk = createClient();
-            zk.create("/stest", new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL);
-            zk.disconnect();
-            Thread.sleep(CONNECTION_TIMEOUT*2);
-            zk = createClient();
-            System.err.println("This test fails");
-            zk.create("/stest", new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL);
-            tearDown();
-            zk.close();
-            zk.disconnect();
-            setUp();
-            zk = createClient();
-            assertTrue(zk.exists("/stest", false) != null);
-            Thread.sleep(CONNECTION_TIMEOUT * 2);
-            assertTrue(zk.exists("/stest", false) == null);
-            zk.close();
-        } finally {
-            CONNECTION_TIMEOUT = oldTimeout;
-        }
+        final int TIMEOUT = 5000;
+        DisconnectableZooKeeper zk = createClient(TIMEOUT);
+        zk.create("/stest", new byte[0], Ids.OPEN_ACL_UNSAFE,
+                CreateMode.EPHEMERAL);
+        zk.disconnect();
+
+        Thread.sleep(TIMEOUT*2);
+
+        zk = createClient(TIMEOUT);
+        zk.create("/stest", new byte[0], Ids.OPEN_ACL_UNSAFE,
+                CreateMode.EPHEMERAL);
+        tearDown();
+        zk.close();
+        zk.disconnect();
+        setUp();
+
+        zk = createClient(TIMEOUT);
+        assertTrue(zk.exists("/stest", false) != null);
+        Thread.sleep(TIMEOUT*2);
+        assertTrue(zk.exists("/stest", false) == null);
+        zk.close();
     }
 
     /**

src/java/test/org/apache/zookeeper/test/TruncateTest.java

@@ -23,6 +23,8 @@ import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.util.HashMap;
 
+import junit.framework.TestCase;
+
 import org.apache.log4j.Logger;
 import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.KeeperException;
@@ -32,13 +34,11 @@ import org.apache.zookeeper.ZooDefs;
 import org.apache.zookeeper.ZooKeeper;
 import org.apache.zookeeper.data.Stat;
 import org.apache.zookeeper.server.NIOServerCnxn;
+import org.apache.zookeeper.server.quorum.QuorumPeer;
+import org.apache.zookeeper.server.quorum.QuorumPeer.QuorumServer;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
-import org.apache.zookeeper.server.quorum.QuorumPeer;
-import org.apache.zookeeper.server.quorum.QuorumPeer.QuorumServer;
-
-import junit.framework.TestCase;
 
 public class TruncateTest extends TestCase {
 	private static final Logger LOG = Logger.getLogger(TruncateTest.class);

src/java/test/org/apache/zookeeper/test/WatcherTest.java

@@ -141,8 +141,8 @@ public class WatcherTest extends ClientBase {
        MyStatCallback cbs[] = new MyStatCallback[COUNT];
        MyWatcher watcher = new MyWatcher();
        int count[] = new int[1];
-       TestableZooKeeper zk = createClient(watcher, hostPort);
-       ZooKeeper zk2 = createClient(watcher, hostPort);
+       TestableZooKeeper zk = createClient(watcher, hostPort, 6000);
+       ZooKeeper zk2 = createClient(watcher, hostPort, 5000);
        zk2.create("/test", new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL);
        for(int i = 0; i < COUNT/2; i++) {
            watches[i] = new MyWatcher();
@@ -150,7 +150,7 @@ public class WatcherTest extends ClientBase {
            zk.exists("/test", watches[i], cbs[i], count);
        }
        zk.exists("/test", false);
-       zk.pauseCnxn(4000);
+       zk.pauseCnxn(3000);
        Thread.sleep(50);
        zk2.close();
        stopServer();
@@ -177,12 +177,14 @@ public class WatcherTest extends ClientBase {
        assertEquals(COUNT, count[0]);
        zk.close();
     }
+    
+    final int TIMEOUT = 5000;
 
     @Test
     public void testWatcherAutoResetWithGlobal() throws Exception {
         ZooKeeper zk = null;
         MyWatcher watcher = new MyWatcher();
-        zk = createClient(watcher, hostPort);
+        zk = createClient(watcher, hostPort, TIMEOUT);
         testWatcherAutoReset(zk, watcher, watcher);
         zk.close();
     }
@@ -191,7 +193,7 @@ public class WatcherTest extends ClientBase {
     public void testWatcherAutoResetWithLocal() throws Exception {
         ZooKeeper zk = null;
         MyWatcher watcher = new MyWatcher();
-        zk = createClient(watcher, hostPort);
+        zk = createClient(watcher, hostPort, TIMEOUT);
         testWatcherAutoReset(zk, watcher, new MyWatcher());
         zk.close();
     }
@@ -212,8 +214,10 @@ public class WatcherTest extends ClientBase {
             MyWatcher localWatcher) throws Exception {
         boolean isGlobal = (localWatcher == globalWatcher);
         // First test to see if the watch survives across reconnects
-        zk.create("/watchtest", new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
-        zk.create("/watchtest/child", new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL);
+        zk.create("/watchtest", new byte[0], Ids.OPEN_ACL_UNSAFE,
+                CreateMode.PERSISTENT);
+        zk.create("/watchtest/child", new byte[0], Ids.OPEN_ACL_UNSAFE,
+                CreateMode.EPHEMERAL);
         if (isGlobal) {
             zk.getChildren("/watchtest", true);
             zk.getData("/watchtest/child", true, new Stat());
@@ -240,56 +244,57 @@ public class WatcherTest extends ClientBase {
         zk.create("/watchtest/child2", new byte[0], Ids.OPEN_ACL_UNSAFE,
                 CreateMode.PERSISTENT);
 
-        WatchedEvent e = localWatcher.events.poll(10, TimeUnit.SECONDS);
+        WatchedEvent e;
         if (!ClientCnxn.getDisableAutoResetWatch()) {
+            e = localWatcher.events.poll(TIMEOUT, TimeUnit.MILLISECONDS);
             assertEquals(e.getPath(), EventType.NodeDataChanged, e.getType());
             assertEquals("/watchtest/child", e.getPath());
         } else {
-            assertNull("unexpected event", e);
+            // we'll catch this later if it does happen after timeout, so
+            // why waste the time on poll
         }
 
-        e = localWatcher.events.poll(10, TimeUnit.SECONDS);
         if (!ClientCnxn.getDisableAutoResetWatch()) {
+            e = localWatcher.events.poll(TIMEOUT, TimeUnit.MILLISECONDS);
             // The create will trigger the get children and the exist
             // watches
             assertEquals(EventType.NodeCreated, e.getType());
             assertEquals("/watchtest/child2", e.getPath());
         } else {
-            assertNull("unexpected event", e);
+            // we'll catch this later if it does happen after timeout, so
+            // why waste the time on poll
         }
 
-        e = localWatcher.events.poll(10, TimeUnit.SECONDS);
         if (!ClientCnxn.getDisableAutoResetWatch()) {
+            e = localWatcher.events.poll(TIMEOUT, TimeUnit.MILLISECONDS);
             assertEquals(EventType.NodeChildrenChanged, e.getType());
             assertEquals("/watchtest", e.getPath());
         } else {
-            assertNull("unexpected event", e);
+            // we'll catch this later if it does happen after timeout, so
+            // why waste the time on poll
         }
 
-        // Make sure PINGs don't screw us up!
-        Thread.sleep(4000);
-
         assertTrue(localWatcher.events.isEmpty()); // ensure no late arrivals
         stopServer();
-        globalWatcher.waitForDisconnected(3000);
+        globalWatcher.waitForDisconnected(TIMEOUT);
         try {
-        try {
-            localWatcher.waitForDisconnected(500);
-            if (!isGlobal && !ClientCnxn.getDisableAutoResetWatch()) {
-                fail("Got an event when I shouldn't have");
-            }
-        } catch(TimeoutException toe) {
-            if (ClientCnxn.getDisableAutoResetWatch()) {
-                fail("Didn't get an event when I should have");
+            try {
+                localWatcher.waitForDisconnected(500);
+                if (!isGlobal && !ClientCnxn.getDisableAutoResetWatch()) {
+                    fail("Got an event when I shouldn't have");
+                }
+            } catch(TimeoutException toe) {
+                if (ClientCnxn.getDisableAutoResetWatch()) {
+                    fail("Didn't get an event when I should have");
+                }
+                // Else what we are expecting since there are no outstanding watches
             }
-            // Else what we are expecting since there are no outstanding watches
-        }
         } catch (Exception e1) {
             LOG.error("bad", e1);
             throw new RuntimeException(e1);
         }
         startServer();
-        globalWatcher.waitForConnected(3000);
+        globalWatcher.waitForConnected(TIMEOUT);
 
         if (isGlobal) {
             zk.getChildren("/watchtest", true);
@@ -305,21 +310,21 @@ public class WatcherTest extends ClientBase {
         // it later
         zk.delete("/watchtest/child2", -1);
 
-        e = localWatcher.events.poll(10, TimeUnit.SECONDS);
+        e = localWatcher.events.poll(TIMEOUT, TimeUnit.MILLISECONDS);
         assertEquals(EventType.NodeDeleted, e.getType());
         assertEquals("/watchtest/child2", e.getPath());
 
-        e = localWatcher.events.poll(10, TimeUnit.SECONDS);
+        e = localWatcher.events.poll(TIMEOUT, TimeUnit.MILLISECONDS);
         assertEquals(EventType.NodeChildrenChanged, e.getType());
         assertEquals("/watchtest", e.getPath());
 
         assertTrue(localWatcher.events.isEmpty());
 
         stopServer();
-        globalWatcher.waitForDisconnected(3000);
+        globalWatcher.waitForDisconnected(TIMEOUT);
         localWatcher.waitForDisconnected(500);
         startServer();
-        globalWatcher.waitForConnected(3000);
+        globalWatcher.waitForConnected(TIMEOUT);
         if (!isGlobal && !ClientCnxn.getDisableAutoResetWatch()) {
             localWatcher.waitForConnected(500);
         }
@@ -327,12 +332,13 @@ public class WatcherTest extends ClientBase {
         zk.delete("/watchtest/child", -1);
         zk.delete("/watchtest", -1);
 
-        e = localWatcher.events.poll(10, TimeUnit.SECONDS);
         if (!ClientCnxn.getDisableAutoResetWatch()) {
+            e = localWatcher.events.poll(TIMEOUT, TimeUnit.MILLISECONDS);
             assertEquals(EventType.NodeDeleted, e.getType());
             assertEquals("/watchtest/child", e.getPath());
         } else {
-            assertNull("unexpected event", e);
+            // we'll catch this later if it does happen after timeout, so
+            // why waste the time on poll
         }
 
         // Make sure nothing is straggling!