ZOOKEEPER-111. Significant cleanup of existing tests. (Patrick Hunt via mahadev)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/zookeeper/trunk@686389 13f79535-47bb-0310-9956-ffa450edef68

src/java/Changes.txt

@@ -1,8 +1,14 @@
 Trunk:
   BUGFIXES:
   
-  ZOOKEEPER-110. Build script relies on svnant, which is not compatible with subversion 1.5 working copies (Jakob Homan)
- 
-  ZOOKEEPER-105. Catch Zookeeper exceptions and print on the stderr. (Anthony Urso via Mahadev)
+  ZOOKEEPER-105. Catch Zookeeper exceptions and print on the stderr. 
+  (Anthony Urso via Mahadev)
 
-  ZOOKEEPER-108. Fix sync operation reordering on a Quorum. (Flavio Paiva Junqueira via Mahadev)
+  ZOOKEEPER-108. Fix sync operation reordering on a Quorum. 
+  (Flavio Paiva Junqueira via Mahadev)
+
+  ZOOKEEPER-110. Build script relies on svnant, which is not compatible 
+  with subversion 1.5 working copies (Jakob Homan)
+
+  ZOOKEEPER-111. Significant cleanup of existing tests. (Patrick Hunt via
+  mahadev)

src/java/jmx/org/apache/zookeeper/jmx/server/ConnectionBean.java

@@ -79,8 +79,8 @@ public class ConnectionBean implements ConnectionMXBean, ZKMBeanInfo {
         try {
             zk.closeSession(connection.getSessionId());
         } catch (Exception e) {
-            LOG.warn("Unable to closeSession() for session: "+getSessionId()+
-                    ", "+e.getMessage());
+            LOG.warn("Unable to closeSession() for session: 0x" 
+                    + getSessionId(), e);
         }
     }
     
@@ -89,7 +89,7 @@ public class ConnectionBean implements ConnectionMXBean, ZKMBeanInfo {
     }
     
     public String toString() {
-        return "ConnectionBean{ClientIP="+getSourceIP()+",SessionId="+getSessionId()+"}";
+        return "ConnectionBean{ClientIP="+getSourceIP()+",SessionId=0x"+getSessionId()+"}";
     }
     
     public long getOutstandingRequests() {

src/java/main/org/apache/zookeeper/ClientCnxn.java

@@ -95,7 +95,8 @@ class ClientCnxn {
      */
     private LinkedList<Packet> pendingQueue = new LinkedList<Packet>();
 
-    private LinkedBlockingQueue waitingEvents = new LinkedBlockingQueue();
+    private LinkedBlockingQueue<Object> waitingEvents = 
+        new LinkedBlockingQueue<Object>();
 
     /**
      * These are the packets that need to be sent.
@@ -125,14 +126,14 @@ class ClientCnxn {
     public long getSessionId() {
         return sessionId;
     }
-
+    
     public byte[] getSessionPasswd() {
         return sessionPasswd;
     }
 
     public String toString() {
         StringBuffer sb = new StringBuffer();
-        sb.append("sessionId: ").append(sessionId).append("\n");
+        sb.append("sessionId: 0x").append(Long.toHexString(getSessionId())).append("\n");
         sb.append("lastZxid: ").append(lastZxid).append("\n");
         sb.append("xid: ").append(xid).append("\n");
         sb.append("nextAddrToTry: ").append(nextAddrToTry).append("\n");
@@ -649,7 +650,7 @@ class ClientCnxn {
                 primeConnection(sockKey);
             }
             initialized = false;
-            
+
             /*
              * Reset incomingBuffer
              */
@@ -729,7 +730,9 @@ class ClientCnxn {
                     }
                     selected.clear();
                 } catch (Exception e) {
-                    LOG.warn("Closing: ", e);
+                    LOG.warn("Closing session 0x" 
+                            + Long.toHexString(getSessionId()),
+                            e);
                     cleanup();
                     if (zooKeeper.state.isAlive()) {
                         waitingEvents.add(new WatcherEvent(Event.EventNone,
@@ -797,11 +800,9 @@ class ClientCnxn {
 
     @SuppressWarnings("unchecked")
     public void close() throws IOException {
-        long traceMask = ZooTrace.SESSION_TRACE_MASK;
-        if (ZooTrace.isTraceEnabled(LOG, traceMask)) {
-            ZooTrace.logTraceMessage(LOG, traceMask,
-                    "Close ClientCnxn for session: " + sessionId + "!");
-        }
+        LOG.info("Closing ClientCnxn for session: 0x" 
+                + Long.toHexString(getSessionId()));
+
         sendThread.close();
         waitingEvents.add(eventOfDeath);
     }

src/java/main/org/apache/zookeeper/ZooKeeper.java

@@ -271,6 +271,13 @@ public class ZooKeeper {
                 sessionPasswd);
     }
 
+    /**
+     * The session id for this ZooKeeper client instance. The value returned
+     * is not valid until the client connects to a server and may change
+     * after a re-connect.
+     * 
+     * @return current session id
+     */
     public long getSessionId() {
         return cnxn.getSessionId();
     }
@@ -303,6 +310,8 @@ public class ZooKeeper {
      * @throws InterruptedException
      */
     public synchronized void close() throws InterruptedException {
+        LOG.info("Closing session: 0x" + Long.toHexString(getSessionId()));
+
         RequestHeader h = new RequestHeader();
         h.setType(ZooDefs.OpCode.closeSession);
         cnxn.submitRequest(h, null, null, null);
@@ -311,6 +320,8 @@ public class ZooKeeper {
         } catch (IOException e) {
             LOG.warn("Unexpected exception", e);
         }
+
+        LOG.info("Session: 0x" + Long.toHexString(getSessionId()) + " closed");
     }
 
     /**

src/java/main/org/apache/zookeeper/server/DataTree.java

@@ -21,17 +21,15 @@ package org.apache.zookeeper.server;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 
-import org.apache.log4j.Logger;
-
 import org.apache.jute.InputArchive;
 import org.apache.jute.OutputArchive;
 import org.apache.jute.Record;
+import org.apache.log4j.Logger;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.Watcher;
 import org.apache.zookeeper.KeeperException.Code;
@@ -443,7 +441,7 @@ public class DataTree {
                     ZooTrace.logTraceMessage(LOG,
                                              ZooTrace.SESSION_TRACE_MASK,
                                              "Deleting ephemeral node "
-                                             + path + " for session "
+                                             + path + " for session 0x"
                                              + Long.toHexString(session));
                 } catch (KeeperException e) {
                     LOG.error("FIXMSG",e);
@@ -538,7 +536,7 @@ public class DataTree {
         StringBuffer sb = new StringBuffer("Sessions with Ephemerals ("
                 + keys.size() + "):\n");
         for (long k : keys) {
-            sb.append(Long.toHexString(k));
+            sb.append("0x" + Long.toHexString(k));
             sb.append(":\n");
             HashSet<String> tmp = ephemerals.get(k);
             synchronized(tmp) {

src/java/main/org/apache/zookeeper/server/LogFormatter.java

@@ -56,11 +56,11 @@ public class LogFormatter {
             hdr.deserialize(ia, "hdr");
             System.out.println(DateFormat.getDateTimeInstance(DateFormat.SHORT,
                     DateFormat.LONG).format(new Date(hdr.getTime()))
-                    + " "
+                    + " session 0x"
                     + Long.toHexString(hdr.getClientId())
                     + ":"
                     + hdr.getCxid()
-                    + " "
+                    + " zxid 0x"
                     + Long.toHexString(hdr.getZxid())
                     + " " + TraceFormatter.op2String(hdr.getType()));
             if (logStream.readByte("EOR") != 'B') {

src/java/main/org/apache/zookeeper/server/NIOServerCnxn.java

@@ -214,6 +214,7 @@ public class NIOServerCnxn implements Watcher, ServerCnxn {
                         try {
                             cnxn.close();
                         } catch (Exception e) {
+                            LOG.warn("exception during session close", e);
                         }
                         break;
                     }
@@ -481,9 +482,9 @@ public class NIOServerCnxn implements Watcher, ServerCnxn {
             throw new IOException("ZooKeeperServer not running");
         }
         if (connReq.getLastZxidSeen() > zk.dataTree.lastProcessedZxid) {
-            LOG.error("Client has seen "
+            LOG.error("Client has seen zxid 0x"
                     + Long.toHexString(connReq.getLastZxidSeen())
-                    + " our last zxid is "
+                    + " our last zxid is 0x"
                     + Long.toHexString(zk.dataTree.lastProcessedZxid));
             throw new IOException("We are out of date");
         }
@@ -501,10 +502,10 @@ public class NIOServerCnxn implements Watcher, ServerCnxn {
         if (connReq.getSessionId() != 0) {
             setSessionId(connReq.getSessionId());
             zk.reopenSession(this, sessionId, passwd, sessionTimeout);
-            LOG.warn("Renewing session " + Long.toHexString(sessionId));
+            LOG.warn("Renewing session 0x" + Long.toHexString(sessionId));
         } else {
             zk.createSession(this, passwd, sessionTimeout);
-            LOG.warn("Creating new session "
+            LOG.warn("Creating new session 0x"
                     + Long.toHexString(sessionId));
         }
         initialized = true;
@@ -759,8 +760,9 @@ public class NIOServerCnxn implements Watcher, ServerCnxn {
     synchronized public void process(WatcherEvent event) {
         ReplyHeader h = new ReplyHeader(-1, -1L, 0);
         ZooTrace.logTraceMessage(LOG, ZooTrace.EVENT_DELIVERY_TRACE_MASK,
-                                 "Deliver event " + event + " to "
-                                 + this.sessionId + " through " + this);
+                                 "Deliver event " + event + " to 0x"
+                                 + Long.toHexString(this.sessionId) 
+                                 + " through " + this);
         sendResponse(h, event, "notification");
     }
 
@@ -778,7 +780,7 @@ public class NIOServerCnxn implements Watcher, ServerCnxn {
             ByteBuffer bb = ByteBuffer.wrap(baos.toByteArray());
             bb.putInt(bb.remaining() - 4).rewind();
             sendBuffer(bb);
-            LOG.warn("Finished init of " + Long.toHexString(sessionId)
+            LOG.warn("Finished init of 0x" + Long.toHexString(sessionId)
                     + ": " + valid);
             if (!valid) {
                 sendBuffer(closeConn);

src/java/main/org/apache/zookeeper/server/PrepRequestProcessor.java

@@ -19,16 +19,14 @@
 package org.apache.zookeeper.server;
 
 import java.nio.ByteBuffer;
-import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.concurrent.LinkedBlockingQueue;
 
-import org.apache.log4j.Logger;
-
 import org.apache.jute.Record;
+import org.apache.log4j.Logger;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.ZooDefs;
 import org.apache.zookeeper.KeeperException.Code;
@@ -177,7 +175,7 @@ public class PrepRequestProcessor extends Thread implements RequestProcessor {
     @SuppressWarnings("unchecked")
     protected void pRequest(Request request) {
         // LOG.info("Prep>>> cxid = " + request.cxid + " type = " +
-        // request.type + " id = " + request.sessionId);
+        // request.type + " id = 0x" + Long.toHexString(request.sessionId));
         TxnHeader txnHeader = null;
         Record txn = null;
         try {
@@ -342,7 +340,7 @@ public class PrepRequestProcessor extends Thread implements RequestProcessor {
                                 path2Delete, null, 0, null));
                     }
                 }
-                LOG.info("Processed session termination request for id: "
+                LOG.info("Processed session termination request for id: 0x"
                         + Long.toHexString(request.sessionId));
                 break;
             case OpCode.sync:

src/java/main/org/apache/zookeeper/server/Request.java

@@ -19,7 +19,6 @@
 package org.apache.zookeeper.server;
 
 import java.nio.ByteBuffer;
-import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.jute.Record;
@@ -160,9 +159,9 @@ public class Request {
 
     public String toString() {
         StringBuffer sb = new StringBuffer();
-        sb.append(Long.toHexString(sessionId)).append(" ");
-        sb.append(Long.toHexString(cxid)).append(" ");
-        sb.append(Long.toHexString((hdr == null ? -2 : hdr.getZxid()))).append(
+        sb.append("session 0x").append(Long.toHexString(sessionId));
+        sb.append(" cxid 0x").append(Long.toHexString(cxid));
+        sb.append("zxid 0x").append(Long.toHexString((hdr == null ? -2 : hdr.getZxid()))).append(
                 " ");
         sb
                 .append(

src/java/main/org/apache/zookeeper/server/SessionTrackerImpl.java

@@ -132,7 +132,7 @@ public class SessionTrackerImpl extends Thread implements SessionTracker {
                 if (set != null) {
                     for (Session s : set.sessions) {
                         sessionsById.remove(s.sessionId);
-                        LOG.warn("Expiring "
+                        LOG.warn("Expiring session 0x"
                                 + Long.toHexString(s.sessionId));
                         expirer.expire(s.sessionId);
                     }
@@ -149,7 +149,7 @@ public class SessionTrackerImpl extends Thread implements SessionTracker {
     synchronized public boolean touchSession(long sessionId, int timeout) {
         ZooTrace.logTraceMessage(LOG,
                                  ZooTrace.CLIENT_PING_TRACE_MASK,
-                                 "SessionTrackerImpl --- Touch session: "
+                                 "SessionTrackerImpl --- Touch session: 0x"
                 + Long.toHexString(sessionId) + " with timeout " + timeout);
         Session s = sessionsById.get(sessionId);
         if (s == null) {
@@ -179,7 +179,7 @@ public class SessionTrackerImpl extends Thread implements SessionTracker {
         Session s = sessionsById.remove(sessionId);
         sessionsWithTimeout.remove(sessionId);
         ZooTrace.logTraceMessage(LOG, ZooTrace.SESSION_TRACE_MASK,
-                                 "SessionTrackerImpl --- Removing "
+                "SessionTrackerImpl --- Removing session 0x"
                 + Long.toHexString(sessionId));
         if (s != null) {
             sessionSets.get(s.tickTime).sessions.remove(s);
@@ -204,12 +204,12 @@ public class SessionTrackerImpl extends Thread implements SessionTracker {
             Session s = new Session(id, 0);
             sessionsById.put(id, s);
             ZooTrace.logTraceMessage(LOG, ZooTrace.SESSION_TRACE_MASK,
-                    "SessionTrackerImpl --- Adding " + Long.toHexString(id)
-                    + " " + sessionTimeout);
+                    "SessionTrackerImpl --- Adding session 0x" 
+                    + Long.toHexString(id) + " " + sessionTimeout);
         } else {
             ZooTrace.logTraceMessage(LOG, ZooTrace.SESSION_TRACE_MASK,
-                    "SessionTrackerImpl --- Existing session " + Long.toHexString(id) + " "
-                            + sessionTimeout);
+                    "SessionTrackerImpl --- Existing session 0x" 
+                    + Long.toHexString(id) + " " + sessionTimeout);
         }
         touchSession(id, sessionTimeout);
     }

src/java/main/org/apache/zookeeper/server/SyncRequestProcessor.java

@@ -54,17 +54,34 @@ public class SyncRequestProcessor extends Thread implements RequestProcessor {
                 "no");
     }
 
-    static long preAllocSize = 65536 * 1024;
+    private static long preAllocSize = 65536 * 1024;
     static {
         String size = System.getProperty("zookeeper.preAllocSize");
         if (size != null) {
             try {
                 preAllocSize = Long.parseLong(size) * 1024;
             } catch (NumberFormatException e) {
-                LOG.warn(size + " is not a valid value for preAllocSize");
+                LOG.warn(size 
+                        + " is not a valid value for zookeeper.preAllocSize");
             }
         }
     }
+    
+    /**
+     * Change the data log pre-allocation size on the fly.
+     * 
+     * You might want to do this on systems (Windows esp) where preallocation
+     * is slow, WARN messages are output the log if preAllocation is taking
+     * too long -- will stall the request pipeline.
+     * 
+     * This value can also be set through the "zookeeper.preAllocSize" (also
+     * in K bytes) environment variable when starting the jvm.
+     * 
+     * @param size size in K bytes to change the log prealloc to
+     */
+    public static void setPreAllocSize(long size) {
+        preAllocSize = size * 1024; 
+    }
 
     /**
      * The number of log entries to log before starting a snapshot

src/java/main/org/apache/zookeeper/server/TraceFormatter.java

@@ -101,13 +101,13 @@ public class TraceFormatter {
                     DateFormat.LONG).format(new Date(time))
                     + ": "
                     + (char) app
-                    + " id="
+                    + " id=0x"
                     + Long.toHexString(id)
                     + " cxid="
                     + cxid
                     + " op="
                     + op2String(type)
-                    + " zxid="
+                    + " zxid=0x"
                     + Long.toHexString(zxid)
                     + " txnType="
                     + txnType

src/java/main/org/apache/zookeeper/server/ZooKeeperServer.java

@@ -105,7 +105,7 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider {
     public int commitLogBuffer = 700;
     public LinkedList<Proposal> committedLog = new LinkedList<Proposal>();
     public long minCommittedLog, maxCommittedLog;
-    private DataTreeBuilder treeBuilder = new BasicDataTreeBuilder();
+    private DataTreeBuilder treeBuilder;
     public DataTree dataTree;
     protected SessionTracker sessionTracker;
     /**
@@ -138,39 +138,54 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider {
     }
 
     /**
+     * Creates a ZooKeeperServer instance. Nothing is setup, use the setX
+     * methods to prepare the instance (eg datadir, datalogdir, ticktime, 
+     * builder, etc...)
      * 
      * @throws IOException
      */
     public ZooKeeperServer() {
         ServerStats.getInstance().setStatsProvider(this);
+        treeBuilder = new BasicDataTreeBuilder();
     }
     /**
      * Creates a ZooKeeperServer instance. It sets everything up, but doesn't
      * actually start listening for clients until run() is invoked.
      *
-     * @param dataDir
-     *            the directory to put the data
+     * @param dataDir the directory to put the data
      * @throws IOException
      */
     public ZooKeeperServer(File dataDir, File dataLogDir, int tickTime,
             DataTreeBuilder treeBuilder) throws IOException {
-        this.treeBuilder = treeBuilder;
         this.dataDir = dataDir;
         this.dataLogDir = dataLogDir;
         this.tickTime = tickTime;
+        this.treeBuilder = treeBuilder;
         ServerStats.getInstance().setStatsProvider(this);
+        
+        LOG.info("Created server with dataDir:" + dataDir 
+                + " dataLogDir:" + dataLogDir
+                + " tickTime:" + tickTime);
     }
 
     /**
-     * This constructor is for backward comaptibility with the existing unit
+     * This constructor is for backward compatibility with the existing unit
      * test code.
      */
     public ZooKeeperServer(File dataDir, File dataLogDir, int tickTime)
-            throws IOException {
-        this();
-        this.dataDir = dataDir;
-        this.dataLogDir = dataLogDir;
-        this.tickTime = tickTime;
+        throws IOException 
+    {
+        this(dataDir, dataLogDir, tickTime, new BasicDataTreeBuilder());
+    }
+
+    /**
+     * Default constructor, relies on the config for its agrument values
+     *
+     * @throws IOException
+     */
+    public ZooKeeperServer(DataTreeBuilder treeBuilder) throws IOException {
+        this(new File(ServerConfig.getDataDir()), new File(ServerConfig
+                .getDataLogDir()), DEFAULT_TICK_TIME, treeBuilder);
     }
 
     public static long getZxidFromName(String name, String prefix) {
@@ -396,7 +411,7 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider {
                             ((CreateSessionTxn) txn).getTimeOut());
                     ZooTrace.logTraceMessage(LOG,
                                              ZooTrace.SESSION_TRACE_MASK,
-                            "playLog --- create session in log: "
+                            "playLog --- create session in log: 0x"
                                     + Long.toHexString(hdr.getClientId())
                                     + " with timeout: "
                                     + ((CreateSessionTxn) txn).getTimeOut());
@@ -407,7 +422,7 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider {
                     sessionsWithTimeouts.remove(hdr.getClientId());
                     ZooTrace.logTraceMessage(LOG,
                             ZooTrace.SESSION_TRACE_MASK,
-                            "playLog --- close session in log: "
+                            "playLog --- close session in log: 0x"
                                     + Long.toHexString(hdr.getClientId()));
                     dataTree.processTxn(hdr, txn);
                     break;
@@ -562,7 +577,7 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider {
         }
         if (truncated == false) {
             // not able to truncate the log
-            LOG.error("Not able to truncate the log "
+            LOG.error("Not able to truncate the log zxid 0x"
                     + Long.toHexString(finalZxid));
             System.exit(13);
         }
@@ -584,7 +599,7 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider {
     public void snapshot() throws InterruptedException {
         long lastZxid = dataTree.lastProcessedZxid;
         ZooTrace.logTraceMessage(LOG, ZooTrace.getTextTraceLevel(),
-                "Snapshotting: " + Long.toHexString(lastZxid));
+                "Snapshotting: zxid 0x" + Long.toHexString(lastZxid));
         try {
             File f =new File(dataDir, "snapshot." + Long.toHexString(lastZxid));
             OutputStream sessOS = new BufferedOutputStream(new FileOutputStream(f));
@@ -593,7 +608,7 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider {
             sessOS.flush();
             sessOS.close();
             ZooTrace.logTraceMessage(LOG, ZooTrace.getTextTraceLevel(),
-                    "Snapshotting finished: " + Long.toHexString(lastZxid));
+                    "Snapshotting finished: zxid 0x" + Long.toHexString(lastZxid));
         } catch (IOException e) {
             LOG.error("Severe error, exiting",e);
             // This is a severe error that we cannot recover from,
@@ -623,7 +638,7 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider {
 
     public void closeSession(long sessionId) throws InterruptedException {
         ZooTrace.logTraceMessage(LOG, ZooTrace.SESSION_TRACE_MASK,
-                                 "ZooKeeperServer --- Session to be closed: "
+                                 "ZooKeeperServer --- Session to be closed: 0x"
                 + Long.toHexString(sessionId));
         // we do not want to wait for a session close. send it as soon as we
         // detect it!
@@ -633,7 +648,7 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider {
     protected void killSession(long sessionId) {
         dataTree.killSession(sessionId);
         ZooTrace.logTraceMessage(LOG, ZooTrace.SESSION_TRACE_MASK,
-                                     "ZooKeeperServer --- killSession: "
+                                     "ZooKeeperServer --- killSession: 0x"
                 + Long.toHexString(sessionId));
         if (sessionTracker != null) {
             sessionTracker.removeSession(sessionId);
@@ -643,8 +658,9 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider {
     public void expire(long sessionId) {
         try {
             ZooTrace.logTraceMessage(LOG,
-                                     ZooTrace.SESSION_TRACE_MASK,
-                    "ZooKeeperServer --- Session to expire: " + Long.toHexString(sessionId));
+                     ZooTrace.SESSION_TRACE_MASK,
+                    "ZooKeeperServer --- Session to expire: 0x" 
+                     + Long.toHexString(sessionId));
             closeSession(sessionId);
         } catch (Exception e) {
             LOG.error("FIXMSG",e);
@@ -658,11 +674,19 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider {
         long id = cnxn.getSessionId();
         int to = cnxn.getSessionTimeout();
         if (!sessionTracker.touchSession(id, to)) {
-            throw new IOException("Missing session " + Long.toHexString(id));
+            throw new IOException("Missing session 0x" + Long.toHexString(id));
         }
     }
 
     public void startup() throws IOException, InterruptedException {
+        if (dataDir == null || !dataDir.isDirectory()) {
+            throw new IOException("data directory does not exist: " + dataDir);
+        }
+        if (dataLogDir == null || !dataLogDir.isDirectory()) {
+            throw new IOException("data log directory does not exist: "
+                    + dataLogDir);
+        }
+
         if (dataTree == null) {
             loadData();
         }
@@ -782,7 +806,7 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider {
             int sessionTimeout) throws IOException, InterruptedException {
         boolean rc = sessionTracker.touchSession(sessionId, sessionTimeout);
         ZooTrace.logTraceMessage(LOG,ZooTrace.SESSION_TRACE_MASK,
-                                 "Session " + Long.toHexString(sessionId) +
+                                 "Session 0x" + Long.toHexString(sessionId) +
                 " is valid: " + rc);
         cnxn.finishSessionInit(rc);
     }
@@ -931,8 +955,11 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider {
     /**
      * Sets directoy for storing the log tnxns
      */
-    public void setDataLogDir(File dataLogDir) {
+    public void setDataLogDir(File dataLogDir) throws IOException {
         this.dataLogDir = dataLogDir;
+        if (!dataLogDir.isDirectory()) {
+            throw new IOException("data log directory does not exist");
+        }
     }
 
     public int getClientPort() {

src/java/main/org/apache/zookeeper/server/ZooTrace.java

@@ -58,7 +58,7 @@ public class ZooTrace {
     public static void setTextTraceLevel(long mask) {
         traceMask = mask;
         Logger LOG = Logger.getLogger(ZooTrace.class);
-        LOG.info("Set text trace mask to " + Long.toHexString(mask));
+        LOG.info("Set text trace mask to 0x" + Long.toHexString(mask));
     }
 
     public static boolean isTraceEnabled(Logger log, long mask) {

src/java/main/org/apache/zookeeper/server/quorum/FastLeaderElection.java

@@ -19,7 +19,6 @@
 
 package org.apache.zookeeper.server.quorum;
 
-import java.net.DatagramSocket;
 import java.net.InetAddress;
 import java.nio.ByteBuffer;
 import java.util.Collection;
@@ -28,7 +27,6 @@ import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeUnit;
 
 import org.apache.log4j.Logger;
-
 import org.apache.zookeeper.server.quorum.QuorumCnxManager.Message;
 import org.apache.zookeeper.server.quorum.QuorumPeer.QuorumServer;
 import org.apache.zookeeper.server.quorum.QuorumPeer.ServerState;

src/java/main/org/apache/zookeeper/server/quorum/Follower.java

@@ -181,7 +181,7 @@ public class Follower {
                     }
                 } else if (qp.getType() == Leader.TRUNC) {
                     //we need to truncate the log to the lastzxid of the leader
-                    LOG.warn("Truncating log to get in sync with the leader "
+                    LOG.warn("Truncating log to get in sync with the leader 0x"
                             + Long.toHexString(qp.getZxid()));
                     zk.truncateLog(qp.getZxid());
                     zk.loadData();
@@ -219,9 +219,9 @@ public class Follower {
                             .getArchive(new ByteArrayInputStream(qp.getData()));
                     Record txn = ZooKeeperServer.deserializeTxn(ia, hdr);
                     if (hdr.getZxid() != lastQueued + 1) {
-                        LOG.warn("Got zxid "
+                        LOG.warn("Got zxid 0x"
                                 + Long.toHexString(hdr.getZxid())
-                                + " expected "
+                                + " expected 0x"
                                 + Long.toHexString(lastQueued + 1));
                     }
                     lastQueued = hdr.getZxid();
@@ -244,7 +244,7 @@ public class Follower {
                         ServerCnxn cnxn = pendingRevalidations
                                 .remove(sessionId);
                         if (cnxn == null) {
-                            LOG.warn("Missing "
+                            LOG.warn("Missing session 0x"
                                     + Long.toHexString(sessionId)
                                     + " for validation");
                         } else {
@@ -252,7 +252,8 @@ public class Follower {
                         }
                     }
                     ZooTrace.logTraceMessage(LOG, ZooTrace.SESSION_TRACE_MASK,
-                                             "Session " + sessionId
+                                             "Session 0x" 
+                                             + Long.toHexString(sessionId)
                                              + " is valid: " + valid);
                     break;
                 case Leader.SYNC:
@@ -303,7 +304,7 @@ public class Follower {
         pendingRevalidations.put(clientId, cnxn);
         ZooTrace.logTraceMessage(LOG,
                                  ZooTrace.SESSION_TRACE_MASK,
-                                 "To validate session "
+                                 "To validate session 0x"
                                  + Long.toHexString(clientId));
         writePacket(qp);
     }
@@ -350,6 +351,7 @@ public class Follower {
                 return zk.getZxid();
             }
         } catch (NullPointerException e) {
+            LOG.warn("error getting zxid", e);
         }
         return -1;
     }

src/java/main/org/apache/zookeeper/server/quorum/FollowerHandler.java

@@ -244,8 +244,9 @@ public class FollowerHandler extends Thread {
             bufferedOutput.flush();
             // only if we are not truncating or fast sycning
             if (packetToSend == Leader.SNAP) {
-                LOG.warn("Sending snapshot last zxid of peer is "
-                        + Long.toHexString(peerLastZxid) + " " + " zxid of leader is "
+                LOG.warn("Sending snapshot last zxid of peer is 0x"
+                        + Long.toHexString(peerLastZxid) + " " 
+                        + " zxid of leader is 0x"
                         + Long.toHexString(leaderLastZxid));
                 // Dump data to follower
                 leader.zk.snapshot(oa);
@@ -316,7 +317,7 @@ public class FollowerHandler extends Thread {
                     boolean valid = leader.zk.touch(id, to);
                     ZooTrace.logTraceMessage(LOG,
                                              ZooTrace.SESSION_TRACE_MASK,
-                                             "Session " + Long.toHexString(id)
+                                             "Session 0x" + Long.toHexString(id)
                                              + " is valid: "+ valid);
                     dos.writeBoolean(valid);
                     qp.setData(bos.toByteArray());

src/java/main/org/apache/zookeeper/server/quorum/FollowerZooKeeperServer.java

@@ -128,8 +128,8 @@ public class FollowerZooKeeperServer extends ZooKeeperServer {
         }
         long firstElementZxid = pendingTxns.element().zxid;
         if (firstElementZxid != zxid) {
-            LOG.error("Committing " + Long.toHexString(zxid)
-                    + " but next pending txn "
+            LOG.error("Committing zxid 0x" + Long.toHexString(zxid)
+                    + " but next pending txn 0x"
                     + Long.toHexString(firstElementZxid));
             System.exit(12);
         }

src/java/main/org/apache/zookeeper/server/quorum/Leader.java

@@ -356,7 +356,7 @@ public class Leader {
                 // Long.toHexString(zxid) + " is " + p.ackCount);
                 if (p.ackCount > self.quorumPeers.size() / 2){
                     if (!first) {
-                        LOG.error("Commiting " + Long.toHexString(zxid)
+                        LOG.error("Commiting zxid 0x" + Long.toHexString(zxid)
                                 + " from " + followerAddr + " not first!");
                         LOG.error("First is "
                                 + outstandingProposals.element().packet);
@@ -385,7 +385,7 @@ public class Leader {
                 first = false;
             }
         }
-        LOG.error("Trying to commit future proposal: "
+        LOG.error("Trying to commit future proposal: zxid 0x"
                 + Long.toHexString(zxid) + " from " + followerAddr);
     }
 

src/java/test/org/apache/zookeeper/server/SerializationPerfTest.java

@@ -29,6 +29,7 @@ import org.apache.zookeeper.KeeperException;
 public class SerializationPerfTest extends TestCase {
     private static class NullOutputStream extends OutputStream {
         public void write(int b) {
+            // do nothing - exclude persistence from perf
         }
     }
 

src/java/test/org/apache/zookeeper/test/AsyncTest.java

@@ -18,20 +18,17 @@
 
 package org.apache.zookeeper.test;
 
-import java.io.File;
+import static org.apache.zookeeper.test.ClientBase.CONNECTION_TIMEOUT;
+import static org.apache.zookeeper.test.ClientBase.verifyThreadTerminated;
+
 import java.io.IOException;
 import java.util.LinkedList;
 import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeUnit;
 
 import junit.framework.TestCase;
 
 import org.apache.log4j.Logger;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.Watcher;
 import org.apache.zookeeper.ZooKeeper;
@@ -43,157 +40,203 @@ import org.apache.zookeeper.ZooDefs.CreateFlags;
 import org.apache.zookeeper.ZooDefs.Ids;
 import org.apache.zookeeper.data.Stat;
 import org.apache.zookeeper.proto.WatcherEvent;
-import org.apache.zookeeper.server.NIOServerCnxn;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
 
-public class AsyncTest extends TestCase implements Watcher, StringCallback, VoidCallback, DataCallback {
+public class AsyncTest extends TestCase
+    implements Watcher, StringCallback, VoidCallback, DataCallback
+{
     private static final Logger LOG = Logger.getLogger(AsyncTest.class);
 
-    private static final int CONNECTION_TIMEOUT=30000;
-    protected static String hostPort = "127.0.0.1:33221";
-    LinkedBlockingQueue<WatcherEvent> events = new LinkedBlockingQueue<WatcherEvent>();
-    static File baseTest = new File(System.getProperty("build.test.dir", "build"));
-    NIOServerCnxn.Factory f = null;
-    QuorumTest qt = new QuorumTest();
+    private QuorumTest quorumTest = new QuorumTest();
     private CountDownLatch clientConnected;
 
+    private volatile boolean bang;
+
     @Before
+    @Override
     protected void setUp() throws Exception {
-        qt.setUp();
-        hostPort = ClientTest.hostPort;
+        LOG.info("STARTING " + getName());
+        
+        ClientBase.setupTestEnv();
+
+        quorumTest.setUp();
     }
 
     protected void restart() throws Exception {
-        qt.startServers();
+        quorumTest.startServers();
     }
 
     @After
+    @Override
     protected void tearDown() throws Exception {
-        qt.tearDown();
-        LOG.error("Client test shutdown");
-        if (f != null) {
-            f.shutdown();
-        }
-        clientConnected=null;
-        LOG.error("Client test shutdown finished");
+        LOG.info("Test clients shutting down");
+        clientConnected = null;
+        quorumTest.tearDown();
+        LOG.info("FINISHED " + getName());
     }
 
-    private ZooKeeper createClient() throws IOException,InterruptedException{
-        clientConnected=new CountDownLatch(1);
-        ZooKeeper zk = new ZooKeeper(hostPort, 30000, this);
+    private ZooKeeper createClient() throws IOException,InterruptedException {
+        return createClient(quorumTest.hostPort);
+    }
+
+    private ZooKeeper createClient(String hp)
+        throws IOException, InterruptedException
+    {
+        clientConnected = new CountDownLatch(1);
+        ZooKeeper zk = new ZooKeeper(hp, 30000, this);
         if(!clientConnected.await(CONNECTION_TIMEOUT, TimeUnit.MILLISECONDS)){
             fail("Unable to connect to server");
         }
         return zk;
     }
 
-    boolean bang;
+    /**
+     * Create /test- sequence nodes asynchronously, max 30 outstanding
+     */
+    class HammerThread extends Thread
+        implements Watcher, StringCallback, VoidCallback
+    {
+        private static final int MAX_OUTSTANDING = 30;
+
+        private ZooKeeper zk;
+        private int outstanding;
+
+        public HammerThread(String name) {
+            super(name);
+        }
 
-    class HammerThread extends Thread implements Watcher, StringCallback, VoidCallback {
-        ZooKeeper zk;
         public void run() {
-        try {
-            zk = new ZooKeeper(hostPort, 30000, this);
-            while(bang) {
-                zk.create("/test-", new byte[0], Ids.OPEN_ACL_UNSAFE, CreateFlags.SEQUENCE, this, null);
-                incOut();
+            try {
+                zk = new ZooKeeper(quorumTest.hostPort, 30000, this);
+                while(bang) {
+                    incOutstanding(); // before create otw race
+                    zk.create("/test-", new byte[0], Ids.OPEN_ACL_UNSAFE,
+                            CreateFlags.SEQUENCE, this, null);
+                }
+            } catch (InterruptedException e) {
+                if (bang) {
+                    LOG.error("sanity check failed!!!"); // sanity check
+                    return;
+                }
+            } catch (Exception e) {
+                LOG.error("Client create operation failed", e);
+                return;
+            } finally {
+                if (zk != null) {
+                    try {
+                        zk.close();
+                    } catch (InterruptedException e) {
+                        LOG.warn("Unexpected", e);
+                    }
+                }
             }
-        } catch (Exception e) {
-            e.printStackTrace();
         }
-        }
-        int outstanding;
-        synchronized void incOut() throws InterruptedException {
+
+        private synchronized void incOutstanding() throws InterruptedException {
             outstanding++;
-            while(outstanding > 30) {
+            while(outstanding > MAX_OUTSTANDING) {
                 wait();
             }
         }
-        synchronized void decOut() {
+
+        private synchronized void decOutstanding() {
             outstanding--;
+            assertTrue("outstanding >= 0", outstanding >= 0);
             notifyAll();
         }
 
         public void process(WatcherEvent event) {
+            // ignore for purposes of this test
         }
 
         public void processResult(int rc, String path, Object ctx, String name) {
             try {
-                decOut();
+                decOutstanding();
                 zk.delete(path, -1, this, null);
             } catch (Exception e) {
-                e.printStackTrace();
+                LOG.error("Client delete failed", e);
             }
         }
 
         public void processResult(int rc, String path, Object ctx) {
+            // ignore for purposes of this test
         }
     }
 
     @Test
     public void testHammer() throws Exception {
-        Thread.sleep(1000);
         bang = true;
-        for (int i = 0; i < 100; i++) {
-            new HammerThread().start();
+        Thread[] hammers = new Thread[100];
+        for (int i = 0; i < hammers.length; i++) {
+            hammers[i] = new HammerThread("HammerThread-" + i);
+            hammers[i].start();
         }
-        Thread.sleep(5000);
-        tearDown();
+        Thread.sleep(5000); // allow the clients to run for max 5sec
         bang = false;
-        restart();
-        Thread.sleep(5000);
-        String parts[] = hostPort.split(",");
-        String prevList[] = null;
-        for (String hp : parts) {
-            ZooKeeper zk = createClient();
-            String list[] = zk.getChildren("/", false).toArray(new String[0]);
-            if (prevList != null) {
-                assertEquals(prevList.length, list.length);
-                for (int i = 0; i < list.length; i++) {
-                    assertEquals(prevList[i], list[i]);
-                }
-            }
-            prevList = list;
+        for (int i = 0; i < hammers.length; i++) {
+            hammers[i].interrupt();
+            verifyThreadTerminated(hammers[i], 60000);
         }
+        // before restart
+        quorumTest.verifyRootOfAllServersMatch(quorumTest.hostPort);
+        tearDown();
+
+        restart();
+
+        // after restart
+        quorumTest.verifyRootOfAllServersMatch(quorumTest.hostPort);
     }
 
     LinkedList<Integer> results = new LinkedList<Integer>();
     @Test
-    public void testAsync() throws IOException,
-            InterruptedException, KeeperException {
+    public void testAsync()
+        throws IOException, InterruptedException, KeeperException
+    {
         ZooKeeper zk = null;
         zk = createClient();
-        zk.addAuthInfo("digest", "ben:passwd".getBytes());
-        zk.create("/ben", new byte[0], Ids.READ_ACL_UNSAFE, 0, this, results);
-        zk.create("/ben/2", new byte[0], Ids.CREATOR_ALL_ACL, 0, this, results);
-        zk.delete("/ben", -1, this, results);
-        zk.create("/ben2", new byte[0], Ids.CREATOR_ALL_ACL, 0, this, results);
-        zk.getData("/ben2", false, this, results);
-        synchronized (results) {
-            while (results.size() < 5) {
-                results.wait();
+        try {
+            zk.addAuthInfo("digest", "ben:passwd".getBytes());
+            zk.create("/ben", new byte[0], Ids.READ_ACL_UNSAFE, 0, this, results);
+            zk.create("/ben/2", new byte[0], Ids.CREATOR_ALL_ACL, 0, this, results);
+            zk.delete("/ben", -1, this, results);
+            zk.create("/ben2", new byte[0], Ids.CREATOR_ALL_ACL, 0, this, results);
+            zk.getData("/ben2", false, this, results);
+            synchronized (results) {
+                while (results.size() < 5) {
+                    results.wait();
+                }
             }
+            assertEquals(0, (int) results.get(0));
+            assertEquals(Code.NoAuth, (int) results.get(1));
+            assertEquals(0, (int) results.get(2));
+            assertEquals(0, (int) results.get(3));
+            assertEquals(0, (int) results.get(4));
+        } finally {
+            zk.close();
         }
-        assertEquals(0, (int) results.get(0));
-        assertEquals(Code.NoAuth, (int) results.get(1));
-        assertEquals(0, (int) results.get(2));
-        assertEquals(0, (int) results.get(3));
-        assertEquals(0, (int) results.get(4));
-        zk.close();
 
-        zk =createClient();
-        zk.addAuthInfo("digest", "ben:passwd2".getBytes());
+        zk = createClient();
         try {
-            zk.getData("/ben2", false, new Stat());
-            fail("Should have received a permission error");
-        } catch (KeeperException e) {
-            assertEquals(Code.NoAuth, e.getCode());
+            zk.addAuthInfo("digest", "ben:passwd2".getBytes());
+            try {
+                zk.getData("/ben2", false, new Stat());
+                fail("Should have received a permission error");
+            } catch (KeeperException e) {
+                assertEquals(Code.NoAuth, e.getCode());
+            }
+        } finally {
+            zk.close();
         }
-        zk.close();
 
-        zk =createClient();
-        zk.addAuthInfo("digest", "ben:passwd".getBytes());
-        zk.getData("/ben2", false, new Stat());
-        zk.close();
+        zk = createClient();
+        try {
+            zk.addAuthInfo("digest", "ben:passwd".getBytes());
+            zk.getData("/ben2", false, new Stat());
+        } finally {
+            zk.close();
+        }
     }
 
     public void process(WatcherEvent event) {
@@ -202,24 +245,27 @@ public class AsyncTest extends TestCase implements Watcher, StringCallback, Void
         }
     }
 
+    @SuppressWarnings("unchecked")
     public void processResult(int rc, String path, Object ctx, String name) {
-        ((LinkedList<Integer>)ctx).add(rc);
         synchronized(ctx) {
+            ((LinkedList<Integer>)ctx).add(rc);
             ctx.notifyAll();
         }
     }
 
+    @SuppressWarnings("unchecked")
     public void processResult(int rc, String path, Object ctx) {
-        ((LinkedList<Integer>)ctx).add(rc);
         synchronized(ctx) {
+            ((LinkedList<Integer>)ctx).add(rc);
             ctx.notifyAll();
         }
     }
 
+    @SuppressWarnings("unchecked")
     public void processResult(int rc, String path, Object ctx, byte[] data,
             Stat stat) {
-        ((LinkedList<Integer>)ctx).add(rc);
         synchronized(ctx) {
+            ((LinkedList<Integer>)ctx).add(rc);
             ctx.notifyAll();
         }
     }

src/java/test/org/apache/zookeeper/test/ClientBase.java

@@ -18,30 +18,33 @@
 
 package org.apache.zookeeper.test;
 
+import java.io.BufferedReader;
 import java.io.File;
 import java.io.IOException;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.net.Socket;
+import java.util.Arrays;
 
 import junit.framework.TestCase;
 
 import org.apache.log4j.Logger;
-
-import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.ZooKeeper;
 import org.apache.zookeeper.server.NIOServerCnxn;
 import org.apache.zookeeper.server.ServerStats;
+import org.apache.zookeeper.server.SyncRequestProcessor;
 import org.apache.zookeeper.server.ZooKeeperServer;
 
 public abstract class ClientBase extends TestCase {
     protected static final Logger LOG = Logger.getLogger(ClientBase.class);
-    protected static String hostPort = "127.0.0.1:33221";
-    protected static final int CONNECTION_TIMEOUT = 30000;
-    protected NIOServerCnxn.Factory f = null;
-    protected File tmpDir = null;
-    protected static File baseTest =
+
+    static final int CONNECTION_TIMEOUT = 30000;
+    static final File BASETEST =
         new File(System.getProperty("build.test.dir", "build"));
 
+    protected String hostPort = "127.0.0.1:33221";
+    protected NIOServerCnxn.Factory serverFactory = null;
+    protected File tmpDir = null;
+
     public ClientBase() {
         super();
     }
@@ -50,39 +53,193 @@ public abstract class ClientBase extends TestCase {
         super(name);
     }
 
+    public static boolean waitForServerUp(String hp, long timeout) {
+        long start = System.currentTimeMillis();
+        String split[] = hp.split(":");
+        String host = split[0];
+        int port = Integer.parseInt(split[1]);
+        while (true) {
+            try {
+                Socket sock = new Socket(host, port);
+                BufferedReader reader = null;
+                try {
+                    OutputStream outstream = sock.getOutputStream();
+                    outstream.write("stat".getBytes());
+                    outstream.flush();
+
+                    reader =
+                        new BufferedReader(
+                                new InputStreamReader(sock.getInputStream()));
+                    String line = reader.readLine();
+                    if (line != null && line.startsWith("Zookeeper version:")) {
+                        return true;
+                    }
+                } finally {
+                    sock.close();
+                    if (reader != null) {
+                        reader.close();
+                    }
+                }
+            } catch (IOException e) {
+                // ignore as this is expected
+                LOG.info("server " + hp + " not up " + e);
+            }
+
+            if (System.currentTimeMillis() > start + timeout) {
+                break;
+            }
+            try {
+                Thread.sleep(250);
+            } catch (InterruptedException e) {
+                // ignore
+            }
+        }
+        return false;
+    }
+    public static boolean waitForServerDown(String hp, long timeout) {
+        long start = System.currentTimeMillis();
+        String split[] = hp.split(":");
+        String host = split[0];
+        int port = Integer.parseInt(split[1]);
+        while (true) {
+            try {
+                Socket sock = new Socket(host, port);
+                try {
+                    OutputStream outstream = sock.getOutputStream();
+                    outstream.write("stat".getBytes());
+                    outstream.flush();
+                } finally {
+                    sock.close();
+                }
+            } catch (IOException e) {
+                return true;
+            }
+
+            if (System.currentTimeMillis() > start + timeout) {
+                break;
+            }
+            try {
+                Thread.sleep(250);
+            } catch (InterruptedException e) {
+                // ignore
+            }
+        }
+        return false;
+    }
+    
+    static void verifyThreadTerminated(Thread thread, long millis)
+        throws InterruptedException
+    {
+        thread.join(millis);
+        if (thread.isAlive()) {
+            LOG.error("Thread " + thread.getName() + " : "
+                    + Arrays.toString(thread.getStackTrace()));
+            assertFalse("thread " + thread.getName() 
+                    + " still alive after join", true);
+        }
+    }
+
+
+    static File createTmpDir() throws IOException {
+        return createTmpDir(BASETEST);
+    }
+    static File createTmpDir(File parentDir) throws IOException {
+        File tmpFile = File.createTempFile("test", ".junit", parentDir);
+        // don't delete tmpFile - this ensures we don't attempt to create
+        // a tmpDir with a duplicate name
+        
+        File tmpDir = new File(tmpFile + ".dir");
+        assertFalse(tmpDir.exists()); // never true if tmpfile does it's job
+        assertTrue(tmpDir.mkdirs());
+        
+        return tmpDir;
+    }
+    
+    static NIOServerCnxn.Factory createNewServerInstance(File dataDir,
+            NIOServerCnxn.Factory factory, String hostPort)
+        throws IOException, InterruptedException 
+    {
+        ZooKeeperServer zks = new ZooKeeperServer(dataDir, dataDir, 3000);
+        final int PORT = Integer.parseInt(hostPort.split(":")[1]);
+        if (factory == null) {
+            factory = new NIOServerCnxn.Factory(PORT);
+        }
+        factory.startup(zks);
+
+        assertTrue("waiting for server up",
+                   ClientBase.waitForServerUp("127.0.0.1:" + PORT,
+                                              CONNECTION_TIMEOUT));
+
+        return factory;
+    }
+    
+    static void shutdownServerInstance(NIOServerCnxn.Factory factory,
+            String hostPort)
+    {
+        if (factory != null) {
+            factory.shutdown();
+            final int PORT = Integer.parseInt(hostPort.split(":")[1]);
+
+            assertTrue("waiting for server down",
+                       ClientBase.waitForServerDown("127.0.0.1:" + PORT,
+                                                    CONNECTION_TIMEOUT));
+        }
+    }
+    
+    /**
+     * Test specific setup
+     */
+    static void setupTestEnv() {
+        // during the tests we run with 100K prealloc in the logs.
+        // on windows systems prealloc of 64M was seen to take ~15seconds
+        // resulting in test failure (client timeout on first session).
+        // set env and directly in order to handle static init/gc issues
+        System.setProperty("zookeeper.preAllocSize", "100");
+        SyncRequestProcessor.setPreAllocSize(100);
+    }
+    
+    @Override
     protected void setUp() throws Exception {
-        LOG.info("Client test setup");
-        tmpDir = File.createTempFile("test", ".junit", baseTest);
-        tmpDir = new File(tmpDir + ".dir");
-        tmpDir.mkdirs();
+        LOG.info("STARTING " + getName());
+
         ServerStats.registerAsConcrete();
-        ZooKeeperServer zks = new ZooKeeperServer(tmpDir, tmpDir, 3000);
-        f = new NIOServerCnxn.Factory(33221);
-        f.startup(zks);
-        Thread.sleep(5000);
+
+        tmpDir = createTmpDir(BASETEST);
+        
+        setupTestEnv();
+        serverFactory =
+            createNewServerInstance(tmpDir, serverFactory, hostPort);
+        
         LOG.info("Client test setup finished");
     }
 
+    @Override
     protected void tearDown() throws Exception {
-        LOG.info("Clent test shutdown");
-        if (f != null) {
-            f.shutdown();
-        }
+        LOG.info("tearDown starting");
+
+        shutdownServerInstance(serverFactory, hostPort);
+        
         if (tmpDir != null) {
+            //assertTrue("delete " + tmpDir.toString(), recursiveDelete(tmpDir));
+            // FIXME see ZOOKEEPER-121 replace following line with previous
             recursiveDelete(tmpDir);
         }
+
         ServerStats.unregister();
-        LOG.info("Client test shutdown finished");
+
+        LOG.info("FINISHED " + getName());
     }
 
-    private static void recursiveDelete(File d) {
+    private static boolean recursiveDelete(File d) {
         if (d.isDirectory()) {
             File children[] = d.listFiles();
             for (File f : children) {
+                //assertTrue("delete " + f.toString(), recursiveDelete(f));
+                // FIXME see ZOOKEEPER-121 replace following line with previous
                 recursiveDelete(f);
             }
         }
-        d.delete();
+        return d.delete();
     }
 
 }

src/java/test/org/apache/zookeeper/test/ClientTest.java

@@ -18,9 +18,9 @@
 
 package org.apache.zookeeper.test;
 
-import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.Date;
 import java.util.List;
@@ -28,13 +28,12 @@ import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeUnit;
 
-import org.junit.Test;
-
+import org.apache.log4j.Logger;
 import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.KeeperException.InvalidACLException;
-import org.apache.zookeeper.KeeperException.Code;
 import org.apache.zookeeper.Watcher;
 import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.KeeperException.Code;
+import org.apache.zookeeper.KeeperException.InvalidACLException;
 import org.apache.zookeeper.ZooDefs.CreateFlags;
 import org.apache.zookeeper.ZooDefs.Ids;
 import org.apache.zookeeper.ZooDefs.Perms;
@@ -42,8 +41,11 @@ import org.apache.zookeeper.data.ACL;
 import org.apache.zookeeper.data.Id;
 import org.apache.zookeeper.data.Stat;
 import org.apache.zookeeper.proto.WatcherEvent;
+import org.junit.Test;
 
 public class ClientTest extends ClientBase implements Watcher {
+    protected static final Logger LOG = Logger.getLogger(ClientBase.class);
+
     LinkedBlockingQueue<WatcherEvent> events =
         new LinkedBlockingQueue<WatcherEvent>();
     protected volatile CountDownLatch clientConnected;
@@ -51,17 +53,25 @@ public class ClientTest extends ClientBase implements Watcher {
     protected ZooKeeper createClient(Watcher watcher)
         throws IOException, InterruptedException
     {
-        clientConnected=new CountDownLatch(1);
-        ZooKeeper zk = new ZooKeeper(hostPort, 20000, watcher);
-        if(!clientConnected.await(CONNECTION_TIMEOUT, TimeUnit.MILLISECONDS)){
+        return createClient(watcher, hostPort);
+    }
+
+    protected ZooKeeper createClient(Watcher watcher, String hp)
+        throws IOException, InterruptedException
+    {
+        clientConnected = new CountDownLatch(1);
+        ZooKeeper zk = new ZooKeeper(hp, 20000, watcher);
+        if (!clientConnected.await(CONNECTION_TIMEOUT, TimeUnit.MILLISECONDS)) {
             fail("Unable to connect to server");
         }
         return zk;
     }
 
+    @Override
     protected void tearDown() throws Exception {
         clientConnected = null;
         super.tearDown();
+        LOG.info("FINISHED " + getName());
     }
 
     @Test
@@ -116,6 +126,7 @@ public class ClientTest extends ClientBase implements Watcher {
                 zk.create("/acltest", new byte[0], Ids.CREATOR_ALL_ACL, 0);
                 fail("Should have received an invalid acl error");
             } catch(InvalidACLException e) {
+                LOG.error("Invalid acl", e);
             }
             try {
                 ArrayList<ACL> testACL = new ArrayList<ACL>();
@@ -124,6 +135,7 @@ public class ClientTest extends ClientBase implements Watcher {
                 zk.create("/acltest", new byte[0], testACL, 0);
                 fail("Should have received an invalid acl error");
             } catch(InvalidACLException e) {
+                LOG.error("Invalid acl", e);
             }
             zk.addAuthInfo("digest", "ben:passwd".getBytes());
             zk.create("/acltest", new byte[0], Ids.CREATOR_ALL_ACL, 0);
@@ -158,10 +170,10 @@ public class ClientTest extends ClientBase implements Watcher {
         ZooKeeper zk = null;
         try {
             zk =createClient(this);
-            //System.out.println("Created client: " + zk.describeCNXN());
-            System.out.println("Before create /benwashere");
+            //LOG.info("Created client: " + zk.describeCNXN());
+            LOG.info("Before create /benwashere");
             zk.create("/benwashere", "".getBytes(), Ids.OPEN_ACL_UNSAFE, 0);
-            System.out.println("After create /benwashere");
+            LOG.info("After create /benwashere");
             try {
                 zk.setData("/benwashere", "hi".getBytes(), 57);
                 fail("Should have gotten BadVersion exception");
@@ -170,15 +182,15 @@ public class ClientTest extends ClientBase implements Watcher {
             } catch (KeeperException e) {
                 fail("Should have gotten BadVersion exception");
             }
-            System.out.println("Before delete /benwashere");
+            LOG.info("Before delete /benwashere");
             zk.delete("/benwashere", 0);
-            System.out.println("Before delete /benwashere");
+            LOG.info("Before delete /benwashere");
             zk.close();
-            //System.out.println("Closed client: " + zk.describeCNXN());
+            //LOG.info("Closed client: " + zk.describeCNXN());
             Thread.sleep(2000);
             zk = createClient(this);
-            //System.out.println("Created a new client: " + zk.describeCNXN());
-            System.out.println("Before delete /");
+            //LOG.info("Created a new client: " + zk.describeCNXN());
+            LOG.info("Before delete /");
 
             try {
                 zk.delete("/", -1);
@@ -188,9 +200,9 @@ public class ClientTest extends ClientBase implements Watcher {
             }
             Stat stat = new Stat();
             // Test basic create, ls, and getData
-            System.out.println("Before create /ben");
+            LOG.info("Before create /ben");
             zk.create("/ben", "Ben was here".getBytes(), Ids.OPEN_ACL_UNSAFE, 0);
-            System.out.println("Before getChildren /");
+            LOG.info("Before getChildren /");
             List<String> children = zk.getChildren("/", false);
             assertEquals(1, children.size());
             assertEquals("ben", children.get(0));
@@ -203,13 +215,13 @@ public class ClientTest extends ClientBase implements Watcher {
                 } else {
                 assertEquals(null, zk.exists("/frog", true));
                 }
-                System.out.println("Comment: asseting passed for frog setting /");
+                LOG.info("Comment: asseting passed for frog setting /");
             } catch (KeeperException.NoNodeException e) {
                 // OK, expected that
             }
             zk.create("/frog", "hi".getBytes(), Ids.OPEN_ACL_UNSAFE, 0);
             // the first poll is just a sesssion delivery
-            System.out.println("Comment: checking for events length " + events.size());
+            LOG.info("Comment: checking for events length " + events.size());
             WatcherEvent event = events.poll(10, TimeUnit.SECONDS);
             assertEquals("/frog", event.getPath());
             assertEquals(Event.EventNodeCreated, event.getType());
@@ -225,7 +237,7 @@ public class ClientTest extends ClientBase implements Watcher {
             assertEquals(10, children.size());
             for (int i = 0; i < 10; i++) {
                 final String name = children.get(i);
-                assertTrue(name.startsWith(i + "-"));
+                assertTrue("starts with -", name.startsWith(i + "-"));
                 byte b[];
                 if (withWatcherObj) {
                     b = zk.getData("/ben/" + name, new MyWatcher(), stat);
@@ -281,9 +293,9 @@ public class ClientTest extends ClientBase implements Watcher {
     // Test that sequential filenames are being created correctly,
     // with 0-padding in the filename
     public void testSequentialNodeNames() throws IOException, InterruptedException, KeeperException {
-    	String path = "/SEQUENCE";
-	String file = "TEST";
-	String filepath = path + "/" + file;
+        String path = "/SEQUENCE";
+    String file = "TEST";
+    String filepath = path + "/" + file;
 
         ZooKeeper zk = null;
         try {
@@ -293,135 +305,213 @@ public class ClientTest extends ClientBase implements Watcher {
             List<String> children = zk.getChildren(path, false);
             assertEquals(1, children.size());
             assertEquals(file + "0000000000", children.get(0));
-            
+
             zk.create(filepath, new byte[0], Ids.OPEN_ACL_UNSAFE, CreateFlags.SEQUENCE);
             children = zk.getChildren(path, false);
             assertEquals(2, children.size());
-            assertTrue(children.contains(file + "0000000001"));
-            
+            assertTrue("contains child 1",
+                       children.contains(file + "0000000001"));
+
             zk.create(filepath, new byte[0], Ids.OPEN_ACL_UNSAFE, CreateFlags.SEQUENCE);
             children = zk.getChildren(path, false);
             assertEquals(3, children.size());
-            assertTrue(children.contains(file + "0000000002"));
-            
+            assertTrue("contains child 2",
+                       children.contains(file + "0000000002"));
+
             // The pattern is holding so far.  Let's run the counter a bit
             // to be sure it continues to spit out the correct answer
             for(int i = children.size(); i < 105; i++)
                zk.create(filepath, new byte[0], Ids.OPEN_ACL_UNSAFE, CreateFlags.SEQUENCE);
 
             children = zk.getChildren(path, false);
-            assertTrue(children.contains(file + "0000000104"));
-            	
+            assertTrue("contains child 104",
+                       children.contains(file + "0000000104"));
+
         }
         finally {
-        	if(zk != null)
-        		zk.close();
+            if(zk != null)
+                zk.close();
         }
     }
-    
-    private void notestConnections() throws IOException, InterruptedException, KeeperException {
-        ZooKeeper zk;
-        for(int i = 0; i < 2000; i++) {
-            if (i % 100 == 0) {
-                System.out.println("Testing " + i + " connections");
-            }
-            // We want to make sure socket descriptors are going away
-            zk = new ZooKeeper(hostPort, 30000, this);
-            zk.getData("/", false, new Stat());
-            zk.close();
+
+//    private void notestConnections()
+//        throws IOException, InterruptedException, KeeperException
+//    {
+//        ZooKeeper zk;
+//        for(int i = 0; i < 2000; i++) {
+//            if (i % 100 == 0) {
+//                LOG.info("Testing " + i + " connections");
+//            }
+//            // We want to make sure socket descriptors are going away
+//            zk = new ZooKeeper(hostPort, 30000, this);
+//            zk.getData("/", false, new Stat());
+//            zk.close();
+//        }
+//    }
+
+    @Test
+    public void testDeleteWithChildren() throws Exception {
+        ZooKeeper zk = createClient(this);
+        zk.create("/parent", new byte[0], Ids.OPEN_ACL_UNSAFE, 0);
+        zk.create("/parent/child", new byte[0], Ids.OPEN_ACL_UNSAFE, 0);
+        try {
+            zk.delete("/parent", -1);
+            fail("Should have received a not equals message");
+        } catch (KeeperException e) {
+            assertEquals(KeeperException.Code.NotEmpty, e.getCode());
         }
+        zk.delete("/parent/child", -1);
+        zk.delete("/parent", -1);
+        zk.close();
     }
 
-    static class HammerThread extends Thread {
-        ZooKeeper zk;
-        String prefix;
-        int count;
+    private static class HammerThread extends Thread {
+        private static final long LATENCY = 5;
+        
+        private final ZooKeeper zk;
+        private final String prefix;
+        private final int count;
+        private volatile int current = 0;
 
-        HammerThread(ZooKeeper zk, String prefix, int count) {
+        HammerThread(String name, ZooKeeper zk, String prefix, int count) {
+            super(name);
             this.zk = zk;
             this.prefix = prefix;
             this.count = count;
-            start();
         }
 
         public void run() {
             byte b[] = new byte[256];
             try {
-                for (int i = 0; i < count; i++) {
+                for (; current < count; current++) {
                     // Simulate a bit of network latency...
-                    Thread.sleep(5);
-                    zk.create(prefix + i, b, Ids.OPEN_ACL_UNSAFE, 0);
+                    Thread.sleep(LATENCY);
+                    zk.create(prefix + current, b, Ids.OPEN_ACL_UNSAFE, 0);
                 }
             } catch (Exception e) {
-                e.printStackTrace();
+                LOG.error("Client create operation failed", e);
+            } finally {
+                if (zk != null) {
+                    try {
+                        zk.close();
+                    } catch (InterruptedException e) {
+                        LOG.warn("Unexpected", e);
+                    }
+                }
             }
         }
+    }
 
-        public void close() throws IOException, InterruptedException {
-            zk.close();
+    /*
+     * Verify that all of the servers see the same number of nodes
+     * at the root
+     */
+    void verifyRootOfAllServersMatch(String hostPort)
+        throws InterruptedException, KeeperException, IOException
+    {
+        String parts[] = hostPort.split(",");
+
+        // run through till the counts no longer change on each server
+        // max 15 tries, with 2 second sleeps, so approx 30 seconds
+        int[] counts = new int[parts.length];
+        for (int j = 0; j < 100; j++) {
+            int newcounts[] = new int[parts.length];
+            int i = 0;
+            for (String hp : parts) {
+                ZooKeeper zk = createClient(this, hp);
+                try {
+                    newcounts[i++] = zk.getChildren("/", false).size();
+                } finally {
+                    zk.close();
+                }
+            }
+
+            if (Arrays.equals(newcounts, counts)) {
+                LOG.info("Found match with array:"
+                        + Arrays.toString(newcounts));
+                counts = newcounts;
+                break;
+            } else {
+                counts = newcounts;
+                Thread.sleep(10000);
+            }
         }
-    }
 
-    @Test
-    public void testDeleteWithChildren() throws Exception {
-        File tmpDir = File.createTempFile("test", ".junit", baseTest);
-        tmpDir = new File(tmpDir + ".dir");
-        tmpDir.mkdirs();
-        ZooKeeper zk = createClient(this);
-        zk.create("/parent", new byte[0], Ids.OPEN_ACL_UNSAFE, 0);
-        zk.create("/parent/child", new byte[0], Ids.OPEN_ACL_UNSAFE, 0);
-        try {
-            zk.delete("/parent", -1);
-            fail("Should have received a not equals message");
-        } catch (KeeperException e) {
-            assertEquals(KeeperException.Code.NotEmpty, e.getCode());
+        // verify all the servers reporting same number of nodes
+        for (int i = 1; i < parts.length; i++) {
+            assertEquals("node count not consistent", counts[i-1], counts[i]);
         }
-        zk.delete("/parent/child", -1);
-        zk.delete("/parent", -1);
-        zk.close();
     }
 
+
     @Test
-    public void testHammer() throws IOException,
-            InterruptedException, KeeperException {
-        File tmpDir = File.createTempFile("test", ".junit", baseTest);
-        tmpDir = new File(tmpDir + ".dir");
-        tmpDir.mkdirs();
-        try {
-            final int threadCount = 10;
-            final int childCount = 1000;
-            ArrayList<HammerThread> threads = new ArrayList<HammerThread>(
-                    threadCount);
-            long start = System.currentTimeMillis();
-            for (int i = 0; i < threadCount; i++) {
-                Thread.sleep(10);
-                ZooKeeper zk = createClient(this);
-                String prefix = "/test-" + i;
-                zk.create(prefix, new byte[0], Ids.OPEN_ACL_UNSAFE, 0);
-                prefix += "/";
-                threads.add(new HammerThread(zk, prefix, childCount));
-            }
+    public void testHammer() 
+        throws IOException, InterruptedException, KeeperException 
+    {
+        final int threadCount = 10;
+        final int childCount = 1000;
+        
+        HammerThread[] threads = new HammerThread[threadCount];
+        long start = System.currentTimeMillis();
+        for (int i = 0; i < threads.length; i++) {
+            Thread.sleep(10);
+            ZooKeeper zk = createClient(this);
+            String prefix = "/test-" + i;
+            zk.create(prefix, new byte[0], Ids.OPEN_ACL_UNSAFE, 0);
+            prefix += "/";
+            HammerThread thread = 
+                new HammerThread("HammerThread-" + i, zk, prefix, childCount);
+            thread.start();
+            
+            threads[i] = thread;
+        }
+        
+        // look for the clients to finish their create operations
+        LOG.info("Starting check for completed hammers");
+        int workingCount = threads.length;
+        for (int i = 0; i < 120; i++) {
+            Thread.sleep(10000);
             for (HammerThread h : threads) {
-                h.join();
-                h.close();
+                if (!h.isAlive() || h.current == h.count) {
+                    workingCount--;
+                }
             }
-            System.err.println(new Date() + " Total time "
-                    + (System.currentTimeMillis() - start));
-            ZooKeeper zk = createClient(this);
-            LOG.error("******************* Connected to ZooKeeper" + new Date());
-            for (int i = 0; i < threadCount; i++) {
-                System.err.println("Doing thread: " + i + " " + new Date());
-                List<String> children =
-                    zk.getChildren("/test-" + i, false);
-                assertEquals(childCount, children.size());
+            if (workingCount == 0) {
+                break;
             }
-            for (int i = 0; i < threadCount; i++) {
-                List<String> children =
-                    zk.getChildren("/test-" + i, false);
-                assertEquals(childCount, children.size());
+            workingCount = threads.length;
+        }
+        if (workingCount > 0) {
+            for (HammerThread h : threads) {
+                LOG.warn(h.getName() + " never finished creation, current:" 
+                        + h.current);
             }
-        } finally {
-            //  recursiveDelete(tmpDir);
+        } else {
+            LOG.info("Hammer threads completed creation operations");
+        }
+        
+        for (HammerThread h : threads) {
+            final int safetyFactor = 3;
+            verifyThreadTerminated(h,
+                    threadCount * childCount 
+                    * HammerThread.LATENCY * safetyFactor);
+        }
+        LOG.info(new Date() + " Total time "
+                + (System.currentTimeMillis() - start));
+        
+        ZooKeeper zk = createClient(this);
+        
+        LOG.info("******************* Connected to ZooKeeper" + new Date());
+        for (int i = 0; i < threadCount; i++) {
+            LOG.info("Doing thread: " + i + " " + new Date());
+            List<String> children =
+                zk.getChildren("/test-" + i, false);
+            assertEquals(childCount, children.size());
+        }
+        for (int i = 0; i < threadCount; i++) {
+            List<String> children =
+                zk.getChildren("/test-" + i, false);
+            assertEquals(childCount, children.size());
         }
     }
 

src/java/test/org/apache/zookeeper/test/DataTreeTest.java

@@ -18,27 +18,29 @@
 
 package org.apache.zookeeper.test;
 
-import static org.junit.Assert.*;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
+import junit.framework.TestCase;
 
+import org.apache.log4j.Logger;
 import org.apache.zookeeper.Watcher;
 import org.apache.zookeeper.data.Stat;
 import org.apache.zookeeper.proto.WatcherEvent;
 import org.apache.zookeeper.server.DataTree;
-import junit.framework.TestCase;
 
 public class DataTreeTest extends TestCase {
-    DataTree dt;
+    protected static final Logger LOG = Logger.getLogger(DataTreeTest.class);
+
+    private DataTree dt;
 
+    @Override
     public void setUp() throws Exception {
+        LOG.info("STARTING " + getName());
         dt=new DataTree();
     }
 
+    @Override
     public void tearDown() throws Exception {
         dt=null;
+        LOG.info("FINISHED " + getName());
     }
 
 
@@ -49,7 +51,7 @@ public class DataTreeTest extends TestCase {
                 if(event.getPath().equals("/"))
                     fired=true;
             }
-        };
+        }
         MyWatcher watcher=new MyWatcher();
         // set a watch on the root node
         dt.getChildren("/", new Stat(), watcher);

src/java/test/org/apache/zookeeper/test/GenerateLoad.java

@@ -37,6 +37,7 @@ import java.util.Map;
 import java.util.Random;
 import java.util.Set;
 
+import org.apache.log4j.Logger;
 import org.apache.zookeeper.AsyncCallback.DataCallback;
 import org.apache.zookeeper.AsyncCallback.StatCallback;
 import org.apache.zookeeper.KeeperException;
@@ -48,6 +49,8 @@ import org.apache.zookeeper.data.Stat;
 import org.apache.zookeeper.proto.WatcherEvent;
 
 public class GenerateLoad {
+    protected static final Logger LOG = Logger.getLogger(GenerateLoad.class);
+
     static ServerSocket ss;
 
     static Set<SlaveThread> slaves = Collections
@@ -113,7 +116,7 @@ public class GenerateLoad {
                         throw new IOException(result);
                     }
                     long time = Long.parseLong(timePercentCount[0]);
-                    int percent = Integer.parseInt(timePercentCount[1]);
+                    //int percent = Integer.parseInt(timePercentCount[1]);
                     int count = Integer.parseInt(timePercentCount[2]);
                     int errs = Integer.parseInt(timePercentCount[3]);
                     if (errs > 0) {
@@ -242,6 +245,7 @@ public class GenerateLoad {
         for (SlaveThread st : slaves.toArray(new SlaveThread[0])) {
             st.send(percentage);
         }
+        now = System.currentTimeMillis();
         long delay = now - start;
         if (delay > 1000) {
             System.out.println("Delay of " + delay + " to send new percentage");
@@ -300,6 +304,7 @@ public class GenerateLoad {
         String path;
 
         ZooKeeper zk;
+
         public void run() {
             try {
                 byte bytes[] = new byte[1024];
@@ -311,6 +316,7 @@ public class GenerateLoad {
                                 CreateFlags.EPHEMERAL|CreateFlags.SEQUENCE);
                         break;
                     } catch(KeeperException e) {
+                        LOG.error("keeper exception thrown", e);
                     }
                 }
                 if (path == null) {

src/java/test/org/apache/zookeeper/test/OOMTest.java

@@ -18,15 +18,17 @@
 
 package org.apache.zookeeper.test;
 
+import static org.apache.zookeeper.test.ClientBase.CONNECTION_TIMEOUT;
+
 import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 
 import junit.framework.TestCase;
 
+import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.Watcher;
 import org.apache.zookeeper.ZooKeeper;
-import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.ZooDefs.Ids;
 import org.apache.zookeeper.data.Stat;
 import org.apache.zookeeper.proto.WatcherEvent;
@@ -42,9 +44,7 @@ public class OOMTest extends TestCase implements Watcher {
         // This test takes too long to run!
         if (true)
             return;
-        File tmpDir = File.createTempFile("test", ".junit");
-        tmpDir = new File(tmpDir + ".dir");
-        tmpDir.mkdirs();
+        File tmpDir = ClientBase.createTmpDir();
         // Grab some memory so that it is easier to cause an
         // OOM condition;
         ArrayList<byte[]> hog = new ArrayList<byte[]>();
@@ -56,10 +56,16 @@ public class OOMTest extends TestCase implements Watcher {
                 break;
             }
         }
+        ClientBase.setupTestEnv();
         ZooKeeperServer zks = new ZooKeeperServer(tmpDir, tmpDir, 3000);
-        NIOServerCnxn.Factory f = new NIOServerCnxn.Factory(33221);
+        
+        final int PORT = 33221;
+        NIOServerCnxn.Factory f = new NIOServerCnxn.Factory(PORT);
         f.startup(zks);
-        Thread.sleep(2000);
+        assertTrue("waiting for server up",
+                   ClientBase.waitForServerUp("127.0.0.1:" + PORT,
+                                              CONNECTION_TIMEOUT));
+
         System.err.println("OOM Stage 0");
         utestPrep();
         System.out.println("Free = " + Runtime.getRuntime().freeMemory()
@@ -90,7 +96,11 @@ public class OOMTest extends TestCase implements Watcher {
                 + " total = " + Runtime.getRuntime().totalMemory() + " max = "
                 + Runtime.getRuntime().maxMemory());
         hog.get(0)[0] = (byte) 1;
+
         f.shutdown();
+        assertTrue("waiting for server down",
+                   ClientBase.waitForServerDown("127.0.0.1:" + PORT,
+                                                CONNECTION_TIMEOUT));
     }
 
     private void utestExists() throws IOException, InterruptedException, KeeperException {

src/java/test/org/apache/zookeeper/test/QuorumTest.java

@@ -24,38 +24,35 @@ import java.net.InetSocketAddress;
 import java.util.ArrayList;
 
 import org.apache.log4j.Logger;
-import org.junit.After;
-import org.junit.Before;
-
 import org.apache.zookeeper.server.quorum.QuorumPeer;
 import org.apache.zookeeper.server.quorum.QuorumStats;
 import org.apache.zookeeper.server.quorum.QuorumPeer.QuorumServer;
+import org.junit.After;
+import org.junit.Before;
 
 public class QuorumTest extends ClientTest {
     private static final Logger LOG = Logger.getLogger(QuorumTest.class);
 
-    static File baseTest = new File(System.getProperty("build.test.dir", "build"));
     File s1dir, s2dir, s3dir, s4dir, s5dir;
     QuorumPeer s1, s2, s3, s4, s5;
     @Before
+    @Override
     protected void setUp() throws Exception {
-        s1dir = File.createTempFile("test", ".junit", baseTest);
-        s1dir = new File(s1dir + ".dir");
-        s1dir.mkdirs();
-        s2dir = File.createTempFile("test", ".junit", baseTest);
-        s2dir = new File(s2dir + ".dir");
-        s2dir.mkdirs();
-        s3dir = File.createTempFile("test", ".junit", baseTest);
-        s3dir = new File(s3dir + ".dir");
-        s3dir.mkdirs();
-        s4dir = File.createTempFile("test", ".junit", baseTest);
-        s4dir = new File(s4dir + ".dir");
-        s4dir.mkdirs();
-        s5dir = File.createTempFile("test", ".junit", baseTest);
-        s5dir = new File(s5dir + ".dir");
-        s5dir.mkdirs();
+        LOG.info("STARTING " + getName());
+
+        setupTestEnv();
+
+        hostPort = "127.0.0.1:2181,127.0.0.1:2182,127.0.0.1:2183,127.0.0.1:2184,127.0.0.1:2185";
+
+        s1dir = ClientBase.createTmpDir();
+        s2dir = ClientBase.createTmpDir();
+        s3dir = ClientBase.createTmpDir();
+        s4dir = ClientBase.createTmpDir();
+        s5dir = ClientBase.createTmpDir();
+        
         startServers();
-        LOG.warn("Setup finished");
+        
+        LOG.info("Setup finished");
     }
     void startServers() throws IOException, InterruptedException {
         QuorumStats.registerAsConcrete();
@@ -63,44 +60,70 @@ public class QuorumTest extends ClientTest {
         int initLimit = 3;
         int syncLimit = 3;
         ArrayList<QuorumServer> peers = new ArrayList<QuorumServer>();
-        hostPort = "127.0.0.1:2181,127.0.0.1:2182,127.0.0.1:2183,127.0.0.1:2184,127.0.0.1:2185";
         peers.add(new QuorumServer(1, new InetSocketAddress("127.0.0.1", 3181)));
         peers.add(new QuorumServer(2, new InetSocketAddress("127.0.0.1", 3182)));
         peers.add(new QuorumServer(3, new InetSocketAddress("127.0.0.1", 3183)));
         peers.add(new QuorumServer(4, new InetSocketAddress("127.0.0.1", 3184)));
         peers.add(new QuorumServer(5, new InetSocketAddress("127.0.0.1", 3185)));
-        LOG.warn("creating QuorumPeer 1");
+        LOG.info("creating QuorumPeer 1");
         s1 = new QuorumPeer(peers, s1dir, s1dir, 2181, 0,  1181, 1, tickTime, initLimit, syncLimit);
-        LOG.warn("creating QuorumPeer 2");
+        LOG.info("creating QuorumPeer 2");
         s2 = new QuorumPeer(peers, s2dir, s2dir, 2182, 0, 1182, 2, tickTime, initLimit, syncLimit);
-        LOG.warn("creating QuorumPeer 3");
+        LOG.info("creating QuorumPeer 3");
         s3 = new QuorumPeer(peers, s3dir, s3dir, 2183, 0, 1183, 3, tickTime, initLimit, syncLimit);
-        LOG.warn("creating QuorumPeer 4");
+        LOG.info("creating QuorumPeer 4");
         s4 = new QuorumPeer(peers, s4dir, s4dir, 2184, 0, 1184, 4, tickTime, initLimit, syncLimit);
-        LOG.warn("creating QuorumPeer 5");
+        LOG.info("creating QuorumPeer 5");
         s5 = new QuorumPeer(peers, s5dir, s5dir, 2185, 0, 1185, 5, tickTime, initLimit, syncLimit);
-        LOG.warn("start QuorumPeer 1");
+        LOG.info("start QuorumPeer 1");
         s1.start();
-        LOG.warn("start QuorumPeer 2");
+        LOG.info("start QuorumPeer 2");
         s2.start();
-        LOG.warn("start QuorumPeer 3");
+        LOG.info("start QuorumPeer 3");
         s3.start();
-        LOG.warn("start QuorumPeer 4");
+        LOG.info("start QuorumPeer 4");
         s4.start();
-        LOG.warn("start QuorumPeer 5");
+        LOG.info("start QuorumPeer 5");
         s5.start();
-        LOG.warn("started QuorumPeer 5");
-        Thread.sleep(5000);
+        LOG.info("started QuorumPeer 5");
+
+        for (String hp : hostPort.split(",")) {
+            assertTrue("waiting for server up",
+                       ClientBase.waitForServerUp(hp,
+                                    CONNECTION_TIMEOUT));
+            LOG.info(hp + " is accepting client connections");
+        }
     }
     @After
+    @Override
     protected void tearDown() throws Exception {
-        LOG.warn("TearDown started");
-        s1.shutdown();
-        s2.shutdown();
-        s3.shutdown();
-        s4.shutdown();
-        s5.shutdown();
-        Thread.sleep(5000);
+        LOG.info("TearDown started");
+        shutdown(s1);
+        shutdown(s2);
+        shutdown(s3);
+        shutdown(s4);
+        shutdown(s5);
+
+        for (String hp : hostPort.split(",")) {
+            assertTrue("waiting for server down",
+                       ClientBase.waitForServerDown(hp,
+                                           ClientBase.CONNECTION_TIMEOUT));
+            LOG.info(hp + " is no longer accepting client connections");
+        }
+
         QuorumStats.unregister();
+        LOG.info("FINISHED " + getName());
+    }
+
+    private void shutdown(QuorumPeer qp) {
+        try {
+            qp.shutdown();
+            qp.join(30000);
+            if (qp.isAlive()) {
+                fail("QP failed to shutdown in 30 seconds");
+            }
+        } catch (InterruptedException e) {
+            LOG.debug("QP interrupted", e);
+        }
     }
 }

src/java/test/org/apache/zookeeper/test/RecoveryTest.java

@@ -18,57 +18,87 @@
 
 package org.apache.zookeeper.test;
 
+import static org.apache.zookeeper.test.ClientBase.CONNECTION_TIMEOUT;
+
 import java.io.File;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
 
 import junit.framework.TestCase;
 
-import org.junit.Test;
-
+import org.apache.log4j.Logger;
 import org.apache.zookeeper.Watcher;
 import org.apache.zookeeper.ZooKeeper;
 import org.apache.zookeeper.ZooDefs.Ids;
 import org.apache.zookeeper.data.Stat;
 import org.apache.zookeeper.proto.WatcherEvent;
+import org.apache.zookeeper.server.NIOServerCnxn;
 import org.apache.zookeeper.server.ServerStats;
 import org.apache.zookeeper.server.SyncRequestProcessor;
-import org.apache.zookeeper.server.NIOServerCnxn;
 import org.apache.zookeeper.server.ZooKeeperServer;
+import org.junit.Test;
 
 /**
  * @author breed
  *
  */
 public class RecoveryTest extends TestCase implements Watcher {
-    static File baseTest = new File(System.getProperty("build.test.dir",
-            "build"));
+    protected static final Logger LOG = Logger.getLogger(RecoveryTest.class);
+
+    private static String HOSTPORT = "127.0.0.1:2344";
+
+    private CountDownLatch startSignal;
+
+    @Override
     protected void setUp() throws Exception {
+        LOG.info("STARTING " + getName());
         ServerStats.registerAsConcrete();
     }
+    @Override
     protected void tearDown() throws Exception {
         ServerStats.unregister();
+        LOG.info("FINISHED " + getName());
     }
 
     @Test
+    /**
+     * Verify that if a server goes down that clients will reconnect
+     * automatically after the server is restarted. Note that this requires the
+     * server to restart within the connection timeout period.
+     *
+     * Also note that the client latches are used to eliminate any chance
+     * of spurrious connectionloss exceptions on the read ops. Specifically
+     * a sync operation will throw this exception if the server goes down
+     * (as recognized by the client) during the operation. If the operation
+     * occurs after the server is down, but before the client recognizes
+     * that the server is down (ping) then the op will throw connectionloss.
+     */
     public void testRecovery() throws Exception {
-        File tmpDir = File.createTempFile("test", ".junit", baseTest);
-        tmpDir = new File(tmpDir + ".dir");
-        tmpDir.mkdirs();
+        File tmpDir = ClientBase.createTmpDir();
+
+        ClientBase.setupTestEnv();
         ZooKeeperServer zs = new ZooKeeperServer(tmpDir, tmpDir, 3000);
+
         int oldSnapCount = SyncRequestProcessor.snapCount;
         SyncRequestProcessor.snapCount = 1000;
         try {
-            NIOServerCnxn.Factory f = new NIOServerCnxn.Factory(2344);
+            final int PORT = Integer.parseInt(HOSTPORT.split(":")[1]);
+            NIOServerCnxn.Factory f = new NIOServerCnxn.Factory(PORT);
             f.startup(zs);
-            System.out.println("starting up the the server -- sleeping");
-            Thread.sleep(1000);
-            ZooKeeper zk = new ZooKeeper("127.0.0.1:2344", 20000, this);
+            LOG.info("starting up the the server, waiting");
+
+            assertTrue("waiting for server up",
+                       ClientBase.waitForServerUp(HOSTPORT,
+                                       CONNECTION_TIMEOUT));
+
+            ZooKeeper zk = new ZooKeeper(HOSTPORT, 20000, this);
             String path;
-            System.out.println("starting creating nodes");
+            LOG.info("starting creating nodes");
             for (int i = 0; i < 10; i++) {
                 path = "/" + i;
-                zk
-                        .create(path, (path + "!").getBytes(),
-                                Ids.OPEN_ACL_UNSAFE, 0);
+                zk.create(path,
+                          (path + "!").getBytes(),
+                          Ids.OPEN_ACL_UNSAFE, 0);
                 for (int j = 0; j < 10; j++) {
                     String subpath = path + "/" + j;
                     zk.create(subpath, (subpath + "!").getBytes(),
@@ -80,16 +110,32 @@ public class RecoveryTest extends TestCase implements Watcher {
                     }
                 }
             }
+
             f.shutdown();
-            Thread.sleep(1000);
+
+            assertTrue("waiting for server down",
+                       ClientBase.waitForServerDown(HOSTPORT,
+                                          CONNECTION_TIMEOUT));
+
             zs = new ZooKeeperServer(tmpDir, tmpDir, 3000);
-            f = new NIOServerCnxn.Factory(2344);
+            f = new NIOServerCnxn.Factory(PORT);
+
+            startSignal = new CountDownLatch(1);
+
             f.startup(zs);
-            Thread.sleep(1000);
+
+            assertTrue("waiting for server up",
+                       ClientBase.waitForServerUp(HOSTPORT,
+                                           CONNECTION_TIMEOUT));
+
+            startSignal.await(CONNECTION_TIMEOUT,
+                    TimeUnit.MILLISECONDS);
+            assertTrue("count == 0", startSignal.getCount() == 0);
+
             Stat stat = new Stat();
             for (int i = 0; i < 10; i++) {
                 path = "/" + i;
-                System.out.println("Checking " + path);
+                LOG.info("Checking " + path);
                 assertEquals(new String(zk.getData(path, false, stat)), path
                         + "!");
                 for (int j = 0; j < 10; j++) {
@@ -104,17 +150,31 @@ public class RecoveryTest extends TestCase implements Watcher {
                 }
             }
             f.shutdown();
-            Thread.sleep(2000);
+
+            assertTrue("waiting for server down",
+                       ClientBase.waitForServerDown(HOSTPORT,
+                                          ClientBase.CONNECTION_TIMEOUT));
+
             zs = new ZooKeeperServer(tmpDir, tmpDir, 3000);
-            f = new NIOServerCnxn.Factory(2344);
+            f = new NIOServerCnxn.Factory(PORT);
+
+            startSignal = new CountDownLatch(1);
+
             f.startup(zs);
-            Thread.sleep(4000);
+
+            assertTrue("waiting for server up",
+                       ClientBase.waitForServerUp(HOSTPORT,
+                               CONNECTION_TIMEOUT));
+
+            startSignal.await(CONNECTION_TIMEOUT, TimeUnit.MILLISECONDS);
+            assertTrue("count == 0", startSignal.getCount() == 0);
+
             stat = new Stat();
-            System.out.println("Check 2");
+            LOG.info("Check 2");
             for (int i = 0; i < 10; i++) {
                 path = "/" + i;
-                assertEquals(new String(zk.getData(path, false, stat)), path
-                        + "!");
+                assertEquals(new String(zk.getData(path, false, stat)),
+                             path + "!");
                 for (int j = 0; j < 10; j++) {
                     String subpath = path + "/" + j;
                     assertEquals(new String(zk.getData(subpath, false, stat)),
@@ -127,6 +187,10 @@ public class RecoveryTest extends TestCase implements Watcher {
                 }
             }
             f.shutdown();
+
+            assertTrue("waiting for server down",
+                       ClientBase.waitForServerDown(HOSTPORT,
+                                                    CONNECTION_TIMEOUT));
         } finally {
             SyncRequestProcessor.snapCount = oldSnapCount;
         }
@@ -138,7 +202,11 @@ public class RecoveryTest extends TestCase implements Watcher {
      * @see org.apache.zookeeper.Watcher#process(org.apache.zookeeper.proto.WatcherEvent)
      */
     public void process(WatcherEvent event) {
-        // TODO Auto-generated method stub
-
+        LOG.info("Event:" + event.getState() + " " + event.getType() + " " + event.getPath());
+        if (event.getState() == Watcher.Event.KeeperStateSyncConnected
+                && startSignal != null && startSignal.getCount() > 0)
+        {
+            startSignal.countDown();
+        }
     }
 }

src/java/test/org/apache/zookeeper/test/SessionTest.java

@@ -18,9 +18,15 @@
 
 package org.apache.zookeeper.test;
 
+import static org.apache.zookeeper.test.ClientBase.CONNECTION_TIMEOUT;
+
 import java.io.File;
 import java.io.IOException;
-import org.junit.Test;
+import java.util.concurrent.CountDownLatch;
+
+import junit.framework.TestCase;
+
+import org.apache.log4j.Logger;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.Watcher;
 import org.apache.zookeeper.ZooKeeper;
@@ -31,88 +37,136 @@ import org.apache.zookeeper.proto.WatcherEvent;
 import org.apache.zookeeper.server.NIOServerCnxn;
 import org.apache.zookeeper.server.ServerStats;
 import org.apache.zookeeper.server.ZooKeeperServer;
-import junit.framework.TestCase;
+import org.junit.Test;
 
 public class SessionTest extends TestCase implements Watcher {
-    static File baseTest = new File(System.getProperty("build.test.dir",
-            "build"));
+    protected static final Logger LOG = Logger.getLogger(SessionTest.class);
+
+    private static final String HOSTPORT = "127.0.0.1:33299";
 
+    private CountDownLatch startSignal;
+
+    private NIOServerCnxn.Factory serverFactory;
+
+    @Override
     protected void setUp() throws Exception {
+        LOG.info("STARTING " + getName());
+
         ServerStats.registerAsConcrete();
+
+        File tmpDir = ClientBase.createTmpDir();
+
+        ClientBase.setupTestEnv();
+        ZooKeeperServer zs = new ZooKeeperServer(tmpDir, tmpDir, 3000);
+        
+        final int PORT = Integer.parseInt(HOSTPORT.split(":")[1]);
+        serverFactory = new NIOServerCnxn.Factory(PORT);
+        serverFactory.startup(zs);
+
+        assertTrue("waiting for server up",
+                   ClientBase.waitForServerUp(HOSTPORT,
+                                              CONNECTION_TIMEOUT));
     }
+    @Override
     protected void tearDown() throws Exception {
+        serverFactory.shutdown();
+        assertTrue("waiting for server down",
+                   ClientBase.waitForServerDown(HOSTPORT,
+                                                CONNECTION_TIMEOUT));
+
         ServerStats.unregister();
+        LOG.info("FINISHED " + getName());
     }
-    /**
-     * this test checks to see if the sessionid that was created for the
-     * first zookeeper client can be reused for the second one immidiately
-     * after the first client closes and the new client resues them.
-     * @throws IOException
-     * @throws InterruptedException
-     * @throws KeeperException
-     */
-    public void testSessionReuse() throws IOException, InterruptedException {
-        File tmpDir = File.createTempFile("test", ".junit", baseTest);
-        tmpDir = new File(tmpDir + ".dir");
-        tmpDir.mkdirs();
-        ZooKeeperServer zs = new ZooKeeperServer(tmpDir, tmpDir, 3000);
-        NIOServerCnxn.Factory f = new NIOServerCnxn.Factory(33299);
-        f.startup(zs);
-        Thread.sleep(4000);
-        ZooKeeper zk = new ZooKeeper("127.0.0.1:33299", 3000, this);
 
-        long sessionId = zk.getSessionId();
-        byte[] passwd = zk.getSessionPasswd();
-        zk.close();
-        zk = new ZooKeeper("127.0.0.1:33299", 3000, this, sessionId, passwd);
-        assertEquals(sessionId, zk.getSessionId());
-        zk.close();
-        zs.shutdown();
-        f.shutdown();
+    private ZooKeeper createClient()
+        throws IOException, InterruptedException
+    {
+        startSignal = new CountDownLatch(1);
+        ZooKeeper zk = new ZooKeeper(HOSTPORT, CONNECTION_TIMEOUT, this);
+        startSignal.await();
 
+        return zk;
     }
+
+// FIXME this test is failing due to client close race condition fixing in separate patch for ZOOKEEPER-63
+//    /**
+//     * this test checks to see if the sessionid that was created for the
+//     * first zookeeper client can be reused for the second one immidiately
+//     * after the first client closes and the new client resues them.
+//     * @throws IOException
+//     * @throws InterruptedException
+//     * @throws KeeperException
+//     */
+//    public void testSessionReuse() throws IOException, InterruptedException {
+//        ZooKeeper zk = createClient();
+//
+//        long sessionId = zk.getSessionId();
+//        byte[] passwd = zk.getSessionPasswd();
+//        zk.close();
+//
+//        zk.close();
+//
+//        LOG.info("Closed first session");
+//
+//        startSignal = new CountDownLatch(1);
+//        zk = new ZooKeeper(HOSTPORT, CONNECTION_TIMEOUT, this,
+//                sessionId, passwd);
+//        startSignal.await();
+//
+//        LOG.info("Opened reuse");
+//
+//        assertEquals(sessionId, zk.getSessionId());
+//
+//        zk.close();
+//    }
+
     @Test
-    public void testSession() throws IOException, InterruptedException, KeeperException {
-        File tmpDir = File.createTempFile("test", ".junit", baseTest);
-        tmpDir = new File(tmpDir + ".dir");
-        tmpDir.mkdirs();
-        ZooKeeperServer zs = new ZooKeeperServer(tmpDir, tmpDir, 3000);
-        NIOServerCnxn.Factory f = new NIOServerCnxn.Factory(33299);
-        f.startup(zs);
-        Thread.sleep(2000);
-        ZooKeeper zk = new ZooKeeper("127.0.0.1:33299", 30000, this);
+    /**
+     * This test verifies that when the session id is reused, and the original
+     * client is disconnected, but not session closed, that the server
+     * will remove ephemeral nodes created by the original session.
+     */
+    public void testSession()
+        throws IOException, InterruptedException, KeeperException
+    {
+        ZooKeeper zk = createClient();
         zk.create("/e", new byte[0], Ids.OPEN_ACL_UNSAFE,
                         CreateFlags.EPHEMERAL);
-        System.out.println("zk with session id " + zk.getSessionId()
+        LOG.info("zk with session id 0x" + Long.toHexString(zk.getSessionId())
                 + " was destroyed!");
-        // zk.close();
+
+        // disconnect the client by killing the socket, not sending the
+        // session disconnect to the server as usual. This allows the test
+        // to verify disconnect handling
+        zk.disconnect();
+
         Stat stat = new Stat();
-        try {
-            zk = new ZooKeeper("127.0.0.1:33299", 30000, this, zk
-                    .getSessionId(), zk.getSessionPasswd());
-            System.out.println("zk with session id " + zk.getSessionId()
-                    + " was created!");
-            zk.getData("/e", false, stat);
-            System.out.println("After get data /e");
-        } catch (KeeperException e) {
-            // the zk.close() above if uncommented will close the session on the
-            // server
-            // in such case we get an exception here because we've tried joining
-            // a closed session
-        }
+        startSignal = new CountDownLatch(1);
+        zk = new ZooKeeper(HOSTPORT, CONNECTION_TIMEOUT, this,
+                           zk.getSessionId(),
+                           zk.getSessionPasswd());
+        startSignal.await();
+
+        LOG.info("zk with session id 0x" + Long.toHexString(zk.getSessionId())
+                 + " was created!");
+        zk.getData("/e", false, stat);
+        LOG.info("After get data /e");
         zk.close();
-        Thread.sleep(10000);
-        zk = new ZooKeeper("127.0.0.1:33299", 30000, this);
+
+        zk = createClient();
         assertEquals(null, zk.exists("/e", false));
-        System.out.println("before close zk with session id "
-                + zk.getSessionId() + "!");
+        LOG.info("before close zk with session id 0x"
+                + Long.toHexString(zk.getSessionId()) + "!");
         zk.close();
-        System.out.println("before shutdown zs!");
-        zs.shutdown();
-        System.out.println("after shutdown zs!");
     }
 
     public void process(WatcherEvent event) {
+        LOG.info("Event:" + event.getState() + " " + event.getType() + " " + event.getPath());
+        if (event.getState() == Watcher.Event.KeeperStateSyncConnected
+                && startSignal.getCount() > 0)
+        {
+            startSignal.countDown();
+        }
     }
 
 }

src/java/test/org/apache/zookeeper/test/SledgeHammer.java

@@ -19,7 +19,6 @@
 package org.apache.zookeeper.test;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 

src/java/test/org/apache/zookeeper/test/TestHammer.java

@@ -39,6 +39,7 @@ public class TestHammer implements VoidCallback {
             } catch (IOException e1) {
                 // TODO Auto-generated catch block
                 e1.printStackTrace();
+                throw new RuntimeException(e1);
             }
             for(int i = 0; i < REPS; i++) {
                 try {

src/java/test/org/apache/zookeeper/test/WatcherFuncTest.java

@@ -79,7 +79,9 @@ public class WatcherFuncTest extends ClientBase {
 
     private List<Integer> expected;
 
+    @Override
     protected void setUp() throws Exception {
+        LOG.info("STARTING " + getName());
         super.setUp();
 
         client_latch = new CountDownLatch(1);
@@ -92,11 +94,12 @@ public class WatcherFuncTest extends ClientBase {
 
         expected = new ArrayList<Integer>();
     }
+    @Override
     protected void tearDown() throws Exception {
         client.close();
         lsnr.close();
-        Thread.sleep(5000);
         super.tearDown();
+        LOG.info("FINISHED " + getName());
     }
 
     protected ZooKeeper createClient(Watcher watcher, CountDownLatch latch)
@@ -132,7 +135,7 @@ public class WatcherFuncTest extends ClientBase {
         try {
             assertNull(lsnr.exists("/car", true));
             client.setData("/car", "missing".getBytes(), -1);
-            assertTrue(false);
+            fail();
         } catch (KeeperException e) {
             assertEquals(KeeperException.Code.NoNode, e.getCode());
         }
@@ -140,7 +143,7 @@ public class WatcherFuncTest extends ClientBase {
         try {
             assertNull(lsnr.exists("/foo/car", true));
             client.setData("/foo/car", "missing".getBytes(), -1);
-            assertTrue(false);
+            fail();
         } catch (KeeperException e) {
             assertEquals(KeeperException.Code.NoNode, e.getCode());
         }
@@ -168,13 +171,13 @@ public class WatcherFuncTest extends ClientBase {
     {
         try {
             lsnr.getData("/foo", true, null);
-            assertTrue(false);
+            fail();
         } catch (KeeperException e) {
             assertEquals(KeeperException.Code.NoNode, e.getCode());
         }
         try {
             lsnr.getData("/foo/bar", true, null);
-            assertTrue(false);
+            fail();
         } catch (KeeperException e) {
             assertEquals(KeeperException.Code.NoNode, e.getCode());
         }
@@ -207,13 +210,13 @@ public class WatcherFuncTest extends ClientBase {
     {
         try {
             lsnr.getChildren("/foo", true);
-            assertTrue(false);
+            fail();
         } catch (KeeperException e) {
             assertEquals(KeeperException.Code.NoNode, e.getCode());
         }
         try {
             lsnr.getChildren("/foo/bar", true);
-            assertTrue(false);
+            fail();
         } catch (KeeperException e) {
             assertEquals(KeeperException.Code.NoNode, e.getCode());
         }
@@ -332,13 +335,13 @@ public class WatcherFuncTest extends ClientBase {
 
         try {
             lsnr.getData("/foo", w1, null);
-            assertTrue(false);
+            fail();
         } catch (KeeperException e) {
             assertEquals(KeeperException.Code.NoNode, e.getCode());
         }
         try {
             lsnr.getData("/foo/bar", w2, null);
-            assertTrue(false);
+            fail();
         } catch (KeeperException e) {
             assertEquals(KeeperException.Code.NoNode, e.getCode());
         }
@@ -398,13 +401,13 @@ public class WatcherFuncTest extends ClientBase {
 
         try {
             lsnr.getChildren("/foo", true);
-            assertTrue(false);
+            fail();
         } catch (KeeperException e) {
             assertEquals(KeeperException.Code.NoNode, e.getCode());
         }
         try {
             lsnr.getChildren("/foo/bar", true);
-            assertTrue(false);
+            fail();
         } catch (KeeperException e) {
             assertEquals(KeeperException.Code.NoNode, e.getCode());
         }

src/java/test/org/apache/zookeeper/test/ZooKeeperTestClient.java

@@ -23,6 +23,7 @@ import java.util.List;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeUnit;
 
+import junit.framework.AssertionFailedError;
 import junit.framework.TestCase;
 
 import org.junit.Test;
@@ -132,9 +133,8 @@ public class ZooKeeperTestClient extends TestCase implements Watcher {
     if (stat == null) {
       fail("node " + nodeName + " should exist");
     }
-    long prevSessionId = zk.getSessionId();
-    byte[] prevSessionPasswd = zk.getSessionPasswd();
-    System.out.println("Closing client with sessioid:  " + prevSessionId);
+    System.out.println("Closing client with sessionid: 0x" 
+            + Long.toHexString(zk.getSessionId()));
     zk.close();
     zk = new ZooKeeper(hostPort, 10000, this);
 
@@ -263,7 +263,7 @@ public class ZooKeeperTestClient extends TestCase implements Watcher {
 
     event = this.getEvent(10);
     if (event == null) {
-      fail("First event was not delivered promptly");
+      throw new AssertionFailedError("First event was not delivered promptly");
     }
     if (!((event.getType() == Watcher.Event.EventNodeChildrenChanged &&
            event.getPath().equalsIgnoreCase(parentName)) ||
@@ -277,7 +277,7 @@ public class ZooKeeperTestClient extends TestCase implements Watcher {
     event = this.getEvent(10);
 
     if (event == null) {
-      fail("Second event was not delivered promptly");
+      throw new AssertionFailedError("Second event was not delivered promptly");
     }
     if (!((event.getType() == Watcher.Event.EventNodeChildrenChanged &&
         event.getPath().equalsIgnoreCase(parentName)) ||