ZOOKEEPER-3439: Observability improvements on client / server connection close.

Currently when server closes a client connection there is not enough information recorded (except few exception logs) which makes it hard to do postmortems. On the other side, having a complete view of the aggregated connection closing reason will provide more signals based on which we can better operate the clusters (e.g. predicate an incident might happen based on the trending of the connection closing reasons).

Server metrics was not added in this PR as we internally use a different metrics system - so some work needed to migrate to ServerMetrics. Want to submit first to get community feedback on this.

Author: Michael Han <lhan@twitter.com>

Reviewers: Enrico Olivelli <eolivelli@gmail.com>, maoling <maoling199210191@sina.com>, enixon

Closes #997 from hanm/twitter/cc6d87505d9745caace2c86acf58d6ffa085281e

zookeeper-server/src/main/java/org/apache/zookeeper/server/DumbWatcher.java

@@ -54,7 +54,7 @@ public class DumbWatcher extends ServerCnxn {
     int getSessionTimeout() { return 0; }
 
     @Override
-    public void close() { }
+    public void close(DisconnectReason reason) { }
 
     @Override
     public void sendResponse(ReplyHeader h, Record r, String tag, String cacheKey, Stat stat) throws IOException { }

zookeeper-server/src/main/java/org/apache/zookeeper/server/FinalRequestProcessor.java

@@ -622,7 +622,7 @@ public class FinalRequestProcessor implements RequestProcessor {
         if (serverCnxnFactory == null) {
             return false;
         }
-        return serverCnxnFactory.closeSession(sessionId);
+        return serverCnxnFactory.closeSession(sessionId, ServerCnxn.DisconnectReason.CLIENT_CLOSED_SESSION);
     }
 
     private boolean connClosedByClient(Request request) {

zookeeper-server/src/main/java/org/apache/zookeeper/server/NIOServerCnxn.java

@@ -159,7 +159,8 @@ public class NIOServerCnxn extends ServerCnxn {
                 throw new EndOfStreamException(
                         "Unable to read additional data from client sessionid 0x"
                         + Long.toHexString(sessionId)
-                        + ", likely client has closed socket");
+                        + ", likely client has closed socket",
+                        DisconnectReason.UNABLE_TO_READ_FROM_CLIENT);
             }
         }
 
@@ -224,7 +225,8 @@ public class NIOServerCnxn extends ServerCnxn {
             ByteBuffer bb;
             while ((bb = outgoingBuffers.peek()) != null) {
                 if (bb == ServerCnxnFactory.closeConn) {
-                    throw new CloseRequestException("close requested");
+                    throw new CloseRequestException("close requested",
+                        DisconnectReason.CLIENT_CLOSED_CONNECTION);
                 }
                 if (bb == packetSentinel) {
                     packetSent();
@@ -274,7 +276,8 @@ public class NIOServerCnxn extends ServerCnxn {
             // Remove the buffers that we have sent
             while ((bb = outgoingBuffers.peek()) != null) {
                 if (bb == ServerCnxnFactory.closeConn) {
-                    throw new CloseRequestException("close requested");
+                    throw new CloseRequestException("close requested",
+                        DisconnectReason.CLIENT_CLOSED_CONNECTION);
                 }
                 if (bb == packetSentinel) {
                     packetSent();
@@ -319,7 +322,8 @@ public class NIOServerCnxn extends ServerCnxn {
                     throw new EndOfStreamException(
                             "Unable to read additional data from client sessionid 0x"
                             + Long.toHexString(sessionId)
-                            + ", likely client has closed socket");
+                            + ", likely client has closed socket",
+                            DisconnectReason.UNABLE_TO_READ_FROM_CLIENT);
                 }
                 if (incomingBuffer.remaining() == 0) {
                     boolean isPayload;
@@ -345,7 +349,8 @@ public class NIOServerCnxn extends ServerCnxn {
                 handleWrite(k);
 
                 if (!initialized && !getReadInterest() && !getWriteInterest()) {
-                    throw new CloseRequestException("responded to info probe");
+                    throw new CloseRequestException("responded to info probe",
+                        DisconnectReason.INFO_PROBE);
                 }
             }
         } catch (CancelledKeyException e) {
@@ -354,14 +359,14 @@ public class NIOServerCnxn extends ServerCnxn {
             if (LOG.isDebugEnabled()) {
                 LOG.debug("CancelledKeyException stack trace", e);
             }
-            close();
+            close(DisconnectReason.CANCELLED_KEY_EXCEPTION);
         } catch (CloseRequestException e) {
             // expecting close to log session closure
             close();
         } catch (EndOfStreamException e) {
             LOG.warn(e.getMessage());
             // expecting close to log session closure
-            close();
+            close(e.getReason());
         } catch (ClientCnxnLimitException e) {
             // Common case exception, print at debug level
             ServerMetrics.getMetrics().CONNECTION_REJECTED.add(1);
@@ -369,14 +374,14 @@ public class NIOServerCnxn extends ServerCnxn {
                 LOG.debug("Exception causing close of session 0x"
                           + Long.toHexString(sessionId) + ": " + e.getMessage());
             }
-            close();
+            close(DisconnectReason.CLIENT_CNX_LIMIT);
         } catch (IOException e) {
             LOG.warn("Exception causing close of session 0x"
                      + Long.toHexString(sessionId) + ": " + e.getMessage());
             if (LOG.isDebugEnabled()) {
                 LOG.debug("IOException stack trace", e);
             }
-            close();
+            close(DisconnectReason.IO_EXCEPTION);
         }
     }
 
@@ -576,11 +581,17 @@ public class NIOServerCnxn extends ServerCnxn {
                " sessionId: 0x" + Long.toHexString(sessionId);
     }
 
+
     /**
      * Close the cnxn and remove it from the factory cnxns list.
      */
     @Override
-    public void close() {
+    public void close(DisconnectReason reason) {
+        disconnectReason = reason;
+        close();
+    }
+
+    private void close() {
         setStale();
         if (!factory.removeCnxn(this)) {
             return;

zookeeper-server/src/main/java/org/apache/zookeeper/server/NIOServerCnxnFactory.java

@@ -404,7 +404,7 @@ public class NIOServerCnxnFactory extends ServerCnxnFactory {
                 for (SelectionKey key : selector.keys()) {
                     NIOServerCnxn cnxn = (NIOServerCnxn) key.attachment();
                     if (cnxn.isSelectable()) {
-                        cnxn.close();
+                        cnxn.close(ServerCnxn.DisconnectReason.SERVER_SHUTDOWN);
                     }
                     cleanupSelectionKey(key);
                 }
@@ -532,7 +532,7 @@ public class NIOServerCnxnFactory extends ServerCnxnFactory {
 
                 // Check if we shutdown or doIO() closed this connection
                 if (stopped) {
-                    cnxn.close();
+                    cnxn.close(ServerCnxn.DisconnectReason.SERVER_SHUTDOWN);
                     return;
                 }
                 if (!key.isValid()) {
@@ -548,13 +548,13 @@ public class NIOServerCnxnFactory extends ServerCnxnFactory {
             // on the current set of interest ops, which may have changed
             // as a result of the I/O operations we just performed.
             if (!selectorThread.addInterestOpsUpdateRequest(key)) {
-                cnxn.close();
+                cnxn.close(ServerCnxn.DisconnectReason.CONNECTION_MODE_CHANGED);
             }
         }
 
         @Override
         public void cleanup() {
-            cnxn.close();
+            cnxn.close(ServerCnxn.DisconnectReason.CLEAN_UP);
         }
     }
 
@@ -577,7 +577,7 @@ public class NIOServerCnxnFactory extends ServerCnxnFactory {
                     }
                     for (NIOServerCnxn conn : cnxnExpiryQueue.poll()) {
                         ServerMetrics.getMetrics().SESSIONLESS_CONNECTIONS_EXPIRED.add(1);
-                        conn.close();
+                        conn.close(ServerCnxn.DisconnectReason.CONNECTION_EXPIRED);
                     }
                 }
 
@@ -867,12 +867,12 @@ public class NIOServerCnxnFactory extends ServerCnxnFactory {
      */
     @Override
     @SuppressWarnings("unchecked")
-    public void closeAll() {
+    public void closeAll(ServerCnxn.DisconnectReason reason) {
         // clear all the connections on which we are selecting
         for (ServerCnxn cnxn : cnxns) {
             try {
                 // This will remove the cnxn from cnxns
-                cnxn.close();
+                cnxn.close(reason);
             } catch (Exception e) {
                 LOG.warn("Ignoring exception closing cnxn sessionid 0x"
                          + Long.toHexString(cnxn.getSessionId()), e);
@@ -921,7 +921,7 @@ public class NIOServerCnxnFactory extends ServerCnxnFactory {
             join();
 
             // close all open connections
-            closeAll();
+            closeAll(ServerCnxn.DisconnectReason.SERVER_SHUTDOWN);
 
             if (login != null) {
                 login.shutdown();

zookeeper-server/src/main/java/org/apache/zookeeper/server/NettyServerCnxn.java

@@ -80,7 +80,15 @@ public class NettyServerCnxn extends ServerCnxn {
         addAuthInfo(new Id("ip", addr.getHostAddress()));
     }
 
+    /**
+     * Close the cnxn and remove it from the factory cnxns list.
+     */
     @Override
+    public void close(DisconnectReason reason) {
+        disconnectReason = reason;
+        close();
+    }
+
     public void close() {
         closingChannel = true;
 
@@ -194,7 +202,7 @@ public class NettyServerCnxn extends ServerCnxn {
     @Override
     public void sendBuffer(ByteBuffer... buffers) {
         if (buffers.length == 1 && buffers[0] == ServerCnxnFactory.closeConn) {
-            close();
+            close(DisconnectReason.CLIENT_CLOSED_CONNECTION);
             return;
         }
         channel.writeAndFlush(Unpooled.wrappedBuffer(buffers)).addListener(onSendBufferDoneListener);
@@ -533,14 +541,14 @@ public class NettyServerCnxn extends ServerCnxn {
             }
         } catch(IOException e) {
             LOG.warn("Closing connection to " + getRemoteSocketAddress(), e);
-            close();
+            close(DisconnectReason.IO_EXCEPTION);
         } catch(ClientCnxnLimitException e) {
             // Common case exception, print at debug level
             ServerMetrics.getMetrics().CONNECTION_REJECTED.add(1);
             if (LOG.isDebugEnabled()) {
                 LOG.debug("Closing connection to " + getRemoteSocketAddress(), e);
             }
-            close();
+            close(DisconnectReason.CLIENT_RATE_LIMIT);
         }
     }
 

zookeeper-server/src/main/java/org/apache/zookeeper/server/NettyServerCnxnFactory.java

@@ -34,7 +34,6 @@ import java.util.concurrent.atomic.AtomicReference;
 
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.SSLEngine;
-import javax.net.ssl.SSLException;
 import javax.net.ssl.SSLPeerUnverifiedException;
 import javax.net.ssl.SSLSession;
 import javax.net.ssl.X509KeyManager;
@@ -58,7 +57,6 @@ import io.netty.channel.group.ChannelGroup;
 import io.netty.channel.group.ChannelGroupFuture;
 import io.netty.channel.group.DefaultChannelGroup;
 import io.netty.channel.socket.SocketChannel;
-import io.netty.handler.ssl.OpenSsl;
 import io.netty.handler.ssl.OptionalSslHandler;
 import io.netty.handler.ssl.SslContext;
 import io.netty.handler.ssl.SslHandler;
@@ -222,7 +220,7 @@ public class NettyServerCnxnFactory extends ServerCnxnFactory {
                 if (LOG.isTraceEnabled()) {
                     LOG.trace("Channel inactive caused close {}", cnxn);
                 }
-                cnxn.close();
+                cnxn.close(ServerCnxn.DisconnectReason.CHANNEL_DISCONNECTED);
             }
         }
 
@@ -234,7 +232,7 @@ public class NettyServerCnxnFactory extends ServerCnxnFactory {
                 if (LOG.isDebugEnabled()) {
                     LOG.debug("Closing {}", cnxn);
                 }
-                cnxn.close();
+                cnxn.close(ServerCnxn.DisconnectReason.CHANNEL_CLOSED_EXCEPTION);
             }
         }
 
@@ -354,7 +352,7 @@ public class NettyServerCnxnFactory extends ServerCnxnFactory {
 
                     if (authProvider == null) {
                         LOG.error("X509 Auth provider not found: {}", authProviderProp);
-                        cnxn.close();
+                        cnxn.close(ServerCnxn.DisconnectReason.AUTH_PROVIDER_NOT_FOUND);
                         return;
                     }
 
@@ -362,7 +360,7 @@ public class NettyServerCnxnFactory extends ServerCnxnFactory {
                             authProvider.handleAuthentication(cnxn, null)) {
                         LOG.error("Authentication failed for session 0x{}",
                                 Long.toHexString(cnxn.getSessionId()));
-                        cnxn.close();
+                        cnxn.close(ServerCnxn.DisconnectReason.SASL_AUTH_FAILURE);
                         return;
                     }
                 }
@@ -373,7 +371,7 @@ public class NettyServerCnxnFactory extends ServerCnxnFactory {
             } else {
                 LOG.error("Unsuccessful handshake with session 0x{}",
                         Long.toHexString(cnxn.getSessionId()));
-                cnxn.close();
+                cnxn.close(ServerCnxn.DisconnectReason.FAILED_HANDSHAKE);
             }
         }
     }
@@ -471,7 +469,7 @@ public class NettyServerCnxnFactory extends ServerCnxnFactory {
     }
 
     @Override
-    public void closeAll() {
+    public void closeAll(ServerCnxn.DisconnectReason reason) {
         if (LOG.isDebugEnabled()) {
             LOG.debug("closeAll()");
         }
@@ -480,7 +478,7 @@ public class NettyServerCnxnFactory extends ServerCnxnFactory {
         for (ServerCnxn cnxn : cnxns) {
             try {
                 // This will remove the cnxn from cnxns
-                cnxn.close();
+                cnxn.close(reason);
             } catch (Exception e) {
                 LOG.warn("Ignoring exception closing cnxn sessionid 0x"
                          + Long.toHexString(cnxn.getSessionId()), e);
@@ -559,7 +557,7 @@ public class NettyServerCnxnFactory extends ServerCnxnFactory {
                     bossGroup.shutdownGracefully();
                 });
             }
-            closeAll();
+            closeAll(ServerCnxn.DisconnectReason.SERVER_SHUTDOWN);
             ChannelGroupFuture allChannelsCloseFuture = allChannels.close();
             if (workerGroup != null) {
                 allChannelsCloseFuture.addListener(future -> {

zookeeper-server/src/main/java/org/apache/zookeeper/server/ServerCnxn.java

@@ -78,6 +78,47 @@ public abstract class ServerCnxn implements Stats, Watcher {
      */
     final ZooKeeperServer zkServer;
 
+    public enum DisconnectReason {
+        UNKNOWN("unknown"),
+        SERVER_SHUTDOWN("server_shutdown"),
+        CLOSE_ALL_CONNECTIONS_FORCED("close_all_connections_forced"),
+        CONNECTION_CLOSE_FORCED("connection_close_forced"),
+        CONNECTION_EXPIRED("connection_expired"),
+        CLIENT_CLOSED_CONNECTION("client_closed_connection"),
+        CLIENT_CLOSED_SESSION("client_closed_session"),
+        UNABLE_TO_READ_FROM_CLIENT("unable_to_read_from_client"),
+        NOT_READ_ONLY_CLIENT("not_read_only_client"),
+        CLIENT_ZXID_AHEAD("client_zxid_ahead"),
+        INFO_PROBE("info_probe"),
+        CLIENT_RECONNECT("client_reconnect"),
+        CANCELLED_KEY_EXCEPTION("cancelled_key_exception"),
+        IO_EXCEPTION("io_exception"),
+        IO_EXCEPTION_IN_SESSION_INIT("io_exception_in_session_init"),
+        BUFFER_UNDERFLOW_EXCEPTION("buffer_underflow_exception"),
+        SASL_AUTH_FAILURE("sasl_auth_failure"),
+        RESET_COMMAND("reset_command"),
+        CLOSE_CONNECTION_COMMAND("close_connection_command"),
+        CLEAN_UP("clean_up"),
+        CONNECTION_MODE_CHANGED("connection_mode_changed"),
+        // Below reasons are NettyServerCnxnFactory only
+        CHANNEL_DISCONNECTED("channel disconnected"),
+        CHANNEL_CLOSED_EXCEPTION("channel_closed_exception"),
+        AUTH_PROVIDER_NOT_FOUND("auth provider not found"),
+        FAILED_HANDSHAKE("Unsuccessful handshake"),
+        CLIENT_RATE_LIMIT("Client hits rate limiting threshold"),
+        CLIENT_CNX_LIMIT("Client hits connection limiting threshold");
+
+        String disconnectReason;
+
+        DisconnectReason(String reason) {
+            this.disconnectReason = reason;
+        }
+
+        public String toDisconnectReasonString() {
+            return disconnectReason;
+        }
+    }
+
     public ServerCnxn(final ZooKeeperServer zkServer) {
         this.zkServer = zkServer;
     }
@@ -103,7 +144,7 @@ public abstract class ServerCnxn implements Stats, Watcher {
         }
     }
 
-    public abstract void close();
+    public abstract void close(DisconnectReason reason);
 
     public abstract void sendResponse(ReplyHeader h, Record r,
             String tag, String cacheKey, Stat stat) throws IOException;
@@ -203,22 +244,28 @@ public abstract class ServerCnxn implements Stats, Watcher {
 
     protected static class CloseRequestException extends IOException {
         private static final long serialVersionUID = -7854505709816442681L;
+        private DisconnectReason reason;
 
-        public CloseRequestException(String msg) {
+        public CloseRequestException(String msg, DisconnectReason reason) {
             super(msg);
+            this.reason = reason;
         }
+        public DisconnectReason getReason() { return reason; }
     }
 
     protected static class EndOfStreamException extends IOException {
         private static final long serialVersionUID = -8255690282104294178L;
+        private DisconnectReason reason;
 
-        public EndOfStreamException(String msg) {
+        public EndOfStreamException(String msg, DisconnectReason reason) {
             super(msg);
+            this.reason = reason;
         }
 
         public String toString() {
             return "EndOfStreamException: " + getMessage();
         }
+        public DisconnectReason getReason() { return reason; }
     }
 
     public boolean isStale() {
@@ -263,8 +310,11 @@ public abstract class ServerCnxn implements Stats, Watcher {
 
     protected long count;
     protected long totalLatency;
+    protected long requestsProcessedCount;
+    protected DisconnectReason disconnectReason = DisconnectReason.UNKNOWN;
 
     public synchronized void resetStats() {
+        disconnectReason = DisconnectReason.RESET_COMMAND;
         packetsReceived.set(0);
         packetsSent.set(0);
         minLatency = Long.MAX_VALUE;
@@ -470,7 +520,7 @@ public abstract class ServerCnxn implements Stats, Watcher {
             LOG.info("Error closing PrintWriter ", e);
         } finally {
             try {
-                close();
+                close(DisconnectReason.CLOSE_CONNECTION_COMMAND);
             } catch (Exception e) {
                 LOG.error("Error closing a command socket ", e);
             }

zookeeper-server/src/main/java/org/apache/zookeeper/server/ServerCnxnFactory.java

@@ -73,11 +73,11 @@ public abstract class ServerCnxnFactory {
      * @return true if the cnxn that contains the sessionId exists in this ServerCnxnFactory
      *         and it's closed. Otherwise false.
      */
-    public boolean closeSession(long sessionId) {
+    public boolean closeSession(long sessionId, ServerCnxn.DisconnectReason reason) {
         ServerCnxn cnxn = sessionMap.remove(sessionId);
         if (cnxn != null) {
             try {
-                cnxn.close();
+                cnxn.close(reason);
             } catch (Exception e) {
                 LOG.warn("exception during session close", e);
             }
@@ -154,7 +154,7 @@ public abstract class ServerCnxnFactory {
         }
     }
 
-    public abstract void closeAll();
+    public abstract void closeAll(ServerCnxn.DisconnectReason reason);
     
     static public ServerCnxnFactory createFactory() throws IOException {
         String serverCnxnFactoryName =

zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServer.java

@@ -887,7 +887,7 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider {
 
         } catch (Exception e) {
             LOG.warn("Exception while establishing session, closing", e);
-            cnxn.close();
+            cnxn.close(ServerCnxn.DisconnectReason.IO_EXCEPTION_IN_SESSION_INIT);
         }
     }
 
@@ -1165,7 +1165,7 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider {
             String msg = "Refusing session request for not-read-only client "
                 + cnxn.getRemoteSocketAddress();
             LOG.info(msg);
-            throw new CloseRequestException(msg);
+            throw new CloseRequestException(msg, ServerCnxn.DisconnectReason.CLIENT_ZXID_AHEAD);
         }
         if (connReq.getLastZxidSeen() > zkDb.dataTree.lastProcessedZxid) {
             String msg = "Refusing session request for client "
@@ -1177,7 +1177,7 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider {
                 + " client must try another server";
 
             LOG.info(msg);
-            throw new CloseRequestException(msg);
+            throw new CloseRequestException(msg, ServerCnxn.DisconnectReason.NOT_READ_ONLY_CLIENT);
         }
         int sessionTimeout = connReq.getTimeOut();
         byte passwd[] = connReq.getPasswd();
@@ -1211,10 +1211,10 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider {
                     connReq.getTimeOut(),
                     cnxn.getRemoteSocketAddress());
             if (serverCnxnFactory != null) {
-                serverCnxnFactory.closeSession(sessionId);
+                serverCnxnFactory.closeSession(sessionId, ServerCnxn.DisconnectReason.CLIENT_RECONNECT);
             }
             if (secureServerCnxnFactory != null) {
-                secureServerCnxnFactory.closeSession(sessionId);
+                secureServerCnxnFactory.closeSession(sessionId, ServerCnxn.DisconnectReason.CLIENT_RECONNECT);
             }
             cnxn.setSessionId(sessionId);
             reopenSession(cnxn, sessionId, passwd, sessionTimeout);
@@ -1370,7 +1370,7 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider {
                     LOG.warn("Maintaining client connection despite SASL authentication failure.");
                 } else {
                     LOG.warn("Closing client connection due to SASL authentication failure.");
-                    cnxn.close();
+                    cnxn.close(ServerCnxn.DisconnectReason.SASL_AUTH_FAILURE);
                 }
             }
         }

zookeeper-server/src/main/java/org/apache/zookeeper/server/auth/EnsembleAuthenticationProvider.java

@@ -113,7 +113,7 @@ public class EnsembleAuthenticationProvider implements AuthenticationProvider {
          * shutdown.
          */
         ServerMetrics.getMetrics().ENSEMBLE_AUTH_FAIL.add(1);
-        cnxn.close();
+        cnxn.close(ServerCnxn.DisconnectReason.FAILED_HANDSHAKE);
         return KeeperException.Code.BADARGUMENTS;
     }
 

zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeer.java

@@ -55,6 +55,7 @@ import org.apache.zookeeper.common.Time;
 import org.apache.zookeeper.common.X509Exception;
 import org.apache.zookeeper.jmx.MBeanRegistry;
 import org.apache.zookeeper.jmx.ZKMBeanInfo;
+import org.apache.zookeeper.server.ServerCnxn;
 import org.apache.zookeeper.server.ServerCnxnFactory;
 import org.apache.zookeeper.server.ServerMetrics;
 import org.apache.zookeeper.server.ZKDatabase;
@@ -1887,10 +1888,10 @@ public class QuorumPeer extends ZooKeeperThread implements QuorumStats.Provider
 
     public void closeAllConnections() {
         if (cnxnFactory != null) {
-            cnxnFactory.closeAll();
+            cnxnFactory.closeAll(ServerCnxn.DisconnectReason.SERVER_SHUTDOWN);
         }
         if (secureCnxnFactory != null) {
-            secureCnxnFactory.closeAll();
+            secureCnxnFactory.closeAll(ServerCnxn.DisconnectReason.SERVER_SHUTDOWN);
         }
     }
 

zookeeper-server/src/test/java/org/apache/zookeeper/server/MockServerCnxn.java

@@ -40,7 +40,7 @@ public class MockServerCnxn extends ServerCnxn {
     }
 
     @Override
-    public void close() {
+    public void close(DisconnectReason reason) {
     }
 
     @Override

zookeeper-server/src/test/java/org/apache/zookeeper/server/NIOServerCnxnTest.java

@@ -63,7 +63,7 @@ public class NIOServerCnxnTest extends ClientBase {
                     serverFactory instanceof NIOServerCnxnFactory);
             Iterable<ServerCnxn> connections = serverFactory.getConnections();
             for (ServerCnxn serverCnxn : connections) {
-                serverCnxn.close();
+                serverCnxn.close(ServerCnxn.DisconnectReason.CHANNEL_CLOSED_EXCEPTION);
                 try {
                     serverCnxn.toString();
                 } catch (Exception e) {

zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/ZabUtils.java

@@ -128,10 +128,12 @@ public class ZabUtils {
                 boolean secure) throws IOException {
         }
 
-        public boolean closeSession(long sessionId) {
+        @Override
+        public boolean closeSession(long sessionId, ServerCnxn.DisconnectReason reason) {
             return false;
         }
-        public void closeAll() {
+        @Override
+        public void closeAll(ServerCnxn.DisconnectReason reason) {
         }
         @Override
         public int getNumAliveConnections() {