ZOOKEEPER-3905: Race condition causes sessions to be created for clients even though their certificate authentication has failed

Netty channel doesn't get closed if authentication fails after a successful SSL handshake. We need a custom authentication provider in order to trigger this, because the default implementation does the same check as for the SSL handshake. Hence it never fails.

Unit test added to make sure client is not able to connect.

Target branches: master, 3.6, 3.5 (will create separate PR for 3.5)

Author: Andor Molnar <andor@apache.org>

Reviewers: Enrico Olivelli <eolivelli@apache.org>, Mate Szalay-Beko <symat@apache.org>

Closes #1422 from anmolnar/ZOOKEEPER-3905

zookeeper-server/src/main/java/org/apache/zookeeper/server/NettyServerCnxn.java

@@ -115,6 +115,9 @@ public class NettyServerCnxn extends ServerCnxn {
         // if this is not in cnxns then it's already closed
         if (!factory.cnxns.remove(this)) {
             LOG.debug("cnxns size:{}", factory.cnxns.size());
+            if (channel.isOpen()) {
+                channel.close();
+            }
             return;
         }
 

zookeeper-server/src/main/java/org/apache/zookeeper/server/auth/ProviderRegistry.java

@@ -29,8 +29,10 @@ public class ProviderRegistry {
 
     private static final Logger LOG = LoggerFactory.getLogger(ProviderRegistry.class);
 
+    public static final String AUTHPROVIDER_PROPERTY_PREFIX = "zookeeper.authProvider.";
+
     private static boolean initialized = false;
-    private static Map<String, AuthenticationProvider> authenticationProviders = new HashMap<>();
+    private static final Map<String, AuthenticationProvider> authenticationProviders = new HashMap<>();
 
     //VisibleForTesting
     public static void reset() {
@@ -49,7 +51,7 @@ public class ProviderRegistry {
             Enumeration<Object> en = System.getProperties().keys();
             while (en.hasMoreElements()) {
                 String k = (String) en.nextElement();
-                if (k.startsWith("zookeeper.authProvider.")) {
+                if (k.startsWith(AUTHPROVIDER_PROPERTY_PREFIX)) {
                     String className = System.getProperty(k);
                     try {
                         Class<?> c = ZooKeeperServer.class.getClassLoader().loadClass(className);

zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeerConfig.java

@@ -46,6 +46,7 @@ import org.apache.zookeeper.common.PathUtils;
 import org.apache.zookeeper.common.StringUtils;
 import org.apache.zookeeper.metrics.impl.DefaultMetricsProvider;
 import org.apache.zookeeper.server.ZooKeeperServer;
+import org.apache.zookeeper.server.auth.ProviderRegistry;
 import org.apache.zookeeper.server.quorum.QuorumPeer.LearnerType;
 import org.apache.zookeeper.server.quorum.QuorumPeer.QuorumServer;
 import org.apache.zookeeper.server.quorum.auth.QuorumAuth;
@@ -505,11 +506,12 @@ public class QuorumPeerConfig {
      */
     public static void configureSSLAuth() throws ConfigException {
         try (ClientX509Util clientX509Util = new ClientX509Util()) {
-            String sslAuthProp = "zookeeper.authProvider."
+            String sslAuthProp = ProviderRegistry.AUTHPROVIDER_PROPERTY_PREFIX
                                  + System.getProperty(clientX509Util.getSslAuthProviderProperty(), "x509");
             if (System.getProperty(sslAuthProp) == null) {
-                if ("zookeeper.authProvider.x509".equals(sslAuthProp)) {
-                    System.setProperty("zookeeper.authProvider.x509", "org.apache.zookeeper.server.auth.X509AuthenticationProvider");
+                if ((ProviderRegistry.AUTHPROVIDER_PROPERTY_PREFIX + "x509").equals(sslAuthProp)) {
+                    System.setProperty(ProviderRegistry.AUTHPROVIDER_PROPERTY_PREFIX + "x509",
+                        "org.apache.zookeeper.server.auth.X509AuthenticationProvider");
                 } else {
                     throw new ConfigException("No auth provider configured for the SSL authentication scheme '"
                                               + System.getProperty(clientX509Util.getSslAuthProviderProperty())

zookeeper-server/src/test/java/org/apache/zookeeper/test/AuthFailX509AuthenticationProvider.java

@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zookeeper.test;
+
+import javax.net.ssl.X509KeyManager;
+import javax.net.ssl.X509TrustManager;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.common.X509Exception;
+import org.apache.zookeeper.server.ServerCnxn;
+import org.apache.zookeeper.server.auth.X509AuthenticationProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class AuthFailX509AuthenticationProvider extends X509AuthenticationProvider {
+  private static final Logger LOG = LoggerFactory.getLogger(AuthFailX509AuthenticationProvider.class);
+
+  public AuthFailX509AuthenticationProvider() throws X509Exception {
+    super();
+  }
+
+  public AuthFailX509AuthenticationProvider(X509TrustManager trustManager, X509KeyManager keyManager) {
+    super(trustManager, keyManager);
+  }
+
+  @Override
+  public KeeperException.Code handleAuthentication(ServerCnxn cnxn, byte[] authData) {
+    LOG.info("Authentication failed");
+    return KeeperException.Code.AUTHFAILED;
+  }
+
+  @Override
+  public String getScheme() {
+    return "authfail";
+  }
+}

zookeeper-server/src/test/java/org/apache/zookeeper/test/ClientBase.java

@@ -735,10 +735,14 @@ public abstract class ClientBase extends ZKTestCase {
      *             in cases of network failure
      */
     public static ZooKeeper createZKClient(String cxnString, int sessionTimeout) throws IOException {
+        return createZKClient(cxnString, sessionTimeout, CONNECTION_TIMEOUT);
+    }
+
+    public static ZooKeeper createZKClient(String cxnString, int sessionTimeout, long connectionTimeout) throws IOException {
         CountdownWatcher watcher = new CountdownWatcher();
         ZooKeeper zk = new ZooKeeper(cxnString, sessionTimeout, watcher);
         try {
-            watcher.waitForConnected(CONNECTION_TIMEOUT);
+            watcher.waitForConnected(connectionTimeout);
         } catch (InterruptedException | TimeoutException e) {
             fail("ZooKeeper client can not connect to " + cxnString);
         }

zookeeper-server/src/test/java/org/apache/zookeeper/test/ClientSSLTest.java

@@ -22,7 +22,11 @@
 
 package org.apache.zookeeper.test;
 
+import static org.hamcrest.CoreMatchers.startsWith;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.junit.Assert.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
+import java.io.IOException;
 import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.PortAssignment;
 import org.apache.zookeeper.ZooDefs;
@@ -31,6 +35,7 @@ import org.apache.zookeeper.client.ZKClientConfig;
 import org.apache.zookeeper.common.ClientX509Util;
 import org.apache.zookeeper.server.NettyServerCnxnFactory;
 import org.apache.zookeeper.server.ServerCnxnFactory;
+import org.apache.zookeeper.server.auth.ProviderRegistry;
 import org.apache.zookeeper.server.quorum.QuorumPeerTestBase;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
@@ -156,4 +161,26 @@ public class ClientSSLTest extends QuorumPeerTestBase {
         mt.shutdown();
     }
 
+    @Test
+    public void testSecureStandaloneServerAuthFail() throws IOException {
+        try {
+            System.setProperty(ProviderRegistry.AUTHPROVIDER_PROPERTY_PREFIX + "authfail",
+                AuthFailX509AuthenticationProvider.class.getName());
+            System.setProperty(clientX509Util.getSslAuthProviderProperty(), "authfail");
+
+            Integer secureClientPort = PortAssignment.unique();
+            MainThread mt = new MainThread(MainThread.UNSET_MYID, "", secureClientPort, false);
+            mt.start();
+
+            AssertionError ex = assertThrows("Client should not able to connect when authentication fails", AssertionError.class,
+                () -> {
+                    ClientBase.createZKClient("localhost:" + secureClientPort, TIMEOUT, 3000);
+                });
+            assertThat("Exception message does not match (different exception caught?)",
+                ex.getMessage(), startsWith("ZooKeeper client can not connect to"));
+        } finally {
+            System.clearProperty(ProviderRegistry.AUTHPROVIDER_PROPERTY_PREFIX + "authfail");
+            System.clearProperty(clientX509Util.getSslAuthProviderProperty());
+        }
+    }
 }