ZOOKEEPER-4798: Secure prometheus support (#2127)

Co-authored-by: purshotam shah <purushah@yahooinc.com>
Co-authored-by: tison <wander4096@gmail.com>

zookeeper-metrics-providers/zookeeper-prometheus-metrics/src/main/java/org/apache/zookeeper/metrics/prometheus/PrometheusMetricsProvider.java

@@ -55,9 +55,12 @@ import org.apache.zookeeper.server.RateLogger;
 import org.eclipse.jetty.security.ConstraintMapping;
 import org.eclipse.jetty.security.ConstraintSecurityHandler;
 import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.ServerConnector;
 import org.eclipse.jetty.servlet.ServletContextHandler;
 import org.eclipse.jetty.servlet.ServletHolder;
 import org.eclipse.jetty.util.security.Constraint;
+import org.eclipse.jetty.util.ssl.KeyStoreScanner;
+import org.eclipse.jetty.util.ssl.SslContextFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -101,7 +104,8 @@ public class PrometheusMetricsProvider implements MetricsProvider {
     private final CollectorRegistry collectorRegistry = CollectorRegistry.defaultRegistry;
     private final RateLogger rateLogger = new RateLogger(LOG, 60 * 1000);
     private String host = "0.0.0.0";
-    private int port = 7000;
+    private int httpPort = -1;
+    private int httpsPort = -1;
     private boolean exportJvmInfo = true;
     private Server server;
     private final MetricsServletImpl servlet = new MetricsServletImpl();
@@ -111,11 +115,48 @@ public class PrometheusMetricsProvider implements MetricsProvider {
     private long workerShutdownTimeoutMs = 1000;
     private Optional<ExecutorService> executorOptional = Optional.empty();
 
+    // Constants for SSL configuration
+    public static final int SCAN_INTERVAL = 60 * 10; // 10 minutes
+    public static final String SSL_KEYSTORE_LOCATION = "ssl.keyStore.location";
+    public static final String SSL_KEYSTORE_PASSWORD = "ssl.keyStore.password";
+    public static final String SSL_KEYSTORE_TYPE = "ssl.keyStore.type";
+    public static final String SSL_TRUSTSTORE_LOCATION = "ssl.trustStore.location";
+    public static final String SSL_TRUSTSTORE_PASSWORD = "ssl.trustStore.password";
+    public static final String SSL_TRUSTSTORE_TYPE = "ssl.trustStore.type";
+    public static final String SSL_X509_CN = "ssl.x509.cn";
+    public static final String SSL_X509_REGEX_CN = "ssl.x509.cn.regex";
+    public static final String SSL_NEED_CLIENT_AUTH = "ssl.need.client.auth";
+    public static final String SSL_WANT_CLIENT_AUTH = "ssl.want.client.auth";
+
+    private String keyStorePath;
+    private String keyStorePassword;
+    private String keyStoreType;
+    private String trustStorePath;
+    private String trustStorePassword;
+    private String trustStoreType;
+    private boolean needClientAuth = true;
+    private boolean wantClientAuth = true;
+
     @Override
     public void configure(Properties configuration) throws MetricsProviderLifeCycleException {
         LOG.info("Initializing metrics, configuration: {}", configuration);
         this.host = configuration.getProperty("httpHost", "0.0.0.0");
-        this.port = Integer.parseInt(configuration.getProperty("httpPort", "7000"));
+        if (configuration.containsKey("httpsPort")) {
+            this.httpsPort = Integer.parseInt(configuration.getProperty("httpsPort"));
+            this.keyStorePath = configuration.getProperty(SSL_KEYSTORE_LOCATION);
+            this.keyStorePassword = configuration.getProperty(SSL_KEYSTORE_PASSWORD);
+            this.keyStoreType = configuration.getProperty(SSL_KEYSTORE_TYPE);
+            this.trustStorePath = configuration.getProperty(SSL_TRUSTSTORE_LOCATION);
+            this.trustStorePassword = configuration.getProperty(SSL_TRUSTSTORE_PASSWORD);
+            this.trustStoreType = configuration.getProperty(SSL_TRUSTSTORE_TYPE);
+            this.needClientAuth = Boolean.parseBoolean(configuration.getProperty(SSL_NEED_CLIENT_AUTH, "true"));
+            this.wantClientAuth = Boolean.parseBoolean(configuration.getProperty(SSL_WANT_CLIENT_AUTH, "true"));
+            //check if httpPort is also configured
+            this.httpPort = Integer.parseInt(configuration.getProperty("httpPort", "-1"));
+        } else {
+            // Use the default HTTP port (7000) or the configured port if HTTPS is not set.
+            this.httpPort = Integer.parseInt(configuration.getProperty("httpPort", "7000"));
+        }
         this.exportJvmInfo = Boolean.parseBoolean(configuration.getProperty("exportJvmInfo", "true"));
         this.numWorkerThreads = Integer.parseInt(
                 configuration.getProperty(NUM_WORKER_THREADS, "1"));
@@ -129,12 +170,29 @@ public class PrometheusMetricsProvider implements MetricsProvider {
     public void start() throws MetricsProviderLifeCycleException {
         this.executorOptional = createExecutor();
         try {
-            LOG.info("Starting /metrics HTTP endpoint at host: {}, port: {}, exportJvmInfo: {}",
-                    host, port, exportJvmInfo);
+            LOG.info("Starting /metrics {} endpoint at HTTP port: {}, HTTPS port: {}, exportJvmInfo: {}",
+                    httpPort > 0 ? httpPort : "disabled",
+                    httpsPort > 0 ? httpsPort : "disabled",
+                    exportJvmInfo);
             if (exportJvmInfo) {
                 DefaultExports.initialize();
             }
-            server = new Server(new InetSocketAddress(host, port));
+            if (httpPort == -1) {
+                server = new Server();
+            } else {
+                server = new Server(new InetSocketAddress(host, httpPort));
+            }
+            if (httpsPort != -1) {
+                SslContextFactory sslServerContextFactory = new SslContextFactory.Server();
+                configureSslContextFactory(sslServerContextFactory);
+                KeyStoreScanner keystoreScanner = new KeyStoreScanner(sslServerContextFactory);
+                keystoreScanner.setScanInterval(SCAN_INTERVAL);
+                server.addBean(keystoreScanner);
+                ServerConnector connector = new ServerConnector(server, sslServerContextFactory);
+                connector.setPort(httpsPort);
+                connector.setHost(host);
+                server.addConnector(connector);
+            }
             ServletContextHandler context = new ServletContextHandler();
             context.setContextPath("/");
             constrainTraceMethod(context);
@@ -156,6 +214,44 @@ public class PrometheusMetricsProvider implements MetricsProvider {
         }
     }
 
+    @SuppressWarnings("deprecation")
+    private void configureSslContextFactory(SslContextFactory sslServerContextFactory) {
+        if (keyStorePath != null) {
+            sslServerContextFactory.setKeyStorePath(keyStorePath);
+        } else {
+            LOG.error("KeyStore configuration is incomplete keyStorePath: {}", keyStorePath);
+            throw new IllegalStateException("KeyStore configuration is incomplete keyStorePath: " + keyStorePath);
+        }
+        if (keyStorePassword != null) {
+            sslServerContextFactory.setKeyStorePassword(keyStorePassword);
+        } else {
+            LOG.error("keyStorePassword configuration is incomplete ");
+            throw new IllegalStateException("keyStorePassword configuration is incomplete ");
+        }
+        if (keyStoreType != null) {
+            sslServerContextFactory.setKeyStoreType(keyStoreType);
+        }
+        if (trustStorePath != null) {
+            sslServerContextFactory.setTrustStorePath(trustStorePath);
+        } else {
+            LOG.error("TrustStore configuration is incomplete trustStorePath: {}", trustStorePath);
+            throw new IllegalStateException("TrustStore configuration is incomplete trustStorePath: " + trustStorePath);
+        }
+        if (trustStorePassword != null) {
+            sslServerContextFactory.setTrustStorePassword(trustStorePassword);
+        } else {
+            LOG.error("trustStorePassword configuration is incomplete");
+            throw new IllegalStateException("trustStorePassword configuration is incomplete");
+        }
+        if (trustStoreType != null) {
+            sslServerContextFactory.setTrustStoreType(trustStoreType);
+        }
+        sslServerContextFactory
+                .setNeedClientAuth(needClientAuth);
+        sslServerContextFactory
+                .setWantClientAuth(wantClientAuth);
+    }
+
     // for tests
     MetricsServletImpl getServlet() {
         return servlet;

zookeeper-metrics-providers/zookeeper-prometheus-metrics/src/test/java/org/apache/zookeeper/metrics/prometheus/PrometheusHttpsMetricsProviderTest.java

@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zookeeper.metrics.prometheus;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.MatcherAssert.assertThat;
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.security.KeyStore;
+import java.util.Properties;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManagerFactory;
+import org.apache.zookeeper.metrics.Counter;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests about Prometheus Metrics Provider. Please note that we are not testing
+ * Prometheus but only our integration.
+ */
+public class PrometheusHttpsMetricsProviderTest extends PrometheusMetricsTestBase {
+
+    private PrometheusMetricsProvider provider;
+    private String httpHost = "127.0.0.1";
+    private int httpsPort = 4443;
+    private int httpPort = 4000;
+    private String testDataPath = System.getProperty("test.data.dir", "src/test/resources/data");
+
+    public void initializeProviderWithCustomConfig(Properties inputConfiguration) throws Exception {
+        provider = new PrometheusMetricsProvider();
+        Properties configuration = new Properties();
+        configuration.setProperty("httpHost", httpHost);
+        configuration.setProperty("exportJvmInfo", "false");
+        configuration.setProperty("ssl.keyStore.location", testDataPath + "/ssl/server_keystore.jks");
+        configuration.setProperty("ssl.keyStore.password", "testpass");
+        configuration.setProperty("ssl.trustStore.location", testDataPath + "/ssl/server_truststore.jks");
+        configuration.setProperty("ssl.trustStore.password", "testpass");
+        configuration.putAll(inputConfiguration);
+        provider.configure(configuration);
+        provider.start();
+    }
+
+    @AfterEach
+    public void tearDown() {
+        if (provider != null) {
+            provider.stop();
+        }
+    }
+
+    @Test
+    void testHttpResponce() throws Exception {
+        Properties configuration = new Properties();
+        configuration.setProperty("httpPort", String.valueOf(httpPort));
+        initializeProviderWithCustomConfig(configuration);
+        simulateMetricIncrement();
+        validateMetricResponse(callHttpServlet("http://" + httpHost + ":" + httpPort + "/metrics"));
+    }
+
+    @Test
+    void testHttpsResponse() throws Exception {
+        Properties configuration = new Properties();
+        configuration.setProperty("httpsPort", String.valueOf(httpsPort));
+        initializeProviderWithCustomConfig(configuration);
+        simulateMetricIncrement();
+        validateMetricResponse(callHttpsServlet("https://" + httpHost + ":" + httpsPort + "/metrics"));
+    }
+
+    @Test
+    void testHttpAndHttpsResponce() throws Exception {
+        Properties configuration = new Properties();
+        configuration.setProperty("httpsPort", String.valueOf(httpsPort));
+        configuration.setProperty("httpPort", String.valueOf(httpPort));
+        initializeProviderWithCustomConfig(configuration);
+        simulateMetricIncrement();
+        validateMetricResponse(callHttpServlet("http://" + httpHost + ":" + httpPort + "/metrics"));
+        validateMetricResponse(callHttpsServlet("https://" + httpHost + ":" + httpsPort + "/metrics"));
+    }
+
+    private String callHttpsServlet(String urlString) throws Exception {
+        // Load and configure the SSL context from the keystore and truststore
+        KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType());
+        try (FileInputStream keystoreStream = new FileInputStream(testDataPath + "/ssl/client_keystore.jks")) {
+            keyStore.load(keystoreStream, "testpass".toCharArray());
+        }
+
+        KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType());
+        try (FileInputStream trustStoreStream = new FileInputStream(testDataPath + "/ssl/client_truststore.jks")) {
+            trustStore.load(trustStoreStream, "testpass".toCharArray());
+        }
+
+        SSLContext sslContext = SSLContext.getInstance("TLS");
+        KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
+        keyManagerFactory.init(keyStore, "testpass".toCharArray());
+        TrustManagerFactory trustManagerFactory = TrustManagerFactory
+                .getInstance(TrustManagerFactory.getDefaultAlgorithm());
+        trustManagerFactory.init(trustStore);
+        sslContext.init(keyManagerFactory.getKeyManagers(), trustManagerFactory.getTrustManagers(),
+                new java.security.SecureRandom());
+
+        HttpsURLConnection.setDefaultSSLSocketFactory(sslContext.getSocketFactory());
+        URL url = new URL(urlString);
+        HttpsURLConnection connection = (HttpsURLConnection) url.openConnection();
+        connection.setRequestMethod("GET");
+
+        return readResponse(connection);
+    }
+
+    private String callHttpServlet(String urlString) throws IOException {
+        URL url = new URL(urlString);
+        HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+        connection.setRequestMethod("GET");
+        return readResponse(connection);
+    }
+
+    private String readResponse(HttpURLConnection connection) throws IOException {
+        int status = connection.getResponseCode();
+        try (BufferedReader reader = new BufferedReader(
+                new InputStreamReader(status > 299 ? connection.getErrorStream() : connection.getInputStream()))) {
+            StringBuilder content = new StringBuilder();
+            String inputLine;
+            while ((inputLine = reader.readLine()) != null) {
+                content.append(inputLine).append("\n");
+            }
+            return content.toString().trim();
+        } finally {
+            connection.disconnect();
+        }
+    }
+
+    public void simulateMetricIncrement() {
+        Counter counter = provider.getRootContext().getCounter("cc");
+        counter.add(10);
+    }
+
+    private void validateMetricResponse(String response) throws IOException {
+        assertThat(response, containsString("# TYPE cc counter"));
+        assertThat(response, containsString("cc 10.0"));
+    }
+}

zookeeper-metrics-providers/zookeeper-prometheus-metrics/src/test/java/org/apache/zookeeper/metrics/prometheus/PrometheusMetricsProviderConfigTest.java

@@ -23,7 +23,6 @@ import java.util.Properties;
 import org.apache.zookeeper.metrics.MetricsProviderLifeCycleException;
 import org.junit.jupiter.api.Test;
 
-
 public class PrometheusMetricsProviderConfigTest extends PrometheusMetricsTestBase {
 
     @Test
@@ -59,4 +58,50 @@ public class PrometheusMetricsProviderConfigTest extends PrometheusMetricsTestBa
         provider.start();
     }
 
+    @Test
+    public void testValidSslConfig() throws MetricsProviderLifeCycleException {
+        PrometheusMetricsProvider provider = new PrometheusMetricsProvider();
+        Properties configuration = new Properties();
+        String testDataPath = System.getProperty("test.data.dir", "src/test/resources/data");
+        configuration.setProperty("httpHost", "127.0.0.1");
+        configuration.setProperty("httpsPort", "50511");
+        configuration.setProperty("ssl.keyStore.location", testDataPath + "/ssl/server_keystore.jks");
+        configuration.setProperty("ssl.keyStore.password", "testpass");
+        configuration.setProperty("ssl.trustStore.location", testDataPath + "/ssl/server_truststore.jks");
+        configuration.setProperty("ssl.trustStore.password", "testpass");
+        provider.configure(configuration);
+        provider.start();
+    }
+
+    @Test
+    public void testValidHttpsAndHttpConfig() throws MetricsProviderLifeCycleException {
+        PrometheusMetricsProvider provider = new PrometheusMetricsProvider();
+        Properties configuration = new Properties();
+        String testDataPath = System.getProperty("test.data.dir", "src/test/resources/data");
+        configuration.setProperty("httpPort", "50512");
+        configuration.setProperty("httpsPort", "50513");
+        configuration.setProperty("ssl.keyStore.location", testDataPath + "/ssl/server_keystore.jks");
+        configuration.setProperty("ssl.keyStore.password", "testpass");
+        configuration.setProperty("ssl.trustStore.location", testDataPath + "/ssl/server_truststore.jks");
+        configuration.setProperty("ssl.trustStore.password", "testpass");
+        provider.configure(configuration);
+        provider.start();
+    }
+
+
+    @Test
+    public void testInvalidSslConfig() throws MetricsProviderLifeCycleException {
+        assertThrows(MetricsProviderLifeCycleException.class, () -> {
+            PrometheusMetricsProvider provider = new PrometheusMetricsProvider();
+            Properties configuration = new Properties();
+            String testDataPath = System.getProperty("test.data.dir", "src/test/resources/data");
+            configuration.setProperty("httpsPort", "50514");
+            //keystore missing
+            configuration.setProperty("ssl.keyStore.password", "testpass");
+            configuration.setProperty("ssl.trustStore.location", testDataPath + "/ssl/server_truststore.jks");
+            configuration.setProperty("ssl.trustStore.password", "testpass");
+            provider.configure(configuration);
+            provider.start();
+        });
+    }
 }

zookeeper-metrics-providers/zookeeper-prometheus-metrics/src/test/resources/data/ssl/README.md

@@ -0,0 +1,5 @@
+SSL test data
+===================
+
+Testing client/server keystore, password is "testpass".
+Testing client/server truststore, password is "testpass".