ZOOKEEPER-2108 ZOOKEEPER-3908 ZOOKEEPER-4491: fixing zktreeutil, adding ssl support

Currently, zktreeutil tool **doesn't compile/work**. This PR brings back `zktreeutil` to life and adds SSL support to connect to TLS enabled ZooKeeper server.

**Following are the issues fixed with this PR.**

1) [ZOOKEEPER-2108](https://issues.apache.org/jira/browse/ZOOKEEPER-2108)  **(Compilation error in ZkAdaptor.cc with GCC 4.7 or later)**
    The compilation error in `ZkAdaptor.cc` is fixed by including the header file. Also, updated the `ZOOKEEPER_PATH` in `configure.ac`

2) [ZOOKEEPER-3908](https://issues.apache.org/jira/browse/ZOOKEEPER-3908) **(zktreeutil multiple issues)**
   Fixed the issues reported in this ticket.

3) [ZOOKEEPER-4491](https://issues.apache.org/jira/browse/ZOOKEEPER-4491) **(Adding SSL support to Zktreeutil)**

    -    Adds the SSL support to `zktreeutil` to talk to TLS enabled ZooKeeper server, If SSL params are passed, invoke `zookeeper_init_ssl` method to connect the server.

    -    Used the same OpenSSL library auto-detect code from the c-client `configure.ac`  (https://github.com/apache/zookeeper/pull/1159)

Attached the testing log.

[zktreeutil_testing_connection.txt](https://github.com/apache/zookeeper/files/8585349/zktreeutil_testing_connection.txt)

Author: Manu Mathew <manu.mathew@netapp.com>
Author: mathewmanu <manmathew@cs.stonybrook.edu>

Reviewers: Enrico Olivelli <eolivelli@apache.org>, Mate Szalay-Beko <symat@apache.org>

Closes #1870 from mathew-manu/ZOOKEEPER-4491

zookeeper-contrib/zookeeper-contrib-zktreeutil/README.txt

@@ -38,7 +38,7 @@ Once ignored, the whole subtree is ignored during DIFF, UPDATE and WRITE.
 Pre-requisites
 --------------
 1. Linux system with 2.6.X kernel.
-2. Zookeeper C client library (locally built at ../../c/.libs) >= 3.X.X
+2. Zookeeper C client library (locally built at ../../zookeeper-client/zookeeper-client-c/target/c/.libs) >= 3.X.X
 3. Development build libraries (rpm packages):
   a. boost-devel >= 1.32.0
   b. libxml2-devel >= 2.7.3
@@ -60,7 +60,7 @@ versions.
 Testing  and usage of zktreeutil
 --------------------------------
 1.  Run Zookeeper server locally on port 2181
-2.  export LD_LIBRARY_PATH=../../c/.libs/:/usr/local/lib/
+2.  export LD_LIBRARY_PATH=../../zookeeper-client/zookeeper-client-c/target/c/.libs
 3.  ./src/zktreeutil --help # show help
 4.  ./src/zktreeutil --zookeeper=localhost:2181 --import --xmlfile=tests/zk_sample.xml 2>/dev/null                 # import sample ZK tree
 5.  ./src/zktreeutil --zookeeper=localhost:2181 --dump --path=/myapp/version-1.0 2>/dev/null                         # dump Zk subtree 
@@ -71,4 +71,4 @@ Testing  and usage of zktreeutil
 9.  ./src/zktreeutil -z localhost:2181 -E 2>/dev/null > zk_sample2.xml                                                         # export the mofied ZK tree
 10. ./src/zktreeutil -z localhost:2181 -U -x zk_sample.xml -p /myapp/version-1.0/distributions 2>/dev/null        # update with incr. changes
 11. ./src/zktreeutil --zookeeper=localhost:2181 --import --force --xmlfile=zk_sample2.xml 2>/dev/null             # re-prime the ZK tree
-
+12. ./src/zktreeutil --zookeeper=localhost:2188 --dump --ssl=/path/certs/root_ca.pem,/path/certs/node.crt,/path/certs/node.key   # connect with ssl params to the secureClientPort

zookeeper-contrib/zookeeper-contrib-zktreeutil/configure.ac

@@ -26,8 +26,8 @@ XML2_INCLUDE="/usr/include/libxml2"
 AC_SUBST(XML2_INCLUDE)
 
 # Zookeeper C client
-ZOOKEEPER_PATH=${BUILD_PATH}/../../c
-AC_CHECK_LIB(zookeeper_mt, main, [ZOOKEEPER="-L${ZOOKEEPER_PATH}/.libs -lzookeeper_mt"],,["-L${ZOOKEEPER_PATH}/.libs"])
+ZOOKEEPER_PATH=${BUILD_PATH}/../../zookeeper-client/zookeeper-client-c
+AC_CHECK_LIB(zookeeper_mt, main, [ZOOKEEPER="-L${ZOOKEEPER_PATH}/target/c/.libs -lzookeeper_mt"],,["-L${ZOOKEEPER_PATH}/target/c/.libs"])
 if test -z "${ZOOKEEPER}"; then
       AC_ERROR("... zookeeper C client not found!")
 fi
@@ -45,6 +45,35 @@ if test -z "${LOG4CXX}"; then
       AC_ERROR("... log4cxx not found!")
 fi
 
+dnl OpenSSL
+AC_ARG_WITH(openssl,
+ [AC_HELP_STRING([--with-openssl[=DIR]], [build with openssl (autodetect openssl library by default) )])],
+ [], [with_openssl=yes])
+AC_MSG_NOTICE([configuring SSL using --with-openssl=$with_openssl])
+saved_CPPFLAGS="$CPPFLAGS"
+saved_LDFLAGS="$LDFLAGS"
+if test "x$with_openssl" != "xno" && test "x$with_openssl" != "xyes" ; then
+        CPPFLAGS="$CPPFLAGS -I$with_openssl/include"
+        LDFLAGS="$LDFLAGS -L$with_openssl/lib"
+fi
+have_openssl=no
+AC_CHECK_HEADER(openssl/ssl.h, [ AC_CHECK_LIB(ssl, SSL_CTX_new, [have_openssl=yes]) ])
+if test "x$with_openssl" != "xno" && test "x$with_openssl" != "xyes" && test "x$have_openssl" != "xyes"; then
+    CPPFLAGS="$saved_CPPFLAGS"
+    LDFLAGS="$saved_LDFLAGS"
+fi
+if test "x$with_openssl" != xno && test "x$have_openssl" = xno; then
+    AC_MSG_WARN([cannot build SSL support -- openssl not found])
+    with_openssl=no
+fi
+if test "x$with_openssl" != xno; then
+    AC_MSG_NOTICE([building with SSL support])
+else
+    AC_MSG_NOTICE([building without SSL support])
+fi
+AM_CONDITIONAL([WANT_OPENSSL],[test "x$with_openssl" != xno])
+
+
 AC_SUBST(LOG4CXX)
 AC_SUBST(LOG4CXX_VERSION)
 AC_SUBST(LOG4CXX_INCLUDE)

zookeeper-contrib/zookeeper-contrib-zktreeutil/src/Makefile.am

@@ -14,9 +14,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+if WANT_OPENSSL
+  OPENSSL_CPPFLAGS = -DHAVE_OPENSSL_H
+endif
+
 AM_CXXFLAGS = -I${ZOOKEEPER_PATH}/include -I${ZOOKEEPER_PATH}/generated \
     -I$(top_srcdir)/include -I${LOG4CXX_INCLUDE} -I/usr/include \
-      -I${XML2_INCLUDE}
+      -I${XML2_INCLUDE} -DTHREADED $(OPENSSL_CPPFLAGS)
 
 bin_PROGRAMS = zktreeutil
 

zookeeper-contrib/zookeeper-contrib-zktreeutil/src/ZkAdaptor.cc

@@ -18,6 +18,7 @@
 
 #include "ZkAdaptor.h"
 #include <string.h>
+#include <unistd.h>
 #include <sstream>
 #include <iostream>
 #include <algorithm>
@@ -172,12 +173,36 @@ namespace zktreeutil
         disconnect();
 
         // Establish a new connection to ZooKeeper
-        mp_zkHandle = zookeeper_init( m_zkConfig.getHosts().c_str(), 
-                NULL, 
+#ifdef HAVE_OPENSSL_H
+        if (!m_zkConfig.getSslParams().empty())
+        {
+            mp_zkHandle = zookeeper_init_ssl( m_zkConfig.getHosts().c_str(),
+                m_zkConfig.getSslParams().c_str(),
+                NULL,
+                m_zkConfig.getLeaseTimeout(),
+                0,
+                NULL,
+                0);
+
+        }
+        else
+        {
+            mp_zkHandle = zookeeper_init( m_zkConfig.getHosts().c_str(),
+                NULL,
                 m_zkConfig.getLeaseTimeout(),
                 0,
                 NULL,
                 0);
+        }
+#else
+        mp_zkHandle = zookeeper_init( m_zkConfig.getHosts().c_str(),
+            NULL,
+            m_zkConfig.getLeaseTimeout(),
+            0,
+            NULL,
+            0);
+#endif
+
         if (mp_zkHandle == NULL)
         {
             // Invalid handle returned
@@ -201,7 +226,7 @@ namespace zktreeutil
                     << std::endl; 
                 return;
             }
-            else if ( state && state != ZOO_CONNECTING_STATE)
+            else if ( state && state != ZOO_NOTCONNECTED_STATE && state != ZOO_CONNECTING_STATE)
             {
                 // Not connecting any more... some other issue
                 std::ostringstream oss;

zookeeper-contrib/zookeeper-contrib-zktreeutil/src/ZkAdaptor.h

@@ -110,15 +110,18 @@ namespace zktreeutil
              * @param leaseTimeout the lease timeout (heartbeat)
              * @param autoReconnect whether to allow for auto-reconnect
              * @param connectTimeout the connect timeout, in milliseconds;
+             * @param certs ssl parameters to initiate SSL connection;
              */
             ZooKeeperConfig(const string &hosts, 
                     int leaseTimeout, 
                     bool autoReconnect = true, 
-                    long long int connectTimeout = 15000)
+                    long long int connectTimeout = 15000,
+                    const string &sslParams = "")
                 : m_hosts(hosts),
                 m_leaseTimeout(leaseTimeout), 
                 m_autoReconnect(autoReconnect),
-                m_connectTimeout(connectTimeout) {}
+                m_connectTimeout(connectTimeout),
+                m_sslParams(sslParams) {}
 
             /**
              * \brief Returns the list of ZK hosts to connect to.
@@ -143,6 +146,11 @@ namespace zktreeutil
              */
             long long int getConnectTimeout() const { return m_connectTimeout; }
 
+            /**
+             * \brief Returns the ssl params
+             */
+            string getSslParams() const { return m_sslParams; }
+
         private:
 
             /**
@@ -166,6 +174,11 @@ namespace zktreeutil
              * is established to ZK.
              */
             const long long int m_connectTimeout;
+
+            /**
+             * comma separated ssl parameters to initiate SSL connection.
+             */
+             const string m_sslParams;
     };
 
     /**

zookeeper-contrib/zookeeper-contrib-zktreeutil/src/ZkTreeUtil.cc

@@ -296,12 +296,12 @@ namespace zktreeutil
         return zkRootSptr;
     }
 
-    ZooKeeperAdapterSptr ZkTreeUtil::get_zkHandle (const string& zkHosts)
+    ZooKeeperAdapterSptr ZkTreeUtil::get_zkHandle (const string& zkHosts, const string& cert)
     {
         try
         {
             // Create an instance of ZK adapter.
-            ZooKeeperConfig config (zkHosts, 10000);
+            ZooKeeperConfig config (zkHosts, 10000, true, 15000, cert);
             ZooKeeperAdapterSptr zkHandleSptr =
                 ZooKeeperAdapterSptr (new ZooKeeperAdapter (config));
             return zkHandleSptr;
@@ -343,7 +343,7 @@ namespace zktreeutil
         }
 
         // Connect to ZK server
-        ZooKeeperAdapterSptr zkHandle = get_zkHandle (zkHosts);
+        ZooKeeperAdapterSptr zkHandle = get_zkHandle (zkHosts, getSslParams());
         std::cerr << "[zktreeutil] connected to ZK serverfor reading"
             << std::endl;
 
@@ -424,7 +424,7 @@ namespace zktreeutil
             bool force) const
     {
         // Connect to ZK server
-        ZooKeeperAdapterSptr zkHandle = get_zkHandle (zkHosts);
+        ZooKeeperAdapterSptr zkHandle = get_zkHandle (zkHosts, getSslParams());
         std::cerr << "[zktreeutil] connected to ZK server for writing"
             << std::endl;
 
@@ -517,7 +517,7 @@ namespace zktreeutil
         }
 
         // Load the rooted subtree from zookeeper
-        ZooKeeperAdapterSptr zkHandle = get_zkHandle (zkHosts);
+        ZooKeeperAdapterSptr zkHandle = get_zkHandle (zkHosts, getSslParams());
         std::cerr << "[zktreeutil] connected to ZK server for reading"
             << std::endl;
         ZkTreeNodeSptr zkLiveRootSptr = loadZkTree_ (zkHandle, path);
@@ -630,7 +630,7 @@ namespace zktreeutil
             if ((execFlags & EXECUTE)
                     || (execFlags & INTERACTIVE))
             {
-                zkHandleSptr = get_zkHandle (zkHosts);
+                zkHandleSptr = get_zkHandle (zkHosts, getSslParams());
                 std::cerr << "[zktreeutil] connected to ZK server for writing"
                     << std::endl;
             }

zookeeper-contrib/zookeeper-contrib-zktreeutil/src/ZkTreeUtil.h

@@ -185,9 +185,10 @@ namespace zktreeutil
              * \brief Connects to zookeeper and returns a valid ZK handle
              *
              * @param zkHosts comma separated list of host:port forming ZK quorum
+             * @param cert certificate file path
              * @param a valid ZK handle
              */
-            static ZooKeeperAdapterSptr get_zkHandle (const string& zkHosts);
+            static ZooKeeperAdapterSptr get_zkHandle (const string& zkHosts, const string& cert="");
 
 
         public:
@@ -252,10 +253,26 @@ namespace zktreeutil
                     const vector< ZkAction >& zkActions,
                     int execFlags) const;
 
+            /**
+             * \brief Sets the ssl params to be used for SSL connection
+             * @param cert ssl params
+             */
+             void setSslParams(const string& cert)
+             {
+                 sslParams_ = cert;
+             }
+
+            /**
+             * \brief Gets the ssl params
+             * @return the cert
+             */
+             string getSslParams() const { return sslParams_; }
+
         private:
 
             ZkTreeNodeSptr zkRootSptr_;     // ZK tree root node
             bool loaded_;                        // Falg indicating whether ZK tree loaded into memory
+            string sslParams_;              // Comma separated parameters to initiate SSL connection
     };
 }
 

zookeeper-contrib/zookeeper-contrib-zktreeutil/src/ZkTreeUtilMain.cc

@@ -43,9 +43,10 @@ static struct option long_options[] = {
     {"path",         required_argument,     0, 'p'},
     {"depth",         required_argument,     0, 'd'},
     {"zookeeper", required_argument,     0, 'z'},
+    {"ssl", required_argument,     0, 's'},
     {0, 0, 0, 0}
 };
-static char *short_options = "IEUFDfx:p:d:hz:";
+static char *short_options = "IEUFDfx:p:d:hz:s:";
 
 static void usage(int argc, char *argv[])
 {
@@ -128,6 +129,13 @@ static void usage(int argc, char *argv[])
         << std::endl
         << "\t  specifies information to connect to zookeeper."
         << std::endl;
+    std::cout
+        << "\t--ssl=<ssl params> or -s <ssl params>: "
+        << std::endl
+        << "\t  Comma separated parameters to initiate SSL connection."
+        << std::endl
+        << "\t  e.g.: server_cert.crt,client_cert.crt,client_priv_key.pem,passwd"
+        << std::endl;
 }
 
 int main(int argc, char **argv)
@@ -143,6 +151,7 @@ int main(int argc, char **argv)
      string zkHosts;
      string xmlFile;
      string path = "/";
+     string cert;
      int depth = 0;
      while (1)
      {
@@ -171,12 +180,16 @@ int main(int argc, char **argv)
                           break;
              case 'z': zkHosts = optarg;
                           break;
+             case 's': cert = optarg;
+                    break;
              case 'h': usage (argc, argv);
                           exit(0);
          }
      }
 
      ZkTreeUtil zkTreeUtil;
+     if (!cert.empty()) zkTreeUtil.setSslParams(cert);
+
      switch (op)
      {
          case 'I':    {