ZOOKEEPER-3567: add SSL support for zkpython

This PR is about adding SSL support for zkPython, based on the C-binding. I also fixed the zkpython ant build to work with the current maven top-level build. I also added a new python test case to try to connect to ZooKeeper with SSL.

You can test this patch in the following way:
```
# cleanup everything, just to be on the safe side:
git clean -xdf

# on ubuntu 16.4 make sure you have the following packages installed
apt-get install -y libcppunit-dev openssl libssl-dev python-setuptools python2.7 python2.7-dev

# make a full build (incl. C-client)
mvn clean install -DskipTests -Pfull-build

# we only support python2, so e.g. on ubuntu 18.4 you need to switch to python2
update-alternatives --install /usr/bin/python python /usr/bin/python2.7 1

# compile and test zkpython
cd zookeeper-contrib/zookeeper-contrib-zkpython/
ant compile
ant test
```

Author: Mate Szalay-Beko <szalay.beko.mate@gmail.com>

Reviewers: andor@apache.org

Closes #1121 from symat/ZOOKEEPER-3567 and squashes the following commits:

a5839cb56 [Mate Szalay-Beko] Merge remote-tracking branch 'apache/master' into ZOOKEEPER-3567
d25d61024 [Mate Szalay-Beko] ZOOKEEPER-3567: fix build issues after top-level ant removal
a8869c969 [Mate Szalay-Beko] Merge remote-tracking branch 'apache/master' into HEAD
b92f686e8 [Mate Szalay-Beko] ZOOKEEPER-3567: fix license check issue
0150986da [Mate Szalay-Beko] ZOOKEEPER-3567: removing code duplication: re-use test SSL certificate generator from C-client tests
7d91359d3 [Mate Szalay-Beko] ZOOKEEPER-3567: add SSL support for zkpython

README_packaging.md

@@ -20,11 +20,11 @@ yum install openssl openssl-devel
 yum install cyrus-sasl-md5 cyrus-sasl-gssapi cyrus-sasl-devel
 ```
 
-On Ubuntu:
+On Ubuntu (in case of 16.4+):
 
 ```
-apt-get install cppunit
-apt-get install python-setuptools
+apt-get install libcppunit-dev
+apt-get install python-setuptools python2.7-dev
 apt-get install openssl libssl-dev
 apt-get install libsasl2-modules-gssapi-mit libsasl2-modules libsasl2-dev
 ```

pom.xml

@@ -855,6 +855,7 @@
             <exclude>zookeeper-contrib-fatjar/src/main/resources/mainClasses</exclude>
             <exclude>zookeeper-contrib-zkperl/Changes</exclude>
             <exclude>zookeeper-contrib-zkperl/MANIFEST</exclude>
+            <exclude>zookeeper-contrib-zkpython/src/test/zoo.cfg</exclude>
             <exclude>zookeeper-contrib-loggraph/src/main/resources/webapp/org/apache/zookeeper/graph/resources/*</exclude>
             <exclude>src/main/resources/webapp/org/apache/zookeeper/graph/resources/*</exclude>
             <exclude>src/main/java/com/nitido/utils/toaster/Toaster.java</exclude>

zookeeper-contrib/build-contrib.xml

@@ -32,7 +32,7 @@
 
   <property name="lib.dir"  location="${zk.root}/zookeeper-server/src/main/resources/lib"/>
 
-  <property name="build.dir" location="${zk.root}/build/zookeeper-contrib/zookeeper-contrib-${name}"/>
+  <property name="build.dir" location="${zk.root}/zookeeper-contrib/zookeeper-contrib-${name}/target"/>
   <property name="build.classes" location="${build.dir}/classes"/>
   <property name="build.test" location="${build.dir}/test"/>
 
@@ -47,7 +47,7 @@
   <property name="ivy.home" value="${user.home}/.ant" />
   <property name="ivy.lib" value="${build.dir}/lib"/>
   <property name="ivy.test.lib" value="${build.test}/lib"/>
-  <property name="ivysettings.xml" value="${zk.root}/ivysettings.xml"/>
+  <property name="ivysettings.xml" value="${zk.root}/zookeeper-contrib/ivysettings.xml"/>
 
   <!-- to be overridden by sub-projects -->
   <target name="check-contrib"/>
@@ -179,7 +179,7 @@
       <!-- we can't use id=classpath, because available fails if fileset directory
            doesn't exist -->
       <classpath>
-        <pathelement location="${zk.root}/build/classes"/>
+        <pathelement location="${zk.root}/zookeeper-server/target/classes"/>
       </classpath>
     </available>
   </target>
@@ -190,7 +190,7 @@
 
 
   <target name="checkMainTestIsAvailable">
-    <available file="${zk.root}/build/test/classes/org/apache/zookeeper/test/ClientBase.class"
+    <available file="${zk.root}/zookeeper-server/target/test-classes/org/apache/zookeeper/test/ClientBase.class"
                property="mainTestIsCompiled">
     </available>
   </target>

zookeeper-contrib/ivysettings.xml

@@ -0,0 +1,41 @@
+<ivysettings>
+
+  <!--
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+ -->
+
+  <property name="repo.maven.org"
+            value="https://repo1.maven.org/maven2/" override="false"/>
+  <property name="repo.jboss.org"
+            value="https://repository.jboss.org/nexus/content/groups/public/" override="false"/>
+  <property name="maven2.pattern"
+            value="[organisation]/[module]/[revision]/[module]-[revision]"/>
+  <property name="maven2.pattern.ext" value="${maven2.pattern}.[ext]"/>
+  <include url="${ivy.default.conf.dir}/ivyconf-local.xml"/>
+  <settings defaultResolver="default"/>
+  <resolvers>
+    <ibiblio name="maven2" root="${repo.maven.org}"
+             pattern="${maven2.pattern.ext}" m2compatible="true"/>
+    <ibiblio name="jboss-maven2" root="${repo.jboss.org}"
+             pattern="${maven2.pattern.ext}" m2compatible="true"/>
+
+    <chain name="default" dual="true">
+      <resolver ref="maven2"/>
+      <resolver ref="jboss-maven2"/>
+    </chain>
+
+  </resolvers>
+</ivysettings>

zookeeper-contrib/zookeeper-contrib-zkpython/README

@@ -7,14 +7,18 @@ DEPENDENCIES:
 
 This has only been tested against SVN (i.e. 3.2.0 in development) but should work against 3.1.1. 
 
-You will need the Python development headers installed to build the module - on many package-management systems, these can be found in python-devel.
+You will need the Python development headers installed to build the module - on many package-management systems, these can be found in python-devel. (On ubuntu 18.4, install python2.7 and python2.7-dev.)
 
 Python >= 2.6 is required. We have tested against 2.6. We have not tested against 3.x. 
 
+E.g. setting up tpyhon and python devel on ubuntu 18.4:
+sudo apt-get install python2.7 python2.7-dev
+sudo update-alternatives --install /usr/bin/python python /usr/bin/python2.7 1
+
 BUILD AND INSTALL:
 -------------------
 
-To install, make sure that the C client has been built and that the libraries are installed in /usr/local/lib (or change this directory in setup.py). Then run:
+To install, make sure that the C client has been built (use `mvn clean install -DskipTests -Pfull-build` in the root folder of zookeeper) or that the zookeeper C libraries are installed in /usr/local/lib (or change this directory in setup.py). Then run:
 
 ant install
 

zookeeper-contrib/zookeeper-contrib-zkpython/build.xml

@@ -20,10 +20,10 @@
 <project name="zkpython" default="install">
   <import file="../build-contrib.xml"/>
   <property name="python.src.dir" value="src/python"/>
-  <property name="test.build.dir" value="build/test/" />
   <property name="test.src.dir" value="src/test"/>
   <property name="test.log.dir" value="${build.test}/logs" />
   <property name="test.output" value="no" />
+  <property name="test.output" value="no" />
   <property name="test.timeout" value="900000" />
 
   <target name="test"

zookeeper-contrib/zookeeper-contrib-zkpython/src/c/pyzk_docstrings.h

@@ -568,6 +568,33 @@ static const char pyzk_init_doc[] =
 " an integer handle. If it fails to create \n"
 " a new zhandle the function throws an exception.\n";
 
+static const char pyzk_init_ssl_doc[] =
+"This method creates a new handle and a zookeeper SSL session that corresponds\n"
+"to that handle. Session establishment is asynchronous, meaning that the\n"
+"session should not be considered established until (and unless) an\n"
+"event of state CONNECTED_STATE is received.\n"
+"PARAMETERS:\n"
+" host: comma separated host:port pairs, each corresponding to a zk\n"
+"  server. e.g. '127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002'\n"
+" cert_str: SSL certificate string e.g. 'server.cert,client.cert,client-priv-key.pom,passwd'\n"
+"\n"
+"(subsequent parameters are optional)\n"
+" fn: the global watcher callback function. When notifications are\n"
+"  triggered this function will be invoked.\n"
+" recv_timeout: \n"
+" (clientid, passwd)\n"
+" clientid the id of a previously established session that this\n"
+"  client will be reconnecting to. Clients can access the session id of an established, valid,\n"
+"  connection by calling zoo_client_id. If\n"
+"  the specified clientid has expired, or if the clientid is invalid for \n"
+"  any reason, the returned zhandle_t will be invalid -- the zhandle_t \n"
+"  state will indicate the reason for failure (typically\n"
+"  EXPIRED_SESSION_STATE).\n"
+"\n"
+"RETURNS:\n"
+" an integer handle. If it fails to create \n"
+" a new zhandle the function throws an exception.\n";
+
 static const char pyzk_get_doc[] = 
 " gets the data associated with a node synchronously.\n"
 "\n"

zookeeper-contrib/zookeeper-contrib-zkpython/src/c/zookeeper.c

@@ -599,12 +599,13 @@ void acl_completion_dispatch(int rc, struct ACL_vector *acl, struct Stat *stat,
 /* ZOOKEEPER API IMPLEMENTATION */
 /* -------------------------------------------------------------------------- */
 
-static PyObject *pyzookeeper_init(PyObject *self, PyObject *args)
-{
+
+static PyObject *pyzookeeper_init_optional_ssl(PyObject *self, PyObject *args, int ssl) {
   const char *host;
+  const char *cert_str;
   PyObject *watcherfn = Py_None;
+  zhandle_t *zh = NULL;
   int recv_timeout = 10000;
-  //  int clientid = -1;
   clientid_t cid;
   cid.client_id = -1;
   const char *passwd;
@@ -621,9 +622,14 @@ static PyObject *pyzookeeper_init(PyObject *self, PyObject *args)
     return NULL;
   }
 
-  if (!PyArg_ParseTuple(args, "s|Oi(Ls)", &host, &watcherfn, &recv_timeout, &cid.client_id, &passwd)) 
-    return NULL;
-  
+  if (ssl) {
+    if (!PyArg_ParseTuple(args, "ss|Oi(Ls)", &host, &cert_str, &watcherfn, &recv_timeout, &cid.client_id, &passwd))
+      return NULL;
+  } else {
+    if (!PyArg_ParseTuple(args, "s|Oi(Ls)", &host, &watcherfn, &recv_timeout, &cid.client_id, &passwd))
+        return NULL;
+  }
+
   if (cid.client_id != -1) {
     strncpy(cid.passwd, passwd, 16*sizeof(char));
   }
@@ -635,14 +641,18 @@ static PyObject *pyzookeeper_init(PyObject *self, PyObject *args)
     }
   }
   watchers[handle] = pyw;
-  zhandle_t *zh = zookeeper_init( host, watcherfn != Py_None ? watcher_dispatch : NULL, 
-                                  recv_timeout, cid.client_id == -1 ? 0 : &cid, 
-                                  pyw,
-                                  0 ); 
+
+  if (ssl) {
+    zh = zookeeper_init_ssl( host, cert_str, watcherfn != Py_None ? watcher_dispatch : NULL,
+                             recv_timeout, cid.client_id == -1 ? 0 : &cid, pyw, 0 );
+  } else {
+    zh = zookeeper_init( host, watcherfn != Py_None ? watcher_dispatch : NULL,
+                         recv_timeout, cid.client_id == -1 ? 0 : &cid, pyw, 0 );
+  }
 
   if (zh == NULL)
     {
-      PyErr_SetString( ZooKeeperException, "Could not internally obtain zookeeper handle" );
+      PyErr_SetString( ZooKeeperException, "Could not internally obtain SSL zookeeper handle" );
       return NULL;
     }
 
@@ -650,6 +660,17 @@ static PyObject *pyzookeeper_init(PyObject *self, PyObject *args)
   return Py_BuildValue( "i", handle);
 }
 
+static PyObject *pyzookeeper_init(PyObject *self, PyObject *args)
+{
+  return pyzookeeper_init_optional_ssl(self, args, 0);
+}
+
+
+static PyObject *pyzookeeper_init_ssl(PyObject *self, PyObject *args)
+{
+  return pyzookeeper_init_optional_ssl(self, args, 1);
+}
+
 
 /* -------------------------------------------------------------------------- */
 /* Asynchronous API implementation */
@@ -1497,6 +1518,7 @@ PyObject *pyzoo_deterministic_conn_order(PyObject *self, PyObject *args)
 
 static PyMethodDef ZooKeeperMethods[] = {
   {"init", pyzookeeper_init, METH_VARARGS, pyzk_init_doc },
+  {"init_ssl", pyzookeeper_init_ssl, METH_VARARGS, pyzk_init_ssl_doc },
   {"create",pyzoo_create, METH_VARARGS, pyzk_create_doc },
   {"delete",pyzoo_delete, METH_VARARGS, pyzk_delete_doc },
   {"get_children", pyzoo_get_children, METH_VARARGS, pyzk_get_children_doc },

zookeeper-contrib/zookeeper-contrib-zkpython/src/python/setup.py

@@ -21,11 +21,11 @@ zookeeper_basedir = "../../"
 zookeepermodule = Extension("zookeeper",
                             sources=["src/c/zookeeper.c"],
                             include_dirs=[zookeeper_basedir + "/zookeeper-client/zookeeper-client-c/include",
-                                          zookeeper_basedir + "/build/c",
+                                          zookeeper_basedir + "/zookeeper-client/zookeeper-client-c/target/c",
                                           zookeeper_basedir + "/zookeeper-client/zookeeper-client-c/generated"],
                             libraries=["zookeeper_mt"],
                             library_dirs=[zookeeper_basedir + "/zookeeper-client/zookeeper-client-c/.libs/",
-                                          zookeeper_basedir + "/build/c/.libs/",
+                                          zookeeper_basedir + "/zookeeper-client/zookeeper-client-c/target/c/.libs/",
                                           zookeeper_basedir + "/build/test/test-cppunit/.libs",
                                           "/usr/local/lib"
                                           ])

zookeeper-contrib/zookeeper-contrib-zkpython/src/test/connection_test.py

@@ -58,6 +58,37 @@ class ConnectionTest(zktestbase.TestBase):
                           self.handle,
                           "/")
 
+
+    def testsslconnection(self):
+        cv = threading.Condition()
+        self.connected = False
+        def connection_watcher(handle, type, state, path):
+            cv.acquire()
+            self.connected = True
+            self.assertEqual(zookeeper.CONNECTED_STATE, state)
+            self.handle = handle
+            cv.notify()
+            cv.release()
+
+        cv.acquire()
+        ret = zookeeper.init_ssl(self.sslhost, self.sslcert, connection_watcher)
+        cv.wait(15.0)
+        cv.release()
+        self.assertEqual(self.connected, True, "SSL Connection timed out to " + self.host)
+        self.assertEqual(zookeeper.CONNECTED_STATE, zookeeper.state(self.handle))
+
+        self.assertEqual(zookeeper.close(self.handle), zookeeper.OK)
+        # Trying to close the same handle twice is an error, and the C library will segfault on it
+        # so make sure this is caught at the Python module layer
+        self.assertRaises(zookeeper.ZooKeeperException,
+                          zookeeper.close,
+                          self.handle)
+
+        self.assertRaises(zookeeper.ZooKeeperException,
+                          zookeeper.get,
+                          self.handle,
+                          "/")
+
     def testhandlereuse(self):
         """
         Test a) multiple concurrent connections b) reuse of closed handles

zookeeper-contrib/zookeeper-contrib-zkpython/src/test/run_tests.sh

@@ -30,11 +30,12 @@ else
 fi
 
 # Find the build directory containing zookeeper.so
-SO_PATH=`find ../../build/ -name "zookeeper.so" | head -1`
+SO_PATH=`find ./target/ -name "zookeeper.so" | head -1`
 PYTHONPATH=`dirname $SO_PATH`
-LIB_PATH=../../build/c/.libs/:../../build/test/test-cppunit/.libs
+LIB_PATH=../../zookeeper-client/zookeeper-client-c/target/c/.libs
 for test in `ls $1/*_test.py`; 
 do
     echo "Running $test"
+    echo "Running LD_LIBRARY_PATH=$LIB_PATH:$LD_LIBRARY_PATH DYLD_LIBRARY_PATH=$LIB_PATH:$DYLD_LIBRARY_PATH PYTHONPATH=$PYTHONPATH python $test"
     LD_LIBRARY_PATH=$LIB_PATH:$LD_LIBRARY_PATH DYLD_LIBRARY_PATH=$LIB_PATH:$DYLD_LIBRARY_PATH PYTHONPATH=$PYTHONPATH python $test
 done

zookeeper-contrib/zookeeper-contrib-zkpython/src/test/zkServer.sh

@@ -18,36 +18,56 @@
 
 if [ "x$1" == "x" ]
 then
-    echo "USAGE: $0 startClean|start|stop hostPorts"
+    echo "USAGE: $0 startClean|start|stop"
     exit 2
 fi
 
-if [ "x$1" == "xstartClean" ]
+if [ "x${base_dir}" == "x" ]
+then
+  PROJECT_ROOT="../../"
+else
+  PROJECT_ROOT=${base_dir}
+fi
+WORK_DIR=${PROJECT_ROOT}/zookeeper-contrib/zookeeper-contrib-zkpython/target/zkpython_tests
+TEST_DIR=${PROJECT_ROOT}/zookeeper-contrib/zookeeper-contrib-zkpython/src/test
+
+
+if [ -r "${WORK_DIR}/../zk.pid" ]
 then
-    if [ "x${base_dir}" == "x" ]
+  pid=`cat "${WORK_DIR}/../zk.pid"`
+  kill -9 $pid
+  rm -f "${WORK_DIR}/../zk.pid"
+fi
+
+which lsof &> /dev/null
+if [ $? -eq 0  ]
+then
+    pid=`lsof -i :22182 | grep LISTEN | awk '{print $2}'`
+    if [ -n "$pid" ]
     then
-    rm -rf /tmp/zkdata
-    else
-    rm -rf ${base_dir}/build/tmp
+        kill -9 $pid
     fi
 fi
 
-if [ "x${base_dir}" == "x" ]	
+
+
+
+if [ "x$1" == "xstartClean" ]
 then
-zk_base="../../"
-else
-zk_base="${base_dir}"
+    rm -rf ${WORK_DIR}
 fi
 
-CLASSPATH="$CLASSPATH:${zk_base}/build/classes"
+
+
+CLASSPATH="$CLASSPATH:${PROJECT_ROOT}/zookeeper-server/target/classes"
 CLASSPATH="$CLASSPATH:${zk_base}/conf"
 
-for i in "${zk_base}"/build/lib/*.jar
+for i in "${PROJECT_ROOT}"/zookeeper-server/target/lib/*.jar
 do
     CLASSPATH="$CLASSPATH:$i"
 done
 
-for i in "${zk_base}"/zookeeper-server/src/main/resource/lib/*.jar
+for i in "${PROJECT_ROOT}"/zookeeper-server/src/main/resource/lib/*.jar
 do
     CLASSPATH="$CLASSPATH:$i"
 done
@@ -57,15 +77,20 @@ done
 
 case $1 in
 start|startClean)
-    if [ "x${base_dir}" == "x" ]
-        then
-        mkdir -p /tmp/zkdata
-        java -cp $CLASSPATH org.apache.zookeeper.server.ZooKeeperServerMain 22182 /tmp/zkdata &> /tmp/zk.log &
-        else
-        mkdir -p ${base_dir}/build/tmp/zkdata
-        java -cp $CLASSPATH org.apache.zookeeper.server.ZooKeeperServerMain 22182 ${base_dir}/build/tmp/zkdata &> ${base_dir}/build/tmp/zk.log &
-    fi
-        sleep 5
+    mkdir -p ${WORK_DIR}/zkdata
+
+    rm -rf ${WORK_DIR}/ssl
+    mkdir -p ${WORK_DIR}/ssl
+    cp ${PROJECT_ROOT}/zookeeper-client/zookeeper-client-c/ssl/gencerts.sh ${WORK_DIR}/ssl/
+    cd ${WORK_DIR}/ssl/
+    ./gencerts.sh
+    cd -
+
+    sed "s#WORKDIR#${WORK_DIR}#g" ${TEST_DIR}/zoo.cfg > "${WORK_DIR}/zoo.cfg"
+    java  -Dzookeeper.extendedTypesEnabled=true -Dznode.container.checkIntervalMs=100 -cp $CLASSPATH org.apache.zookeeper.server.ZooKeeperServerMain "${WORK_DIR}/zoo.cfg" &> "${WORK_DIR}/zoo.log" &
+    pid=$!
+    echo -n $! > ${WORK_DIR}/../zk.pid
+    sleep 5
     ;;
 stop)
     # Already killed above

zookeeper-contrib/zookeeper-contrib-zkpython/src/test/zktestbase.py

@@ -22,10 +22,13 @@ ZOO_OPEN_ACL_UNSAFE = {"perms":0x1f, "scheme":"world", "id" :"anyone"}
 
 class TestBase(unittest.TestCase):
     SERVER_PORT = 22182
-    
+    SERVER_SSL_PORT = 22183
+
     def __init__(self,methodName='runTest'):
         unittest.TestCase.__init__(self,methodName)
         self.host = "localhost:%d" % self.SERVER_PORT
+        self.sslhost = "localhost:%d" % self.SERVER_SSL_PORT
+        self.sslcert =  "./target/zkpython_tests/ssl/server.crt,./target/zkpython_tests/ssl/client.crt,./target/zkpython_tests/ssl/clientkey.pem,password"
         self.connected = False
         self.handle = -1
         logdir = os.environ.get("ZKPY_LOG_DIR")

zookeeper-contrib/zookeeper-contrib-zkpython/src/test/zoo.cfg

@@ -0,0 +1,14 @@
+tickTime=500
+initLimit=10
+syncLimit=5
+dataDir=WORKDIR/zkdata
+maxClientCnxns=200
+
+clientPort=22182
+secureClientPort=22183
+serverCnxnFactory=org.apache.zookeeper.server.NettyServerCnxnFactory
+ssl.keyStore.location=WORKDIR/ssl/server.jks
+ssl.keyStore.password=password
+ssl.trustStore.location=WORKDIR/ssl/servertrust.jks
+ssl.trustStore.password=password
+