ZOOKEEPER-627. zkpython arbitrarily restricts the size of a \'get\' to 512 bytes (henry robinson via mahadev)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/zookeeper/trunk@891474 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -176,6 +176,9 @@ BUGFIXES:
   ZOOKEEPER-630. Trunk has duplicate ObserverTest.java files
   (henry robinson via phunt)
 
+  ZOOKEEPER-627. zkpython arbitrarily restricts the size of a 'get' to 512
+  bytes (henry robinson via mahadev)
+
 IMPROVEMENTS:
   ZOOKEEPER-473. cleanup junit tests to eliminate false positives due to
   "socket reuse" and failure to close client (phunt via mahadev)

src/contrib/zkpython/src/c/pyzk_docstrings.h

@@ -579,7 +579,9 @@ static const char pyzk_get_doc[] =
 "\n"
 "(subsequent parameters are optional)\n"
 " watcher if not None, a watch will be set at the server to notify \n"
-"the client if the node changes.\n"
+" the client if the node changes.\n"
+" bufferlen: This value defaults to 1024*1024 - 1Mb. This method returns \n"
+" the minimum of bufferlen and the true length of the znode's data. \n"
 "RETURNS:\n"
 " the data associated with the node\n"
 "OK operation completed succesfully\n"

src/contrib/zkpython/src/c/zookeeper.c

@@ -848,33 +848,49 @@ static PyObject *pyzoo_set2(PyObject *self, PyObject *args)
   return build_stat(stat);
 }
 
+// As per ZK documentation, datanodes are limited
+// to 1Mb. Why not do a stat followed by a get, to 
+// determine how big the buffer should be? Because the znode
+// may get updated between calls, so we can't guarantee a 
+// complete get anyhow. 
+#define GET_BUFFER_SIZE 1024*1024
+
+// pyzoo_get has an extra parameter over the java/C equivalents.
+// If you set the fourth integer parameter buffer_len, we return 
+// min(buffer_len, datalength) bytes. This is set by default to 
+// GET_BUFFER_SIZE
 static PyObject *pyzoo_get(PyObject *self, PyObject *args)
 {
   int zkhid;
   char *path;
-  char buffer[512];
-  memset(buffer,0,sizeof(char)*512);
-  int buffer_len=512;
+  char *buffer; 
+  int buffer_len=GET_BUFFER_SIZE;
   struct Stat stat;
   PyObject *watcherfn = Py_None;
   pywatcher_t *pw = NULL;
-  if (!PyArg_ParseTuple(args, "is|O", &zkhid, &path, &watcherfn))
+  if (!PyArg_ParseTuple(args, "is|Oi", &zkhid, &path, &watcherfn, &buffer_len))
     return NULL;
   CHECK_ZHANDLE(zkhid);
-  if (watcherfn != Py_None)
-    pw = create_pywatcher( zkhid, watcherfn,0 );
+  if (watcherfn != Py_None) 
+		{
+			pw = create_pywatcher( zkhid, watcherfn,0 );
+		}
+	buffer = malloc(sizeof(char)*buffer_len);
   int err = zoo_wget(zhandles[zkhid], path, 
 		     watcherfn != Py_None ? watcher_dispatch : NULL, 
 		     pw, buffer, 
 		     &buffer_len, &stat);
+	
   PyObject *stat_dict = build_stat( &stat );
+
   if (err != ZOK)
-    {
+		{
       PyErr_SetString(err_to_exception(err), zerror(err));
       return NULL;
     }
-
-  return Py_BuildValue( "(s#,N)", buffer,buffer_len, stat_dict );
+	PyObject *ret = Py_BuildValue( "(s#,N)", buffer,buffer_len, stat_dict );
+	free(buffer);
+  return ret;
 }
 
 PyObject *pyzoo_get_acl(PyObject *self, PyObject *args)

src/contrib/zkpython/src/test/get_set_test.py

@@ -71,6 +71,25 @@ class GetSetTest(zktestbase.TestBase):
         self.assertEqual(self.stat, None, "Stat should be none!")
         self.assertEqual(self.value, None, "Value should be none!")
 
+    def test_sync_get_large_datanode(self):
+        """
+        Test that we can retrieve datanode sizes up to
+        1Mb with default parameters (depends on ZooKeeper server).
+        """
+
+        data = ''.join(["A" for x in xrange(1024*1023)])
+        self.ensureDeleted("/zk-python-test-large-datanode")
+        zookeeper.create(self.handle, "/zk-python-test-large-datanode", data,
+                         [{"perms":0x1f, "scheme":"world", "id" :"anyone"}])
+        (ret,stat) = zookeeper.get(self.handle, "/zk-python-test-large-datanode")
+        self.assertEqual(len(ret), 1024*1023,
+                         "Should have got 1Mb returned, instead got %s" % len(ret))
+        (ret,stat) = zookeeper.get(self.handle, "/zk-python-test-large-datanode",None,500)
+        self.assertEqual(len(ret), 500,
+                         "Should have got 500 bytes returned, instead got %s" % len(ret))
+
+
+
     def test_async_getset(self):
         self.cv = threading.Condition()
         def get_callback(handle, rc, value, stat):