HADOOP-459. Fix memory leaks and a host of other issues with libhdfs. Contributed by Sameer.

git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk@472684 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -28,6 +28,9 @@ Trunk (unreleased changes)
  8. HADOOP-604.  Fix some synchronization issues and a
     NullPointerException in DFS datanode.  (Raghu Angadi via cutting)
 
+ 9. HADOOP-459.  Fix memory leaks and a host of other issues with
+    libhdfs.  (Sameer Paranjpye via cutting)
+
 
 Release 0.8.0 - 2006-11-03
 

build.xml

@@ -46,7 +46,7 @@
   <property name="test.junit.output.format" value="plain"/>
 
   <property name="libhdfs.test.conf.dir" value="${libhdfs.src}/tests/conf"/>
-  <property name="libhdfs.test.log.dir" value="${build.libhdfs}/tests/logs"/>
+  <property name="libhdfs.test.dir" value="${test.build.dir}/libhdfs"/>
 
   <property name="web.src.dir" value="${basedir}/src/web"/>
   <property name="src.webapps" value="${basedir}/src/webapps"/>
@@ -504,19 +504,20 @@
     </exec>
   </target>
 	
-  <target name="test-libhdfs" depends="compile-libhdfs, jar">
-    <delete dir="${libhdfs.test.log.dir}"/>
-    <mkdir dir="${libhdfs.test.log.dir}"/>
+  <target name="test-libhdfs" depends="compile-libhdfs, compile-core">
+    <delete dir="${libhdfs.test.dir}"/>
+    <mkdir dir="${libhdfs.test.dir}"/>
+    <mkdir dir="${libhdfs.test.dir}/logs"/>
+    <mkdir dir="${libhdfs.test.dir}/dfs/name"/>
 
     <exec dir="${libhdfs.src}" executable="make" failonerror="true">
         <env key="OS_NAME" value="${os.name}"/>
         <env key="OS_ARCH" value="${os.arch}"/>
         <env key="SHLIB_VERSION" value="${libhdfs.version}"/>
         <env key="LIBHDFS_BUILD_DIR" value="${build.libhdfs}"/>
-        <env key="CLASSPATH" value="${build.dir}/${final.name}.jar"/>
         <env key="HADOOP_HOME" value="${basedir}"/>
         <env key="HADOOP_CONF_DIR" value="${libhdfs.test.conf.dir}"/>
-        <env key="HADOOP_LOG_DIR" value="${libhdfs.test.log.dir}"/>
+        <env key="HADOOP_LOG_DIR" value="${libhdfs.test.dir}/logs"/>
 		<arg value="test"/>
     </exec>
   </target>

src/c++/libhdfs/Makefile

@@ -25,10 +25,10 @@
 
 CC = gcc
 LD = gcc
-CFLAGS =  -g -W -fPIC
-LDFLAGS = -L$(JAVA_HOME)/jre/lib/$(OS_ARCH)/server -ljvm -m32 -shared -Wl,-x 
+CFLAGS =  -g -Wall -O2 -fPIC
+LDFLAGS = -L$(JAVA_HOME)/jre/lib/$(OS_ARCH)/server -ljvm -shared -m32 -Wl,-x 
 PLATFORM = $(shell echo $$OS_NAME | tr [A-Z] [a-z])
-CPPFLAGS = -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/$(PLATFORM)
+CPPFLAGS = -m32 -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/$(PLATFORM)
 
 LIB_NAME = hdfs
 SO_TARGET = $(LIBHDFS_BUILD_DIR)/lib$(LIB_NAME).so.$(SHLIB_VERSION)
@@ -40,6 +40,7 @@ DOXYGEN = doxygen
 
 CSRC = \
 	hdfs.c \
+	hdfsJniHelper.c \
 	$(NONE)
 
 COBJS = $(addprefix $(LIBHDFS_BUILD_DIR)/,$(patsubst %,%.o,$(basename $(CSRC))))

src/c++/libhdfs/hdfs.h

@@ -19,6 +19,10 @@
 #ifndef LIBHDFS_HDFS_H
 #define LIBHDFS_HDFS_H
 
+#include <sys/types.h>
+#include <sys/stat.h>
+
+#include <fcntl.h>
 #include <stdio.h>
 #include <stdint.h>
 #include <string.h>
@@ -28,18 +32,28 @@
 
 #include <jni.h>
 
-#define O_RDONLY 1 
+#ifndef O_RDONLY
+#define O_RDONLY 1
+#endif
+
+#ifndef O_WRONLY 
 #define O_WRONLY 2
+#endif
+
+#ifndef EINTERNAL
 #define EINTERNAL 255 
+#endif
+
 
 /** All APIs set errno to meaningful values */
 #ifdef __cplusplus
 extern  "C" {
 #endif
 
-	/**
-	 * Some utility decls used in libhdfs.
-	 */
+    /**
+     * Some utility decls used in libhdfs.
+     */
+
     typedef int32_t   tSize; /// size of data for read/write io ops 
     typedef time_t    tTime; /// time type
     typedef int64_t   tOffset;/// offset within the file
@@ -54,6 +68,7 @@ extern  "C" {
      * The C reflection of org.apache.org.hadoop.FileSystem .
      */
     typedef void* hdfsFS;
+
     
     /**
      * The C equivalent of org.apache.org.hadoop.FSData(Input|Output)Stream .
@@ -64,6 +79,7 @@ extern  "C" {
         INPUT = 1,
         OUTPUT = 2,
     };
+
     
     /**
      * The 'file-handle' to a file in hdfs.
@@ -74,18 +90,21 @@ extern  "C" {
     };
     typedef struct hdfsFile_internal* hdfsFile;
       
+
     /** 
      * hdfsConnect - Connect to a hdfs file system.
      * Connect to the hdfs.
-     * @param host A string containing either a host name, or an ip address of the namenode of a hdfs cluster. 'host' should be passed as NULL if you want to connect to local filesystem. 'host' should be passed as 'default' (and port as 0) to used the 'configured' filesystem (hadoop-site/hadoop-default.xml).
+     * @param host A string containing either a host name, or an ip address
+     * of the namenode of a hdfs cluster. 'host' should be passed as NULL if
+     * you want to connect to local filesystem. 'host' should be passed as
+     * 'default' (and port as 0) to used the 'configured' filesystem
+     * (hadoop-site/hadoop-default.xml).
      * @param port The port on which the server is listening.
      * @return Returns a handle to the filesystem or NULL on error.
      */
     hdfsFS hdfsConnect(const char* host, tPort port);
 
-	/**
-	 * Disconnects
-	 */
+
     /** 
      * hdfsDisconnect - Disconnect from the hdfs file system.
      * Disconnect from hdfs.
@@ -94,18 +113,23 @@ extern  "C" {
      */
     int hdfsDisconnect(hdfsFS fs);
         
+
     /** 
      * hdfsOpenFile - Open a hdfs file in given mode.
      * @param fs The configured filesystem handle.
      * @param path The full path to the file.
      * @param flags Either O_RDONLY or O_WRONLY, for read-only or write-only.
-     * @param bufferSize Size of buffer for read/write - pass 0 if you want to use the default configured values.
-     * @param replication Block replication - pass 0 if you want to use the default configured values.
-     * @param blocksize Size of block - pass 0 if you want to use the default configured values.
+     * @param bufferSize Size of buffer for read/write - pass 0 if you want
+     * to use the default configured values.
+     * @param replication Block replication - pass 0 if you want to use
+     * the default configured values.
+     * @param blocksize Size of block - pass 0 if you want to use the
+     * default configured values.
      * @return Returns the handle to the open file or NULL on error.
      */
     hdfsFile hdfsOpenFile(hdfsFS fs, const char* path, int flags,
-            int bufferSize, short replication, tSize blocksize);
+                          int bufferSize, short replication, tSize blocksize);
+
 
     /** 
      * hdfsCloseFile - Close an open file. 
@@ -115,6 +139,16 @@ extern  "C" {
      */
     int hdfsCloseFile(hdfsFS fs, hdfsFile file);
 
+
+    /** 
+     * hdfsExists - Checks if a given path exsits on the filesystem 
+     * @param fs The configured filesystem handle.
+     * @param path The path to look for
+     * @return Returns 0 on success, -1 on error.  
+     */
+    int hdfsExists(hdfsFS fs, const char *path);
+
+
     /** 
      * hdfsSeek - Seek to given offset in file. 
      * This works only for files opened in read-only mode. 
@@ -125,6 +159,7 @@ extern  "C" {
      */
     int hdfsSeek(hdfsFS fs, hdfsFile file, tOffset desiredPos); 
 
+
     /** 
      * hdfsTell - Get the current offset in the file, in bytes.
      * @param fs The configured filesystem handle.
@@ -133,16 +168,19 @@ extern  "C" {
      */
     tOffset hdfsTell(hdfsFS fs, hdfsFile file);
 
+
     /** 
      * hdfsRead - Read data from an open file.
      * @param fs The configured filesystem handle.
      * @param file The file handle.
      * @param buffer The buffer to copy read bytes into.
      * @param length The length of the buffer.
-     * @return Returns the number of bytes actually read, possibly less than than length;-1 on error.
+     * @return Returns the number of bytes actually read, possibly less
+     * than than length;-1 on error.
      */
     tSize hdfsRead(hdfsFS fs, hdfsFile file, void* buffer, tSize length);
 
+
     /** 
      * hdfsPread - Positional read of data from an open file.
      * @param fs The configured filesystem handle.
@@ -150,9 +188,12 @@ extern  "C" {
      * @param position Position from which to read
      * @param buffer The buffer to copy read bytes into.
      * @param length The length of the buffer.
-     * @return Returns the number of bytes actually read, possibly less than than length;-1 on error.
+     * @return Returns the number of bytes actually read, possibly less than
+     * than length;-1 on error.
      */
-    tSize hdfsPread(hdfsFS fs, hdfsFile file, tOffset position, void* buffer, tSize length);
+    tSize hdfsPread(hdfsFS fs, hdfsFile file, tOffset position,
+                    void* buffer, tSize length);
+
 
     /** 
      * hdfsWrite - Write data into an open file.
@@ -162,7 +203,9 @@ extern  "C" {
      * @param length The no. of bytes to write. 
      * @return Returns the number of bytes written, -1 on error.
      */
-    tSize hdfsWrite(hdfsFS fs, hdfsFile file, const void* buffer, tSize length);
+    tSize hdfsWrite(hdfsFS fs, hdfsFile file, const void* buffer,
+                    tSize length);
+
 
     /** 
      * hdfsWrite - Flush the data. 
@@ -172,14 +215,17 @@ extern  "C" {
      */
     int hdfsFlush(hdfsFS fs, hdfsFile file);
 
+
     /**
-     * hdfsAvailable - Number of bytes that can be read from this input stream without blocking.
+     * hdfsAvailable - Number of bytes that can be read from this
+     * input stream without blocking.
      * @param fs The configured filesystem handle.
      * @param file The file handle.
      * @return Returns available bytes; -1 on error. 
      */
     int hdfsAvailable(hdfsFS fs, hdfsFile file);
 
+
     /**
      * hdfsCopy - Copy file from one filesystem to another.
      * @param srcFS The handle to source filesystem.
@@ -190,6 +236,7 @@ extern  "C" {
      */
     int hdfsCopy(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst);
 
+
     /**
      * hdfsMove - Move file from one filesystem to another.
      * @param srcFS The handle to source filesystem.
@@ -200,6 +247,7 @@ extern  "C" {
      */
     int hdfsMove(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst);
 
+
     /**
      * hdfsDelete - Delete file. 
      * @param fs The configured filesystem handle.
@@ -208,8 +256,9 @@ extern  "C" {
      */
     int hdfsDelete(hdfsFS fs, const char* path);
 
+
     /**
-     * hdfsDelete - Rename file. 
+     * hdfsRename - Rename file. 
      * @param fs The configured filesystem handle.
      * @param oldPath The path of the source file. 
      * @param newPath The path of the destination file. 
@@ -217,6 +266,7 @@ extern  "C" {
      */
     int hdfsRename(hdfsFS fs, const char* oldPath, const char* newPath);
 
+
     /**
      * hdfsLock - Obtain a lock on the file.
      * @param fs The configured filesystem handle.
@@ -226,6 +276,7 @@ extern  "C" {
      */
     int hdfsLock(hdfsFS fs, const char* path, int shared);
 
+
     /**
      * hdfsReleaseLock - Release the lock.
      * @param fs The configured filesystem handle.
@@ -234,8 +285,10 @@ extern  "C" {
      */
     int hdfsReleaseLock(hdfsFS fs, const char* path);
 
+
     /** 
-     * hdfsGetWorkingDirectory - Get the current working directory for the given filesystem.
+     * hdfsGetWorkingDirectory - Get the current working directory for
+     * the given filesystem.
      * @param fs The configured filesystem handle.
      * @param buffer The user-buffer to copy path of cwd into. 
      * @param bufferSize The length of user-buffer.
@@ -243,68 +296,98 @@ extern  "C" {
      */
     char* hdfsGetWorkingDirectory(hdfsFS fs, char *buffer, size_t bufferSize);
 
+
     /** 
-     * hdfsSetWorkingDirectory - Set the working directory. All relative paths will be resolved relative to it.
+     * hdfsSetWorkingDirectory - Set the working directory. All relative
+     * paths will be resolved relative to it.
      * @param fs The configured filesystem handle.
      * @param path The path of the new 'cwd'. 
      * @return Returns 0 on success, -1 on error. 
      */
     int hdfsSetWorkingDirectory(hdfsFS fs, const char* path);
 
+
     /** 
-     * hdfsCreateDirectory - Make the given file and all non-existent parents into directories.
+     * hdfsCreateDirectory - Make the given file and all non-existent
+     * parents into directories.
      * @param fs The configured filesystem handle.
      * @param path The path of the directory. 
      * @return Returns 0 on success, -1 on error. 
      */
     int hdfsCreateDirectory(hdfsFS fs, const char* path);
 
+
     /** 
      * hdfsFileInfo - Information about a file/directory.
      */
     typedef struct  {
-        tObjectKind mKind; /*file or directory */
-        char *mName; /*the name of the file */
-        tTime mCreationTime; /*the creation time for the file*/
-        tOffset mSize; /*the size of the file in bytes */
-        int replicaCount; /*the count of replicas */
+        tObjectKind mKind;   /* file or directory */
+        char *mName;         /* the name of the file */
+        tTime mCreationTime; /* the creation time for the file*/
+        tOffset mSize;       /* the size of the file in bytes */
+        int replicaCount;    /* the count of replicas */
     } hdfsFileInfo;
 
+
     /** 
-     * hdfsListDirectory - Get list of files/directories for a given directory-path. freehdfsFileInfo should be called to deallocate memory. 
+     * hdfsListDirectory - Get list of files/directories for a given
+     * directory-path. freehdfsFileInfo should be called to deallocate memory. 
      * @param fs The configured filesystem handle.
      * @param path The path of the directory. 
      * @param numEntries Set to the number of files/directories in path.
-     * @return Returns a dynamically-allocated array of hdfsFileInfo objects; NULL on error.
+     * @return Returns a dynamically-allocated array of hdfsFileInfo
+     * objects; NULL on error.
      */
-    hdfsFileInfo *hdfsListDirectory(hdfsFS fs, const char* path, int *numEntries);
+    hdfsFileInfo *hdfsListDirectory(hdfsFS fs, const char* path,
+                                    int *numEntries);
+
 
     /** 
-     * hdfsGetPathInfo - Get information about a path as a (dynamically allocated) single hdfsFileInfo struct. freehdfsFileInfo should be called when the pointer is no longer needed.
+     * hdfsGetPathInfo - Get information about a path as a (dynamically
+     * allocated) single hdfsFileInfo struct. freehdfsFileInfo should be
+     * called when the pointer is no longer needed.
      * @param fs The configured filesystem handle.
      * @param path The path of the file. 
-     * @return Returns a dynamically-allocated hdfsFileInfo object; NULL on error.
+     * @return Returns a dynamically-allocated hdfsFileInfo object;
+     * NULL on error.
      */
     hdfsFileInfo *hdfsGetPathInfo(hdfsFS fs, const char* path);
 
+
     /** 
-     * hdfsFreeFileInfo - Free up the hdfsFileInfo array (including the fields) 
-     * @param hdfsFileInfo The array of dynamically-allocated hdfsFileInfo objects.
+     * hdfsFreeFileInfo - Free up the hdfsFileInfo array (including fields) 
+     * @param hdfsFileInfo The array of dynamically-allocated hdfsFileInfo
+     * objects.
      * @param numEntries The size of the array.
      */
     void hdfsFreeFileInfo(hdfsFileInfo *hdfsFileInfo, int numEntries);
 
+
     /** 
-     * hdfsGetHosts - Get hostnames where a particular block (determined by pos & blocksize) of a file is stored. The last element in the array is NULL. Due to replication, a single block could be present on multiple hosts.
+     * hdfsGetHosts - Get hostnames where a particular block (determined by
+     * pos & blocksize) of a file is stored. The last element in the array
+     * is NULL. Due to replication, a single block could be present on
+     * multiple hosts.
      * @param fs The configured filesystem handle.
      * @param path The path of the file. 
      * @param start The start of the block.
      * @param length The length of the block.
-     * @return Returns a dynamically-allocated 2-d array of blocks-hosts; NULL on error.
+     * @return Returns a dynamically-allocated 2-d array of blocks-hosts;
+     * NULL on error.
      */
     char*** hdfsGetHosts(hdfsFS fs, const char* path, 
             tOffset start, tOffset length);
 
+
+    /** 
+     * hdfsFreeHosts - Free up the structure returned by hdfsGetHosts
+     * @param hdfsFileInfo The array of dynamically-allocated hdfsFileInfo
+     * objects.
+     * @param numEntries The size of the array.
+     */
+    void hdfsFreeHosts(char ***blockHosts);
+
+
     /** 
      * hdfsGetDefaultBlockSize - Get the optimum blocksize.
      * @param fs The configured filesystem handle.
@@ -312,6 +395,7 @@ extern  "C" {
      */
     tOffset hdfsGetDefaultBlockSize(hdfsFS fs);
 
+
     /** 
      * hdfsGetCapacity - Return the raw capacity of the filesystem.  
      * @param fs The configured filesystem handle.
@@ -319,30 +403,13 @@ extern  "C" {
      */
     tOffset hdfsGetCapacity(hdfsFS fs);
 
+
     /** 
      * hdfsGetUsed - Return the total raw size of all files in the filesystem.
      * @param fs The configured filesystem handle.
      * @return Returns the total-size; -1 on error. 
      */
     tOffset hdfsGetUsed(hdfsFS fs);
-
-    /**
-     * hdfsConvertToGlobalRef - Return a global reference for the jobject.
-     * The user needs to explicitly call this to share the jobject between
-     * multiple threads! This function automatically deletes the local reference
-     * if it succesfully converted it to a global reference.
-     * @param localRef The local reference which needs to be globalized.
-     * @return Returns the global reference; NULL on error.
-     */
-    jobject hdfsConvertToGlobalRef(jobject localRef);
-    
-    /**
-     * hdfsDeleteGlobalRef - Destroy a global reference.
-     * multiple threads!
-     * @param globalRef The global reference to be destroyed.
-     * @return None. 
-     */
-    void hdfsDeleteGlobalRef(jobject globalRef);
     
 #ifdef __cplusplus
 }

src/c++/libhdfs/hdfsJniHelper.c

@@ -0,0 +1,386 @@
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <string.h> 
+#include "hdfsJniHelper.h"
+
+static pthread_mutex_t hdfsHashMutex = PTHREAD_MUTEX_INITIALIZER;
+static volatile int hashTableInited = 0;
+
+#define LOCK_HASH_TABLE() pthread_mutex_lock(&hdfsHashMutex)
+#define UNLOCK_HASH_TABLE() pthread_mutex_unlock(&hdfsHashMutex)
+
+
+/** The Native return types that methods could return */
+#define VOID          'V'
+#define JOBJECT       'L'
+#define JARRAYOBJECT  '['
+#define JBOOLEAN      'Z'
+#define JBYTE         'B'
+#define JCHAR         'C'
+#define JSHORT        'S'
+#define JINT          'I'
+#define JLONG         'J'
+#define JFLOAT        'F'
+#define JDOUBLE       'D'
+
+
+/**
+ * Helpful macro to convert a pthread_t to a string
+ */
+#define GET_threadID(threadID, key, keySize) \
+    snprintf(key, keySize, "__hdfs_threadID__%u", (unsigned)(threadID)); 
+#define threadID_SIZE 32
+
+
+/**
+ * MAX_HASH_TABLE_ELEM: The maximum no. of entries in the hashtable.
+ * It's set to 4096 to account for (classNames + No. of threads)
+ */
+#define MAX_HASH_TABLE_ELEM 4096
+
+
+#define CHECK_EXCEPTION_IN_METH_INVOC \
+    if ((*env)->ExceptionCheck(env)) {\
+        (*env)->ExceptionDescribe(env);\
+        va_end(args);\
+        return -1;\
+    }\
+
+
+static void validateMethodType(MethType methType)
+{
+    if (methType != STATIC && methType != INSTANCE) {
+        fprintf(stderr, "Unimplemented method type\n");
+        exit(1);
+    }
+    return;
+}
+
+
+static void hashTableInit(void)
+{
+    if (!hashTableInited) {
+        LOCK_HASH_TABLE();
+        if (!hashTableInited) {
+            if (hcreate(MAX_HASH_TABLE_ELEM) == 0) {
+                fprintf(stderr, "error creating hashtable, <%d>: %s\n",
+                        errno, strerror(errno));
+                exit(1);
+            } 
+            hashTableInited = 1;
+        }
+        UNLOCK_HASH_TABLE();
+    }
+}
+
+
+static void insertEntryIntoTable(const char *key, void *data)
+{
+    ENTRY e, *ep;
+    if (key == NULL || data == NULL) {
+        return;
+    }
+    hashTableInit();
+    e.data = data;
+    e.key = (char*)key;
+    LOCK_HASH_TABLE();
+    ep = hsearch(e, ENTER);
+    UNLOCK_HASH_TABLE();
+    if (ep == NULL) {
+        fprintf(stderr, "error adding key (%s) to hash table, <%d>: %s\n",
+                key, errno, strerror(errno));
+        exit(1);
+    }  
+}
+
+
+
+static void* searchEntryFromTable(const char *key)
+{
+    ENTRY e,*ep;
+    if (key == NULL) {
+        return NULL;
+    }
+    hashTableInit();
+    e.key = (char*)key;
+    LOCK_HASH_TABLE();
+    ep = hsearch(e, FIND);
+    UNLOCK_HASH_TABLE();
+    if (ep != NULL) {
+        return ep->data;
+    }
+    return NULL;
+}
+
+
+
+int invokeMethod(JNIEnv *env, RetVal *retval, MethType methType,
+                 jobject instObj, const char *className,
+                 const char *methName, const char *methSignature, ...)
+{
+    va_list args;
+    jclass cls;
+    jmethodID mid;
+    const char *str; 
+    char returnType;
+    
+    validateMethodType(methType);
+    cls = globalClassReference(className, env);
+    mid = methodIdFromClass(className, methName, methSignature, 
+                            methType, env);
+    if (mid == NULL) {
+        (*env)->ExceptionDescribe(env);
+        exit(1);
+    }
+   
+    str = methSignature;
+    while (*str != ')') str++;
+    str++;
+    returnType = *str;
+    va_start(args, methSignature);
+    if (returnType == JOBJECT || returnType == JARRAYOBJECT) {
+        jobject jobj = NULL;
+        if (methType == STATIC) {
+            jobj = (*env)->CallStaticObjectMethodV(env, cls, mid, args);
+        }
+        else if (methType == INSTANCE) {
+            jobj = (*env)->CallObjectMethodV(env, instObj, mid, args);
+        }
+        CHECK_EXCEPTION_IN_METH_INVOC
+        retval->l = jobj;
+    }
+    else if (returnType == VOID) {
+        if (methType == STATIC) {
+            (*env)->CallStaticVoidMethodV(env, cls, mid, args);
+        }
+        else if (methType == INSTANCE) {
+            (*env)->CallVoidMethodV(env, instObj, mid, args);
+        }
+       CHECK_EXCEPTION_IN_METH_INVOC
+    }
+    else if (returnType == JBOOLEAN) {
+        jboolean jbool = 0;
+        if (methType == STATIC) {
+            jbool = (*env)->CallStaticBooleanMethodV(env, cls, mid, args);
+        }
+        else if (methType == INSTANCE) {
+            jbool = (*env)->CallBooleanMethodV(env, instObj, mid, args);
+        }
+        CHECK_EXCEPTION_IN_METH_INVOC
+        retval->z = jbool;
+    }
+    else if (returnType == JLONG) {
+        jlong jl = -1;
+        if (methType == STATIC) {
+            jl = (*env)->CallStaticLongMethodV(env, cls, mid, args);
+        }
+        else if (methType == INSTANCE) {
+            jl = (*env)->CallLongMethodV(env, instObj, mid, args);
+        }
+        CHECK_EXCEPTION_IN_METH_INVOC
+        retval->j = jl;
+    }
+    else if (returnType == JINT) {
+        jint ji = -1;
+        if (methType == STATIC) {
+            ji = (*env)->CallStaticIntMethodV(env, cls, mid, args);
+        }
+        else if (methType == INSTANCE) {
+            ji = (*env)->CallIntMethodV(env, instObj, mid, args);
+        }
+        CHECK_EXCEPTION_IN_METH_INVOC
+        retval->i = ji;
+    }
+    va_end(args);
+    return 0;
+}
+
+
+jobject constructNewObjectOfClass(JNIEnv *env, const char *className, 
+                                  const char *ctorSignature, ...)
+{
+    va_list args;
+    jclass cls;
+    jmethodID mid; 
+    jobject jobj;
+
+    cls = globalClassReference(className, env);
+    mid = methodIdFromClass(className, "<init>", ctorSignature, 
+                            INSTANCE, env);
+    if (mid == NULL) {
+        (*env)->ExceptionDescribe(env);
+        exit(1);
+    } 
+    va_start(args, ctorSignature);
+    jobj = (*env)->NewObjectV(env, cls, mid, args);
+    va_end(args);
+    if ((*env)->ExceptionCheck(env)) {
+        (*env)->ExceptionDescribe(env);
+    }
+    return jobj;
+}
+
+
+
+
+jmethodID methodIdFromClass(const char *className, const char *methName, 
+                            const char *methSignature, MethType methType, 
+                            JNIEnv *env)
+{
+    jclass cls = globalClassReference(className, env);
+    jmethodID mid = 0;
+    validateMethodType(methType);
+    if (methType == STATIC) {
+        mid = (*env)->GetStaticMethodID(env, cls, methName, methSignature);
+    }
+    else if (methType == INSTANCE) {
+        mid = (*env)->GetMethodID(env, cls, methName, methSignature);
+    }
+    return mid;
+}
+
+
+jclass globalClassReference(const char *className, JNIEnv *env)
+{
+    jclass clsLocalRef;
+    jclass cls = searchEntryFromTable(className);
+    if (cls) {
+        return cls; 
+    }
+
+    clsLocalRef = (*env)->FindClass(env,className);
+    if (clsLocalRef == NULL) {
+        (*env)->ExceptionDescribe(env);
+        exit(1);
+    }
+    cls = (*env)->NewGlobalRef(env, clsLocalRef);
+    if (cls == NULL) {
+        (*env)->ExceptionDescribe(env);
+        exit(1);
+    }
+    (*env)->DeleteLocalRef(env, clsLocalRef);
+    insertEntryIntoTable(className, cls);
+    return cls;
+}
+
+
+
+
+/**
+ * getJNIEnv: A helper function to get the JNIEnv* for the given thread.
+ * @param: None.
+ * @return The JNIEnv* corresponding to the thread.
+ */
+JNIEnv* getJNIEnv(void)
+{
+    char threadID[threadID_SIZE];
+
+    const jsize vmBufLength = 1;
+    JavaVM* vmBuf[vmBufLength]; 
+    JNIEnv *env;
+    jint rv = 0; 
+    jint noVMs = 0;
+
+    //Get the threadID and stringize it 
+    GET_threadID(pthread_self(), threadID, sizeof(threadID));
+
+    //See if you already have the JNIEnv* cached...
+    env = (JNIEnv*)searchEntryFromTable(threadID);
+    if (env != NULL) {
+        return env; 
+    }
+
+    //All right... some serious work required here!
+    //1. Initialize the HashTable
+    //2. LOCK!
+    //3. Check if any JVMs have been created here
+    //      Yes: Use it (we should only have 1 VM)
+    //      No: Create the JVM
+    //4. UNLOCK
+
+    hashTableInit();
+
+    LOCK_HASH_TABLE();
+
+    rv = JNI_GetCreatedJavaVMs(&(vmBuf[0]), vmBufLength, &noVMs);
+    if (rv != 0) {
+        fprintf(stderr, "JNI_GetCreatedJavaVMs failed with error: %d\n", rv);
+        exit(1);
+    }
+
+    if (noVMs == 0) {
+        //Get the environment variables for initializing the JVM
+        char *hadoopClassPath = getenv("CLASSPATH");
+        if (hadoopClassPath == NULL) {
+            fprintf(stderr, "Environment variable CLASSPATH not set!\n");
+            exit(-1);
+        } 
+        char *hadoopClassPathVMArg = "-Djava.class.path=";
+        size_t optHadoopClassPathLen = strlen(hadoopClassPath) + 
+          strlen(hadoopClassPathVMArg) + 1;
+        char *optHadoopClassPath = malloc(sizeof(char)*optHadoopClassPathLen);
+        snprintf(optHadoopClassPath, optHadoopClassPathLen,
+        	"%s%s", hadoopClassPathVMArg, hadoopClassPath);
+
+        //Create the VM
+        JavaVMInitArgs vm_args;
+        JavaVMOption options[1];
+        JavaVM *vm;
+        
+        // User classes
+        options[0].optionString = optHadoopClassPath;
+        // Print JNI-related messages      
+        //options[2].optionString = "-verbose:jni";
+
+        vm_args.version = JNI_VERSION_1_2;
+        vm_args.options = options;
+        vm_args.nOptions = 1; 
+        vm_args.ignoreUnrecognized = 1;
+
+        rv = JNI_CreateJavaVM(&vm, (void*)&env, &vm_args);
+        if (rv != 0) {
+            fprintf(stderr, "Call to JNI_CreateJavaVM failed "
+                    "with error: %d\n", rv);
+            exit(1);
+        }
+
+        free(optHadoopClassPath);
+    }
+    else {
+        //Attach this thread to the VM
+        JavaVM* vm = vmBuf[0];
+        rv = (*vm)->AttachCurrentThread(vm, (void*)&env, 0);
+        if (rv != 0) {
+            fprintf(stderr, "Call to AttachCurrentThread "
+                    "failed with error: %d\n", rv);
+            exit(1);
+        }
+    }
+
+    //Save the threadID -> env mapping
+    ENTRY e, *ep;
+    e.key = threadID;
+    e.data = (void*)(env);
+    if ((ep = hsearch(e, ENTER)) == NULL) {
+        fprintf(stderr, "Call to hsearch(ENTER) failed\n");
+        exit(1);
+    }
+
+    UNLOCK_HASH_TABLE();
+
+    return env;
+}

src/c++/libhdfs/hdfsJniHelper.h

@@ -26,46 +26,25 @@
 #include <stdarg.h>
 #include <search.h>
 #include <pthread.h>
-
-pthread_mutex_t hashTableMutex = PTHREAD_MUTEX_INITIALIZER; 
-
-#define LOCK_HASH_TABLE() pthread_mutex_lock(&hashTableMutex)
-#define UNLOCK_HASH_TABLE() pthread_mutex_unlock(&hashTableMutex)
+#include <errno.h>
 
 #define PATH_SEPARATOR ':'
 
 #define USER_CLASSPATH "/home/y/libexec/hadoop/conf:/home/y/libexec/hadoop/lib/hadoop-0.1.0.jar"
 
-/** The Native return types that methods could return */
-#define VOID          'V'
-#define JOBJECT       'L'
-#define JARRAYOBJECT  '['
-#define JBOOLEAN      'Z'
-#define JBYTE         'B'
-#define JCHAR         'C'
-#define JSHORT        'S'
-#define JINT          'I'
-#define JLONG         'J'
-#define JFLOAT        'F'
-#define JDOUBLE       'D'
 
 /** Denote the method we want to invoke as STATIC or INSTANCE */
-typedef enum { 
+typedef enum {
     STATIC,
     INSTANCE
 } MethType;
 
+
 /** Used for returning an appropriate return value after invoking
  * a method
  */
 typedef jvalue RetVal;
 
-/**
- * MAX_HASH_TABLE_ELEM: The maximum no. of entries in the hashtable.
- * It's set to 4096 to account for (classNames + No. of threads)
- */
-#define MAX_HASH_TABLE_ELEM 4096
-
 /** invokeMethod: Invoke a Static or Instance method.
  * className: Name of the class where the method can be found
  * methName: Name of the method
@@ -83,9 +62,8 @@ typedef jvalue RetVal;
  * RETURNS: -1 on error and 0 on success. If -1 is returned, exc will have 
    a valid exception reference.
  */
-int invokeMethod(JNIEnv *env, RetVal *retval, jthrowable *exc,
-                 MethType methType, jobject instObj,
-                 const char *className, const char *methName, 
+int invokeMethod(JNIEnv *env, RetVal *retval, MethType methType,
+                 jobject instObj, const char *className, const char *methName, 
                  const char *methSignature, ...);
 
 /** constructNewObjectOfClass: Invoke a constructor.
@@ -95,239 +73,16 @@ int invokeMethod(JNIEnv *env, RetVal *retval, jthrowable *exc,
  * exc: If the ctor throws any exception, this will contain the reference
  * Arguments to the ctor must be passed after ctorSignature 
  */
-jobject constructNewObjectOfClass(JNIEnv *env, jthrowable *exc, 
-                                  const char *className, 
+jobject constructNewObjectOfClass(JNIEnv *env, const char *className, 
                                   const char *ctorSignature, ...);
 
-void validateMethodType(MethType methType);
-
 jmethodID methodIdFromClass(const char *className, const char *methName, 
                             const char *methSignature, MethType methType, 
                             JNIEnv *env);
 
 jclass globalClassReference(const char *className, JNIEnv *env);
 
-void insertEntryIntoTable(const char *key, void *data);
-void *searchEntryFromTable(const char *key);
-void hashTableInit();
-
-#define CHECK_EXCEPTION_IN_METH_INVOC {\
-    jthrowable _exc_;\
-    if((_exc_ = (*env)->ExceptionOccurred(env))) {\
-        (*env)->ExceptionDescribe(env);\
-        *exc = _exc_;\
-        (*env)->ExceptionClear(env);\
-        va_end(args);\
-        return -1;\
-    }\
-}
-
-int invokeMethod(JNIEnv *env, RetVal *retval, jthrowable *exc,
-                 MethType methType, jobject instObj,
-                 const char *className, const char *methName, 
-                 const char *methSignature, ...)
-{
-    va_list args;
-    jclass cls;
-    jmethodID mid;
-    const char *str; 
-    char returnType;
-    
-    validateMethodType(methType);
-    cls = globalClassReference(className, env);
-    mid = methodIdFromClass(className, methName, methSignature, 
-            methType, env);
-    if(mid == NULL) {
-        (*env)->ExceptionDescribe(env);
-        exit(1);
-    }
-   
-    str = methSignature;
-    while(*str != ')') str++;
-    str++;
-    returnType = *str;
-    va_start(args, methSignature);
-    if (returnType == JOBJECT || returnType == JARRAYOBJECT) {
-        jobject jobj;
-        if (methType == STATIC) {
-            jobj = (*env)->CallStaticObjectMethodV(env, cls, mid, args);
-        }
-        else if (methType == INSTANCE) {
-            jobj = (*env)->CallObjectMethodV(env, instObj, mid, args);
-        }
-        CHECK_EXCEPTION_IN_METH_INVOC
-        retval->l = jobj;
-    }
-    else if (returnType == VOID) {
-        if (methType == STATIC) {
-            (*env)->CallStaticVoidMethodV(env, cls, mid, args);
-        }
-        else if (methType == INSTANCE) {
-            (*env)->CallVoidMethodV(env, instObj, mid, args);
-        }
-        CHECK_EXCEPTION_IN_METH_INVOC
-    }
-    else if (returnType == JBOOLEAN) {
-        jboolean jbool;
-        if (methType == STATIC) {
-            jbool = (*env)->CallStaticBooleanMethodV(env, cls, mid, args);
-        }
-        else if (methType == INSTANCE) {
-            jbool = (*env)->CallBooleanMethodV(env, instObj, mid, args);
-        }
-        CHECK_EXCEPTION_IN_METH_INVOC
-        retval->z = jbool;
-    }
-    else if (returnType == JLONG) {
-        jlong jl;
-        if (methType == STATIC) {
-            jl = (*env)->CallStaticLongMethodV(env, cls, mid, args);
-        }
-        else if (methType == INSTANCE) {
-            jl = (*env)->CallLongMethodV(env, instObj, mid, args);
-        }
-        CHECK_EXCEPTION_IN_METH_INVOC
-        retval->j = jl;
-    }
-    else if (returnType == JINT) {
-        jint ji;
-        if (methType == STATIC) {
-            ji = (*env)->CallStaticIntMethodV(env, cls, mid, args);
-        }
-        else if (methType == INSTANCE) {
-            ji = (*env)->CallIntMethodV(env, instObj, mid, args);
-        }
-        CHECK_EXCEPTION_IN_METH_INVOC
-        retval->i = ji;
-    }
-    va_end(args);
-    return 0;
-}
-
-void validateMethodType(MethType methType)
-{
-    if (methType != STATIC && methType != INSTANCE) {
-        fprintf(stderr,"Unimplemented method type\n");
-        exit(1);
-    }
-    return;
-}
-
-jobject constructNewObjectOfClass(JNIEnv *env, jthrowable *exc, 
-                                  const char *className, 
-                                  const char *ctorSignature, ...)
-{
-    va_list args;
-    jclass cls;
-    jmethodID mid; 
-    jobject jobj;
-    jthrowable _exc;
-
-    cls = globalClassReference(className, env);
-    mid = methodIdFromClass(className, "<init>", ctorSignature, 
-            INSTANCE, env);
-    if(mid == NULL) {
-        (*env)->ExceptionDescribe(env);
-        exit(1);
-    } 
-    va_start(args, ctorSignature);
-    jobj = (*env)->NewObjectV(env, cls, mid, args);
-    va_end(args);
-    if((_exc = (*env)->ExceptionOccurred(env))) {
-        (*env)->ExceptionDescribe(env);
-        *exc = _exc;
-        (*env)->ExceptionClear(env);
-    }
-    return jobj;
-}
-
-jmethodID methodIdFromClass(const char *className, const char *methName, 
-                            const char *methSignature, MethType methType, 
-                            JNIEnv *env)
-{
-    jclass cls = globalClassReference(className, env);
-    jmethodID mid;
-    validateMethodType(methType);
-    if(methType == STATIC) {
-        mid = (*env)->GetStaticMethodID(env, cls, methName, methSignature);
-    }
-    else if(methType == INSTANCE) {
-        mid = (*env)->GetMethodID(env, cls, methName, methSignature);
-    }
-    return mid;
-}
-
-jclass globalClassReference(const char *className, JNIEnv *env)
-{
-    jclass clsLocalRef;
-    jclass cls = searchEntryFromTable(className);
-    if(cls) {
-        return cls; 
-    }
-
-    clsLocalRef = (*env)->FindClass(env,className);
-    if(clsLocalRef == NULL) {
-        (*env)->ExceptionDescribe(env);
-        exit(1);
-    }
-    cls = (*env)->NewGlobalRef(env, clsLocalRef);
-    if(cls == NULL) {
-        (*env)->ExceptionDescribe(env);
-        exit(1);
-    }
-    (*env)->DeleteLocalRef(env, clsLocalRef);
-    insertEntryIntoTable(className, cls);
-    return cls;
-}
-
-void insertEntryIntoTable(const char *key, void *data)
-{
-    ENTRY e, *ep;
-    if(key == NULL || data == NULL) {
-        return;
-    }
-    hashTableInit();
-    e.data = data;
-    e.key = (char*)key;
-    LOCK_HASH_TABLE();
-    ep = hsearch(e, ENTER);
-    UNLOCK_HASH_TABLE();
-    if(ep == NULL) {
-        fprintf(stderr,"hsearch(ENTER) returned error\n");
-        exit(1);
-    }  
-}
-
-void *searchEntryFromTable(const char *key)
-{
-    ENTRY e,*ep;
-    if(key == NULL) {
-        return NULL;
-    }
-    hashTableInit();
-    e.key = (char*)key;
-    LOCK_HASH_TABLE();
-    ep = hsearch(e, FIND);
-    UNLOCK_HASH_TABLE();
-    if(ep != NULL) {
-        return ep->data;
-    }
-    return NULL;
-}
-
-void hashTableInit()
-{
-    static int hash_table_inited = 0;
-    LOCK_HASH_TABLE();
-    if(!hash_table_inited) {
-        if (hcreate(MAX_HASH_TABLE_ELEM) == 0) {
-            fprintf(stderr,"hcreate returned error\n");
-            exit(1);
-        } 
-        hash_table_inited = 1;
-    }  
-    UNLOCK_HASH_TABLE();
-}
+JNIEnv* getJNIEnv(void);
 
 #endif /*LIBHDFS_JNI_HELPER_H*/
 

src/c++/libhdfs/hdfs_test.c

@@ -70,6 +70,13 @@ int main(int argc, char **argv) {
         //Read tests
         
         const char* readPath = "/tmp/testfile.txt";
+        int exists = hdfsExists(fs, readPath);
+
+        if (exists) {
+          fprintf(stderr, "Failed to validate existence of %s\n", readPath);
+          exit(-1);
+        }
+
         hdfsFile readFile = hdfsOpenFile(fs, readPath, O_RDONLY, 0, 0, 0);
         if (!readFile) {
             fprintf(stderr, "Failed to open %s for reading!\n", readPath);
@@ -188,6 +195,7 @@ int main(int argc, char **argv) {
         fprintf(stderr, "hdfsDelete: %s\n", (hdfsDelete(fs, srcPath) ? "Failed!" : "Success!"));
         fprintf(stderr, "hdfsDelete: %s\n", (hdfsDelete(lfs, srcPath) ? "Failed!" : "Success!"));
         fprintf(stderr, "hdfsDelete: %s\n", (hdfsDelete(lfs, dstPath) ? "Failed!" : "Success!"));
+        fprintf(stderr, "hdfsExists: %s\n", (hdfsExists(fs, newDirectory) ? "Success!" : "Failed!"));
     }
 
     return 0;

src/c++/libhdfs/tests/conf/hadoop-site.xml

@@ -8,28 +8,17 @@
 <configuration>
 
 <property>
-  <name>fs.default.name</name>
-  <value>localhost:23000</value>
-  <description>The name of the default file system.  Either the
-  literal string "local" or a host:port for DFS.</description>
+  <name>hadoop.tmp.dir</name>
+  <value>../../../build/test/libhdfs</value>
+  <description>A base for other temporary directories.</description>
 </property>
 
-<property>
-  <name>dfs.name.dir</name>
-  <value>/tmp/test-libhdfs/dfs/name</value>
-  <description>Determines where on the local filesystem the DFS name node
-      should store the name table.</description>
-</property>
 
 <property>
-  <name>dfs.data.dir</name>
-  <value>/tmp/test-libhdfs/dfs/data</value>
-  <description>Determines where on the local filesystem an DFS data node
-  should store its blocks.  If this is a comma-delimited
-  list of directories, then data will be stored in all named
-  directories, typically on different devices.
-  Directories that do not exist are ignored.
-  </description>
+  <name>fs.default.name</name>
+  <value>localhost:23000</value>
+  <description>The name of the default file system.  Either the
+  literal string "local" or a host:port for DFS.</description>
 </property>
 
 <property>

src/c++/libhdfs/tests/test-libhdfs.sh

@@ -28,14 +28,41 @@ HDFS_TEST=hdfs_test
 HADOOP_LIB_DIR=$HADOOP_HOME/lib
 HADOOP_BIN_DIR=$HADOOP_HOME/bin
 
-# Add libs to CLASSPATH for libhdfs (jni)
-CLASSPATH=`for f in $HADOOP_LIB_DIR/*.jar; do CLASSPATH=$CLASSPATH:$f; done; echo $CLASSPATH;`
-
-# Manipulate HADOOP_CONF_DIR so as to include 
-# HADOOP_HOME/conf/hadoop-default.xml too
+## Manipulate HADOOP_CONF_DIR so as to include 
+## HADOOP_HOME/conf/hadoop-default.xml too
 # which is necessary to circumvent bin/hadoop
 HADOOP_CONF_DIR=$HADOOP_CONF_DIR:$HADOOP_HOME/conf
 
+# CLASSPATH initially contains $HADOOP_CONF_DIR
+CLASSPATH="${HADOOP_CONF_DIR}"
+CLASSPATH=${CLASSPATH}:$JAVA_HOME/lib/tools.jar
+
+# for developers, add Hadoop classes to CLASSPATH
+if [ -d "$HADOOP_HOME/build/classes" ]; then
+  CLASSPATH=${CLASSPATH}:$HADOOP_HOME/build/classes
+fi
+if [ -d "$HADOOP_HOME/build/webapps" ]; then
+  CLASSPATH=${CLASSPATH}:$HADOOP_HOME/build
+fi
+if [ -d "$HADOOP_HOME/build/test/classes" ]; then
+  CLASSPATH=${CLASSPATH}:$HADOOP_HOME/build/test/classes
+fi
+
+# so that filenames w/ spaces are handled correctly in loops below
+IFS=
+
+# add libs to CLASSPATH
+for f in $HADOOP_HOME/lib/*.jar; do
+  CLASSPATH=${CLASSPATH}:$f;
+done
+
+for f in $HADOOP_HOME/lib/jsp-2.0/*.jar; do
+  CLASSPATH=${CLASSPATH}:$f;
+done
+
+# restore ordinary behaviour
+unset IFS
+
 # Put delays to ensure hdfs is up and running and also shuts down 
 # after the tests are complete
 echo Y | $HADOOP_BIN_DIR/hadoop namenode -format &&