HADOOP-10640. Implement Namenode RPCs in HDFS native client (cmccabe)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HADOOP-10388@1602280 13f79535-47bb-0310-9956-ffa450edef68

hadoop-native-core/src/main/native/CMakeLists.txt

@@ -3,6 +3,9 @@ set(CMAKE_BUILD_TYPE, Release) # Default to release builds
 enable_testing()
 MESSAGE(STATUS "Building hadoop-native-core, the native Hadoop core libraries.")
 
+include(../../../../hadoop-common-project/hadoop-common/src/JNIFlags.cmake NO_POLICY_SCOPE)
+GET_FILENAME_COMPONENT(JNI_LIBRARY_NAME ${JAVA_JVM_LIBRARY} NAME)
+
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -Wall -Wextra -O2 -fno-strict-aliasing")
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_REENTRANT -D_GNU_SOURCE")
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64")
@@ -12,18 +15,24 @@ macro(add_utest utest)
     add_test(${utest} ${CMAKE_CURRENT_BINARY_DIR}/${utest} ${utest})
 endmacro(add_utest)
 
+# Check to see if our compiler and linker support the __thread attribute.
+# On Linux and some other operating systems, this is a more efficient
+# alternative to POSIX thread local storage.
+INCLUDE(CheckCSourceCompiles)
+CHECK_C_SOURCE_COMPILES("int main(void) { static __thread int i = 0; return 0; }" HAVE_BETTER_TLS)
+
 # Find libuv
 find_library(LIBUV_LIB NAMES uv PATHS lib/libuv)
 find_path(LIBUV_HEADER_PATH NAMES uv.h PATHS lib/libuv/include uv/include)
 if (NOT (LIBUV_LIB AND LIBUV_HEADER_PATH))
-    MESSAGE(FATAL_ERROR "Failed to find libuv.  Please install libuv.  LIBUV_LIB=${LIBUV_LIB}, LIBUV_HEADER_PATH=${LIBUV_HEADER_PATH}") 
+    MESSAGE(FATAL_ERROR "Failed to find libuv.  Please install libuv.  LIBUV_LIB=${LIBUV_LIB}, LIBUV_HEADER_PATH=${LIBUV_HEADER_PATH}")
 endif ()
 include_directories(
     lib/libuv/include
     uv/include)
 
 # Find protobuf-c
-find_library(PROTOBUFC_LIB NAMES protobuf-c 
+find_library(PROTOBUFC_LIB NAMES protobuf-c
     HINTS /usr/lib64 /usr/lib)
 find_program(PROTOBUFC_EXE NAMES protoc-c)
 if (NOT (PROTOBUFC_LIB AND PROTOBUFC_EXE))
@@ -31,33 +40,57 @@ if (NOT (PROTOBUFC_LIB AND PROTOBUFC_EXE))
 endif()
 
 # Find protobuf
-find_library(PROTOC_LIB NAMES protoc
+find_library(PROTOC_LIB NAMES libprotoc.a protoc
     HINTS /usr/lib /usr/lib64)
-find_library(PROTOBUF_LIB NAMES protobuf
+find_library(PROTOBUF_LIB NAMES libprotobuf.a protobuf
     HINTS /usr/lib /usr/lib64)
 find_program(PROTOC_EXE NAMES protoc)
-find_path(PROTOC_HEADER_PATH NAMES 
+find_path(PROTOC_HEADER_PATH NAMES
     google/protobuf/compiler/command_line_interface.h
     HINTS /usr/include)
 if (NOT (PROTOC_LIB AND PROTOBUF_LIB AND PROTOC_EXE AND PROTOC_HEADER_PATH))
     MESSAGE(FATAL_ERROR "Failed to find the C++ protobuf libraries, which are needed for RPC code generation.  PROTOC_LIB=${PROTOC_LIB}, PROTOBUF_LIB=${PROTOBUF_LIB}, PROTOC_EXE=${PROTOC_EXE}, PROTOC_HEADER_PATH=${PROTOC_HEADER_PATH}")
 endif ()
 
+# Find libexpat
+find_library(EXPAT_LIB NAMES expat
+    HINTS /usr/lib /usr/lib64)
+find_path(EXPAT_HEADER_PATH NAMES expat.h
+    HINTS /usr/include)
+if (NOT (EXPAT_LIB AND EXPAT_HEADER_PATH))
+    MESSAGE(FATAL_ERROR "Failed to find libexpat, which is needed for parsing configuration XML files. EXPAT_LIB=${XEXPAT_LIB}, EXPAT_HEADER_PATH=${EXPAT_HEADER_PATH}")
+endif ()
+
+# Find liburiparser
+find_library(URIPARSER_LIB NAMES uriparser
+    HINTS /usr/lib /usr/lib64)
+find_path(URIPARSER_HEADER_PATH NAMES uriparser/Uri.h
+    HINTS /usr/include)
+if (NOT (URIPARSER_LIB AND URIPARSER_HEADER_PATH))
+    MESSAGE(FATAL_ERROR "Failed to find liburiparser, which is needed for parsing URIs. URIPARSER_LIB=${URIPARSER_LIB}, URIPARSER_HEADER_PATH=${URIPARSER_HEADER_PATH}")
+endif ()
+
 include_directories(
-    ${CMAKE_CURRENT_SOURCE_DIR}
     ${CMAKE_CURRENT_BINARY_DIR}
-    ${PROTOBUF_HEADER_PATH})
+    ${CMAKE_CURRENT_SOURCE_DIR}
+    ${EXPAT_HEADER_PATH}
+    ${JNI_INCLUDE_DIRS}
+    ${PROTOBUF_HEADER_PATH}
+    ${URIPARSER_HEADER_PATH})
 
 include(GenerateProtobufs.cmake NO_POLICY_SCOPE)
 
 set(COMMON_SRCS
     common/hadoop_err.c
+    common/hconf.c
+    common/htable.c
     common/net.c
     common/string.c
-    common/test.c
     common/user.c
+    common/uri.c
 )
 set(COMMON_DEPS
+    ${EXPAT_LIB}
     pthread
 )
 
@@ -76,53 +109,142 @@ set(RPC_DEPS
     ${PROTOBUFC_LIB}
 )
 
+set(FS_SRCS
+    fs/common.c
+    fs/fs.c
+    ndfs/ndfs.c
+    ${HDFS_PROTOBUF_SRCS}
+)
+
+set(FS_DEPS
+    ${URIPARSER_LIB}
+)
+
+set(JNI_SRCS
+    jni/exception.c
+    jni/jni_helper.c
+    jni/jnifs.c
+)
+set(JNI_DEPS
+    ${JAVA_JVM_LIBRARY}
+    uv
+    pthread
+)
+
+set(HCONF_XML_TEST_PATH "${CMAKE_SOURCE_DIR}/test/common/conf")
+CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
+
+add_library(fstest STATIC
+    test/native_mini_dfs.c
+    test/posix_util.c
+    test/test.c
+    ${COMMON_SRCS}
+    ${RPC_SRCS}
+    ${JNI_SRCS}
+    ${FS_SRCS}
+)
+target_link_libraries(fstest
+    ${COMMON_DEPS}
+    ${RPC_DEPS}
+    ${JNI_DEPS}
+    ${FS_DEPS}
+)
+
 add_executable(varint-unit rpc/varint-unit.c
-    rpc/varint.c common/test.c)
+    rpc/varint.c test/test.c)
 add_utest(varint-unit)
 
-add_executable(net-unit common/net-unit.c
-    common/net.c common/test.c)
-add_utest(net-unit)
-target_link_libraries(net-unit uv)
+add_executable(string-unit common/string-unit.c
+    common/string.c test/test.c)
+add_utest(string-unit)
+
+add_executable(htable-unit common/htable-unit.c
+    common/htable.c test/test.c)
+add_utest(htable-unit)
+
+add_executable(hconf-unit common/hconf-unit.c
+    common/hconf.c common/htable.c common/hadoop_err.c test/test.c)
+add_utest(hconf-unit)
+target_link_libraries(hconf-unit ${EXPAT_LIB} ${LIBUV_LIB})
 
 add_executable(hadoop_err-unit common/hadoop_err-unit.c
-    common/hadoop_err.c common/test.c)
+    common/hadoop_err.c test/test.c)
 add_utest(hadoop_err-unit)
-target_link_libraries(hadoop_err-unit uv)
+target_link_libraries(hadoop_err-unit ${LIBUV_LIB})
+
+add_executable(uri-unit common/uri-unit.c
+    common/uri.c common/hadoop_err.c test/test.c)
+add_utest(uri-unit)
+target_link_libraries(uri-unit ${URIPARSER_LIB} ${LIBUV_LIB})
+
+add_executable(namenode-rpc-unit
+    ndfs/namenode-rpc-unit.c)
+target_link_libraries(namenode-rpc-unit fstest)
+
+add_executable(test_libhdfs_threaded
+    test/fs/test_libhdfs_threaded.c
+)
+target_link_libraries(test_libhdfs_threaded
+    fstest
+)
+
+add_executable(test_libhdfs_zerocopy
+    test/fs/test_libhdfs_zerocopy.c
+)
+target_link_libraries(test_libhdfs_zerocopy
+    fstest
+)
+
+add_executable(test_libhdfs_meta_ops
+    test/fs/test_libhdfs_meta_ops.c
+)
+target_link_libraries(test_libhdfs_meta_ops
+    fstest
+)
 
-add_executable(namenode-rpc-unit hdfs/namenode-rpc-unit.c)
-target_link_libraries(namenode-rpc-unit hdfs-core)
+# When we generate our shared libraries, we want to hide symbols by default,
+# exporting only a few carefully chosen symbols.  This prevents symbol name
+# conflicts between our library and client programs.  It also prevents client
+# programs from calling internal APIs they shouldn't.
+# TODO: figure out what flag should be used here for Windows.
+IF(UNIX)
+    SET(VISIBILITY_FLAGS "-fvisibility=hidden")
+ENDIF(UNIX)
 
-add_library(hdfs-core SHARED
+add_library(hdfs SHARED
     ${COMMON_SRCS}
+    ${FS_SRCS}
+    ${JNI_SRCS}
     ${RPC_SRCS}
-    ${HDFS_PROTOBUF_SRCS}
 )
-target_link_libraries(hdfs-core 
+target_link_libraries(hdfs
     ${COMMON_DEPS}
+    ${FS_DEPS}
+    ${LIB_DL}
     ${RPC_DEPS}
 )
 set(HDFS_CORE_VERSION_MAJOR 1)
 set(HDFS_CORE_VERSION_MINOR 0)
 set(HDFS_CORE_VERSION_PATCH 0)
 set(HDFS_CORE_VERSION_STRING "${HDFS_CORE_VERSION_MAJOR}.${HDFS_CORE_VERSION_MINOR}.${HDFS_CORE_VERSION_PATCH}")
-set_target_properties(hdfs-core PROPERTIES
+set_target_properties(hdfs PROPERTIES
     VERSION ${HDFS_CORE_VERSION_STRING}
     SOVERSION ${HDFS_CORE_VERSION_MAJOR})
+SET_TARGET_PROPERTIES(hdfs PROPERTIES COMPILE_FLAGS ${VISIBILITY_FLAGS})
 
-add_library(yarn-core SHARED
-    ${COMMON_SRCS}
-    ${RPC_SRCS}
-    ${YARN_PROTOBUF_SRCS}
-)
-target_link_libraries(yarn-core 
-    ${COMMON_DEPS}
-    ${RPC_DEPS}
-)
-set(YARN_CORE_VERSION_MAJOR 1)
-set(YARN_CORE_VERSION_MINOR 0)
-set(YARN_CORE_VERSION_PATCH 0)
-set(YARN_CORE_VERSION_STRING ${YARN_CORE_VERSION_MAJOR}.${YARN_CORE_VERSION_MINOR}.${YARN_CORE_VERSION_PATCH})
-set_target_properties(yarn-core PROPERTIES
-    VERSION ${YARN_CORE_VERSION_STRING}
-    SOVERSION ${YARN_CORE_VERSION_MAJOR})
+#add_library(yarn SHARED
+#    ${COMMON_SRCS}
+#    ${RPC_SRCS}
+#    ${YARN_PROTOBUF_SRCS}
+#)
+#target_link_libraries(yarn
+#    ${COMMON_DEPS}
+#    ${RPC_DEPS}
+#)
+#set(YARN_VERSION_MAJOR 1)
+#set(YARN_VERSION_MINOR 0)
+#set(YARN_VERSION_PATCH 0)
+#set(YARN_VERSION_STRING ${YARN_VERSION_MAJOR}.${YARN_VERSION_MINOR}.${YARN_VERSION_PATCH})
+#set_target_properties(yarn PROPERTIES
+#    VERSION ${YARN_VERSION_STRING}
+#    SOVERSION ${YARN_VERSION_MAJOR})

hadoop-native-core/src/main/native/common/hadoop_err-unit.c

@@ -28,101 +28,127 @@
 
 #define RUNTIME_EXCEPTION_ERROR_CODE EFAULT
 
-static int hadoop_lerr_alloc_test(int code, char *verMsg, char *fmt) {
-	struct hadoop_err *err;
-	err = hadoop_lerr_alloc(code, fmt);
-	EXPECT_STR_EQ(verMsg, hadoop_err_msg(err));
-	EXPECT_INT_EQ(code, hadoop_err_code(err));
-	hadoop_err_free(err);
-	return 0;
+static int hadoop_lerr_alloc_test(int code, char *verMsg, char *fmt)
+{
+    struct hadoop_err *err;
+    err = hadoop_lerr_alloc(code, fmt);
+    EXPECT_STR_EQ(verMsg, hadoop_err_msg(err));
+    EXPECT_INT_EQ(code, hadoop_err_code(err));
+    hadoop_err_free(err);
+    return 0;
 }
 
-static int hadoop_lerr_alloc_test2(int code, char *verMsg) {
-	struct hadoop_err *err;
-	char msg[100];
-	memset(msg, 0, 100);
-	strcat(msg, verMsg);
-	err = hadoop_lerr_alloc(code, "foo bar baz %d", 101);
-	EXPECT_STR_EQ(strcat(msg, "foo bar baz 101"), hadoop_err_msg(err));
-	EXPECT_INT_EQ(code, hadoop_err_code(err));
-	hadoop_err_free(err);
-	return 0;
+static int hadoop_lerr_alloc_test2(int code, char *verMsg)
+{
+    struct hadoop_err *err;
+    char msg[100];
+    memset(msg, 0, 100);
+    strcat(msg, verMsg);
+    err = hadoop_lerr_alloc(code, "foo bar baz %d", 101);
+    EXPECT_STR_EQ(strcat(msg, "foo bar baz 101"), hadoop_err_msg(err));
+    EXPECT_INT_EQ(code, hadoop_err_code(err));
+    hadoop_err_free(err);
+    return 0;
 }
 
-static int hadoop_uverr_alloc_test(int code, char *verMsg, char *fmt) {
-	struct hadoop_err *err;
-	err = hadoop_uverr_alloc(code, fmt);
-	EXPECT_STR_EQ(verMsg, hadoop_err_msg(err));
-	EXPECT_INT_EQ(code, hadoop_err_code(err));
-	hadoop_err_free(err);
-	return 0;
+static int hadoop_uverr_alloc_test(int code, char *verMsg, char *fmt)
+{
+    struct hadoop_err *err;
+    err = hadoop_uverr_alloc(code, fmt);
+    EXPECT_STR_EQ(verMsg, hadoop_err_msg(err));
+    EXPECT_INT_EQ(code, hadoop_err_code(err));
+    hadoop_err_free(err);
+    return 0;
 }
 
-static int hadoop_uverr_alloc_test2(int code, char *verMsg) {
-	struct hadoop_err *err;
-	char msg[100];
-	memset(msg, 0, 100);
-	strcat(msg, verMsg);
-	err = hadoop_uverr_alloc(code, "foo bar baz %d", 101);
-	EXPECT_STR_EQ(strcat(msg, "foo bar baz 101"), hadoop_err_msg(err));
-	EXPECT_INT_EQ(code, hadoop_err_code(err));
-	hadoop_err_free(err);
-	return 0;
+static int hadoop_uverr_alloc_test2(int code, char *verMsg) 
+{
+    struct hadoop_err *err;
+    char msg[100];
+    memset(msg, 0, 100);
+    strcat(msg, verMsg);
+    err = hadoop_uverr_alloc(code, "foo bar baz %d", 101);
+    EXPECT_STR_EQ(strcat(msg, "foo bar baz 101"), hadoop_err_msg(err));
+    EXPECT_INT_EQ(code, hadoop_err_code(err));
+    hadoop_err_free(err);
+    return 0;
 }
 
-int main(void) {
-	hadoop_lerr_alloc_test(RUNTIME_EXCEPTION_ERROR_CODE,
-			"org.apache.hadoop.native.HadoopCore.RuntimeException: "
-					"test RUNTIME_EXCEPTION_ERROR_CODE",
-			"test RUNTIME_EXCEPTION_ERROR_CODE");
-	hadoop_lerr_alloc_test(EINVAL,
-			"org.apache.hadoop.native.HadoopCore.InvalidRequestException: "
-					"test EINVAL", "test EINVAL");
-	hadoop_lerr_alloc_test(ENOMEM,
-			"org.apache.hadoop.native.HadoopCore.OutOfMemoryException: "
-					"test ENOMEM", "test ENOMEM");
-	hadoop_lerr_alloc_test(0,
-			"org.apache.hadoop.native.HadoopCore.IOException: "
-					"test default", "test default");
-	hadoop_uverr_alloc_test(UV_EOF,
-			"org.apache.hadoop.native.HadoopCore.EOFException: end of file: "
-					"test UV_EOF", "test UV_EOF");
-	hadoop_uverr_alloc_test(UV_EINVAL,
-			"org.apache.hadoop.native.HadoopCore.InvalidRequestException: "
-					"invalid argument: test UV_EINVAL", "test UV_EINVAL");
-	hadoop_uverr_alloc_test(UV_ECONNREFUSED,
-			"org.apache.hadoop.native.HadoopCore.ConnectionRefusedException: "
-					"connection refused: test UV_ECONNREFUSED",
-			"test UV_ECONNREFUSED");
-	hadoop_uverr_alloc_test(UV_ENOMEM,
-			"org.apache.hadoop.native.HadoopCore.OutOfMemoryException: "
-					"not enough memory: test UV_ENOMEM", "test UV_ENOMEM");
-	hadoop_uverr_alloc_test(0,
-			"org.apache.hadoop.native.HadoopCore.IOException: "
-					"Unknown system error: test default", "test default");
-	hadoop_lerr_alloc_test2(EINVAL,
-			"org.apache.hadoop.native.HadoopCore.InvalidRequestException: ");
-	hadoop_lerr_alloc_test2(RUNTIME_EXCEPTION_ERROR_CODE,
-			"org.apache.hadoop.native.HadoopCore.RuntimeException: ");
-	hadoop_lerr_alloc_test2(ENOMEM,
-			"org.apache.hadoop.native.HadoopCore.OutOfMemoryException: ");
-	hadoop_lerr_alloc_test2(0,
-			"org.apache.hadoop.native.HadoopCore.IOException: ");
-	hadoop_uverr_alloc_test2(UV_EOF,
-			"org.apache.hadoop.native.HadoopCore.EOFException: end of file: ");
-	hadoop_uverr_alloc_test2(UV_EINVAL,
-			"org.apache.hadoop.native.HadoopCore.InvalidRequestException: "
-					"invalid argument: ");
-	hadoop_uverr_alloc_test2(UV_ECONNREFUSED,
-			"org.apache.hadoop.native.HadoopCore.ConnectionRefusedException: "
-					"connection refused: ");
-	hadoop_uverr_alloc_test2(UV_ENOMEM,
-			"org.apache.hadoop.native.HadoopCore.OutOfMemoryException: "
-					"not enough memory: ");
-	hadoop_uverr_alloc_test2(0,
-			"org.apache.hadoop.native.HadoopCore.IOException: "
-					"Unknown system error: ");
-	return EXIT_SUCCESS;
+static int hadoop_err_copy_test(void)
+{
+    struct hadoop_err *err, *err2, *err3;
+ 
+    err = hadoop_lerr_alloc(EINVAL, "foo bar baz %d", 101);
+    err2 = hadoop_err_copy(err);
+    hadoop_err_free(err);
+    EXPECT_INT_EQ(EINVAL, hadoop_err_code(err2));
+    EXPECT_STR_EQ("org.apache.hadoop.native.HadoopCore.InvalidRequestException: "
+              "foo bar baz 101", hadoop_err_msg(err2));
+    err3 = hadoop_err_prepend(err2, EIO, "Turboencabulator error");
+    EXPECT_INT_EQ(EIO, hadoop_err_code(err3));
+    EXPECT_STR_EQ("Turboencabulator error: "
+        "org.apache.hadoop.native.HadoopCore.InvalidRequestException: "
+        "foo bar baz 101", hadoop_err_msg(err3));
+    hadoop_err_free(err3);
+    return 0;
+}
+
+int main(void)
+{
+    EXPECT_INT_ZERO(hadoop_lerr_alloc_test(RUNTIME_EXCEPTION_ERROR_CODE,
+            "org.apache.hadoop.native.HadoopCore.RuntimeException: "
+                    "test RUNTIME_EXCEPTION_ERROR_CODE",
+            "test RUNTIME_EXCEPTION_ERROR_CODE"));
+    EXPECT_INT_ZERO(hadoop_lerr_alloc_test(EINVAL,
+            "org.apache.hadoop.native.HadoopCore.InvalidRequestException: "
+                    "test EINVAL", "test EINVAL"));
+    EXPECT_INT_ZERO(hadoop_lerr_alloc_test(ENOMEM,
+            "org.apache.hadoop.native.HadoopCore.OutOfMemoryException: "
+                    "test ENOMEM", "test ENOMEM"));
+    EXPECT_INT_ZERO(hadoop_lerr_alloc_test(0,
+            "org.apache.hadoop.native.HadoopCore.IOException: "
+                    "test default", "test default"));
+    EXPECT_INT_ZERO(hadoop_uverr_alloc_test(UV_EOF,
+            "org.apache.hadoop.native.HadoopCore.EOFException: end of file: "
+                    "test UV_EOF", "test UV_EOF"));
+    EXPECT_INT_ZERO(hadoop_uverr_alloc_test(UV_EINVAL,
+            "org.apache.hadoop.native.HadoopCore.InvalidRequestException: "
+                    "invalid argument: test UV_EINVAL", "test UV_EINVAL"));
+    EXPECT_INT_ZERO(hadoop_uverr_alloc_test(UV_ECONNREFUSED,
+            "org.apache.hadoop.native.HadoopCore.ConnectionRefusedException: "
+                    "connection refused: test UV_ECONNREFUSED",
+            "test UV_ECONNREFUSED"));
+    EXPECT_INT_ZERO(hadoop_uverr_alloc_test(UV_ENOMEM,
+            "org.apache.hadoop.native.HadoopCore.OutOfMemoryException: "
+                    "not enough memory: test UV_ENOMEM", "test UV_ENOMEM"));
+    EXPECT_INT_ZERO(hadoop_uverr_alloc_test(0,
+            "org.apache.hadoop.native.HadoopCore.IOException: "
+                    "Unknown system error: test default", "test default"));
+    EXPECT_INT_ZERO(hadoop_lerr_alloc_test2(EINVAL,
+            "org.apache.hadoop.native.HadoopCore.InvalidRequestException: "));
+    EXPECT_INT_ZERO(hadoop_lerr_alloc_test2(RUNTIME_EXCEPTION_ERROR_CODE,
+            "org.apache.hadoop.native.HadoopCore.RuntimeException: "));
+    EXPECT_INT_ZERO(hadoop_lerr_alloc_test2(ENOMEM,
+            "org.apache.hadoop.native.HadoopCore.OutOfMemoryException: "));
+    EXPECT_INT_ZERO(hadoop_lerr_alloc_test2(0,
+            "org.apache.hadoop.native.HadoopCore.IOException: "));
+    EXPECT_INT_ZERO(hadoop_uverr_alloc_test2(UV_EOF,
+            "org.apache.hadoop.native.HadoopCore.EOFException: "
+            "end of file: "));
+    EXPECT_INT_ZERO(hadoop_uverr_alloc_test2(UV_EINVAL,
+            "org.apache.hadoop.native.HadoopCore.InvalidRequestException: "
+                    "invalid argument: "));
+    EXPECT_INT_ZERO(hadoop_uverr_alloc_test2(UV_ECONNREFUSED,
+            "org.apache.hadoop.native.HadoopCore.ConnectionRefusedException: "
+                    "connection refused: "));
+    EXPECT_INT_ZERO(hadoop_uverr_alloc_test2(UV_ENOMEM,
+            "org.apache.hadoop.native.HadoopCore.OutOfMemoryException: "
+                    "not enough memory: "));
+    EXPECT_INT_ZERO(hadoop_uverr_alloc_test2(0,
+            "org.apache.hadoop.native.HadoopCore.IOException: "
+                    "Unknown system error: "));
+    EXPECT_INT_ZERO(hadoop_err_copy_test());
+    return EXIT_SUCCESS;
 }
 
 // vim: ts=4:sw=4:tw=79:et

hadoop-native-core/src/main/native/common/hadoop_err.c

@@ -196,4 +196,64 @@ const char *hadoop_err_msg(const struct hadoop_err *err)
     return err->msg;
 }
 
+struct hadoop_err *hadoop_err_prepend(struct hadoop_err *err,
+        int code, const char *fmt, ...)
+{
+    struct hadoop_err *nerr = NULL;
+    va_list ap;
+    char *nmsg = NULL, *prepend_str = NULL;
+
+    va_start(ap, fmt);
+    if (vasprintf(&prepend_str, fmt, ap) < 0) {
+        prepend_str = NULL;
+        va_end(ap);
+        return err;
+    }
+    va_end(ap);
+    if (asprintf(&nmsg, "%s: %s", prepend_str, err->msg) < 0) {
+        free(prepend_str);
+        return (struct hadoop_err*)err;
+    }
+    free(prepend_str);
+    nerr = calloc(1, sizeof(*nerr));
+    if (!nerr) {
+        free(nmsg);
+        return err;
+    }
+    nerr->code = code ? code : err->code;
+    hadoop_err_free(err);
+    nerr->malloced = 1;
+    nerr->msg = nmsg;
+    return nerr;
+}
+
+struct hadoop_err *hadoop_err_copy(const struct hadoop_err *err)
+{
+    struct hadoop_err *nerr;
+
+    if (!err->malloced) {
+        return (struct hadoop_err*)err;
+    }
+    nerr = malloc(sizeof(*nerr));
+    if (!nerr) {
+        return (struct hadoop_err*)&HADOOP_OOM_ERR;
+    }
+    nerr->code = err->code;
+    nerr->msg = strdup(err->msg);
+    nerr->malloced = 1;
+    if (!nerr->msg) {
+        free(nerr);
+        return (struct hadoop_err*)&HADOOP_OOM_ERR;
+    }
+    return nerr;
+}
+
+const char* terror(int errnum)
+{
+    if ((errnum < 0) || (errnum >= sys_nerr)) {
+        return "unknown error.";
+    }
+    return sys_errlist[errnum];
+}
+
 // vim: ts=4:sw=4:tw=79:et

hadoop-native-core/src/main/native/common/hadoop_err.h

@@ -41,7 +41,7 @@ struct hadoop_err *hadoop_lerr_alloc(int code, const char *fmt, ...)
 /**
  * Allocate a new error object based on a libuv error.
  *
- * @param loop          The libuv loop to check.
+ * @param code          The libuv error code to check.
  * @param fmt           printf-style format.
  * @param ...           printf-style arguments.
  *
@@ -76,6 +76,38 @@ const char *hadoop_err_msg(const struct hadoop_err *err);
  */
 void hadoop_err_free(struct hadoop_err *err);
 
+/**
+ * Prepend an error message to an existing hadoop error.
+ *
+ * @param err       The hadoop error to prepend to.
+ * @param code      0 to use the existing code; the new code otherwise.
+ * @param fmt       printf-style format string.
+ * @param ...       printf-style arguments.
+ *
+ * @return          The error message.  Valid until the hadoop_err
+ *                  object is freed.
+ */
+struct hadoop_err *hadoop_err_prepend(struct hadoop_err *err,
+        int code, const char *fmt, ...) __attribute__((format(printf, 3, 4)));
+
+/**
+ * Copy a hadoop error.
+ *
+ * @param err       The hadoop error.
+ *
+ * @return          A copy of the hadoop error.
+ */
+struct hadoop_err *hadoop_err_copy(const struct hadoop_err *err);
+
+/**
+ * A thread-safe version of strerror.
+ *
+ * @param errnum    The POSIX errno.
+ *
+ * @return          The error string.
+ */
+const char* terror(int errnum);
+
 #endif
 
 // vim: ts=4:sw=4:tw=79:et

hadoop-native-core/src/main/native/common/hconf-unit.c

@@ -0,0 +1,150 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "common/hadoop_err.h"
+#include "common/hconf.h"
+#include "config.h"
+#include "test/test.h"
+
+#include <errno.h>
+#include <inttypes.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+const char* const TEST_XML_NAMES[] = {
+    "core-default.xml",
+    "core-site.xml",
+    "hdfs-default.xml",
+    "hdfs-site.xml",
+    NULL
+};
+
+static int test_hconf_builder_free(void)
+{
+    struct hconf_builder *bld = NULL;
+
+    EXPECT_NULL(hconf_builder_alloc(&bld));
+    EXPECT_NONNULL(bld);
+    hconf_builder_free(bld);
+    bld = NULL;
+    EXPECT_NULL(hconf_builder_alloc(&bld));
+    EXPECT_NONNULL(bld);
+    hconf_builder_set(bld, "foo", "bar");
+    hconf_builder_free(bld);
+    return 0;
+}
+
+static int test_hconf_create(void)
+{
+    struct hconf_builder *bld = NULL;
+    struct hconf *conf = NULL;
+    int32_t i32 = 0;
+    int64_t i64 = 0;
+    double g = 0;
+
+    EXPECT_NULL(hconf_builder_alloc(&bld));
+    EXPECT_NONNULL(bld);
+    hconf_builder_set(bld, "foo", "foo_val");
+    hconf_builder_set(bld, "bar", "123");
+    hconf_builder_set(bld, "baz", "1.25");
+    hconf_builder_set(bld, "foo", "foo_val2");
+    hconf_builder_set(bld, "nothing", "");
+    EXPECT_NO_HADOOP_ERR(hconf_build(bld, &conf));
+    EXPECT_NONNULL(conf);
+    EXPECT_STR_EQ("foo_val2", hconf_get(conf, "foo"));
+    EXPECT_INT_ZERO(hconf_get_int32(conf, "bar", &i32));
+    EXPECT_INT_EQ(123, i32);
+    EXPECT_INT_ZERO(hconf_get_int64(conf, "bar", &i64));
+    EXPECT_INT64_EQ((int64_t)123, i64);
+    EXPECT_INT_ZERO(hconf_get_float64(conf, "baz", &g));
+    EXPECT_NULL(hconf_get(conf, "nothing"));
+    EXPECT_NULL(hconf_get(conf, "nada"));
+    if (g != 1.25) {
+        fail("got bad value for baz: expected %g; got %g", 1.25, g);
+    }
+    hconf_free(conf);
+    return 0;
+}
+
+static int test_hconf_substitutions(void)
+{
+    struct hconf_builder *bld = NULL;
+    struct hconf *conf = NULL;
+
+    EXPECT_NULL(hconf_builder_alloc(&bld));
+    EXPECT_NONNULL(bld);
+    hconf_builder_set(bld, "foo.bar", "foobar");
+    hconf_builder_set(bld, "foo.bar.indirect", "3${foo.bar}");
+    hconf_builder_set(bld, "foo.bar.double.indirect", "2${foo.bar.indirect}");
+    hconf_builder_set(bld, "foo.bar.triple.indirect", "1${foo.bar.double.indirect}");
+    hconf_builder_set(bld, "foo.baz", "${foo.bar}");
+    hconf_builder_set(bld, "foo.unresolved", "${foo.nonexistent}");
+    hconf_builder_set(bld, "double.foo.bar", "${foo.bar}${foo.bar}");
+    hconf_builder_set(bld, "double.foo.bar.two", "Now ${foo.bar} and ${foo.bar}");
+    hconf_builder_set(bld, "expander", "a${expander}");
+    hconf_builder_set(bld, "tweedledee", "${tweedledum}");
+    hconf_builder_set(bld, "tweedledum", "${tweedledee}");
+    hconf_builder_set(bld, "bling", "{$$$${$$${$$$$$$$");
+    EXPECT_NO_HADOOP_ERR(hconf_build(bld, &conf));
+    EXPECT_NONNULL(conf);
+    EXPECT_STR_EQ("foobar", hconf_get(conf, "foo.bar"));
+    EXPECT_STR_EQ("123foobar", hconf_get(conf, "foo.bar.triple.indirect"));
+    EXPECT_STR_EQ("aaaaaaaaaaaaaaaaaaaaa${expander}",
+                  hconf_get(conf, "expander"));
+    EXPECT_STR_EQ("foobar", hconf_get(conf, "foo.baz"));
+    EXPECT_STR_EQ("${foo.nonexistent}", hconf_get(conf, "foo.unresolved"));
+    EXPECT_STR_EQ("foobarfoobar", hconf_get(conf, "double.foo.bar"));
+    EXPECT_STR_EQ("Now foobar and foobar",
+                  hconf_get(conf, "double.foo.bar.two"));
+    EXPECT_STR_EQ("${tweedledee}", hconf_get(conf, "tweedledee"));
+    EXPECT_STR_EQ("{$$$${$$${$$$$$$$", hconf_get(conf, "bling"));
+    hconf_free(conf);
+    return 0;
+}
+
+static int test_hconf_xml(void)
+{
+    struct hconf_builder *bld = NULL;
+    struct hconf *conf = NULL;
+
+    EXPECT_NULL(hconf_builder_alloc(&bld));
+    EXPECT_NONNULL(bld);
+    EXPECT_NO_HADOOP_ERR(hconf_builder_load_xmls(bld, TEST_XML_NAMES,
+            HCONF_XML_TEST_PATH ":" HCONF_XML_TEST_PATH "/.."));
+    EXPECT_NO_HADOOP_ERR(hconf_build(bld, &conf));
+    EXPECT_NONNULL(conf);
+    EXPECT_NULL(hconf_get(conf, "foo.empty"));
+    EXPECT_STR_EQ("1", hconf_get(conf, "foo.final"));
+    EXPECT_STR_EQ("hdfs-site-val", hconf_get(conf, "foo.overridden"));
+    EXPECT_STR_EQ("woo:hdfs-default.woo:hdfs-default.hdfs-default.",
+                  hconf_get(conf, "triple.foo.bar"));
+    hconf_free(conf);
+    return 0;
+}
+
+int main(void)
+{
+    EXPECT_INT_ZERO(test_hconf_builder_free());
+    EXPECT_INT_ZERO(test_hconf_create());
+    EXPECT_INT_ZERO(test_hconf_substitutions());
+    EXPECT_INT_ZERO(test_hconf_xml());
+
+    return EXIT_SUCCESS;
+}
+
+// vim: ts=4:sw=4:tw=79:et

hadoop-native-core/src/main/native/common/hconf.c

@@ -0,0 +1,808 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "common/hadoop_err.h"
+#include "common/hconf.h"
+#include "common/htable.h"
+
+#include <errno.h>
+#include <expat.h>
+#include <inttypes.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+/** Size of the buffer to use when reading files. */
+#define XML_PARSE_BUF_LEN 16384
+
+/** The maximum number of times we'll attempt to expand a config value. */
+#define MAX_EXPANSIONS 20
+
+struct hconf_builder_entry {
+    /**
+     * A dynamically allocated string with the text of the entry.
+     */
+    char *text;
+
+    /**
+     * Nonzero if this entry is final.
+     * Final entries cannot be overridden during loading, although they can be
+     * overriden manually by calling hconf_builder_set.
+     */
+    int final;
+};
+
+struct hconf_builder {
+    /**
+     * Non-zero if we encountered an out-of-memory error during
+     * hconf_builder_set, and will report it later during hconf_build.
+     */
+    int oom;
+
+    /**
+     * A hash table mapping malloced C strings to malloced hconf_builder_entry
+     * objects.
+     */
+    struct htable *table;
+
+    /**
+     * During hconf_build, the hconf object we're in the process of building.
+     */
+    struct hconf *conf;
+};
+
+/**
+ * A Hadoop configuration.  This is immutable once it's fully constructed.
+ */
+struct hconf {
+    /**
+     * A hash table mapping malloced C strings to malloced C strings.
+     */
+    struct htable *table;
+};
+
+/**
+ * A hash table mapping static C strings to static C strings.
+ * Protected by g_deprecation_table_once.
+ */
+static struct htable *DEPRECATION_TABLE;
+
+static uv_once_t g_deprecation_table_once = UV_ONCE_INIT;
+
+/**
+ * The error we encountered when loading the deprecation table, or NULL if the
+ * loading succeeded.  Protected by g_deprecation_table_once.
+ */
+static struct hadoop_err *g_deprecation_table_err;
+
+/**
+ * Deprecations.
+ * 
+ * The pattern here is:
+ * [modern-key-name-a] [deprecated-alias-a-1] [deprecated-alias-a-2] ... NULL
+ * [modern-key-name-b] [deprecated-alias-b-1] [deprecated-alias-b-2] ... NULL
+ * ...
+ * NULL NULL
+ */
+static const char* const DEPRECATIONS[] = {
+    "fs.defaultFS", "fs.default.name", NULL,
+    "dfs.client.socket-timeout", "dfs.socket.timeout", NULL,
+    "dfs.client-write-packet-size", "dfs.write.packet.size", NULL,
+    "dfs.client.file-block-storage-locations.timeout.millis",
+                "dfs.client.file-block-storage-locations.timeout", NULL,
+    "dfs.client-write-packet-size", "dfs.write.packet.size", NULL,
+    NULL, NULL
+};
+
+enum xml_parse_state {
+    HCXML_PARSE_INIT = 0,
+    HCXML_PARSE_IN_CONFIG,
+    HCXML_PARSE_IN_PROPERTY,
+    HCXML_PARSE_IN_NAME,
+    HCXML_PARSE_IN_VALUE,
+    HCXML_PARSE_IN_FINAL,
+};
+
+struct xml_parse_ctx {
+    /** Path of the current XML file we're parsing. */
+    const char *path;
+
+    /** The Hadoop configuration builder we're populating. */
+    struct hconf_builder *bld;
+
+    /** XML parse state. */
+    enum xml_parse_state state;
+
+    /** The number of parent elements we are ignoring. */
+    int ignored_parents;
+
+    /** Malloced key, if we saw one. */
+    char *name;
+
+    /** Malloced value, if we saw one. */
+    char *value;
+
+    /** Nonzero if the current property is final. */
+    int final;
+
+    /** The XML parser we're using. */
+    XML_Parser parser;
+};
+/**
+ * Initialize DEPRECATION_TABLE from DEPRECATIONS.
+ */
+static void init_deprecation_table(void)
+{
+    const char *modern_name;
+    size_t i = 0;
+    struct htable *table = NULL;
+
+    // Allocate the deprecation table.
+    table = htable_alloc(16, ht_hash_string, ht_compare_string);
+    if (!table) {
+        g_deprecation_table_err = hadoop_lerr_alloc(ENOMEM,
+                "init_deprecation_table: out of memory.");
+        return;
+    }
+    // Populate the deprecation table.
+    while ((modern_name = DEPRECATIONS[i])) {
+        const char *old_name;
+        while ((old_name = DEPRECATIONS[++i])) {
+            int ret = htable_put(table, (void*)old_name, (void*)modern_name);
+            if (ret) {
+                g_deprecation_table_err = hadoop_lerr_alloc(ret,
+                                "init_deprecation_table: htable put of %s "
+                                "failed.\n", old_name);
+                htable_free(table);
+                return;
+            }
+        }
+        i++;
+    }
+    DEPRECATION_TABLE = table;
+}
+
+struct hadoop_err *hconf_builder_alloc(struct hconf_builder **out)
+{
+    struct hconf_builder *bld = NULL;
+    struct hadoop_err *err = NULL;
+
+    uv_once(&g_deprecation_table_once, init_deprecation_table);
+    if (g_deprecation_table_err) {
+        err = hadoop_err_copy(g_deprecation_table_err);
+        goto done;
+    }
+    bld = calloc(1, sizeof(*bld));
+    if (!bld) {
+        err = hadoop_lerr_alloc(ENOMEM, "hconf_builder_alloc: OOM");
+        goto done;
+    }
+    bld->table = htable_alloc(128, ht_hash_string, ht_compare_string);
+    if (!bld->table) {
+        err = hadoop_lerr_alloc(ENOMEM, "hconf_builder_alloc: OOM");
+        goto done;
+    }
+done:
+    if (err) {
+        if (bld) {
+            htable_free(bld->table);
+            free(bld);
+        }
+        return err;
+    }
+    *out = bld;
+    return NULL;
+}
+
+static void hconf_builder_free_cb(void *ctx __attribute__((unused)),
+                                  void *k, void *v)
+{
+    struct hconf_builder_entry *entry;
+
+    free(k);
+    entry = v;
+    free(entry->text);
+    free(entry);
+}
+
+void hconf_builder_free(struct hconf_builder *bld)
+{
+    if (!bld)
+        return;
+    htable_visit(bld->table, hconf_builder_free_cb, NULL);
+    htable_free(bld->table);
+    hconf_free(bld->conf);
+    free(bld);
+}
+
+/**
+ * Get the most modern version of the given key.
+ *
+ * @param key           The key
+ *
+ * @return              The most modern version of the key.
+ */
+static const char *get_modern_key(const char *key)
+{
+    const char *ekey;
+
+    ekey = htable_get(DEPRECATION_TABLE, key);
+    return ekey ? ekey : key;
+}
+
+static struct hadoop_err *hconf_builder_set_internal(struct hconf_builder *bld,
+                const char *key, const char *val,
+                int set_final, int honor_final)
+{
+    struct hadoop_err *err = NULL;
+    const char *ekey;
+    struct hconf_builder_entry *entry;
+    char *nkey = NULL;
+    struct hconf_builder_entry *nentry = NULL;
+
+    ekey = get_modern_key(key);
+    if (val && val[0]) {
+        nentry = calloc(1, sizeof(*nentry));
+        if (!nentry)
+            goto oom;
+        nentry->text = strdup(val);
+        if (!nentry->text)
+            goto oom;
+        nentry->final = set_final;
+    }
+    entry = htable_get(bld->table, ekey);
+    if (entry) {
+        void *old_key;
+
+        if (honor_final && entry->final) {
+            err = hadoop_lerr_alloc(EINVAL, "attempted to override "
+                                    "final key %s", key);
+            goto error;
+        }
+        htable_pop(bld->table, ekey, &old_key, (void**)&entry);
+        free(old_key);
+        free(entry->text);
+        free(entry);
+    }
+    // Now that we've removed any old entry that might have existed, insert a
+    // new entry if the val supplied is non-null and non-empty.  Hadoop's
+    // configuration treats values that are empty strings the same as values
+    // that are not present.
+    if (nentry) {
+        nkey = strdup(ekey);
+        if (!nkey)
+            goto oom;
+        if (htable_put(bld->table, nkey, nentry))
+            goto oom;
+    }
+    return NULL;
+
+oom:
+    bld->oom = 1;
+    err = hadoop_lerr_alloc(ENOMEM, "out of memory.");
+error:
+    free(nkey);
+    if (nentry) {
+        free(nentry->text);
+        free(nentry);
+    }
+    return err;
+}
+
+void hconf_builder_set(struct hconf_builder *bld,
+                const char *key, const char *val)
+{
+    struct hadoop_err *err =
+        hconf_builder_set_internal(bld, key, val, 0, 0);
+    if (err) {
+        fprintf(stderr, "hconf_builder_set(key=%s, val=%s): %s",
+                key, val, hadoop_err_msg(err));
+        hadoop_err_free(err);
+    }
+}
+
+/**
+ * Translate an hconf_builder entry into an hconf entry.
+ * To do this, we need to resolve all the variable references.
+ *
+ * When we see a reference of the form ${variable.name}, we replace it with
+ * the value of that variable within the configuration builder.
+ * To prevent infinite expansions, we have two limits.  First of all, we
+ * will only expand 20 times.  Second of all, we detect cycles where the entire
+ * state of the string has repeated.  This could be done a bit smarter, but
+ * it's nice to be compatible.
+ */
+static void hconf_builder_translate_cb(void *ctx, void *k, void *v)
+{
+    int i, j;
+    struct hconf_builder *bld = ctx;
+    char *key = NULL, *text = NULL;
+    struct hconf_builder_entry *entry = v;
+    int num_expansions = 0;
+    char *prev_expansions[MAX_EXPANSIONS];
+
+    key = strdup(k);
+    text = strdup(entry->text);
+    if ((!key) || (!text)) {
+        bld->oom = 1;
+        goto done;
+    }
+    i = 0;
+    while (1) {
+        char *ntext;
+        int repeat;
+        struct hconf_builder_entry *nentry;
+        size_t slen, rem_len, nentry_len, nlen;
+
+
+        // Look for the beginning of a variable substitution segment
+        i += strcspn(text + i, "$");
+        if (text[i] == '\0') {
+            // We reached the end of the string without finding the beginning
+            // of a substitution.
+            break;
+        }
+        if (text[i + 1] != '{') {
+            // We found a dollar sign, but it was not followed by an open
+            // bracket.
+            i++;
+            continue;
+        }
+        slen = strcspn(text + i + 2, "}");
+        if (text[i + 2 + slen] == '\0') {
+            // We reached the end of the string without finding a close
+            // bracket.
+            break;
+        }
+        if (num_expansions == MAX_EXPANSIONS) {
+            // We reached the limit on the maximum number of expansions we'll
+            // perform.
+            break;
+        }
+        // Try to expand the text inside the ${ } block.
+        text[i + 2 + slen] = '\0';
+        nentry = htable_get(bld->table, get_modern_key(text + i + 2));
+        text[i + 2 + slen] = '}';
+        if (!nentry) {
+            // There was no entry corresponding to the text inside the block.
+            i += slen + 1;
+            continue;
+        }
+        // Resize the string to fit the new contents.
+        rem_len = strlen(text + i + 2 + slen + 1);
+        nentry_len = strlen(nentry->text);
+        nlen = i + nentry_len + rem_len + 1;
+        if (nlen > i + 2 + slen + 1 + rem_len) {
+            ntext = realloc(text, i + nentry_len + rem_len + 1);
+            if (!ntext) {
+                bld->oom = 1;
+                goto done;
+            }
+            text = ntext;
+        }
+        // First, copy the part after the variable expansion to its new
+        // location.  Then, copy the newly expanded text into the position it
+        // belongs in.
+        memmove(text + i + nentry_len, text + i + 2 + slen + 1, rem_len);
+        memcpy(text + i, nentry->text, nentry_len);
+        text[i + nentry_len + rem_len] = '\0';
+        // Check if we've expanded something to this pattern before.
+        // If so, we stop the expansion immediately.
+        repeat = 0;
+        for (j = 0; j < num_expansions; j++) {
+            if (strcmp(prev_expansions[j], text) == 0) {
+                repeat = 1;
+                break;
+            }
+        }
+        if (repeat) {
+            break;
+        }
+        // Keep track of this expansion in prev_expansions.
+        prev_expansions[num_expansions] = strdup(text);
+        if (!prev_expansions[num_expansions]) {
+            bld->oom = 1;
+            goto done;
+        }
+        num_expansions++;
+    }
+done:
+    for (j = 0; j < num_expansions; j++) {
+        free(prev_expansions[j]);
+    }
+    if (bld->oom || htable_put(bld->conf->table, key, text)) {
+        bld->oom = 1;
+        free(key);
+        free(text);
+        return;
+    }
+}
+
+struct hadoop_err *hconf_build(struct hconf_builder *bld,
+                struct hconf **out)
+{
+    struct hconf *conf = NULL;
+    struct hadoop_err *err = NULL;
+
+    if (bld->oom) {
+        err = hadoop_lerr_alloc(ENOMEM, "hconf_build: out of memory.");
+        goto done;
+    }
+    conf = calloc(1, sizeof(struct hconf));
+    if (!conf) {
+        err = hadoop_lerr_alloc(ENOMEM, "hconf_build: out of memory.");
+        goto done;
+    }
+    bld->conf = conf;
+    conf->table = htable_alloc(htable_capacity(bld->table),
+                            ht_hash_string, ht_compare_string);
+    if (!conf->table) {
+        err = hadoop_lerr_alloc(ENOMEM, "hconf_build: out of memory.");
+        goto done;
+    }
+    // Translate builder entries into configuration entries.
+    htable_visit(bld->table, hconf_builder_translate_cb, bld);
+    if (bld->oom) {
+        err = hadoop_lerr_alloc(ENOMEM, "hconf_build: out of memory.");
+        goto done;
+    }
+    *out = bld->conf;
+    bld->conf = NULL;
+    err = NULL;
+done:
+    hconf_builder_free(bld);
+    return err;
+}
+
+static void hconf_free_cb(void *ctx __attribute__((unused)), void *k, void *v)
+{
+    free(k);
+    free(v);
+}
+
+void hconf_free(struct hconf *conf)
+{
+    if (!conf)
+        return;
+    htable_visit(conf->table, hconf_free_cb, NULL);
+    htable_free(conf->table);
+    free(conf);
+}
+
+const char *hconf_get(struct hconf *conf, const char *key)
+{
+    const char *ekey;
+    const char *val;
+
+    ekey = get_modern_key(key);
+    val = htable_get(conf->table, ekey);
+    if (!val) {
+        return NULL;
+    }
+    return val;
+}
+
+int hconf_get_int32(struct hconf *conf, const char *key,
+                            int32_t *out)
+{
+    const char *val = hconf_get(conf, key);
+    if (!val)
+        return -ENOENT;
+    *out = atoi(val);
+    return 0;
+}
+
+int hconf_get_int64(struct hconf *conf, const char *key,
+                            int64_t *out)
+{
+    const char *val = hconf_get(conf, key);
+    if (!val)
+        return -ENOENT;
+    *out = atoll(val);
+    return 0;
+}
+
+int hconf_get_float64(struct hconf *conf, const char *key,
+                              double *out)
+{
+    const char *val = hconf_get(conf, key);
+    if (!val)
+        return -ENOENT;
+    *out = atof(val);
+    return 0;
+}
+
+static int xml_parse_bool(const char *path, XML_Size line_no,
+                      const char *str)
+{
+    if (strcasecmp(str, "false") == 0) {
+        return 0;
+    } else if (strcasecmp(str, "true") == 0) {
+        return 1;
+    }
+    fprintf(stderr, "hconf_builder_load_xml(%s): on line %lld, "
+            "failed to parse '%s' as a boolean.  Assuming false.\n",
+            path, (long long)line_no, str);
+    return 0;
+}
+
+/* first when start element is encountered */
+static void xml_start_element(void *data, const char *element,
+                const char **attribute __attribute__((unused)))
+{
+    struct xml_parse_ctx *ctx = data;
+
+    if (ctx->ignored_parents > 0) {
+        ctx->ignored_parents++;
+        return;
+    }
+    switch (ctx->state) {
+    case HCXML_PARSE_INIT:
+        if (!strcmp(element, "configuration")) {
+            ctx->state = HCXML_PARSE_IN_CONFIG;
+            return;
+        }
+        break;
+    case HCXML_PARSE_IN_CONFIG:
+        if (!strcmp(element, "property")) {
+            ctx->state = HCXML_PARSE_IN_PROPERTY;
+            return;
+        }
+        break;
+    case HCXML_PARSE_IN_PROPERTY:
+        if (!strcmp(element, "name")) {
+            ctx->state = HCXML_PARSE_IN_NAME;
+            return;
+        } else if (!strcmp(element, "value")) {
+            ctx->state = HCXML_PARSE_IN_VALUE;
+            return;
+        } else if (!strcmp(element, "final")) {
+            ctx->state = HCXML_PARSE_IN_FINAL;
+            return;
+        }
+        break;
+    default:
+        break;
+    }
+    fprintf(stderr, "hconf_builder_load_xml(%s): ignoring "
+            "element '%s'\n", ctx->path, element);
+    ctx->ignored_parents++;
+}
+
+/* decrement the current level of the tree */
+static void xml_end_element(void *data, const char *el __attribute__((unused)))
+{
+    struct xml_parse_ctx *ctx = data;
+    struct hadoop_err *err = NULL;
+
+    if (ctx->ignored_parents > 0) {
+        ctx->ignored_parents--;
+        return;
+    }
+    switch (ctx->state) {
+    case HCXML_PARSE_IN_CONFIG:
+        ctx->state = HCXML_PARSE_INIT;
+        break;
+    case HCXML_PARSE_IN_PROPERTY:
+        ctx->state = HCXML_PARSE_IN_CONFIG;
+        if (!ctx->name) {
+            fprintf(stderr, "hconf_builder_load_xml(%s): property "
+                    "tag is missing <name> on line %lld\n", ctx->path,
+                    (long long)XML_GetCurrentLineNumber(ctx->parser));
+        } else if (!ctx->value) {
+            fprintf(stderr, "hconf_builder_load_xml(%s): property "
+                    "tag is missing <value> on line %lld\n", ctx->path,
+                    (long long)XML_GetCurrentLineNumber(ctx->parser));
+        } else {
+            err = hconf_builder_set_internal(ctx->bld,
+                    ctx->name, ctx->value, ctx->final, 1);
+            if (err) {
+                fprintf(stderr, "hconf_builder_load_xml(%s): on line "
+                        "%lld, %s\n", ctx->path,
+                        (long long)XML_GetCurrentLineNumber(ctx->parser),
+                        hadoop_err_msg(err));
+                hadoop_err_free(err);
+            }
+        }
+        free(ctx->name);
+        ctx->name = NULL;
+        free(ctx->value);
+        ctx->value = NULL;
+        ctx->final = 0;
+        break;
+    case HCXML_PARSE_IN_NAME:
+        ctx->state = HCXML_PARSE_IN_PROPERTY;
+        break;
+    case HCXML_PARSE_IN_VALUE:
+        ctx->state = HCXML_PARSE_IN_PROPERTY;
+        break;
+    case HCXML_PARSE_IN_FINAL:
+        ctx->state = HCXML_PARSE_IN_PROPERTY;
+        break;
+    default:
+        break;
+    }
+}
+
+static char *ltstrdup(const char *src, int length)
+{
+    char *dst = malloc(length + 1);
+    if (!dst)
+        return NULL;
+    memcpy(dst, src, length);
+    dst[length] = 0;
+    return dst;
+}
+
+static void xml_handle_data(void *data, const char *content, int length)
+{
+    struct xml_parse_ctx *ctx = data;
+    char *bool_str;
+
+    switch (ctx->state) {
+    case HCXML_PARSE_IN_NAME:
+        if (ctx->name) {
+            fprintf(stderr, "hconf_builder_load_xml(%s): duplicate "
+                    "<name> tag on line %lld\n", ctx->path,
+                    (long long)XML_GetCurrentLineNumber(ctx->parser));
+        } else {
+            ctx->name = ltstrdup(content, length);
+            if (!ctx->name) {
+                ctx->bld->oom = 1;
+            }
+        }
+        break;
+    case HCXML_PARSE_IN_VALUE:
+        if (ctx->value) {
+            fprintf(stderr, "hconf_builder_load_xml(%s): duplicate "
+                    "<value> tag on line %lld\n", ctx->path,
+                    (long long)XML_GetCurrentLineNumber(ctx->parser));
+        } else {
+            ctx->value = ltstrdup(content, length);
+            if (!ctx->value) {
+                ctx->bld->oom = 1;
+            }
+        }
+        break;
+    case HCXML_PARSE_IN_FINAL:
+        bool_str = ltstrdup(content, length);
+        if (!bool_str) {
+            ctx->bld->oom = 1;
+        } else {
+            ctx->final = xml_parse_bool(ctx->path,
+                XML_GetCurrentLineNumber(ctx->parser), bool_str);
+            free(bool_str);
+        }
+        break;
+    default:
+        break;
+    }
+}
+
+static struct hadoop_err *hconf_builder_load_xml(struct hconf_builder *bld,
+                            const char *path, FILE *fp)
+{
+    struct hadoop_err *err = NULL;
+    struct xml_parse_ctx ctx;
+    char *buf = NULL;
+    enum XML_Status status;
+    int res = 0;
+
+    memset(&ctx, 0, sizeof(ctx));
+    ctx.bld = bld;
+    ctx.path = path;
+    ctx.parser = XML_ParserCreate("UTF-8");
+    if (!ctx.parser) {
+        err = hadoop_lerr_alloc(ENOMEM, "hconf_builder_load_xml: failed "
+                                "to create libexpat XML parser.");
+        goto done;
+    }
+    XML_SetUserData(ctx.parser, &ctx);
+    XML_SetElementHandler(ctx.parser, xml_start_element, xml_end_element);
+    XML_SetCharacterDataHandler(ctx.parser, xml_handle_data);
+    buf = malloc(XML_PARSE_BUF_LEN);
+    if (!buf) {
+        err = hadoop_lerr_alloc(ENOMEM, "hconf_builder_load_xml: OOM");
+        goto done;
+    }
+    do {
+        res = fread(buf, 1, XML_PARSE_BUF_LEN, fp);
+        if (res <= 0) {
+            if (feof(fp)) {
+                res = 0;
+            } else {
+                int e = errno;
+                err = hadoop_lerr_alloc(e, "hconf_builder_load_xml(%s): failed "
+                            "to read from file: error %d", ctx.path, e);
+                goto done;
+            }
+        }
+        status = XML_Parse(ctx.parser, buf, res, res ? XML_FALSE : XML_TRUE);
+        if (status != XML_STATUS_OK) {
+            enum XML_Error error = XML_GetErrorCode(ctx.parser);
+            err = hadoop_lerr_alloc(EINVAL, "hconf_builder_load_xml(%s): "
+                                    "parse error: %s",
+                                    ctx.path, XML_ErrorString(error));
+            goto done;
+        }
+    } while (res);
+done:
+    free(buf);
+    free(ctx.name);
+    free(ctx.value);
+    if (ctx.parser) {
+        XML_ParserFree(ctx.parser);
+    }
+    return err; 
+}
+
+struct hadoop_err *hconf_builder_load_xmls(struct hconf_builder *bld,
+            const char * const* XMLS, const char *pathlist)
+{
+    struct hadoop_err *err = NULL;
+    char *npathlist = NULL, *dir;
+    char *ptr = NULL, *path = NULL;
+    int ret, i;
+    FILE *fp = NULL;
+
+    npathlist = strdup(pathlist);
+    if (!npathlist)
+        goto oom;
+    // We need to read XML files in a certain order.  For example,
+    // core-site.xml must be read in before hdfs-site.xml in libhdfs.
+    for (i = 0; XMLS[i]; i++) {
+        for (dir = strtok_r(npathlist, ":", &ptr); dir;
+                    dir = strtok_r(NULL, ":", &ptr)) {
+            if (asprintf(&path, "%s/%s", dir, XMLS[i]) < 0) {
+                path = NULL;
+                goto oom;
+            }
+            fp = fopen(path, "r");
+            if (!fp) {
+                ret = errno;
+                if ((ret != ENOTDIR) && (ret != ENOENT)) {
+                    fprintf(stderr, "hconf_builder_load_xmls: failed to "
+                            "open %s: error %d\n", path, ret);
+                }
+            } else {
+                err = hconf_builder_load_xml(bld, path, fp);
+                if (err)
+                    goto done;
+                fclose(fp);
+                fp = NULL;
+            }
+            free(path);
+            path = NULL;
+        }
+    }
+    err = NULL;
+    goto done;
+
+oom:
+    err = hadoop_lerr_alloc(ENOMEM, "hconf_builder_load_xmls: OOM");
+done:
+    if (fp) {
+        fclose(fp);
+    }
+    free(npathlist);
+    free(path);
+    return err;
+}
+
+// vim: ts=4:sw=4:tw=79:et

hadoop-native-core/src/main/native/common/hconf.h

@@ -0,0 +1,140 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef HADOOP_CORE_COMMON_hconf
+#define HADOOP_CORE_COMMON_hconf
+
+#include <stdint.h>
+
+struct hconf;
+struct hconf_builder;
+
+extern const char* const HDFS_XML_NAMES[];
+
+/**
+ * Allocate a new Hadoop configuration build object.
+ *
+ * @param out       (out param) The new Hadoop configuration builder object
+ *
+ * @return          NULL on success; the error otherwise.
+ */
+struct hadoop_err *hconf_builder_alloc(struct hconf_builder **out);
+
+/**
+ * Free a Hadoop configuration builder object.
+ *
+ * @param bld       The configuration builder object to free.
+ */
+void hconf_builder_free(struct hconf_builder *bld);
+
+/**
+ * Free a Hadoop configuration object.
+ *
+ * @param conf      The object to free.
+ */
+void hconf_free(struct hconf *conf);
+
+/**
+ * Set a Hadoop configuration string value.
+ *
+ * @param bld       The configuration builder object.
+ * @param key       The configuration key.  Will be shallow-copied.
+ * @param val       The configuration value.  Will be shallow-copied.
+ */
+void hconf_builder_set(struct hconf_builder *bld,
+                const char *key, const char *val);
+
+/**
+ * Load a set of configuration XML files into the builder.
+ *
+ * @param bld       The configuration builder object.
+ * @param XMLS      A NULL-terminated list of configuration XML files to read.
+ * @param path      A semicolon-separated list of paths to search for the files
+ *                      in XMLS.  This is essentially a JNI-style CLASSPATH.
+ *                      (Like the JNI version, it doesn't support wildcards.)
+ */
+struct hadoop_err *hconf_builder_load_xmls(struct hconf_builder *bld,
+                            const char * const* XMLS, const char *path);
+
+/**
+ * Build a hadoop configuration object.
+ * Hadoop configuration objects are immutable.
+ *
+ * @param bld       The configuration builder object.  Will be freed, whether
+ *                      or not the function succeeds.
+ * @param conf      (out param) on success, the configuration object.
+ *
+ * @return          NULL on success; the hadoop error otherwise.
+ */
+struct hadoop_err *hconf_build(struct hconf_builder *bld,
+                struct hconf **conf);
+
+/**
+ * Get a Hadoop configuration string value.
+ *
+ * @param conf      The configuration object.
+ * @param key       The configuration key.
+ *
+ * @return          NULL if there was no such value.  The configuration value
+ *                      otherwise.  This pointer will remain valid until the
+ *                      enclosing configuration is freed.
+ */
+const char *hconf_get(struct hconf *conf, const char *key);
+
+/**
+ * Get a Hadoop configuration int32 value.
+ *
+ * @param conf      The configuration object.
+ * @param key       The configuration key.
+ * @param out       (out param) On success, the 32-bit value.
+ *
+ * @return          0 on success.
+ *                  -ENOENT if there was no such key.
+ */
+int hconf_get_int32(struct hconf *conf, const char *key,
+                            int32_t *out);
+
+/**
+ * Get a Hadoop configuration int64 value.
+ *
+ * @param conf      The configuration object.
+ * @param key       The configuration key.
+ * @param out       (out param) On success, the 64-bit value.
+ *
+ * @return          0 on success.
+ *                  -ENOENT if there was no such key.
+ */
+int hconf_get_int64(struct hconf *conf, const char *key,
+                            int64_t *out);
+
+/**
+ * Get a Hadoop configuration 64-bit float value.
+ *
+ * @param conf      The configuration object.
+ * @param key       The configuration key.
+ * @param out       (out param) On success, the 64-bit float value.
+ *
+ * @return          0 on success.
+ *                  -ENOENT if there was no such key.
+ */
+int hconf_get_float64(struct hconf *conf, const char *key,
+                              double *out);
+
+#endif
+
+// vim: ts=4:sw=4:tw=79:et

hadoop-native-core/src/main/native/common/htable-unit.c

@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "common/htable.h"
+#include "test/test.h"
+
+#include <errno.h>
+#include <inttypes.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+static uint32_t simple_hash(const void *key, uint32_t size)
+{
+    uintptr_t k = (uintptr_t)key;
+    return ((13 + k) * 6367) % size;
+}
+
+static int simple_compare(const void *a, const void *b)
+{
+    return a == b;
+}
+
+static void expect_102(void *f, void *k, void *v)
+{
+    int *found_102 = f;
+    uintptr_t key = (uintptr_t)k;
+    uintptr_t val = (uintptr_t)v;
+
+    if ((key == 2) && (val == 102)) {
+        *found_102 = 1;
+    } else {
+        abort();
+    }
+}
+
+static void *htable_pop_val(struct htable *ht, void *key)
+{
+    void *old_key, *old_val;
+
+    htable_pop(ht, key, &old_key, &old_val);
+    return old_val;
+}
+
+int main(void)
+{
+    struct htable *ht;
+    int found_102 = 0;
+
+    ht = htable_alloc(4, simple_hash, simple_compare);
+    EXPECT_INT_EQ(0, htable_used(ht));
+    EXPECT_INT_EQ(4, htable_capacity(ht));
+    EXPECT_NULL(htable_get(ht, (void*)123));
+    EXPECT_NULL(htable_pop_val(ht, (void*)123));
+    EXPECT_INT_ZERO(htable_put(ht, (void*)123, (void*)456));
+    EXPECT_INT_EQ(456, (uintptr_t)htable_get(ht, (void*)123));
+    EXPECT_INT_EQ(456, (uintptr_t)htable_pop_val(ht, (void*)123));
+    EXPECT_NULL(htable_pop_val(ht, (void*)123));
+
+    // Enlarge the hash table
+    EXPECT_INT_ZERO(htable_put(ht, (void*)1, (void*)101));
+    EXPECT_INT_ZERO(htable_put(ht, (void*)2, (void*)102));
+    EXPECT_INT_ZERO(htable_put(ht, (void*)3, (void*)103));
+    EXPECT_INT_EQ(3, htable_used(ht));
+    EXPECT_INT_EQ(8, htable_capacity(ht));
+    EXPECT_INT_EQ(102, (uintptr_t)htable_get(ht, (void*)2));
+    EXPECT_INT_EQ(101, (uintptr_t)htable_pop_val(ht, (void*)1));
+    EXPECT_INT_EQ(103, (uintptr_t)htable_pop_val(ht, (void*)3));
+    EXPECT_INT_EQ(1, htable_used(ht));
+    htable_visit(ht, expect_102, &found_102);
+    EXPECT_INT_EQ(1, found_102);
+    htable_free(ht);
+
+    return EXIT_SUCCESS;
+}
+
+// vim: ts=4:sw=4:tw=79:et

hadoop-native-core/src/main/native/common/htable.c

@@ -0,0 +1,271 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "common/htable.h"
+
+#include <errno.h>
+#include <inttypes.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+struct htable_pair {
+    void *key;
+    void *val;
+};
+
+/**
+ * A hash table which uses linear probing.
+ */
+struct htable {
+    uint32_t capacity;
+    uint32_t used;
+    htable_hash_fn_t hash_fun;
+    htable_eq_fn_t eq_fun;
+    struct htable_pair *elem;
+};
+
+/**
+ * An internal function for inserting a value into the hash table.
+ *
+ * Note: this function assumes that you have made enough space in the table.
+ *
+ * @param nelem         The new element to insert.
+ * @param capacity      The capacity of the hash table.
+ * @param hash_fun      The hash function to use.
+ * @param key           The key to insert.
+ * @param val           The value to insert.
+ */
+static void htable_insert_internal(struct htable_pair *nelem, 
+        uint32_t capacity, htable_hash_fn_t hash_fun, void *key,
+        void *val)
+{
+    uint32_t i;
+
+    i = hash_fun(key, capacity);
+    while (1) {
+        if (!nelem[i].key) {
+            nelem[i].key = key;
+            nelem[i].val = val;
+            return;
+        }
+        i++;
+        if (i == capacity) {
+            i = 0;
+        }
+    }
+}
+
+static int htable_realloc(struct htable *htable, uint32_t new_capacity)
+{
+    struct htable_pair *nelem;
+    uint32_t i, old_capacity = htable->capacity;
+    htable_hash_fn_t hash_fun = htable->hash_fun;
+
+    nelem = calloc(new_capacity, sizeof(struct htable_pair));
+    if (!nelem) {
+        return ENOMEM;
+    }
+    for (i = 0; i < old_capacity; i++) {
+        struct htable_pair *pair = htable->elem + i;
+        htable_insert_internal(nelem, new_capacity, hash_fun,
+                               pair->key, pair->val);
+    }
+    free(htable->elem);
+    htable->elem = nelem;
+    htable->capacity = new_capacity;
+    return 0;
+}
+
+struct htable *htable_alloc(uint32_t size,
+                htable_hash_fn_t hash_fun, htable_eq_fn_t eq_fun)
+{
+    struct htable *htable;
+
+    htable = calloc(1, sizeof(*htable));
+    if (!htable) {
+        return NULL;
+    }
+    size = (size + 1) >> 1;
+    size = size << 1;
+    if (size < HTABLE_MIN_SIZE) {
+        size = HTABLE_MIN_SIZE;
+    }
+    htable->hash_fun = hash_fun;
+    htable->eq_fun = eq_fun;
+    htable->used = 0;
+    if (htable_realloc(htable, size)) {
+        free(htable);
+        return NULL;
+    }
+    return htable;
+}
+
+void htable_visit(struct htable *htable, visitor_fn_t fun, void *ctx)
+{
+    uint32_t i;
+
+    for (i = 0; i != htable->capacity; ++i) {
+        struct htable_pair *elem = htable->elem + i;
+        if (elem->key) {
+            fun(ctx, elem->key, elem->val);
+        }
+    }
+}
+
+void htable_free(struct htable *htable)
+{
+    if (htable) {
+        free(htable->elem);
+        free(htable);
+    }
+}
+
+int htable_put(struct htable *htable, void *key, void *val)
+{
+    int ret;
+    uint32_t nused;
+
+    // NULL is not a valid key value.
+    // This helps us implement htable_get_internal efficiently, since we know
+    // that we can stop when we encounter the first NULL key.
+    if (!key) {
+        return EINVAL;
+    }
+    // NULL is not a valid value.  Otherwise the results of htable_get would
+    // be confusing (does a NULL return mean entry not found, or that the
+    // entry was found and was NULL?) 
+    if (!val) {
+        return EINVAL;
+    }
+    // Re-hash if we have used more than half of the hash table
+    nused = htable->used + 1;
+    if (nused >= (htable->capacity / 2)) {
+        ret = htable_realloc(htable, htable->capacity * 2);
+        if (ret)
+            return ret;
+    }
+    htable_insert_internal(htable->elem, htable->capacity,
+                                htable->hash_fun, key, val);
+    htable->used++;
+    return 0;
+}
+
+static int htable_get_internal(const struct htable *htable,
+                               const void *key, uint32_t *out)
+{
+    uint32_t start_idx, idx;
+
+    start_idx = htable->hash_fun(key, htable->capacity);
+    idx = start_idx;
+    while (1) {
+        struct htable_pair *pair = htable->elem + idx;
+        if (!pair->key) {
+            // We always maintain the invariant that the entries corresponding
+            // to a given key are stored in a contiguous block, not separated
+            // by any NULLs.  So if we encounter a NULL, our search is over.
+            return ENOENT;
+        } else if (htable->eq_fun(pair->key, key)) {
+            *out = idx;
+            return 0;
+        }
+        idx++;
+        if (idx == htable->capacity) {
+            idx = 0;
+        }
+        if (idx == start_idx) {
+            return ENOENT;
+        }
+    }
+}
+
+void *htable_get(const struct htable *htable, const void *key)
+{
+    uint32_t idx;
+
+    if (htable_get_internal(htable, key, &idx)) {
+        return NULL;
+    }
+    return htable->elem[idx].val;
+}
+
+void htable_pop(struct htable *htable, const void *key,
+                void **found_key, void **found_val)
+{
+    uint32_t hole, i;
+    const void *nkey;
+
+    if (htable_get_internal(htable, key, &hole)) {
+        *found_key = NULL;
+        *found_val = NULL;
+        return;
+    }
+    i = hole;
+    htable->used--;
+    // We need to maintain the compactness invariant used in
+    // htable_get_internal.  This invariant specifies that the entries for any
+    // given key are never separated by NULLs (although they may be separated
+    // by entries for other keys.)
+    while (1) {
+        i++;
+        if (i == htable->capacity) {
+            i = 0;
+        }
+        nkey = htable->elem[i].key;
+        if (!nkey) {
+            *found_key = htable->elem[hole].key;
+            *found_val = htable->elem[hole].val;
+            htable->elem[hole].key = NULL;
+            htable->elem[hole].val = NULL;
+            return;
+        } else if (htable->eq_fun(key, nkey)) {
+            htable->elem[hole].key = htable->elem[i].key;
+            htable->elem[hole].val = htable->elem[i].val;
+            hole = i;
+        }
+    }
+}
+
+uint32_t htable_used(const struct htable *htable)
+{
+    return htable->used;
+}
+
+uint32_t htable_capacity(const struct htable *htable)
+{
+    return htable->capacity;
+}
+
+uint32_t ht_hash_string(const void *str, uint32_t max)
+{
+    const char *s = str;
+    uint32_t hash = 0;
+
+    while (*s) {
+        hash = (hash * 31) + *s;
+        s++;
+    }
+    return hash % max;
+}
+
+int ht_compare_string(const void *a, const void *b)
+{
+    return strcmp(a, b) == 0;
+}
+
+// vim: ts=4:sw=4:tw=79:et

hadoop-native-core/src/main/native/common/htable.h

@@ -0,0 +1,161 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef HADOOP_CORE_COMMON_HASH_TABLE
+#define HADOOP_CORE_COMMON_HASH_TABLE
+
+#include <inttypes.h>
+#include <stdio.h>
+#include <stdint.h>
+
+#define HTABLE_MIN_SIZE 4
+
+struct htable;
+
+/**
+ * An HTable hash function.
+ *
+ * @param key       The key.
+ * @param capacity  The total capacity.
+ *
+ * @return          The hash slot.  Must be less than the capacity.
+ */
+typedef uint32_t (*htable_hash_fn_t)(const void *key, uint32_t capacity);
+
+/**
+ * An HTable equality function.  Compares two keys.
+ *
+ * @param a         First key.
+ * @param b         Second key.
+ *
+ * @return          nonzero if the keys are equal.
+ */
+typedef int (*htable_eq_fn_t)(const void *a, const void *b);
+
+/**
+ * Allocate a new hash table.
+ *
+ * @param capacity  The minimum suggested starting capacity.
+ * @param hash_fun  The hash function to use in this hash table.
+ * @param eq_fun    The equals function to use in this hash table.
+ *
+ * @return          The new hash table on success; NULL on OOM.
+ */
+struct htable *htable_alloc(uint32_t capacity, htable_hash_fn_t hash_fun,
+                            htable_eq_fn_t eq_fun);
+
+typedef void (*visitor_fn_t)(void *ctx, void *key, void *val);
+
+/**
+ * Visit all of the entries in the hash table.
+ *
+ * @param htable    The hash table.
+ * @param fun       The callback function to invoke on each key and value.
+ * @param ctx       Context pointer to pass to the callback.
+ */
+void htable_visit(struct htable *htable, visitor_fn_t fun, void *ctx);
+
+/**
+ * Free the hash table.
+ *
+ * It is up the calling code to ensure that the keys and values inside the
+ * table are de-allocated, if that is necessary.
+ *
+ * @param htable    The hash table.
+ */
+void htable_free(struct htable *htable);
+
+/**
+ * Add an entry to the hash table.
+ *
+ * @param htable    The hash table.
+ * @param key       The key to add.  This cannot be NULL.
+ * @param fun       The value to add.  This cannot be NULL.
+ *
+ * @return          0 on success;
+ *                  EEXIST if the value already exists in the table;
+ *                  ENOMEM if there is not enough memory to add the element.
+ *                  EFBIG if the hash table has too many entries to fit in 32
+ *                      bits.
+ */
+int htable_put(struct htable *htable, void *key, void *val);
+
+/**
+ * Get an entry from the hash table.
+ *
+ * @param htable    The hash table.
+ * @param key       The key to find.
+ *
+ * @return          NULL if there is no such entry; the entry otherwise.
+ */
+void *htable_get(const struct htable *htable, const void *key);
+
+/**
+ * Get an entry from the hash table and remove it.
+ *
+ * @param htable    The hash table.
+ * @param key       The key for the entry find and remove.
+ * @param found_key (out param) NULL if the entry was not found; the found key
+ *                      otherwise.
+ * @param found_val (out param) NULL if the entry was not found; the found
+ *                      value otherwise.
+ */
+void htable_pop(struct htable *htable, const void *key,
+                void **found_key, void **found_val);
+
+/**
+ * Get the number of entries used in the hash table.
+ *
+ * @param htable    The hash table.
+ *
+ * @return          The number of entries used in the hash table.
+ */
+uint32_t htable_used(const struct htable *htable);
+
+/**
+ * Get the capacity of the hash table.
+ *
+ * @param htable    The hash table.
+ *
+ * @return          The capacity of the hash table.
+ */
+uint32_t htable_capacity(const struct htable *htable);
+
+/**
+ * Hash a string.
+ *
+ * @param str       The string.
+ * @param max       Maximum hash value
+ *
+ * @return          A number less than max.
+ */
+uint32_t ht_hash_string(const void *str, uint32_t max);
+
+/**
+ * Compare two strings.
+ *
+ * @param a         The first string.
+ * @param b         The second string.
+ *
+ * @return          1 if the strings are identical; 0 otherwise.
+ */
+int ht_compare_string(const void *a, const void *b);
+
+#endif
+
+// vim: ts=4:sw=4:tw=79:et

hadoop-native-core/src/main/native/common/net.c

@@ -16,16 +16,77 @@
  * limitations under the License.
  */
 
+#include "common/hadoop_err.h"
+#include "common/net.h"
+
 #include <netinet/in.h>
+#include <stdio.h>
+#include <string.h>
 #include <unistd.h>
 #include <uv.h>
 
+static const char * const NET_IPV4_NAME_ERROR = "(uv_ip4_name error)";
+
 const char *net_ipv4_name(struct sockaddr_in *src, char *dst, size_t size)
 {
   if (uv_ip4_name(src, dst, size) < 0) {
-    return "(uv_ip4_name error)";
+    return NET_IPV4_NAME_ERROR;
   }
   return dst;
 }
 
+const char *net_ipv4_name_and_port(struct sockaddr_in *src,
+                                   char *dst, size_t size)
+{
+    size_t len;
+
+    if (net_ipv4_name(src, dst, size) == NET_IPV4_NAME_ERROR)
+        return NET_IPV4_NAME_ERROR;
+    len = strlen(dst);
+    snprintf(dst + len, size - len + 1, ":%d", 
+             htons(src->sin_port));
+    return dst;
+}
+
+struct hadoop_err *get_first_ipv4_addr(const char *hostname, uint32_t *out)
+{
+    struct hadoop_err *err = NULL;
+    uint32_t addr = 0;
+    int ret;
+    struct addrinfo hints, *list, *info;
+
+    memset(&hints, 0, sizeof(hints));
+    hints.ai_family = AF_INET;
+    hints.ai_socktype = SOCK_STREAM;
+    hints.ai_flags |= AI_CANONNAME;
+    ret = getaddrinfo(hostname, NULL, &hints, &list);
+    if (ret) {
+        if (ret == EAI_SYSTEM) {
+            ret = errno;
+            err = hadoop_lerr_alloc(ret, "getaddrinfo(%s): %s",
+                                    hostname, terror(ret));
+        } else {
+            // TODO: gai_strerror is not thread-safe on Windows, need
+            // workaround
+            err = hadoop_lerr_alloc(ENOENT, "getaddrinfo(%s): %s",
+                                    hostname, gai_strerror(ret));
+        }
+        list = NULL;
+        goto done;
+    }
+    for (info = list; info; info = info->ai_next) {
+        if (info->ai_family != AF_INET)
+            continue;
+        addr = ((struct sockaddr_in*)list->ai_addr)->sin_addr.s_addr;
+        err = NULL;
+        goto done;
+    }
+    err = hadoop_lerr_alloc(ENOENT, "getaddrinfo(%s): no IPv4 addresses "
+                            "found for hostname.", hostname);
+done:
+    freeaddrinfo(list);
+    *out = ntohl(addr);
+    return err;
+}
+
 // vim: ts=4:sw=4:tw=79:et

hadoop-native-core/src/main/native/common/net.h

@@ -19,12 +19,18 @@
 #ifndef HADOOP_CORE_COMMON_NET
 #define HADOOP_CORE_COMMON_NET
 
+#include <stdint.h>
 #include <stddef.h>
 
 struct sockaddr_in;
 
 const char *net_ipv4_name(struct sockaddr_in *src, char *dst, size_t size);
 
+const char *net_ipv4_name_and_port(struct sockaddr_in *src,
+                                   char *dst, size_t size);
+
+struct hadoop_err *get_first_ipv4_addr(const char *hostname, uint32_t *out);
+
 #endif
 
 // vim: ts=4:sw=4:tw=79:et

hadoop-native-core/src/main/native/common/string-unit.c

@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "common/string.h"
+#include "test/test.h"
+
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+static int test_strdupto(void)
+{
+    char *dst = NULL;
+    EXPECT_INT_ZERO(strdupto(&dst, "FOO"));
+    EXPECT_INT_ZERO(strcmp(dst, "FOO"));
+    EXPECT_INT_ZERO(strdupto(&dst, NULL));
+    EXPECT_NULL(dst);
+    EXPECT_INT_ZERO(strdupto(&dst, "BAR"));
+    EXPECT_INT_ZERO(strcmp(dst, "BAR"));
+    EXPECT_INT_ZERO(strdupto(&dst, "BAZ"));
+    EXPECT_INT_ZERO(strcmp(dst, "BAZ"));
+    EXPECT_INT_ZERO(strdupto(&dst, NULL));
+    EXPECT_NULL(dst);
+    return 0;
+}
+
+int main(void)
+{
+    EXPECT_INT_ZERO(test_strdupto());
+
+    return EXIT_SUCCESS;
+}
+
+// vim: ts=4:sw=4:tw=79:et

hadoop-native-core/src/main/native/common/string.c

@@ -18,9 +18,11 @@
 
 #include "common/string.h"
 
+#include <errno.h>
 #include <stdarg.h>
 #include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
 
 void hex_buf_print(FILE *fp, const void *buf, int32_t buf_len,
                    const char *fmt, ...)
@@ -43,4 +45,24 @@ void hex_buf_print(FILE *fp, const void *buf, int32_t buf_len,
     }
 }
 
+int strdupto(char **dst, const char *src)
+{
+    char *ndst;
+    size_t src_len;
+
+    if (!src) {
+        free(*dst);
+        *dst = NULL;
+        return 0;
+    }
+    src_len = strlen(src);
+    ndst = realloc(*dst, src_len + 1);
+    if (!ndst) {
+        return ENOMEM;
+    }
+    strcpy(ndst, src);
+    *dst = ndst;
+    return 0;
+}
+
 // vim: ts=4:sw=4:tw=79:et

hadoop-native-core/src/main/native/common/string.h

@@ -34,6 +34,19 @@
 void hex_buf_print(FILE *fp, const void *buf, int32_t buf_len,
         const char *fmt, ...) __attribute__((format(printf, 4, 5)));
 
+/**
+ * Duplicate a string. 
+ *
+ * @param dst       (out param) the destination address.
+ *                    We will put either NULL (if src == NULL) or a malloc'ed
+ *                    string here.  If a malloc'ed string is here, we will free
+ *                    or realloc it.
+ * @param src       The string to duplicate, or NULL to set *dst to NULL.
+ *
+ * @return          0 on success; ENOMEM on OOM.
+ */
+int strdupto(char **dst, const char *src);
+
 #endif
 
 // vim: ts=4:sw=4:tw=79:et

hadoop-native-core/src/main/native/common/uri-unit.c

@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "common/hadoop_err.h"
+#include "common/uri.h"
+#include "test/test.h"
+
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <uv.h>
+
+static int check_uri_params(const char *scheme, const char *escheme,
+                            const char *user, const char *euser,
+                            const char *auth, const char *eauth,
+                            int port, int eport,
+                            const char *path, const char *epath)
+{
+    EXPECT_STR_EQ(escheme, scheme);
+    EXPECT_STR_EQ(euser, user);
+    EXPECT_STR_EQ(eauth, auth);
+    EXPECT_STR_EQ(epath, path);
+    EXPECT_INT_EQ(eport, port);
+    return 0;
+}
+
+static struct hadoop_err *test_parse_uri(const char *uri_str,
+            const char *escheme, const char *euser, const char *eauth,
+            int eport, const char *epath)
+{
+    UriParserStateA base_uri_state, uri_state;
+    UriUriA base_uri, uri;
+    struct hadoop_err *err = NULL;
+    char *scheme = NULL, *user = NULL, *auth = NULL;
+    char *path = NULL;
+    uint16_t port;
+
+    memset(&uri_state, 0, sizeof(uri_state));
+    err = uri_parse_abs("hdfs:///home/cmccabe/", &base_uri_state,
+            &base_uri, "hdfs");
+    if (err)
+        goto done;
+    err = uri_parse(uri_str, &uri_state, &uri, &base_uri);
+    if (err)
+        goto done;
+    err = uri_get_scheme(&uri, &scheme);
+    if (err)
+        goto done;
+    err = uri_get_user_info(&uri, &user);
+    if (err)
+        goto done;
+    // Get the authority, which we typically treat as a hostname.
+    err = uri_get_authority(&uri, &auth);
+    if (err)
+        goto done;
+    err = uri_get_path(&uri, &path);
+    if (err)
+        goto done;
+    err = uri_get_port(&uri, &port);
+    if (err)
+        goto done;
+//    fprintf(stderr, "test_parse_uri(%s): "
+//            "scheme=%s, user=%s, auth=%s, path=%s\n",
+//            uri_str, scheme, user, auth, path);
+    err = NULL;
+    if (check_uri_params(scheme, escheme,
+                         user, euser,
+                         auth, eauth,
+                        port, eport,
+                         path, epath)) {
+        err = hadoop_lerr_alloc(EINVAL, "check_uri_params: failed.");
+        if (err)
+            goto done;
+    }
+
+done:
+    if (base_uri_state.uri) {
+        uriFreeUriMembersA(&base_uri);
+    }
+    if (uri_state.uri) {
+        uriFreeUriMembersA(&uri);
+    }
+    free(scheme);
+    free(user);
+    free(auth);
+    free(path);
+    return err;
+}
+
+int main(void)
+{
+    struct hadoop_err *err;
+
+    //EXPECT_NO_HADOOP_ERR(test_parse_uri("localhost:6000"));
+    EXPECT_NO_HADOOP_ERR(test_parse_uri("a/b/c/d",
+                    "hdfs", "", "", 0, "/home/cmccabe/a/b/c/d"));
+    EXPECT_NO_HADOOP_ERR(test_parse_uri("hdfs://localhost:6000",
+                    "hdfs", "", "localhost", 6000, ""));
+    EXPECT_NO_HADOOP_ERR(test_parse_uri("file:///a/b/c/d",
+                    "file", "", "", 0, "/a/b/c/d"));
+    EXPECT_NO_HADOOP_ERR(test_parse_uri("s3://jeffbezos:6000",
+                    "s3", "", "jeffbezos", 6000, ""));
+    EXPECT_NO_HADOOP_ERR(test_parse_uri("s3:///foo",
+                    "s3", "", "", 0, "/foo"));
+    EXPECT_NO_HADOOP_ERR(test_parse_uri("hdfs://nn1.example.com/foo/bar",
+                    "hdfs", "", "nn1.example.com", 0, "/foo/bar"));
+    EXPECT_NO_HADOOP_ERR(test_parse_uri("hdfs://user:password@hdfshost:9000/a/b/c",
+                    "hdfs", "user:password", "hdfshost", 9000, "/a/b/c"));
+    err = test_parse_uri("://user:password@hdfshost:9000/a/b/c",
+                    "", "", "", 0, "");
+    EXPECT_NONNULL(strstr(hadoop_err_msg(err), "failed to parse"));
+    hadoop_err_free(err);
+
+    return EXIT_SUCCESS;
+}
+
+// vim: ts=4:sw=4:tw=79:et

hadoop-native-core/src/main/native/common/uri.c

@@ -0,0 +1,298 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "common/hadoop_err.h"
+#include "common/uri.h"
+
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+static struct hadoop_err *uri_err_to_hadoop_err(int err)
+{
+    switch (err) {
+    case URI_SUCCESS:
+        return NULL;
+    case URI_ERROR_SYNTAX:
+        return hadoop_lerr_alloc(EINVAL, "invalid URI format");
+    case URI_ERROR_NULL:
+        return hadoop_lerr_alloc(EINVAL, "unexpected NULL pointer "
+                                 "passed as parameter to uriparse");
+    case URI_ERROR_MALLOC:
+        return hadoop_lerr_alloc(ENOMEM, "out of memory");
+    case URI_ERROR_OUTPUT_TOO_LARGE:
+        return hadoop_lerr_alloc(ENAMETOOLONG, "data too big for uriparse "
+                                 "buffer");
+    case URI_ERROR_NOT_IMPLEMENTED:
+        return hadoop_lerr_alloc(ENOTSUP, "uriparse function not "
+                                 "implemented.");
+    case URI_ERROR_ADDBASE_REL_BASE:
+        return hadoop_lerr_alloc(EINVAL, "given add base is not absolute");
+    case URI_ERROR_REMOVEBASE_REL_BASE:
+        return hadoop_lerr_alloc(EINVAL, "given remove base is not absolute");
+    case URI_ERROR_REMOVEBASE_REL_SOURCE:
+        return hadoop_lerr_alloc(EINVAL, "given remove source is not "
+                                 "absolute");
+    case URI_ERROR_RANGE_INVALID:
+        return hadoop_lerr_alloc(ERANGE, "invalid range in uriparse.");
+    default:
+        return hadoop_lerr_alloc(EIO, "unknown uri error.");
+    }
+}
+
+struct hadoop_err *uri_parse_abs(const char *str, UriParserStateA *state,
+            UriUriA *uri, const char *def_scheme)
+{
+    int ret;
+    struct hadoop_err *err = NULL;
+    size_t str_len;
+    const char *effective_str = NULL;
+    char *malloced_str = NULL, *nmalloced_str;
+
+    // If the URI doesn't end with a slash, append one.
+    // This is necessary to get AddBaseUri to act like we expect when using
+    // this absolute URI as a base.
+    state->uri = NULL;
+    str_len = strlen(str);
+    if ((str_len == 0) || (str[str_len - 1] != '/')) {
+        if (asprintf(&malloced_str, "%s/", str) < 0) {
+            err = hadoop_lerr_alloc(ENOMEM, "uri_parse_abs: OOM");
+            malloced_str = NULL;
+            goto done;
+        }
+        effective_str = malloced_str;
+    } else {
+        effective_str = str;
+    }
+    state->uri = uri;
+    ret = uriParseUriA(state, effective_str);
+    if (ret) {
+        state->uri = NULL;
+        err = hadoop_err_prepend(uri_err_to_hadoop_err(ret),
+            0, "uri_parse: failed to parse '%s' as URI",
+            effective_str);
+        goto done;
+    }
+    if (uri->scheme.first == NULL) {
+        // If the URI doesn't have a scheme, prepend the default one to the
+        // string, and re-parse.  This is necessary because AddBaseUri refuses
+        // to rebase URIs on absolute URIs without a scheme.
+        if (asprintf(&nmalloced_str, "%s://%s", def_scheme,
+                     effective_str) < 0) {
+            err = hadoop_lerr_alloc(ENOMEM, "uri_parse_abs: OOM");
+            goto done;
+        }
+        free(malloced_str);
+        malloced_str = nmalloced_str;
+        effective_str = malloced_str;
+        uriFreeUriMembersA(uri);
+        state->uri = uri;
+        ret = uriParseUriA(state, effective_str);
+        if (ret) {
+            state->uri = NULL;
+            err = hadoop_err_prepend(uri_err_to_hadoop_err(ret),
+                0, "uri_parse: failed to parse '%s' as URI",
+                effective_str);
+            goto done;
+        }
+    }
+    err = NULL;
+done:
+    if (err) {
+        if (state->uri) {
+            uriFreeUriMembersA(state->uri);
+            state->uri = NULL;
+        }
+    }
+    return err;
+}
+
+struct hadoop_err *uri_parse(const char *str, UriParserStateA *state,
+            UriUriA *uri, UriUriA *base_uri)
+{
+    int ret;
+    struct hadoop_err *err = NULL;
+    UriUriA first_uri;
+
+    state->uri = &first_uri;
+    ret = uriParseUriA(state, str);
+    if (ret) {
+        state->uri = NULL;
+        err = hadoop_err_prepend(uri_err_to_hadoop_err(ret),
+            0, "uri_parse: failed to parse '%s' as a URI", str);
+        goto done;
+    }
+//    fprintf(stderr, "str=%s, base_path=%s, base_uri->absolutePath=%d\n",
+//            str, base_path, base_uri.absolutePath);
+//        fprintf(stderr, "uriAddBaseUriA base_path=%s, str=%s, ret %d\n", base_path, str, ret); 
+    ret = uriAddBaseUriA(uri, &first_uri, base_uri);
+    if (ret) {
+        err = hadoop_err_prepend(uri_err_to_hadoop_err(ret),
+            0, "uri_parse: failed to add base URI");
+        goto done;
+    }
+    uriFreeUriMembersA(&first_uri);
+    state->uri = uri;
+    ret = uriNormalizeSyntaxA(uri);
+    if (ret) {
+        err = hadoop_err_prepend(uri_err_to_hadoop_err(ret),
+            0, "uri_parse: failed to normalize URI");
+        goto done;
+    }
+done:
+    if (err) {
+        if (state->uri) {
+            uriFreeUriMembersA(uri);
+            state->uri = NULL;
+        }
+    }
+    return err;
+}
+
+static struct hadoop_err *text_range_to_str(struct UriTextRangeStructA *text,
+                                            char **out, const char *def)
+{
+    struct hadoop_err *err = NULL;
+    char *str = NULL;
+    const char *c;
+    size_t len = 0;
+
+    if (!text->first) {
+        str = strdup(def);
+        if (!str) {
+            err = hadoop_lerr_alloc(ENOMEM, "text_range_to_str: out of memory "
+                "trying to allocate a %zd-byte default string.",
+                strlen(def) + 1);
+        }
+        goto done;
+    }
+    for (c = text->first; c != text->afterLast; c++) {
+        ++len;
+    }
+    str = malloc(len + 1);
+    if (!str) {
+        err = hadoop_lerr_alloc(ENOMEM, "text_range_to_str: out of memory "
+            "trying to allocate a %zd-byte string.", len + 1);
+        goto done;
+    }
+    memcpy(str, text->first, len);
+    str[len] = '\0';
+    err = NULL;
+
+done:
+    if (err) {
+        free(str);
+        return err;
+    }
+    *out = str;
+    return NULL;
+}
+
+struct hadoop_err *uri_get_scheme(UriUriA *uri, char **out)
+{
+    struct hadoop_err *err;
+    char *scheme = NULL;
+
+    err = text_range_to_str(&uri->scheme, &scheme, "");
+    if (err)
+        return err;
+    *out = scheme;
+    return NULL;
+}
+
+struct hadoop_err *uri_get_user_info(UriUriA *uri, char **user_info)
+{
+    return text_range_to_str(&uri->userInfo, user_info, "");
+}
+
+struct hadoop_err *uri_get_authority(UriUriA *uri, char **authority)
+{
+    return text_range_to_str(&uri->hostText, authority, "");
+}
+
+struct hadoop_err *uri_get_port(UriUriA *uri, uint16_t *out)
+{
+    struct hadoop_err *err;
+    char *port_str = NULL;
+    int port;
+
+    err = text_range_to_str(&uri->portText, &port_str, "");
+    if (err)
+        return err;
+    port = atoi(port_str);
+    free(port_str);
+    if (port < 0 || port > 0xffff) {
+        return hadoop_lerr_alloc(EINVAL, "uri_get_port: invalid "
+                                 "port number %d\n", port);
+    }
+    *out = port;
+    return NULL;
+}
+
+struct hadoop_err *uri_get_path(UriUriA *uri, char **out)
+{
+    struct UriPathSegmentStructA *cur;
+    size_t i = 0, path_len = 0;
+    char *path = NULL;
+    int absolute = 0;
+
+    if (uri->absolutePath) {
+        absolute = 1;
+    } else if (uri->pathHead && uri->scheme.first) {
+        // Hadoop treats all URIs with a path as absolute, if they have a
+        // non-empty path.
+        // So hdfs://mynamenode/ maps to the root path, for example.  But as a
+        // special case, hdfs://mynamenode (no terminating slash) maps to "."
+        absolute = 1;
+    }
+    // The URI parser library splits paths up into lots of PathSegment
+    // structures-- one per path segment.  We need to reconstruct the full
+    // path.  The first step is figuring out the upper bound on the path
+    // length.
+    for (cur = uri->pathHead; cur; cur = cur->next) {
+        const char *c;
+        path_len++; // +1 for the leading slash.
+        for (c = cur->text.first; c != cur->text.afterLast; c++) {
+            path_len++;
+        }
+    }
+    path = malloc(path_len + 1); // +1 for the NULL terminator
+    if (!path) {
+        return hadoop_lerr_alloc(ENOMEM, "uri_get_path: OOM copying "
+                                 "%zd byte path.", path_len);
+    }
+    // The next step is copying over the path.
+    for (cur = uri->pathHead; cur; cur = cur->next) {
+        const char *c;
+        size_t copy_len = 0;
+        if ((i != 0) || absolute) {
+            path[i++] = '/';
+        }
+        for (c = cur->text.first; c != cur->text.afterLast; c++) {
+            copy_len++;
+        }
+        memcpy(path + i, cur->text.first, copy_len);
+        i += copy_len;
+    }
+    path[i] = '\0';
+    *out = path;
+    return NULL;
+}
+
+// vim: ts=4:sw=4:tw=79:et

hadoop-native-core/src/main/native/common/uri.h

@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef HADOOP_CORE_COMMON_URI
+#define HADOOP_CORE_COMMON_URI
+
+#include <uriparser/Uri.h>
+
+/**
+ * Parse an absolute URI.
+ *
+ * @param str           The string to parse.  If there is not a slash at the
+ *                          end, one will be added.
+ * @param state         (inout) The URI parser state to use.
+ *                          On success, state->uri will be set to a non-NULL
+ *                          value.
+ * @param uri           (out param) The URI object to fill.
+ * @param def_scheme    The default scheme to add if there is no scheme.
+ *
+ * @return              NULL on success; the URI parsing problem otherwise.
+ */
+struct hadoop_err *uri_parse_abs(const char *str, UriParserStateA *state,
+            UriUriA *uri, const char *def_scheme);
+
+/**
+ * Parse a relative or absolute URI.
+ *
+ * @param str           The string to parse.
+ * @param state         (inout) The URI parser state to use.
+ *                          On success, state->uri will be set to a non-NULL
+ *                          value.
+ * @param uri           (out param) The URI object to fill.
+ *
+ * @return              NULL on success; the URI parsing problem otherwise.
+ */
+struct hadoop_err *uri_parse(const char *str, UriParserStateA *state,
+            UriUriA *uri, UriUriA *base_uri);
+
+/**
+ * Get the scheme of a URI.
+ *
+ * We disallow schemes with non-ASCII characters.
+ *
+ * @param uri           The Uri object.
+ * @param scheme        (out param) the scheme.
+ *
+ * @return              NULL on success; the URI parsing problem otherwise.
+ */
+struct hadoop_err *uri_get_scheme(UriUriA *uri, char **scheme);
+
+/**
+ * Get the user_info of a URI.
+ *
+ * @param uri           The Uri object.
+ * @param user_info     (out param) the user_info.
+ *
+ * @return              NULL on success; the URI parsing problem otherwise.
+ */
+struct hadoop_err *uri_get_user_info(UriUriA *uri, char **user_info);
+
+/**
+ * Get the authority of a URI.
+ *
+ * @param uri           The Uri object.
+ * @param authority     (out param) the authority.
+ *
+ * @return              NULL on success; the URI parsing problem otherwise.
+ */
+struct hadoop_err *uri_get_authority(UriUriA *uri, char **authority);
+
+/**
+ * Get the port of a URI.
+ *
+ * @param uri           The Uri object.
+ * @param port          (out param) the port, or 0 if there was no port.
+ *
+ * @return              NULL on success; the URI parsing problem otherwise.
+ */
+struct hadoop_err *uri_get_port(UriUriA *uri, uint16_t *port);
+
+/**
+ * Get the path of a URI.
+ *
+ * @param uri       The Uri object.
+ * @param path      (out param) the path.
+ *
+ * @return          NULL on success; the URI parsing problem otherwise.
+ */
+struct hadoop_err *uri_get_path(UriUriA *uri, char **path);
+
+#endif
+
+// vim: ts=4:sw=4:tw=79:et

hadoop-native-core/src/main/native/config.h.cmake

@@ -0,0 +1,40 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+#ifndef CONFIG_H
+#define CONFIG_H
+
+/**
+ * Defined if we can use the __thread keyword to get faster thread-local 
+ * storage.
+ */
+#cmakedefine HAVE_BETTER_TLS
+
+/**
+ * Short version of JNI library name.  
+ * This varies by platform.
+ *
+ * Example: "libvjm.so"
+ */
+#cmakedefine JNI_LIBRARY_NAME "@JNI_LIBRARY_NAME@"
+
+/**
+ * Where to find the test XML files we use in hconf-unit.
+ */
+#cmakedefine HCONF_XML_TEST_PATH "@HCONF_XML_TEST_PATH@"
+
+#endif

hadoop-native-core/src/main/native/fs/common.c

@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "common/hadoop_err.h"
+#include "common/string.h"
+#include "common/uri.h"
+#include "common/user.h"
+#include "fs/fs.h"
+#include "fs/hdfs.h"
+
+#include <errno.h>
+#include <fcntl.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include <uriparser/Uri.h>
+
+void release_file_info_entry(hdfsFileInfo *hdfsFileInfo)
+{
+    free(hdfsFileInfo->mName);
+    free(hdfsFileInfo->mOwner);
+    free(hdfsFileInfo->mGroup);
+    memset(&hdfsFileInfo, 0, sizeof(hdfsFileInfo));
+}
+
+int hadoopfs_errno_and_retcode(struct hadoop_err *err)
+{
+    if (err) {
+        fputs(hadoop_err_msg(err), stderr);
+        errno = hadoop_err_code(err);
+        hadoop_err_free(err);
+        return -1;
+    }
+    return 0;
+}
+
+void *hadoopfs_errno_and_retptr(struct hadoop_err *err, void *ptr)
+{
+    if (err) {
+        fputs(hadoop_err_msg(err), stderr);
+        errno = hadoop_err_code(err);
+        hadoop_err_free(err);
+        return NULL;
+    }
+    return ptr;
+}
+
+// vim: ts=4:sw=4:et

hadoop-native-core/src/main/native/fs/common.h

@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef HADOOP_NATIVE_CORE_FS_COMMON_H
+#define HADOOP_NATIVE_CORE_FS_COMMON_H
+
+struct file_info;
+struct hadoop_err;
+struct hdfsBuilder;
+
+/**
+ * Release the memory used inside an hdfsFileInfo structure.
+ * Does not free the structure itself.
+ *
+ * @param hdfsFileInfo          The hdfsFileInfo structure.
+ */
+void release_file_info_entry(struct file_info *hdfsFileInfo);
+
+/**
+ * Sets errno and logs a message appropriately on encountering a Hadoop error.
+ *
+ * @param err                   The hadoop error, or NULL if there is none.
+ *
+ * @return                      -1 on error; 0 otherwise.  Errno will be set on
+ *                              error.
+ */
+int hadoopfs_errno_and_retcode(struct hadoop_err *err);
+
+/**
+ * Sets errno and logs a message appropriately on encountering a Hadoop error.
+ *
+ * @param err                   The hadoop error, or NULL if there is none.
+ * @param ptr                   The pointer to return if err is NULL.
+ *
+ * @return                      NULL on error; ptr otherwise.  Errno will be set
+ *                              on error.
+ */
+void *hadoopfs_errno_and_retptr(struct hadoop_err *err, void *ptr);
+
+#endif
+
+// vim: ts=4:sw=4:et

hadoop-native-core/src/main/native/fs/fs.c

@@ -0,0 +1,734 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "common/hadoop_err.h"
+#include "common/hconf.h"
+#include "common/string.h"
+#include "common/uri.h"
+#include "common/user.h"
+#include "fs/common.h"
+#include "fs/fs.h"
+#include "fs/hdfs.h"
+
+#include <errno.h>
+#include <fcntl.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include <uriparser/Uri.h>
+
+#define DEFAULT_SCHEME "hdfs"
+
+#define DEFAULT_NATIVE_HANDLERS "ndfs,jnifs"
+
+const char* const HDFS_XML_NAMES[] = {
+    "core-default.xml",
+    "core-site.xml",
+    "hdfs-default.xml",
+    "hdfs-site.xml",
+    NULL
+};
+
+const struct hadoop_fs_ops g_jni_ops;
+const struct hadoop_fs_ops g_ndfs_ops;
+
+static const struct hadoop_fs_ops *g_ops[] = {
+    &g_jni_ops,
+    &g_ndfs_ops,
+    NULL
+};
+
+int hdfsFileIsOpenForRead(hdfsFile file)
+{
+    struct hadoop_file_base *base = (struct hadoop_file_base*)file;
+    return g_ops[base->ty]->file_is_open_for_read(file);
+}
+
+int hdfsFileIsOpenForWrite(hdfsFile file)
+{
+    struct hadoop_file_base *base = (struct hadoop_file_base*)file;
+    return g_ops[base->ty]->file_is_open_for_write(file);
+}
+
+int hdfsFileGetReadStatistics(hdfsFile file, struct hdfsReadStatistics **stats)
+{
+    struct hadoop_file_base *base = (struct hadoop_file_base*)file;
+    return g_ops[base->ty]->get_read_statistics(file, stats);
+}
+
+int64_t hdfsReadStatisticsGetRemoteBytesRead(
+                            const struct hdfsReadStatistics *stats)
+{
+    return stats->totalBytesRead - stats->totalLocalBytesRead;
+}
+
+void hdfsFileFreeReadStatistics(struct hdfsReadStatistics *stats)
+{
+    free(stats);
+}
+
+hdfsFS hdfsConnect(const char* host, tPort port)
+{
+    struct hdfsBuilder *bld = hdfsNewBuilder();
+    if (!bld)
+        return NULL;
+    hdfsBuilderSetNameNode(bld, host);
+    hdfsBuilderSetNameNodePort(bld, port);
+    return hdfsBuilderConnect(bld);
+}
+
+hdfsFS hdfsConnectNewInstance(const char* host, tPort port)
+{
+    struct hdfsBuilder *bld = hdfsNewBuilder();
+    if (!bld)
+        return NULL;
+    hdfsBuilderSetNameNode(bld, host);
+    hdfsBuilderSetNameNodePort(bld, port);
+    hdfsBuilderSetForceNewInstance(bld);
+    return hdfsBuilderConnect(bld);
+}
+
+hdfsFS hdfsConnectAsUser(const char* host, tPort port, const char *user)
+{
+    struct hdfsBuilder *bld = hdfsNewBuilder();
+    if (!bld)
+        return NULL;
+    hdfsBuilderSetNameNode(bld, host);
+    hdfsBuilderSetNameNodePort(bld, port);
+    hdfsBuilderSetUserName(bld, user);
+    return hdfsBuilderConnect(bld);
+}
+
+hdfsFS hdfsConnectAsUserNewInstance(const char* host, tPort port,
+        const char *user)
+{
+    struct hdfsBuilder *bld = hdfsNewBuilder();
+    if (!bld)
+        return NULL;
+    hdfsBuilderSetNameNode(bld, host);
+    hdfsBuilderSetNameNodePort(bld, port);
+    hdfsBuilderSetForceNewInstance(bld);
+    hdfsBuilderSetUserName(bld, user);
+    return hdfsBuilderConnect(bld);
+}
+
+static const struct hadoop_fs_ops *find_fs_impl_by_name(const char *name)
+{
+    const struct hadoop_fs_ops *ops;
+    int i = 0;
+
+    while (1) {
+        ops = g_ops[i++];
+        if (!ops) {
+            fprintf(stderr, "hdfsBuilderConnect: we don't support the '%s' "
+                    "native fs implementation.\n", name);
+            return NULL;
+        }
+        if (strcmp(ops->name, name) == 0) {
+            return ops;
+        }
+    }
+}
+
+static struct hadoop_err *hdfs_builder_load_conf(struct hdfsBuilder *hdfs_bld)
+{
+    struct hadoop_err *err;
+    struct hconf_builder *conf_bld = NULL;
+    const char *classpath;
+    struct hdfsBuilderConfOpt *opt;
+
+    err = hconf_builder_alloc(&conf_bld);
+    if (err) {
+        goto done;
+    }
+
+    // Load the XML files.
+    classpath = getenv("CLASSPATH");
+    if (!classpath) {
+        classpath = ".";
+    }
+    err = hconf_builder_load_xmls(conf_bld, HDFS_XML_NAMES, classpath);
+    if (err) {
+        goto done;
+    }
+
+    // Add the options that were specified by hdfsBuilderConfSetStr.
+    for (opt = hdfs_bld->opts; opt; opt = opt->next) {
+        hconf_builder_set(conf_bld, opt->key, opt->val);
+    }
+
+    // Create the conf object.
+    err = hconf_build(conf_bld, &hdfs_bld->hconf);
+    conf_bld = NULL;
+    err = NULL;
+done:
+    if (conf_bld) {
+        hconf_builder_free(conf_bld);
+    }
+    if (err)
+        return err;
+    return NULL;
+}
+
+static struct hadoop_err *hdfs_builder_parse_conn_uri(
+                                    struct hdfsBuilder *hdfs_bld)
+{
+    int ret;
+    uint16_t port;
+    const char *uri_str;
+    char *malloced_uri_str = NULL;
+    UriParserStateA uri_state;
+    UriUriA uri;
+    struct hadoop_err *err = NULL;
+
+    memset(&uri_state, 0, sizeof(uri_state));
+    uri_str = hdfs_bld->nn;
+    if (uri_str) {
+        // If the connection URI was set via hdfsBuilderSetNameNode, it may
+        // not be a real URI, but just a <hostname>:<port> pair.  This won't
+        // parse correctly unless we add a hdfs:// scheme in front of it.
+        if ((!index(uri_str, '/')) && (index(uri_str, ':'))) {
+            if (asprintf(&malloced_uri_str, "hdfs://%s", uri_str) < 0) {
+                malloced_uri_str = NULL;
+                err = hadoop_lerr_alloc(ENOMEM, "uri_parse: OOM "
+                                        "adding default scheme");
+                goto done;
+            }
+            uri_str = malloced_uri_str;
+        }
+    } else {
+        uri_str = hconf_get(hdfs_bld->hconf, "fs.defaultFS");
+        if (!uri_str) {
+            uri_str = "file:///";
+        }
+    }
+    err = uri_parse_abs(uri_str, &uri_state, &uri, DEFAULT_SCHEME);
+    if (err)
+        goto done;
+    err = uri_get_scheme(&uri, &hdfs_bld->uri_scheme);
+    if (err)
+        goto done; 
+    // Get the user_info.  We default to the userName passed in to the hdfs
+    // builder.
+    err = uri_get_user_info(&uri, &hdfs_bld->uri_user_info);
+    if (err)
+        goto done;
+    if (hdfs_bld->uri_user_info[0] == '\0') {
+        // If we still don't have an authority, fill in the authority from the
+        // current user name.
+        free(hdfs_bld->uri_user_info);
+        hdfs_bld->uri_user_info = NULL;
+        ret = geteuid_string(&hdfs_bld->uri_user_info);
+        if (ret) {
+            err = hadoop_lerr_alloc(ret, "geteuid_string failed: error "
+                                    "%d", ret);
+            goto done;
+        }
+    }
+    // Get the authority, which we typically treat as a hostname.
+    err = uri_get_authority(&uri, &hdfs_bld->uri_authority);
+    if (err)
+        goto done;
+    // Get the port, or 0.
+    err = uri_get_port(&uri, &port);
+    if (err)
+        goto done;
+    fprintf(stderr, "hdfs_builder_parse_conn_uri: "
+            "uri_scheme=%s, uri_user_info=%s, "
+            "uri_authority=%s, port=%d\n",
+            hdfs_bld->uri_scheme, hdfs_bld->uri_user_info,
+            hdfs_bld->uri_authority, port);
+    // The URI's port overrides the port supplied via
+    // hdfsBuilderSetNameNodePort.
+    if (port) {
+        hdfs_bld->port = port;
+    }
+    err = NULL;
+
+done:
+    free(malloced_uri_str);
+    if (uri_state.uri) {
+        uriFreeUriMembersA(&uri);
+    }
+    return err;
+}
+
+hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld)
+{
+    struct hadoop_err *err;
+    hdfsFS fs = NULL;
+    const char *fs_list_val;
+    char *fs_list_key = NULL, *fs_list_val_copy = NULL, *ptr;
+    char *fs_impl_name;
+
+    //
+    // Load the configuration from XML.
+    //
+    // The hconf object will be available to all native FS implementations to
+    // use in their connect methods.
+    //
+    err = hdfs_builder_load_conf(bld);
+    if (err)
+        goto done;
+
+    //
+    // Determine the URI we should connect to.  It gets a bit complicated
+    // because of all the defaults.
+    //
+    err = hdfs_builder_parse_conn_uri(bld);
+    if (err)
+        goto done;
+
+    // Find out the native filesystems we should use for this URI.
+    if (asprintf(&fs_list_key, "%s.native.handler.", bld->uri_scheme) < 0) {
+        fs_list_key = NULL;
+        err = hadoop_lerr_alloc(ENOMEM, "hdfsBuilderConnect: OOM");
+        goto done;
+    }
+    fs_list_val = hconf_get(bld->hconf, fs_list_key);
+    if (!fs_list_val) {
+        fs_list_val = hconf_get(bld->hconf, "default.native.handler");
+        if (!fs_list_val) {
+            fs_list_val = DEFAULT_NATIVE_HANDLERS;
+        }
+    }
+    fs_list_val_copy = strdup(fs_list_val);
+    if (!fs_list_val_copy) {
+        err = hadoop_lerr_alloc(ENOMEM, "hdfsBuilderConnect: OOM");
+        goto done;
+    }
+    // Give each native filesystem implementation a shot at connecting.
+    for (fs_impl_name = strtok_r(fs_list_val_copy, ",", &ptr); fs_impl_name;
+                fs_impl_name = strtok_r(NULL, ",", &ptr)) {
+        const struct hadoop_fs_ops *ops = find_fs_impl_by_name(fs_impl_name);
+        if (!ops)
+            continue;
+        if (err)
+            hadoop_err_free(err);
+        err = ops->connect(bld, &fs);
+        if (!err) {
+            break;
+        }
+        fprintf(stderr, "hdfsBuilderConnect: %s failed to connect: "
+                "%s (error %d)\n", fs_impl_name, hadoop_err_msg(err),
+                hadoop_err_code(err));
+    }
+
+done:
+    hdfsFreeBuilder(bld);
+    free(fs_list_key);
+    free(fs_list_val_copy);
+    return hadoopfs_errno_and_retptr(err, fs);
+}
+
+struct hdfsBuilder *hdfsNewBuilder(void)
+{
+    struct hdfsBuilder *bld = calloc(1, sizeof(struct hdfsBuilder));
+    if (!bld) {
+        errno = ENOMEM;
+        return NULL;
+    }
+    return bld;
+}
+
+void hdfsBuilderSetForceNewInstance(struct hdfsBuilder *bld __attribute__((unused)))
+{
+    // Does nothing-- present only for compatibility
+}
+
+void hdfsBuilderSetNameNode(struct hdfsBuilder *bld, const char *nn)
+{
+    bld->nn = nn;
+}
+
+void hdfsBuilderSetNameNodePort(struct hdfsBuilder *bld, tPort port)
+{
+    bld->port = port;
+}
+
+void hdfsBuilderSetUserName(struct hdfsBuilder *bld, const char *userName)
+{
+    bld->userName = userName;
+}
+
+void hdfsBuilderSetKerbTicketCachePath(struct hdfsBuilder *bld,
+                                       const char *kerbTicketCachePath)
+{
+    bld->kerbTicketCachePath = kerbTicketCachePath;
+}
+
+void hdfsFreeBuilder(struct hdfsBuilder *bld)
+{
+    struct hdfsBuilderConfOpt *cur, *next;
+
+    if (!bld)
+        return;
+    cur = bld->opts;
+    for (cur = bld->opts; cur; ) {
+        next = cur->next;
+        free(cur);
+        cur = next;
+    }
+    hconf_free(bld->hconf);
+    free(bld);
+}
+
+int hdfsBuilderConfSetStr(struct hdfsBuilder *bld, const char *key,
+                          const char *val)
+{
+    struct hdfsBuilderConfOpt *opt, *next;
+
+    opt = calloc(1, sizeof(struct hdfsBuilderConfOpt));
+    if (!opt)
+        return -ENOMEM;
+    next = bld->opts;
+    bld->opts = opt;
+    opt->next = next;
+    opt->key = key;
+    opt->val = val;
+    return 0;
+}
+
+int hdfsConfGetStr(const char *key __attribute__((unused)),
+                   char **val __attribute__((unused)))
+{
+    // FIXME: add configuration stuff
+    errno = ENOTSUP;
+    return -1;
+}
+
+int hdfsConfGetInt(const char *key, int32_t *val)
+{
+    char *str = NULL;
+    int ret;
+
+    ret = hdfsConfGetStr(key, &str);
+    if (ret)
+        return ret;
+    *val = atoi(str);
+    hdfsConfStrFree(str);
+    return 0;
+}
+
+void hdfsConfStrFree(char *val)
+{
+    free(val);
+}
+
+int hdfsDisconnect(hdfsFS fs)
+{
+    struct hadoop_fs_base *base = (struct hadoop_fs_base*)fs;
+    return g_ops[base->ty]->disconnect(fs);
+}
+
+hdfsFile hdfsOpenFile(hdfsFS fs, const char *path, int flags,
+                      int bufferSize, short replication, tSize blocksize)
+{
+    struct hadoop_fs_base *base = (struct hadoop_fs_base*)fs;
+    return g_ops[base->ty]->open(fs, path, flags, bufferSize,
+                                 replication, blocksize);
+}
+
+int hdfsCloseFile(hdfsFS fs, hdfsFile file)
+{
+    struct hadoop_fs_base *base = (struct hadoop_fs_base*)fs;
+    return g_ops[base->ty]->close(fs, file);
+}
+
+int hdfsExists(hdfsFS fs, const char *path)
+{
+    struct hadoop_fs_base *base = (struct hadoop_fs_base*)fs;
+    return g_ops[base->ty]->exists(fs, path);
+}
+
+int hdfsSeek(hdfsFS fs, hdfsFile file, tOffset desiredPos)
+{
+    struct hadoop_fs_base *base = (struct hadoop_fs_base*)fs;
+    return g_ops[base->ty]->seek(fs, file, desiredPos);
+}
+
+tOffset hdfsTell(hdfsFS fs, hdfsFile file)
+{
+    struct hadoop_fs_base *base = (struct hadoop_fs_base*)fs;
+    return g_ops[base->ty]->tell(fs, file);
+}
+
+tSize hdfsRead(hdfsFS fs, hdfsFile file, void* buffer, tSize length)
+{
+    struct hadoop_fs_base *base = (struct hadoop_fs_base*)fs;
+    return g_ops[base->ty]->read(fs, file, buffer, length);
+}
+
+tSize hdfsPread(hdfsFS fs, hdfsFile file, tOffset position,
+                    void* buffer, tSize length)
+{
+    struct hadoop_fs_base *base = (struct hadoop_fs_base*)fs;
+    return g_ops[base->ty]->pread(fs, file, position, buffer, length);
+}
+
+tSize hdfsWrite(hdfsFS fs, hdfsFile file, const void* buffer,
+                tSize length)
+{
+    struct hadoop_fs_base *base = (struct hadoop_fs_base*)fs;
+    return g_ops[base->ty]->write(fs, file, buffer, length);
+}
+
+int hdfsFlush(hdfsFS fs, hdfsFile file)
+{
+    struct hadoop_fs_base *base = (struct hadoop_fs_base*)fs;
+    return g_ops[base->ty]->flush(fs, file);
+}
+
+int hdfsHFlush(hdfsFS fs, hdfsFile file)
+{
+    struct hadoop_fs_base *base = (struct hadoop_fs_base*)fs;
+    return g_ops[base->ty]->hflush(fs, file);
+}
+
+int hdfsHSync(hdfsFS fs, hdfsFile file)
+{
+    struct hadoop_fs_base *base = (struct hadoop_fs_base*)fs;
+    return g_ops[base->ty]->hsync(fs, file);
+}
+
+int hdfsAvailable(hdfsFS fs, hdfsFile file)
+{
+    struct hadoop_fs_base *base = (struct hadoop_fs_base*)fs;
+    return g_ops[base->ty]->available(fs, file);
+}
+
+int hdfsCopy(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst)
+{
+    struct hadoop_fs_base *src_base = (struct hadoop_fs_base*)srcFS;
+    struct hadoop_fs_base *dst_base = (struct hadoop_fs_base*)dstFS;
+
+    if (src_base->ty != dst_base->ty) {
+        errno = EINVAL;
+        return -1;
+    }
+    return g_ops[src_base->ty]->copy(srcFS, src, dstFS, dst);
+}
+
+int hdfsMove(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst)
+{
+    struct hadoop_fs_base *src_base = (struct hadoop_fs_base*)srcFS;
+    struct hadoop_fs_base *dst_base = (struct hadoop_fs_base*)dstFS;
+
+    if (src_base->ty != dst_base->ty) {
+        errno = EINVAL;
+        return -1;
+    }
+    return g_ops[src_base->ty]->move(srcFS, src, dstFS, dst);
+}
+
+int hdfsDelete(hdfsFS fs, const char* path, int recursive)
+{
+    struct hadoop_fs_base *base = (struct hadoop_fs_base*)fs;
+    return g_ops[base->ty]->unlink(fs, path, recursive);
+}
+
+int hdfsRename(hdfsFS fs, const char* oldPath, const char* newPath)
+{
+    struct hadoop_fs_base *base = (struct hadoop_fs_base*)fs;
+    return g_ops[base->ty]->rename(fs, oldPath, newPath);
+}
+
+char* hdfsGetWorkingDirectory(hdfsFS fs, char *buffer, size_t bufferSize)
+{
+    struct hadoop_fs_base *base = (struct hadoop_fs_base*)fs;
+    return g_ops[base->ty]->get_working_directory(fs, buffer, bufferSize);
+}
+
+int hdfsSetWorkingDirectory(hdfsFS fs, const char* path)
+{
+    struct hadoop_fs_base *base = (struct hadoop_fs_base*)fs;
+    return g_ops[base->ty]->set_working_directory(fs, path);
+}
+
+int hdfsCreateDirectory(hdfsFS fs, const char* path)
+{
+    struct hadoop_fs_base *base = (struct hadoop_fs_base*)fs;
+    return g_ops[base->ty]->mkdir(fs, path);
+}
+
+int hdfsSetReplication(hdfsFS fs, const char* path, int16_t replication)
+{
+    struct hadoop_fs_base *base = (struct hadoop_fs_base*)fs;
+    return g_ops[base->ty]->set_replication(fs, path, replication);
+}
+
+hdfsFileInfo *hdfsListDirectory(hdfsFS fs, const char* path, int *numEntries)
+{
+    struct hadoop_fs_base *base = (struct hadoop_fs_base*)fs;
+    return g_ops[base->ty]->list_directory(fs, path, numEntries);
+}
+
+hdfsFileInfo *hdfsGetPathInfo(hdfsFS fs, const char* path)
+{
+    struct hadoop_fs_base *base = (struct hadoop_fs_base*)fs;
+    return g_ops[base->ty]->get_path_info(fs, path);
+}
+
+void hdfsFreeFileInfo(hdfsFileInfo *hdfsFileInfo, int numEntries)
+{
+    //Free the mName, mOwner, and mGroup
+    int i;
+    for (i=0; i < numEntries; ++i) {
+        release_file_info_entry(hdfsFileInfo + i);
+    }
+
+    //Free entire block
+    free(hdfsFileInfo);
+}
+
+char*** hdfsGetHosts(hdfsFS fs, const char* path, 
+          tOffset start, tOffset length)
+{
+    struct hadoop_fs_base *base = (struct hadoop_fs_base*)fs;
+    return g_ops[base->ty]->get_hosts(fs, path, start, length);
+}
+
+void hdfsFreeHosts(char ***blockHosts)
+{
+    int i, j;
+    for (i=0; blockHosts[i]; i++) {
+        for (j=0; blockHosts[i][j]; j++) {
+            free(blockHosts[i][j]);
+        }
+        free(blockHosts[i]);
+    }
+    free(blockHosts);
+}
+
+tOffset hdfsGetDefaultBlockSize(hdfsFS fs)
+{
+    struct hadoop_fs_base *base = (struct hadoop_fs_base*)fs;
+    return g_ops[base->ty]->get_default_block_size(fs);
+}
+
+tOffset hdfsGetDefaultBlockSizeAtPath(hdfsFS fs, const char *path)
+{
+    struct hadoop_fs_base *base = (struct hadoop_fs_base*)fs;
+    return g_ops[base->ty]->get_default_block_size_at_path(fs, path);
+}
+
+tOffset hdfsGetCapacity(hdfsFS fs)
+{
+    struct hadoop_fs_base *base = (struct hadoop_fs_base*)fs;
+    return g_ops[base->ty]->get_capacity(fs);
+}
+
+tOffset hdfsGetUsed(hdfsFS fs)
+{
+    struct hadoop_fs_base *base = (struct hadoop_fs_base*)fs;
+    return g_ops[base->ty]->get_used(fs);
+}
+
+int hdfsChown(hdfsFS fs, const char* path, const char *owner,
+              const char *group)
+{
+    struct hadoop_fs_base *base = (struct hadoop_fs_base*)fs;
+    return g_ops[base->ty]->chown(fs, path, owner, group);
+}
+
+int hdfsChmod(hdfsFS fs, const char* path, short mode)
+{
+    struct hadoop_fs_base *base = (struct hadoop_fs_base*)fs;
+    return g_ops[base->ty]->chmod(fs, path, mode);
+}
+
+int hdfsUtime(hdfsFS fs, const char* path, tTime mtime, tTime atime)
+{
+    struct hadoop_fs_base *base = (struct hadoop_fs_base*)fs;
+    return g_ops[base->ty]->utime(fs, path, mtime, atime);
+}
+
+struct hadoopRzOptions *hadoopRzOptionsAlloc(void)
+{
+    struct hadoopRzOptions *opts;
+    opts = calloc(1, sizeof(*opts));
+    if (!opts) {
+        errno = ENOMEM;
+        return NULL;
+    }
+    return opts;
+}
+
+int hadoopRzOptionsSetSkipChecksum(struct hadoopRzOptions *opts, int skip)
+{
+    opts->skip_checksums = skip;
+    return 0;
+}
+
+int hadoopRzOptionsSetByteBufferPool(
+            struct hadoopRzOptions *opts, const char *className)
+{
+    return strdupto(&opts->pool_name, className);
+}
+
+void hadoopRzOptionsFree(struct hadoopRzOptions *opts)
+{
+    if (opts) {
+        if (opts->cache_teardown_cb) {
+            opts->cache_teardown_cb(opts->cache);
+            opts->cache = NULL;
+        }
+        free(opts->pool_name);
+        free(opts);
+    }
+}
+
+struct hadoopRzBuffer* hadoopReadZero(hdfsFile file,
+            struct hadoopRzOptions *opts, int32_t maxLength)
+{
+    struct hadoop_file_base *base = (struct hadoop_file_base*)file;
+    return g_ops[base->ty]->read_zero(file, opts, maxLength);
+}
+
+int32_t hadoopRzBufferLength(const struct hadoopRzBuffer *buf)
+{
+    struct hadoop_rz_buffer_base *bbuf = (struct hadoop_rz_buffer_base *)buf;
+    return bbuf->length;
+}
+
+const void *hadoopRzBufferGet(const struct hadoopRzBuffer *buf)
+{
+    struct hadoop_rz_buffer_base *bbuf = (struct hadoop_rz_buffer_base *)buf;
+    return bbuf->ptr;
+}
+
+void hadoopRzBufferFree(hdfsFile file, struct hadoopRzBuffer *buffer)
+{
+    struct hadoop_file_base *base = (struct hadoop_file_base*)file;
+    g_ops[base->ty]->rz_buffer_free(file, buffer);
+}
+
+int hdfsFileUsesDirectRead(struct hdfsFile_internal *file)
+{
+    struct hadoop_file_base *base = (struct hadoop_file_base*)file;
+    return g_ops[base->ty]->file_uses_direct_read(file);
+}
+
+void hdfsFileDisableDirectRead(struct hdfsFile_internal *file)
+{
+    struct hadoop_file_base *base = (struct hadoop_file_base*)file;
+    return g_ops[base->ty]->file_disable_direct_read(file);
+}
+
+// vim: ts=4:sw=4:et

hadoop-native-core/src/main/native/fs/fs.h

@@ -0,0 +1,203 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef HADOOP_NATIVE_CORE_FS_H
+#define HADOOP_NATIVE_CORE_FS_H
+
+#include "fs/hdfs.h"
+
+#include <inttypes.h>
+
+struct hadoop_err;
+struct hconf;
+
+/**
+ * fs.h
+ *
+ * This is the __private__ API for native Hadoop filesystems.  (The public API
+ * is in hdfs.h.) Native Hadoop filesystems such as JniFS or NDFS implement the
+ * APIs in this file to provide a uniform experience to users.
+ *
+ * The general pattern here is:
+ * 1. The client makes a call to libhdfs API
+ * 2. fs.c locates the appropriate function in hadoop_fs_ops and calls it.
+ * 3. Some filesystem-specific code implements the operation.
+ *
+ * In C, it is always safe to typecast a structure to the type of the first
+ * element.  This allows fs.c to treat hdfsFS instances as if they were
+ * instances of struct hadoop_file_base.  Other structures with "base" in the
+ * name are intended to be used similarly.  This functionality is similar in
+ * many ways to how "base classes" operate in Java.  The derived class contains
+ * all the elements of the base class, plus some more.
+ *
+ * The structure definitions in this file are private, and users of this library
+ * will not be able to access them.  This file will not be packaged or
+ * distributed... only hdfs.h will.  Thus, it is safe to change any of the APIs
+ * or types in this file without creating compatibility problems.
+ */
+
+/**
+ * Hadoop filesystem types.
+ */
+enum hadoop_fs_ty {
+    HADOOP_FS_TY_JNI = 0,
+    HADOOP_FS_TY_NDFS = 1,
+    HADOOP_FS_TY_NUM,
+};
+
+/**
+ * Base data for Hadoop files.
+ */
+struct hadoop_file_base {
+    // The type of filesystem this file was created by.
+    enum hadoop_fs_ty ty;
+};
+
+/**
+ * Base data for Hadoop FileSystem objects.
+ */
+struct hadoop_fs_base {
+    // The type of this filesystem.
+    enum hadoop_fs_ty ty;
+};
+
+/**
+ * Base data for Hadoop Zero-Copy Read objects.
+ */
+struct hadoopRzOptions {
+    // The name of the ByteBufferPool class we should use when doing a zero-copy
+    // read.
+    char *pool_name;
+
+    // Non-zero to always skip checksums.
+    int skip_checksums;
+
+    // If non-null, this callback will be invoked to tear down the cached data
+    // inside this options structure during hadoopRzOptionsFree.
+    void (*cache_teardown_cb)(void *);
+
+    // The cached data inside this options structure. 
+    void *cache;
+};
+
+/**
+ * Base data for Hadoop Zero-Copy Read buffers.
+ */
+struct hadoop_rz_buffer_base {
+    // The base address the client can start reading at.
+    void *ptr;
+
+    // The maximum valid length of this buffer.
+    int32_t length;
+};
+
+struct hdfsBuilderConfOpt {
+    struct hdfsBuilderConfOpt *next;
+    const char *key;
+    const char *val;
+};
+
+/**
+ * A builder used to create Hadoop filesystem instances.
+ */
+struct hdfsBuilder {
+    const char *nn;
+    uint16_t port;
+    const char *kerbTicketCachePath;
+    const char *userName;
+    struct hdfsBuilderConfOpt *opts;
+    struct hconf *hconf;
+    char *uri_scheme;
+    char *uri_user_info;
+    char *uri_authority;
+    uint16_t uri_port;
+};
+
+/**
+ * Operations which a libhadoopfs filesystem must implement.
+ */
+struct hadoop_fs_ops {
+    const char * const name;
+    int (*file_is_open_for_read)(struct hdfsFile_internal *file);
+    int (*file_is_open_for_write)(struct hdfsFile_internal * file);
+    int (*get_read_statistics)(struct hdfsFile_internal *file, 
+            struct hdfsReadStatistics **stats);
+    struct hadoop_err *(*connect)(struct hdfsBuilder *bld,
+                                  struct hdfs_internal **fs);
+    int (*disconnect)(struct hdfs_internal *fs);
+    struct hdfsFile_internal *(*open)(struct hdfs_internal *fs,
+            const char* uri, int flags, int bufferSize, short replication,
+            int32_t blocksize);
+    int (*close)(struct hdfs_internal *fs, struct hdfsFile_internal *file);
+    int (*exists)(struct hdfs_internal *fs, const char *uri);
+    int (*seek)(struct hdfs_internal *fs, struct hdfsFile_internal *file, 
+            int64_t desiredPos);
+    int64_t (*tell)(struct hdfs_internal *fs, struct hdfsFile_internal *file);
+    int32_t (*read)(struct hdfs_internal *fs, struct hdfsFile_internal *file,
+            void* buffer, int32_t length);
+    int32_t (*pread)(struct hdfs_internal *fs, struct hdfsFile_internal *file,
+            int64_t position, void *buffer, int32_t length);
+    int32_t (*write)(struct hdfs_internal *fs, struct hdfsFile_internal *file,
+            const void* buffer, int32_t length);
+    int (*flush)(struct hdfs_internal *fs, struct hdfsFile_internal *file);
+    int (*hflush)(struct hdfs_internal *fs, struct hdfsFile_internal *file);
+    int (*hsync)(struct hdfs_internal *fs, struct hdfsFile_internal *file);
+    int (*available)(struct hdfs_internal * fs, struct hdfsFile_internal *file);
+    int (*copy)(struct hdfs_internal *srcFS, const char *src,
+            struct hdfs_internal *dstFS, const char *dst);
+    int (*move)(struct hdfs_internal *srcFS, const char *src,
+            struct hdfs_internal *dstFS, const char *dst);
+    int (*unlink)(struct hdfs_internal *fs, const char *path, int recursive);
+    int (*rename)(struct hdfs_internal *fs, const char *old_uri,
+            const char* new_uri);
+    char* (*get_working_directory)(struct hdfs_internal *fs, char *buffer,
+            size_t bufferSize);
+    int (*set_working_directory)(struct hdfs_internal *fs, const char* uri);
+    int (*mkdir)(struct hdfs_internal *fs, const char* uri);
+    int (*set_replication)(struct hdfs_internal *fs, const char* uri,
+            int16_t replication);
+    hdfsFileInfo *(*list_directory)(struct hdfs_internal *fs,
+            const char* uri, int *numEntries);
+    hdfsFileInfo *(*get_path_info)(struct hdfs_internal *fs, const char* uri);
+    hdfsFileInfo *(*stat)(struct hdfs_internal *fs, const char* uri);
+    void (*free_file_info)(hdfsFileInfo *, int numEntries);
+    char*** (*get_hosts)(struct hdfs_internal *fs, const char* uri, 
+            int64_t start, int64_t length);
+    int64_t (*get_default_block_size)(struct hdfs_internal *fs);
+    int64_t (*get_default_block_size_at_path)(struct hdfs_internal *fs,
+            const char *uri);
+    int64_t (*get_capacity)(struct hdfs_internal *fs);
+    int64_t (*get_used)(struct hdfs_internal *fs);
+    int (*chown)(struct hdfs_internal *fs, const char *uri, const char *owner,
+            const char *group);
+    int (*chmod)(struct hdfs_internal *fs, const char* uri, short mode);
+    int (*utime)(struct hdfs_internal *fs, const char* uri,
+            int64_t mtime, int64_t atime);
+    struct hadoopRzBuffer* (*read_zero)(struct hdfsFile_internal *file,
+                struct hadoopRzOptions *opts, int32_t maxLength);
+    void (*rz_buffer_free)(struct hdfsFile_internal *file,
+                        struct hadoopRzBuffer *buffer);
+
+    // For testing
+    int (*file_uses_direct_read)(struct hdfsFile_internal *fs);
+    void (*file_disable_direct_read)(struct hdfsFile_internal *file);
+};
+
+#endif
+
+// vim: ts=4:sw=4:et

hadoop-native-core/src/main/native/fs/hdfs.h

@@ -0,0 +1,872 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFS_HDFS_H
+#define LIBHDFS_HDFS_H
+
+#include <errno.h> /* for EINTERNAL, etc. */
+#include <fcntl.h> /* for O_RDONLY, O_WRONLY */
+#include <stdint.h> /* for uint64_t, etc. */
+#include <time.h> /* for time_t */
+
+/**
+ * This is the public API for libhdfs.
+ *
+ * libhdfs allows C programs to interact with Hadoop filesystems.  There are
+ * several existing libhdfs filesystems.
+ *
+ * IMPLEMENTATIONS
+ * NDFS: the pure native HDFS client.  NDFS is an all-C implementation of the
+ * Hadoop Distributed Filesystem client.  Because it does not use a Java virtual
+ * machine, its startup time is very quick.  However, it can only talk to HDFS,
+ * not other filesystems.
+ *
+ * jniFS: JNIfs uses the Hadoop FileSystem.java code to interact with
+ * filesystems.  One way of looking at jniFS is as a window into Hadoop's Java
+ * code.  The advantage of jniFS is that it can do anything that Hadoop's Java
+ * code can do.  The moment someone writes a subclass of
+ * org.apache.hadoop.fs.FileSystem, it is accessible through jniFS.  However,
+ * JNI has many disadvantages.  It takes a while to start up, and may use a lot
+ * of memory, since it needs to start a JVM.  It is difficult to debug a C
+ * program which uses a lot of JNI code.  Partly this is because you don't have
+ * a lot of visibility into the Java part of the program.  Another reason is
+ * because JNI uses signals internally very heavily, which confuses debuggers
+ * like gdb.  jniFS requires the CLASSPATH environment variable to be set up
+ * correctly.  Keep in mind that JNI cannot handle wildcards in CLASSPATH.
+ *
+ * ERROR HANDLING
+ * Most functions here follow the error reporting pattern laid down by POSIX: on
+ * error, they return -1 or NULL, and set errno.  Errno is a thread-local
+ * variable.  It should be checked immediately after the failure is detected,
+ * since many functions from the C standard library overwrite errno-- even
+ * functions like printf.  Also keep in mind that libhdfs will not set errno on
+ * success, so you cannot simply always check errno.  You must only check it if
+ * you know there is a failure.
+ *
+ * TODO: we ought to create a new version of this API which doesn't use errno.
+ * It is difficult for clients correctly deal with errno, and it makes the code
+ * harder to read.
+ *
+ * THREAD-SAFETY
+ * In general, it is thread-safe to use the same filesystem (hdfsFS)  object
+ * from multiple threads at once, as long as you don't call setWorkingDirectory.
+ * You must not use the same file object from more than one thread at once.
+ * In other words, file objects are not thread-safe.
+ *
+ * These guarantees are the same as provided by Filesystem.java implementations
+ * in Java.
+ *
+ * CLEANUP
+ * Every FileSystem object you create with hdfsBuilderConnect should be
+ * eventually cleaned up with hdfsDisconnect.  Similarly, every file you open
+ * should be closed eventually.
+ *
+ * IMPLEMENTER NOTES
+ * Since this is a public API, we have to be careful about making
+ * non-backwards-compatible changes.  It is also important to provide a
+ * deallocation function for each structure that we provide a way to allocate.
+ * We should not assume that the client's implementation of "malloc" and "free"
+ * is the same as that used by our library.  In general, it is often better to
+ * use a forward declaration and accessor functions, rather than declaring
+ * structures in this file.  This leaves us more flexibility for the future.
+ */
+
+#if defined(unix) || defined(__MACH__)
+// Make all functions and structures inside this block externally visible.
+#pragma GCC visibility push(default)
+#endif
+
+#ifndef O_RDONLY
+#define O_RDONLY 1
+#endif
+
+#ifndef O_WRONLY 
+#define O_WRONLY 2
+#endif
+
+#ifndef EINTERNAL
+#define EINTERNAL 255 
+#endif
+
+#define ELASTIC_BYTE_BUFFER_POOL_CLASS \
+  "org/apache/hadoop/io/ElasticByteBufferPool"
+
+#ifdef __cplusplus
+extern  "C" {
+#endif
+    /**
+     * Some utility decls used in libhdfs.
+     */
+    struct hdfsBuilder;
+    typedef int32_t   tSize; /// size of data for read/write io ops 
+    typedef time_t    tTime; /// time type in seconds
+    typedef int64_t   tOffset;/// offset within the file
+    typedef uint16_t  tPort; /// port
+    typedef enum tObjectKind {
+        kObjectKindFile = 'F',
+        kObjectKindDirectory = 'D',
+    } tObjectKind;
+
+
+    /**
+     * The C reflection of org.apache.org.hadoop.FileSystem .
+     */
+    struct hdfs_internal;
+    typedef struct hdfs_internal* hdfsFS;
+    
+    struct hdfsFile_internal;
+    typedef struct hdfsFile_internal* hdfsFile;
+
+    struct hadoopRzOptions;
+
+    struct hadoopRzBuffer;
+
+    /**
+     * Determine if a file is open for read.
+     *
+     * @param file     The HDFS file
+     * @return         1 if the file is open for read; 0 otherwise
+     */
+    int hdfsFileIsOpenForRead(hdfsFile file);
+
+    /**
+     * Determine if a file is open for write.
+     *
+     * @param file     The HDFS file
+     * @return         1 if the file is open for write; 0 otherwise
+     */
+    int hdfsFileIsOpenForWrite(hdfsFile file);
+
+    struct hdfsReadStatistics {
+      uint64_t totalBytesRead;
+      uint64_t totalLocalBytesRead;
+      uint64_t totalShortCircuitBytesRead;
+      uint64_t totalZeroCopyBytesRead;
+    };
+
+    /**
+     * Get read statistics about a file.  This is only applicable to files
+     * opened for reading.
+     *
+     * @param file     The HDFS file
+     * @param stats    (out parameter) on a successful return, the read
+     *                 statistics.  Unchanged otherwise.  You must free the
+     *                 returned statistics with hdfsFileFreeReadStatistics.
+     * @return         0 if the statistics were successfully returned,
+     *                 -1 otherwise.  On a failure, please check errno against
+     *                 ENOTSUP.  webhdfs, LocalFilesystem, and so forth may
+     *                 not support read statistics.
+     */
+    int hdfsFileGetReadStatistics(hdfsFile file,
+                                  struct hdfsReadStatistics **stats);
+
+    /**
+     * @param stats    HDFS read statistics for a file.
+     *
+     * @return the number of remote bytes read.
+     */
+    int64_t hdfsReadStatisticsGetRemoteBytesRead(
+                            const struct hdfsReadStatistics *stats);
+
+    /**
+     * Free some HDFS read statistics.
+     *
+     * @param stats    The HDFS read statistics to free.
+     */
+    void hdfsFileFreeReadStatistics(struct hdfsReadStatistics *stats);
+
+    /** 
+     * hdfsConnectAsUser - Connect to a hdfs file system as a specific user
+     * Connect to the hdfs.
+     * @param nn   The NameNode.  See hdfsBuilderSetNameNode for details.
+     * @param port The port on which the server is listening.
+     * @param user the user name (this is hadoop domain user). Or NULL is equivelant to hhdfsConnect(host, port)
+     * @return Returns a handle to the filesystem or NULL on error.
+     * @deprecated Use hdfsBuilderConnect instead. 
+     */
+     hdfsFS hdfsConnectAsUser(const char* nn, tPort port, const char *user);
+
+    /** 
+     * hdfsConnect - Connect to a hdfs file system.
+     * Connect to the hdfs.
+     * @param nn   The NameNode.  See hdfsBuilderSetNameNode for details.
+     * @param port The port on which the server is listening.
+     * @return Returns a handle to the filesystem or NULL on error.
+     * @deprecated Use hdfsBuilderConnect instead. 
+     */
+     hdfsFS hdfsConnect(const char* nn, tPort port);
+
+    /** 
+     * hdfsConnect - Connect to an hdfs file system.
+     *
+     * Forces a new instance to be created
+     *
+     * @param nn     The NameNode.  See hdfsBuilderSetNameNode for details.
+     * @param port   The port on which the server is listening.
+     * @param user   The user name to use when connecting
+     * @return       Returns a handle to the filesystem or NULL on error.
+     * @deprecated   Use hdfsBuilderConnect instead. 
+     */
+     hdfsFS hdfsConnectAsUserNewInstance(const char* nn, tPort port, const char *user );
+
+    /** 
+     * hdfsConnect - Connect to an hdfs file system.
+     *
+     * Forces a new instance to be created
+     *
+     * @param nn     The NameNode.  See hdfsBuilderSetNameNode for details.
+     * @param port   The port on which the server is listening.
+     * @return       Returns a handle to the filesystem or NULL on error.
+     * @deprecated   Use hdfsBuilderConnect instead. 
+     */
+     hdfsFS hdfsConnectNewInstance(const char* nn, tPort port);
+
+    /** 
+     * Connect to HDFS using the parameters defined by the builder.
+     *
+     * The HDFS builder will be freed, whether or not the connection was
+     * successful.
+     *
+     * Every successful call to hdfsBuilderConnect should be matched with a call
+     * to hdfsDisconnect, when the hdfsFS is no longer needed.
+     *
+     * @param bld    The HDFS builder
+     * @return       Returns a handle to the filesystem, or NULL on error.
+     */
+     hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld);
+
+    /**
+     * Create an HDFS builder.
+     *
+     * @return The HDFS builder, or NULL on error.
+     */
+    struct hdfsBuilder *hdfsNewBuilder(void);
+
+    /**
+     * Force the builder to always create a new instance of the FileSystem,
+     * rather than possibly finding one in the cache.
+     *
+     * @param bld The HDFS builder
+     */
+    void hdfsBuilderSetForceNewInstance(struct hdfsBuilder *bld);
+
+    /**
+     * Set the HDFS NameNode to connect to.
+     *
+     * @param bld  The HDFS builder
+     * @param nn   The NameNode to use.
+     *
+     *             If the string given is 'default', the default NameNode
+     *             configuration will be used (from the XML configuration files)
+     *
+     *             If NULL is given, a LocalFileSystem will be created.
+     *
+     *             If the string starts with a protocol type such as file:// or
+     *             hdfs://, this protocol type will be used.  If not, the
+     *             hdfs:// protocol type will be used.
+     *
+     *             You may specify a NameNode port in the usual way by 
+     *             passing a string of the format hdfs://<hostname>:<port>.
+     *             Alternately, you may set the port with
+     *             hdfsBuilderSetNameNodePort.  However, you must not pass the
+     *             port in two different ways.
+     */
+    void hdfsBuilderSetNameNode(struct hdfsBuilder *bld, const char *nn);
+
+    /**
+     * Set the port of the HDFS NameNode to connect to.
+     *
+     * @param bld The HDFS builder
+     * @param port The port.
+     */
+    void hdfsBuilderSetNameNodePort(struct hdfsBuilder *bld, tPort port);
+
+    /**
+     * Set the username to use when connecting to the HDFS cluster.
+     *
+     * @param bld The HDFS builder
+     * @param userName The user name.  The string will be shallow-copied.
+     */
+    void hdfsBuilderSetUserName(struct hdfsBuilder *bld, const char *userName);
+
+    /**
+     * Set the path to the Kerberos ticket cache to use when connecting to
+     * the HDFS cluster.
+     *
+     * @param bld The HDFS builder
+     * @param kerbTicketCachePath The Kerberos ticket cache path.  The string
+     *                            will be shallow-copied.
+     */
+    void hdfsBuilderSetKerbTicketCachePath(struct hdfsBuilder *bld,
+                                   const char *kerbTicketCachePath);
+
+    /**
+     * Free an HDFS builder.
+     *
+     * It is normally not necessary to call this function since
+     * hdfsBuilderConnect frees the builder.
+     *
+     * @param bld The HDFS builder
+     */
+    void hdfsFreeBuilder(struct hdfsBuilder *bld);
+
+    /**
+     * Set a configuration string for an HdfsBuilder.
+     *
+     * @param key      The key to set.
+     * @param val      The value, or NULL to set no value.
+     *                 This will be shallow-copied.  You are responsible for
+     *                 ensuring that it remains valid until the builder is
+     *                 freed.
+     *
+     * @return         0 on success; nonzero error code otherwise.
+     */
+    int hdfsBuilderConfSetStr(struct hdfsBuilder *bld, const char *key,
+                              const char *val);
+
+    /**
+     * Get a configuration string.
+     *
+     * @param key      The key to find
+     * @param val      (out param) The value.  This will be set to NULL if the
+     *                 key isn't found.  You must free this string with
+     *                 hdfsConfStrFree.
+     *
+     * @return         0 on success; nonzero error code otherwise.
+     *                 Failure to find the key is not an error.
+     */
+    int hdfsConfGetStr(const char *key, char **val);
+
+    /**
+     * Get a configuration integer.
+     *
+     * @param key      The key to find
+     * @param val      (out param) The value.  This will NOT be changed if the
+     *                 key isn't found.
+     *
+     * @return         0 on success; nonzero error code otherwise.
+     *                 Failure to find the key is not an error.
+     */
+    int hdfsConfGetInt(const char *key, int32_t *val);
+
+    /**
+     * Free a configuration string found with hdfsConfGetStr. 
+     *
+     * @param val      A configuration string obtained from hdfsConfGetStr
+     */
+    void hdfsConfStrFree(char *val);
+
+    /** 
+     * hdfsDisconnect - Disconnect from the hdfs file system.
+     * Disconnect from hdfs.
+     * @param fs The configured filesystem handle.
+     * @return Returns 0 on success, -1 on error.
+     *         Even if there is an error, the resources associated with the
+     *         hdfsFS will be freed.
+     */
+    int hdfsDisconnect(hdfsFS fs);
+        
+
+    /** 
+     * hdfsOpenFile - Open a hdfs file in given mode.
+     * @param fs The configured filesystem handle.
+     * @param path The full path to the file.
+     * @param flags - an | of bits/fcntl.h file flags - supported flags are O_RDONLY, O_WRONLY (meaning create or overwrite i.e., implies O_TRUNCAT), 
+     * O_WRONLY|O_APPEND. Other flags are generally ignored other than (O_RDWR || (O_EXCL & O_CREAT)) which return NULL and set errno equal ENOTSUP.
+     * @param bufferSize Size of buffer for read/write - pass 0 if you want
+     * to use the default configured values.
+     * @param replication Block replication - pass 0 if you want to use
+     * the default configured values.
+     * @param blocksize Size of block - pass 0 if you want to use the
+     * default configured values.
+     * @return Returns the handle to the open file or NULL on error.
+     */
+    hdfsFile hdfsOpenFile(hdfsFS fs, const char* path, int flags,
+                          int bufferSize, short replication, tSize blocksize);
+
+
+    /** 
+     * hdfsCloseFile - Close an open file. 
+     * @param fs The configured filesystem handle.
+     * @param file The file handle.
+     * @return Returns 0 on success, -1 on error.  
+     *         On error, errno will be set appropriately.
+     *         If the hdfs file was valid, the memory associated with it will
+     *         be freed at the end of this call, even if there was an I/O
+     *         error.
+     */
+    int hdfsCloseFile(hdfsFS fs, hdfsFile file);
+
+
+    /** 
+     * hdfsExists - Checks if a given path exsits on the filesystem 
+     * @param fs The configured filesystem handle.
+     * @param path The path to look for
+     * @return Returns 0 on success, -1 on error.  
+     */
+    int hdfsExists(hdfsFS fs, const char *path);
+
+
+    /** 
+     * hdfsSeek - Seek to given offset in file. 
+     * This works only for files opened in read-only mode. 
+     * @param fs The configured filesystem handle.
+     * @param file The file handle.
+     * @param desiredPos Offset into the file to seek into.
+     * @return Returns 0 on success, -1 on error.  
+     */
+    int hdfsSeek(hdfsFS fs, hdfsFile file, tOffset desiredPos); 
+
+
+    /** 
+     * hdfsTell - Get the current offset in the file, in bytes.
+     * @param fs The configured filesystem handle.
+     * @param file The file handle.
+     * @return Current offset, -1 on error.
+     */
+    tOffset hdfsTell(hdfsFS fs, hdfsFile file);
+
+
+    /** 
+     * hdfsRead - Read data from an open file.
+     * @param fs The configured filesystem handle.
+     * @param file The file handle.
+     * @param buffer The buffer to copy read bytes into.
+     * @param length The length of the buffer.
+     * @return      On success, a positive number indicating how many bytes
+     *              were read.
+     *              On end-of-file, 0.
+     *              On error, -1.  Errno will be set to the error code.
+     *              Just like the POSIX read function, hdfsRead will return -1
+     *              and set errno to EINTR if data is temporarily unavailable,
+     *              but we are not yet at the end of the file.
+     */
+    tSize hdfsRead(hdfsFS fs, hdfsFile file, void* buffer, tSize length);
+
+    /** 
+     * hdfsPread - Positional read of data from an open file.
+     * @param fs The configured filesystem handle.
+     * @param file The file handle.
+     * @param position Position from which to read
+     * @param buffer The buffer to copy read bytes into.
+     * @param length The length of the buffer.
+     * @return      See hdfsRead
+     */
+    tSize hdfsPread(hdfsFS fs, hdfsFile file, tOffset position,
+                    void* buffer, tSize length);
+
+
+    /** 
+     * hdfsWrite - Write data into an open file.
+     * @param fs The configured filesystem handle.
+     * @param file The file handle.
+     * @param buffer The data.
+     * @param length The no. of bytes to write. 
+     * @return Returns the number of bytes written, -1 on error.
+     */
+    tSize hdfsWrite(hdfsFS fs, hdfsFile file, const void* buffer,
+                    tSize length);
+
+
+    /** 
+     * hdfsWrite - Flush the data. 
+     * @param fs The configured filesystem handle.
+     * @param file The file handle.
+     * @return Returns 0 on success, -1 on error. 
+     */
+    int hdfsFlush(hdfsFS fs, hdfsFile file);
+
+
+    /**
+     * hdfsHFlush - Flush out the data in client's user buffer. After the
+     * return of this call, new readers will see the data.
+     * @param fs configured filesystem handle
+     * @param file file handle
+     * @return 0 on success, -1 on error and sets errno
+     */
+    int hdfsHFlush(hdfsFS fs, hdfsFile file);
+
+
+    /**
+     * hdfsHSync - Similar to posix fsync, Flush out the data in client's 
+     * user buffer. all the way to the disk device (but the disk may have 
+     * it in its cache).
+     * @param fs configured filesystem handle
+     * @param file file handle
+     * @return 0 on success, -1 on error and sets errno
+     */
+    int hdfsHSync(hdfsFS fs, hdfsFile file);
+
+
+    /**
+     * hdfsAvailable - Number of bytes that can be read from this
+     * input stream without blocking.
+     * @param fs The configured filesystem handle.
+     * @param file The file handle.
+     * @return Returns available bytes; -1 on error. 
+     */
+    int hdfsAvailable(hdfsFS fs, hdfsFile file);
+
+
+    /**
+     * hdfsCopy - Copy file from one filesystem to another.
+     * @param srcFS The handle to source filesystem.
+     * @param src The path of source file. 
+     * @param dstFS The handle to destination filesystem.
+     * @param dst The path of destination file. 
+     * @return Returns 0 on success, -1 on error. 
+     */
+    int hdfsCopy(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst);
+
+
+    /**
+     * hdfsMove - Move file from one filesystem to another.
+     * @param srcFS The handle to source filesystem.
+     * @param src The path of source file. 
+     * @param dstFS The handle to destination filesystem.
+     * @param dst The path of destination file. 
+     * @return Returns 0 on success, -1 on error. 
+     */
+    int hdfsMove(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst);
+
+
+    /**
+     * hdfsDelete - Delete file. 
+     * @param fs The configured filesystem handle.
+     * @param path The path of the file. 
+     * @param recursive if path is a directory and set to 
+     * non-zero, the directory is deleted else throws an exception. In
+     * case of a file the recursive argument is irrelevant.
+     * @return Returns 0 on success, -1 on error. 
+     */
+    int hdfsDelete(hdfsFS fs, const char* path, int recursive);
+
+    /**
+     * hdfsRename - Rename file. 
+     * @param fs The configured filesystem handle.
+     * @param oldPath The path of the source file. 
+     * @param newPath The path of the destination file. 
+     * @return Returns 0 on success, -1 on error. 
+     */
+    int hdfsRename(hdfsFS fs, const char* oldPath, const char* newPath);
+
+
+    /** 
+     * hdfsGetWorkingDirectory - Get the current working directory for
+     * the given filesystem.
+     * @param fs The configured filesystem handle.
+     * @param buffer The user-buffer to copy path of cwd into. 
+     * @param bufferSize The length of user-buffer.
+     * @return Returns buffer, NULL on error.
+     */
+    char* hdfsGetWorkingDirectory(hdfsFS fs, char *buffer, size_t bufferSize);
+
+
+    /** 
+     * hdfsSetWorkingDirectory - Set the working directory. All relative
+     * paths will be resolved relative to it.
+     * @param fs The configured filesystem handle.
+     * @param path The path of the new 'cwd'. 
+     * @return Returns 0 on success, -1 on error. 
+     */
+    int hdfsSetWorkingDirectory(hdfsFS fs, const char* path);
+
+
+    /** 
+     * hdfsCreateDirectory - Make the given file and all non-existent
+     * parents into directories.
+     * @param fs The configured filesystem handle.
+     * @param path The path of the directory. 
+     * @return Returns 0 on success, -1 on error. 
+     */
+    int hdfsCreateDirectory(hdfsFS fs, const char* path);
+
+
+    /** 
+     * hdfsSetReplication - Set the replication of the specified
+     * file to the supplied value
+     * @param fs The configured filesystem handle.
+     * @param path The path of the file. 
+     * @return Returns 0 on success, -1 on error. 
+     */
+    int hdfsSetReplication(hdfsFS fs, const char* path, int16_t replication);
+
+
+    /** 
+     * hdfsFileInfo - Information about a file/directory.
+     */
+    typedef struct file_info  {
+        tObjectKind mKind;   /* file or directory */
+        char *mName;         /* the name of the file */
+        tTime mLastMod;      /* the last modification time for the file in seconds */
+        tOffset mSize;       /* the size of the file in bytes */
+        short mReplication;    /* the count of replicas */
+        tOffset mBlockSize;  /* the block size for the file */
+        char *mOwner;        /* the owner of the file */
+        char *mGroup;        /* the group associated with the file */
+        short mPermissions;  /* the permissions associated with the file */
+        tTime mLastAccess;    /* the last access time for the file in seconds */
+    } hdfsFileInfo;
+
+
+    /** 
+     * hdfsListDirectory - Get list of files/directories for a given
+     * directory-path. hdfsFreeFileInfo should be called to deallocate memory. 
+     * @param fs The configured filesystem handle.
+     * @param path The path of the directory. 
+     * @param numEntries Set to the number of files/directories in path.
+     * @return Returns a dynamically-allocated array of hdfsFileInfo
+     * objects; NULL on error.
+     */
+    hdfsFileInfo *hdfsListDirectory(hdfsFS fs, const char* path,
+                                    int *numEntries);
+
+
+    /** 
+     * hdfsGetPathInfo - Get information about a path as a (dynamically
+     * allocated) single hdfsFileInfo struct. hdfsFreeFileInfo should be
+     * called when the pointer is no longer needed.
+     * @param fs The configured filesystem handle.
+     * @param path The path of the file. 
+     * @return Returns a dynamically-allocated hdfsFileInfo object;
+     * NULL on error.
+     */
+    hdfsFileInfo *hdfsGetPathInfo(hdfsFS fs, const char* path);
+
+
+    /** 
+     * hdfsFreeFileInfo - Free up the hdfsFileInfo array (including fields) 
+     * @param hdfsFileInfo The array of dynamically-allocated hdfsFileInfo
+     * objects.
+     * @param numEntries The size of the array.
+     */
+    void hdfsFreeFileInfo(hdfsFileInfo *hdfsFileInfo, int numEntries);
+
+
+    /** 
+     * hdfsGetHosts - Get hostnames where a particular block (determined by
+     * pos & blocksize) of a file is stored. The last element in the array
+     * is NULL. Due to replication, a single block could be present on
+     * multiple hosts.
+     * @param fs The configured filesystem handle.
+     * @param path The path of the file. 
+     * @param start The start of the block.
+     * @param length The length of the block.
+     * @return Returns a dynamically-allocated 2-d array of blocks-hosts;
+     * NULL on error.
+     */
+    char*** hdfsGetHosts(hdfsFS fs, const char* path, 
+            tOffset start, tOffset length);
+
+
+    /** 
+     * hdfsFreeHosts - Free up the structure returned by hdfsGetHosts
+     * @param hdfsFileInfo The array of dynamically-allocated hdfsFileInfo
+     * objects.
+     * @param numEntries The size of the array.
+     */
+    void hdfsFreeHosts(char ***blockHosts);
+
+
+    /** 
+     * hdfsGetDefaultBlockSize - Get the default blocksize.
+     *
+     * @param fs            The configured filesystem handle.
+     * @deprecated          Use hdfsGetDefaultBlockSizeAtPath instead.
+     *
+     * @return              Returns the default blocksize, or -1 on error.
+     */
+    tOffset hdfsGetDefaultBlockSize(hdfsFS fs);
+
+
+    /** 
+     * hdfsGetDefaultBlockSizeAtPath - Get the default blocksize at the
+     * filesystem indicated by a given path.
+     *
+     * @param fs            The configured filesystem handle.
+     * @param path          The given path will be used to locate the actual
+     *                      filesystem.  The full path does not have to exist.
+     *
+     * @return              Returns the default blocksize, or -1 on error.
+     */
+    tOffset hdfsGetDefaultBlockSizeAtPath(hdfsFS fs, const char *path);
+
+
+    /** 
+     * hdfsGetCapacity - Return the raw capacity of the filesystem.  
+     * @param fs The configured filesystem handle.
+     * @return Returns the raw-capacity; -1 on error. 
+     */
+    tOffset hdfsGetCapacity(hdfsFS fs);
+
+
+    /** 
+     * hdfsGetUsed - Return the total raw size of all files in the filesystem.
+     * @param fs The configured filesystem handle.
+     * @return Returns the total-size; -1 on error. 
+     */
+    tOffset hdfsGetUsed(hdfsFS fs);
+
+    /** 
+     * Change the user and/or group of a file or directory.
+     *
+     * @param fs            The configured filesystem handle.
+     * @param path          the path to the file or directory
+     * @param owner         User string.  Set to NULL for 'no change'
+     * @param group         Group string.  Set to NULL for 'no change'
+     * @return              0 on success else -1
+     */
+    int hdfsChown(hdfsFS fs, const char* path, const char *owner,
+                  const char *group);
+
+    /** 
+     * hdfsChmod
+     * @param fs The configured filesystem handle.
+     * @param path the path to the file or directory
+     * @param mode the bitmask to set it to
+     * @return 0 on success else -1
+     */
+      int hdfsChmod(hdfsFS fs, const char* path, short mode);
+
+    /** 
+     * hdfsUtime
+     * @param fs The configured filesystem handle.
+     * @param path the path to the file or directory
+     * @param mtime new modification time or -1 for no change
+     * @param atime new access time or -1 for no change
+     * @return 0 on success else -1
+     */
+    int hdfsUtime(hdfsFS fs, const char* path, tTime mtime, tTime atime);
+
+    /**
+     * Allocate a zero-copy options structure.
+     *
+     * You must free all options structures allocated with this function using
+     * hadoopRzOptionsFree.
+     *
+     * @return            A zero-copy options structure, or NULL if one could
+     *                    not be allocated.  If NULL is returned, errno will
+     *                    contain the error number.
+     */
+    struct hadoopRzOptions *hadoopRzOptionsAlloc(void);
+
+    /**
+     * Determine whether we should skip checksums in read0.
+     *
+     * @param opts        The options structure.
+     * @param skip        Nonzero to skip checksums sometimes; zero to always
+     *                    check them.
+     *
+     * @return            0 on success; -1 plus errno on failure.
+     */
+    int hadoopRzOptionsSetSkipChecksum(
+            struct hadoopRzOptions *opts, int skip);
+
+    /**
+     * Set the ByteBufferPool to use with read0.
+     *
+     * @param opts        The options structure.
+     * @param className   If this is NULL, we will not use any
+     *                    ByteBufferPool.  If this is non-NULL, it will be
+     *                    treated as the name of the pool class to use.
+     *                    For example, you can use
+     *                    ELASTIC_BYTE_BUFFER_POOL_CLASS.
+     *
+     * @return            0 if the ByteBufferPool class was found and
+     *                    instantiated;
+     *                    -1 plus errno otherwise.
+     */
+    int hadoopRzOptionsSetByteBufferPool(
+            struct hadoopRzOptions *opts, const char *className);
+
+    /**
+     * Free a hadoopRzOptionsFree structure.
+     *
+     * @param opts        The options structure to free.
+     *                    Any associated ByteBufferPool will also be freed.
+     */
+    void hadoopRzOptionsFree(struct hadoopRzOptions *opts);
+
+    /**
+     * Perform a byte buffer read.
+     * If possible, this will be a zero-copy (mmap) read.
+     *
+     * @param file       The file to read from.
+     * @param opts       An options structure created by hadoopRzOptionsAlloc.
+     * @param maxLength  The maximum length to read.  We may read fewer bytes
+     *                   than this length.
+     *
+     * @return           On success, we will return a new hadoopRzBuffer.
+     *                   This buffer will continue to be valid and readable
+     *                   until it is released by readZeroBufferFree.  Failure to
+     *                   release a buffer will lead to a memory leak.
+     *                   You can access the data within the hadoopRzBuffer with
+     *                   hadoopRzBufferGet.  If you have reached EOF, the data
+     *                   within the hadoopRzBuffer will be NULL.  You must still
+     *                   free hadoopRzBuffer instances containing NULL.
+     *
+     *                   On failure, we will return NULL plus an errno code.
+     *                   errno = EOPNOTSUPP indicates that we could not do a
+     *                   zero-copy read, and there was no ByteBufferPool
+     *                   supplied.
+     */
+    struct hadoopRzBuffer* hadoopReadZero(hdfsFile file,
+            struct hadoopRzOptions *opts, int32_t maxLength);
+
+    /**
+     * Determine the length of the buffer returned from readZero.
+     *
+     * @param buffer     a buffer returned from readZero.
+     * @return           the length of the buffer.
+     */
+    int32_t hadoopRzBufferLength(const struct hadoopRzBuffer *buffer);
+
+    /**
+     * Get a pointer to the raw buffer returned from readZero.
+     *
+     * To find out how many bytes this buffer contains, call
+     * hadoopRzBufferLength.
+     *
+     * @param buffer     a buffer returned from readZero.
+     * @return           a pointer to the start of the buffer.  This will be
+     *                   NULL when end-of-file has been reached.
+     */
+    const void *hadoopRzBufferGet(const struct hadoopRzBuffer *buffer);
+
+    /**
+     * Release a buffer obtained through readZero.
+     *
+     * @param file       The hdfs stream that created this buffer.  This must be
+     *                   the same stream you called hadoopReadZero on.
+     * @param buffer     The buffer to release.
+     */
+    void hadoopRzBufferFree(hdfsFile file, struct hadoopRzBuffer *buffer);
+
+#ifdef __cplusplus
+}
+#endif
+
+#if defined(unix) || defined(__MACH__)
+#pragma GCC visibility pop
+#endif
+
+#endif /*LIBHDFS_HDFS_H*/
+
+/**
+ * vim: ts=4: sw=4: et
+ */

hadoop-native-core/src/main/native/fs/hdfs_test.h

@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFS_HDFS_TEST_H
+#define LIBHDFS_HDFS_TEST_H
+
+struct hdfsFile_internal;
+
+/**
+ * Some functions that are visible only for testing.
+ *
+ * This header is not meant to be exported or used outside of the libhdfs unit
+ * tests.
+ */
+
+#ifdef __cplusplus
+extern  "C" {
+#endif
+    /**
+     * Determine if a file is using the "direct read" optimization.
+     *
+     * @param file     The HDFS file
+     * @return         1 if the file is using the direct read optimization,
+     *                 0 otherwise.
+     */
+    int hdfsFileUsesDirectRead(struct hdfsFile_internal *file);
+
+    /**
+     * Disable the direct read optimization for a file.
+     *
+     * This is mainly provided for unit testing purposes.
+     *
+     * @param file     The HDFS file
+     */
+    void hdfsFileDisableDirectRead(struct hdfsFile_internal *file);
+#ifdef __cplusplus
+}
+#endif
+
+#endif

hadoop-native-core/src/main/native/jni/exception.c

@@ -0,0 +1,234 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fs/hdfs.h"
+#include "jni/exception.h"
+#include "jni/jni_helper.h"
+
+#include <inttypes.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define EXCEPTION_INFO_LEN (sizeof(gExceptionInfo)/sizeof(gExceptionInfo[0]))
+
+struct ExceptionInfo {
+    const char * const name;
+    int noPrintFlag;
+    int excErrno;
+};
+
+static const struct ExceptionInfo gExceptionInfo[] = {
+    {
+        .name = "java.io.FileNotFoundException",
+        .noPrintFlag = NOPRINT_EXC_FILE_NOT_FOUND,
+        .excErrno = ENOENT,
+    },
+    {
+        .name = "org.apache.hadoop.security.AccessControlException",
+        .noPrintFlag = NOPRINT_EXC_ACCESS_CONTROL,
+        .excErrno = EACCES,
+    },
+    {
+        .name = "org.apache.hadoop.fs.UnresolvedLinkException",
+        .noPrintFlag = NOPRINT_EXC_UNRESOLVED_LINK,
+        .excErrno = ENOLINK,
+    },
+    {
+        .name = "org.apache.hadoop.fs.ParentNotDirectoryException",
+        .noPrintFlag = NOPRINT_EXC_PARENT_NOT_DIRECTORY,
+        .excErrno = ENOTDIR,
+    },
+    {
+        .name = "java.lang.IllegalArgumentException",
+        .noPrintFlag = NOPRINT_EXC_ILLEGAL_ARGUMENT,
+        .excErrno = EINVAL,
+    },
+    {
+        .name = "java.lang.OutOfMemoryError",
+        .noPrintFlag = 0,
+        .excErrno = ENOMEM,
+    },
+    {
+        .name = "org.apache.hadoop.hdfs.server.namenode.SafeModeException",
+        .noPrintFlag = 0,
+        .excErrno = EROFS,
+    },
+    {
+        .name = "org.apache.hadoop.fs.FileAlreadyExistsException",
+        .noPrintFlag = 0,
+        .excErrno = EEXIST,
+    },
+    {
+        .name = "org.apache.hadoop.hdfs.protocol.QuotaExceededException",
+        .noPrintFlag = 0,
+        .excErrno = EDQUOT,
+    },
+    {
+        .name = "org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException",
+        .noPrintFlag = 0,
+        .excErrno = ESTALE,
+    },
+};
+
+void getExceptionInfo(const char *excName, int noPrintFlags,
+                      int *excErrno, int *shouldPrint)
+{
+    size_t i;
+
+    for (i = 0; i < EXCEPTION_INFO_LEN; i++) {
+        if (strstr(gExceptionInfo[i].name, excName)) {
+            break;
+        }
+    }
+    if (i < EXCEPTION_INFO_LEN) {
+        *shouldPrint = !(gExceptionInfo[i].noPrintFlag & noPrintFlags);
+        *excErrno = gExceptionInfo[i].excErrno;
+    } else {
+        *shouldPrint = 1;
+        *excErrno = EINTERNAL;
+    }
+}
+
+int printExceptionAndFreeV(JNIEnv *env, jthrowable exc, int noPrintFlags,
+        const char *fmt, va_list ap)
+{
+    size_t i;
+    int noPrint, excErrno;
+    char *className = NULL;
+    jstring jStr = NULL;
+    jvalue jVal;
+    jthrowable jthr;
+
+    jthr = classNameOfObject(exc, env, &className);
+    if (jthr) {
+        fprintf(stderr, "PrintExceptionAndFree: error determining class name "
+            "of exception.\n");
+        className = strdup("(unknown)");
+        destroyLocalReference(env, jthr);
+    }
+    for (i = 0; i < EXCEPTION_INFO_LEN; i++) {
+        if (!strcmp(gExceptionInfo[i].name, className)) {
+            break;
+        }
+    }
+    if (i < EXCEPTION_INFO_LEN) {
+        noPrint = (gExceptionInfo[i].noPrintFlag & noPrintFlags);
+        excErrno = gExceptionInfo[i].excErrno;
+    } else {
+        noPrint = 0;
+        excErrno = EINTERNAL;
+    }
+    if (!noPrint) {
+        vfprintf(stderr, fmt, ap);
+        fprintf(stderr, " error:\n");
+
+        // We don't want to  use ExceptionDescribe here, because that requires a
+        // pending exception.  Instead, use ExceptionUtils.
+        jthr = invokeMethod(env, &jVal, STATIC, NULL, 
+            "org/apache/commons/lang/exception/ExceptionUtils",
+            "getStackTrace", "(Ljava/lang/Throwable;)Ljava/lang/String;", exc);
+        if (jthr) {
+            fprintf(stderr, "(unable to get stack trace for %s exception: "
+                    "ExceptionUtils::getStackTrace error.)\n", className);
+            destroyLocalReference(env, jthr);
+        } else {
+            jStr = jVal.l;
+            const char *stackTrace = (*env)->GetStringUTFChars(env, jStr, NULL);
+            if (!stackTrace) {
+                fprintf(stderr, "(unable to get stack trace for %s exception: "
+                        "GetStringUTFChars error.)\n", className);
+            } else {
+                fprintf(stderr, "%s", stackTrace);
+                (*env)->ReleaseStringUTFChars(env, jStr, stackTrace);
+            }
+        }
+    }
+    destroyLocalReference(env, jStr);
+    destroyLocalReference(env, exc);
+    free(className);
+    return excErrno;
+}
+
+int printExceptionAndFree(JNIEnv *env, jthrowable exc, int noPrintFlags,
+        const char *fmt, ...)
+{
+    va_list ap;
+    int ret;
+
+    va_start(ap, fmt);
+    ret = printExceptionAndFreeV(env, exc, noPrintFlags, fmt, ap);
+    va_end(ap);
+    return ret;
+}
+
+int printPendingExceptionAndFree(JNIEnv *env, int noPrintFlags,
+        const char *fmt, ...)
+{
+    va_list ap;
+    int ret;
+    jthrowable exc;
+
+    exc = (*env)->ExceptionOccurred(env);
+    if (!exc) {
+        va_start(ap, fmt);
+        vfprintf(stderr, fmt, ap);
+        va_end(ap);
+        fprintf(stderr, " error: (no exception)");
+        ret = 0;
+    } else {
+        (*env)->ExceptionClear(env);
+        va_start(ap, fmt);
+        ret = printExceptionAndFreeV(env, exc, noPrintFlags, fmt, ap);
+        va_end(ap);
+    }
+    return ret;
+}
+
+jthrowable getPendingExceptionAndClear(JNIEnv *env)
+{
+    jthrowable jthr = (*env)->ExceptionOccurred(env);
+    if (!jthr)
+        return NULL;
+    (*env)->ExceptionClear(env);
+    return jthr;
+}
+
+jthrowable newRuntimeError(JNIEnv *env, const char *fmt, ...)
+{
+    char buf[512];
+    jobject out, exc;
+    jstring jstr;
+    va_list ap;
+
+    va_start(ap, fmt);
+    vsnprintf(buf, sizeof(buf), fmt, ap);
+    va_end(ap);
+    jstr = (*env)->NewStringUTF(env, buf);
+    if (!jstr) {
+        // We got an out of memory exception rather than a RuntimeException.
+        // Too bad...
+        return getPendingExceptionAndClear(env);
+    }
+    exc = constructNewObjectOfClass(env, &out, "RuntimeException",
+        "(java/lang/String;)V", jstr);
+    (*env)->DeleteLocalRef(env, jstr);
+    // Again, we'll either get an out of memory exception or the
+    // RuntimeException we wanted.
+    return (exc) ? exc : out;
+}

hadoop-native-core/src/main/native/jni/exception.h

@@ -0,0 +1,155 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFS_EXCEPTION_H
+#define LIBHDFS_EXCEPTION_H
+
+/**
+ * Exception handling routines for libhdfs.
+ *
+ * The convention we follow here is to clear pending exceptions as soon as they
+ * are raised.  Never assume that the caller of your function will clean up
+ * after you-- do it yourself.  Unhandled exceptions can lead to memory leaks
+ * and other undefined behavior.
+ *
+ * If you encounter an exception, return a local reference to it.  The caller is
+ * responsible for freeing the local reference, by calling a function like
+ * PrintExceptionAndFree.  (You can also free exceptions directly by calling
+ * DeleteLocalRef.  However, that would not produce an error message, so it's
+ * usually not what you want.)
+ */
+
+#include <jni.h>
+#include <stdio.h>
+
+#include <stdlib.h>
+#include <stdarg.h>
+#include <search.h>
+#include <pthread.h>
+#include <errno.h>
+
+/**
+ * Exception noprint flags
+ *
+ * Theses flags determine which exceptions should NOT be printed to stderr by
+ * the exception printing routines.  For example, if you expect to see
+ * FileNotFound, you might use NOPRINT_EXC_FILE_NOT_FOUND, to avoid filling the
+ * logs with messages about routine events.
+ *
+ * On the other hand, if you don't expect any failures, you might pass
+ * PRINT_EXC_ALL.
+ *
+ * You can OR these flags together to avoid printing multiple classes of
+ * exceptions.
+ */
+#define PRINT_EXC_ALL                           0x00
+#define NOPRINT_EXC_FILE_NOT_FOUND              0x01
+#define NOPRINT_EXC_ACCESS_CONTROL              0x02
+#define NOPRINT_EXC_UNRESOLVED_LINK             0x04
+#define NOPRINT_EXC_PARENT_NOT_DIRECTORY        0x08
+#define NOPRINT_EXC_ILLEGAL_ARGUMENT            0x10
+
+/**
+ * Get information about an exception.
+ *
+ * @param excName         The Exception name.
+ *                        This is a Java class name in JNI format.
+ * @param noPrintFlags    Flags which determine which exceptions we should NOT
+ *                        print.
+ * @param excErrno        (out param) The POSIX error number associated with the
+ *                        exception.
+ * @param shouldPrint     (out param) Nonzero if we should print this exception,
+ *                        based on the noPrintFlags and its name. 
+ */
+void getExceptionInfo(const char *excName, int noPrintFlags,
+                      int *excErrno, int *shouldPrint);
+
+/**
+ * Print out information about an exception and free it.
+ *
+ * @param env             The JNI environment
+ * @param exc             The exception to print and free
+ * @param noPrintFlags    Flags which determine which exceptions we should NOT
+ *                        print.
+ * @param fmt             Printf-style format list
+ * @param ap              Printf-style varargs
+ *
+ * @return                The POSIX error number associated with the exception
+ *                        object.
+ */
+int printExceptionAndFreeV(JNIEnv *env, jthrowable exc, int noPrintFlags,
+        const char *fmt, va_list ap);
+
+/**
+ * Print out information about an exception and free it.
+ *
+ * @param env             The JNI environment
+ * @param exc             The exception to print and free
+ * @param noPrintFlags    Flags which determine which exceptions we should NOT
+ *                        print.
+ * @param fmt             Printf-style format list
+ * @param ...             Printf-style varargs
+ *
+ * @return                The POSIX error number associated with the exception
+ *                        object.
+ */
+int printExceptionAndFree(JNIEnv *env, jthrowable exc, int noPrintFlags,
+        const char *fmt, ...) __attribute__((format(printf, 4, 5)));  
+
+/**
+ * Print out information about the pending exception and free it.
+ *
+ * @param env             The JNI environment
+ * @param noPrintFlags    Flags which determine which exceptions we should NOT
+ *                        print.
+ * @param fmt             Printf-style format list
+ * @param ...             Printf-style varargs
+ *
+ * @return                The POSIX error number associated with the exception
+ *                        object.
+ */
+int printPendingExceptionAndFree(JNIEnv *env, int noPrintFlags,
+        const char *fmt, ...) __attribute__((format(printf, 3, 4)));  
+
+/**
+ * Get a local reference to the pending exception and clear it.
+ *
+ * Once it is cleared, the exception will no longer be pending.  The caller will
+ * have to decide what to do with the exception object.
+ *
+ * @param env             The JNI environment
+ *
+ * @return                The exception, or NULL if there was no exception
+ */
+jthrowable getPendingExceptionAndClear(JNIEnv *env);
+
+/**
+ * Create a new runtime error.
+ *
+ * This creates (but does not throw) a new RuntimeError.
+ *
+ * @param env             The JNI environment
+ * @param fmt             Printf-style format list
+ * @param ...             Printf-style varargs
+ *
+ * @return                A local reference to a RuntimeError
+ */
+jthrowable newRuntimeError(JNIEnv *env, const char *fmt, ...)
+        __attribute__((format(printf, 2, 3)));
+
+#endif

hadoop-native-core/src/main/native/jni/jni_helper.c

@@ -0,0 +1,739 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "common/htable.h"
+#include "config.h"
+#include "jni/exception.h"
+#include "jni/jni_helper.h"
+
+#include <stdio.h> 
+#include <string.h> 
+#include <uv.h> 
+
+/**
+ * JNI Helper functions.
+ *
+ * These are a bunch of functions to make using JNI more bearable.
+ * They should be portable, since we use libuv for most platform-specific
+ * things.
+ *
+ * Currently, once JNI is initialized, it cannot be uninitialized.  This is
+ * fine for most apps, but at some point, we should implement a shutdown
+ * function.
+ */
+
+/**
+ * Non-zero if jni_helper.c is initialized.
+ */
+static int g_jni_init;
+
+/**
+ * Once structure initializing jni_helper.c.
+ */
+static uv_once_t g_jni_init_once = UV_ONCE_INIT;
+
+/**
+ * The JNI library.
+ */
+static uv_lib_t g_jni_lib;
+
+/**
+ * The JNI_GetCreatedJavaVMs function.
+ */
+jint (*GetCreatedJavaVMs)(JavaVM **vmBuf, jsize bufLen, jsize *nVMs);
+
+/**
+ * The JNI_CreateJavaVM function.
+ */
+jint (*CreateJavaVM)(JavaVM **p_vm, void **p_env, void *vm_args);
+
+/**
+ * A hash table mapping class names to jclass objects.
+ * The keys are strings, and the values are jclass objects.
+ * We do not ever delete entries from this table.
+ *
+ * This may only be accessed under g_name_to_classref_htable_lock.
+ */
+static struct htable *g_name_to_classref_htable;
+
+/**
+ * The lock protecting g_name_to_classref_htable.
+ */
+static uv_mutex_t g_name_to_classref_htable_lock;
+
+/**
+ * Key that allows us to retrieve thread-local storage
+ *
+ * TODO: use the libuv local storage wrappers, once they supports thread
+ * destructors.  (See https://github.com/joyent/libuv/issues/1216)
+ **/
+static pthread_key_t g_tls_key;
+
+/** The Native return types that methods could return */
+#define VOID          'V'
+#define JOBJECT       'L'
+#define JARRAYOBJECT  '['
+#define JBOOLEAN      'Z'
+#define JBYTE         'B'
+#define JCHAR         'C'
+#define JSHORT        'S'
+#define JINT          'I'
+#define JLONG         'J'
+#define JFLOAT        'F'
+#define JDOUBLE       'D'
+
+/** Pthreads thread-local storage for each library thread. */
+struct hdfsTls {
+    JNIEnv *env;
+};
+
+static void hdfsThreadDestructor(void *v);
+static JNIEnv* getGlobalJNIEnv(void);
+static uint32_t jni_helper_hash_string(const void *key, uint32_t capacity);
+static int jni_helper_compare_string(const void *a, const void *b);
+
+static void jni_helper_init(void)
+{
+    int ret;
+    JNIEnv *env;
+    jthrowable jthr;
+
+    if (uv_dlopen(JNI_LIBRARY_NAME, &g_jni_lib) < 0) {
+        fprintf(stderr, "jni_helper_init: failed to load " JNI_LIBRARY_NAME
+                ": %s\n", uv_dlerror(&g_jni_lib));
+        return;
+    }
+    if (uv_dlsym(&g_jni_lib, "JNI_GetCreatedJavaVMs",
+                 (void**)&GetCreatedJavaVMs) < 0) {
+        fprintf(stderr, "jni_helper_init: failed to find "
+                "JNI_GetCreatedJavaVMs function in "
+                JNI_LIBRARY_NAME "\n");
+        return;
+    }
+    if (uv_dlsym(&g_jni_lib, "JNI_CreateJavaVM",
+                 (void**)&CreateJavaVM) < 0) {
+        fprintf(stderr, "jni_helper_init: failed to find "
+                "JNI_CreateJavaVM function in "
+                JNI_LIBRARY_NAME "\n");
+        return;
+    }
+    env = getGlobalJNIEnv();
+    if (!env) {
+        fprintf(stderr, "jni_helper_init: getGlobalJNIENv failed.\n");
+        return;
+    }
+    g_name_to_classref_htable = htable_alloc(128,
+            jni_helper_hash_string, jni_helper_compare_string);
+    if (!g_name_to_classref_htable) {
+        fprintf(stderr, "jni_helper_init: failed to allocate "
+                "name-to-classref hash table.\n");
+        return;
+    }
+    if (uv_mutex_init(&g_name_to_classref_htable_lock) < 0) {
+        fprintf(stderr, "jni_helper_init: failed to init mutex for "
+                "name-to-classref hash table.\n");
+        return;
+    }
+    ret = pthread_key_create(&g_tls_key, hdfsThreadDestructor);
+    if (ret) {
+        fprintf(stderr, "jni_helper_init: pthread_key_create failed with "
+            "error %d\n", ret);
+        return;
+    }
+    jthr = invokeMethod(env, NULL, STATIC, NULL,
+                     "org/apache/hadoop/fs/FileSystem",
+                     "loadFileSystems", "()V");
+    if (jthr) {
+        printExceptionAndFree(env, jthr, PRINT_EXC_ALL, "loadFileSystems");
+        return;
+    }
+    g_jni_init = 1;
+}
+
+/**
+ * The function that is called whenever a thread with libhdfs thread local data
+ * is destroyed.
+ *
+ * @param v         The thread-local data
+ */
+static void hdfsThreadDestructor(void *v)
+{
+    struct hdfsTls *tls = v;
+    JavaVM *vm;
+    JNIEnv *env = tls->env;
+    jint ret;
+
+    ret = (*env)->GetJavaVM(env, &vm);
+    if (ret) {
+        fprintf(stderr, "hdfsThreadDestructor: GetJavaVM failed with "
+                "error %d\n", ret);
+        (*env)->ExceptionDescribe(env);
+    } else {
+        (*vm)->DetachCurrentThread(vm);
+    }
+    free(tls);
+}
+
+void destroyLocalReference(JNIEnv *env, jobject jObject)
+{
+  if (jObject)
+    (*env)->DeleteLocalRef(env, jObject);
+}
+
+static jthrowable validateMethodType(JNIEnv *env, MethType methType)
+{
+    if (methType != STATIC && methType != INSTANCE) {
+        return newRuntimeError(env, "validateMethodType(methType=%d): "
+            "illegal method type.\n", methType);
+    }
+    return NULL;
+}
+
+jthrowable newJavaStr(JNIEnv *env, const char *str, jstring *out)
+{
+    jstring jstr;
+
+    if (!str) {
+        /* Can't pass NULL to NewStringUTF: the result would be
+         * implementation-defined. */
+        *out = NULL;
+        return NULL;
+    }
+    jstr = (*env)->NewStringUTF(env, str);
+    if (!jstr) {
+        /* If NewStringUTF returns NULL, an exception has been thrown,
+         * which we need to handle.  Probaly an OOM. */
+        return getPendingExceptionAndClear(env);
+    }
+    *out = jstr;
+    return NULL;
+}
+
+jthrowable newCStr(JNIEnv *env, jstring jstr, char **out)
+{
+    const char *tmp;
+
+    if (!jstr) {
+        *out = NULL;
+        return NULL;
+    }
+    tmp = (*env)->GetStringUTFChars(env, jstr, NULL);
+    if (!tmp) {
+        return getPendingExceptionAndClear(env);
+    }
+    *out = strdup(tmp);
+    (*env)->ReleaseStringUTFChars(env, jstr, tmp);
+    return NULL;
+}
+
+static uint32_t jni_helper_hash_string(const void *key, uint32_t capacity)
+{
+    const char *str = key;
+    uint32_t hash = 0;
+
+    while (*str) {
+        hash = (hash * 31) + *str;
+        str++;
+    }
+    return hash % capacity;
+}
+
+static int jni_helper_compare_string(const void *a, const void *b)
+{
+    return strcmp(a, b) == 0;
+}
+
+jthrowable invokeMethod(JNIEnv *env, jvalue *retval, MethType methType,
+                 jobject instObj, const char *className,
+                 const char *methName, const char *methSignature, ...)
+{
+    va_list args;
+    jclass cls;
+    jmethodID mid;
+    jthrowable jthr;
+    const char *str; 
+    char returnType;
+    
+    jthr = validateMethodType(env, methType);
+    if (jthr)
+        return jthr;
+    jthr = globalClassReference(className, env, &cls);
+    if (jthr)
+        return jthr;
+    jthr = methodIdFromClass(className, methName, methSignature, 
+                            methType, env, &mid);
+    if (jthr)
+        return jthr;
+    str = methSignature;
+    while (*str != ')') str++;
+    str++;
+    returnType = *str;
+    va_start(args, methSignature);
+    if (returnType == JOBJECT || returnType == JARRAYOBJECT) {
+        jobject jobj = NULL;
+        if (methType == STATIC) {
+            jobj = (*env)->CallStaticObjectMethodV(env, cls, mid, args);
+        }
+        else if (methType == INSTANCE) {
+            jobj = (*env)->CallObjectMethodV(env, instObj, mid, args);
+        }
+        retval->l = jobj;
+    }
+    else if (returnType == VOID) {
+        if (methType == STATIC) {
+            (*env)->CallStaticVoidMethodV(env, cls, mid, args);
+        }
+        else if (methType == INSTANCE) {
+            (*env)->CallVoidMethodV(env, instObj, mid, args);
+        }
+    }
+    else if (returnType == JBOOLEAN) {
+        jboolean jbool = 0;
+        if (methType == STATIC) {
+            jbool = (*env)->CallStaticBooleanMethodV(env, cls, mid, args);
+        }
+        else if (methType == INSTANCE) {
+            jbool = (*env)->CallBooleanMethodV(env, instObj, mid, args);
+        }
+        retval->z = jbool;
+    }
+    else if (returnType == JSHORT) {
+        jshort js = 0;
+        if (methType == STATIC) {
+            js = (*env)->CallStaticShortMethodV(env, cls, mid, args);
+        }
+        else if (methType == INSTANCE) {
+            js = (*env)->CallShortMethodV(env, instObj, mid, args);
+        }
+        retval->s = js;
+    }
+    else if (returnType == JLONG) {
+        jlong jl = -1;
+        if (methType == STATIC) {
+            jl = (*env)->CallStaticLongMethodV(env, cls, mid, args);
+        }
+        else if (methType == INSTANCE) {
+            jl = (*env)->CallLongMethodV(env, instObj, mid, args);
+        }
+        retval->j = jl;
+    }
+    else if (returnType == JINT) {
+        jint ji = -1;
+        if (methType == STATIC) {
+            ji = (*env)->CallStaticIntMethodV(env, cls, mid, args);
+        }
+        else if (methType == INSTANCE) {
+            ji = (*env)->CallIntMethodV(env, instObj, mid, args);
+        }
+        retval->i = ji;
+    }
+    va_end(args);
+
+    jthr = (*env)->ExceptionOccurred(env);
+    if (jthr) {
+        (*env)->ExceptionClear(env);
+        return jthr;
+    }
+    return NULL;
+}
+
+jthrowable constructNewObjectOfClass(JNIEnv *env, jobject *out, const char *className, 
+                                  const char *ctorSignature, ...)
+{
+    va_list args;
+    jclass cls;
+    jmethodID mid; 
+    jobject jobj;
+    jthrowable jthr;
+
+    jthr = globalClassReference(className, env, &cls);
+    if (jthr)
+        return jthr;
+    jthr = methodIdFromClass(className, "<init>", ctorSignature, 
+                            INSTANCE, env, &mid);
+    if (jthr)
+        return jthr;
+    va_start(args, ctorSignature);
+    jobj = (*env)->NewObjectV(env, cls, mid, args);
+    va_end(args);
+    if (!jobj)
+        return getPendingExceptionAndClear(env);
+    *out = jobj;
+    return NULL;
+}
+
+
+jthrowable methodIdFromClass(const char *className, const char *methName, 
+                            const char *methSignature, MethType methType, 
+                            JNIEnv *env, jmethodID *out)
+{
+    jclass cls;
+    jthrowable jthr;
+
+    jthr = globalClassReference(className, env, &cls);
+    if (jthr)
+        return jthr;
+    jmethodID mid = 0;
+    jthr = validateMethodType(env, methType);
+    if (jthr)
+        return jthr;
+    if (methType == STATIC) {
+        mid = (*env)->GetStaticMethodID(env, cls, methName, methSignature);
+    }
+    else if (methType == INSTANCE) {
+        mid = (*env)->GetMethodID(env, cls, methName, methSignature);
+    }
+    if (mid == NULL) {
+        fprintf(stderr, "could not find method %s from class %s with "
+            "signature %s\n", methName, className, methSignature);
+        return getPendingExceptionAndClear(env);
+    }
+    *out = mid;
+    return NULL;
+}
+
+jthrowable globalClassReference(const char *className, JNIEnv *env, jclass *out)
+{
+    jthrowable jthr = NULL;
+    jclass local_clazz = NULL;
+    jclass clazz = NULL;
+    int ret;
+
+    uv_mutex_lock(&g_name_to_classref_htable_lock);
+    clazz = htable_get(g_name_to_classref_htable, className);
+    if (clazz) {
+        *out = clazz;
+        goto done;
+    }
+    local_clazz = (*env)->FindClass(env,className);
+    if (!local_clazz) {
+        jthr = getPendingExceptionAndClear(env);
+        goto done;
+    }
+    clazz = (*env)->NewGlobalRef(env, local_clazz);
+    if (!clazz) {
+        jthr = getPendingExceptionAndClear(env);
+        goto done;
+    }
+    ret = htable_put(g_name_to_classref_htable, (char*)className, clazz); 
+    if (ret) {
+        jthr = newRuntimeError(env, "htable_put failed with error "
+                               "code %d\n", ret);
+        goto done;
+    }
+    *out = clazz;
+    jthr = NULL;
+done:
+    uv_mutex_unlock(&g_name_to_classref_htable_lock);
+    (*env)->DeleteLocalRef(env, local_clazz);
+    if (jthr && clazz) {
+        (*env)->DeleteGlobalRef(env, clazz);
+    }
+    return jthr;
+}
+
+jthrowable classNameOfObject(jobject jobj, JNIEnv *env, char **name)
+{
+    jthrowable jthr;
+    jclass cls, clsClass = NULL;
+    jmethodID mid;
+    jstring str = NULL;
+    const char *cstr = NULL;
+    char *newstr;
+
+    cls = (*env)->GetObjectClass(env, jobj);
+    if (cls == NULL) {
+        jthr = getPendingExceptionAndClear(env);
+        goto done;
+    }
+    clsClass = (*env)->FindClass(env, "java/lang/Class");
+    if (clsClass == NULL) {
+        jthr = getPendingExceptionAndClear(env);
+        goto done;
+    }
+    mid = (*env)->GetMethodID(env, clsClass, "getName", "()Ljava/lang/String;");
+    if (mid == NULL) {
+        jthr = getPendingExceptionAndClear(env);
+        goto done;
+    }
+    str = (*env)->CallObjectMethod(env, cls, mid);
+    if (str == NULL) {
+        jthr = getPendingExceptionAndClear(env);
+        goto done;
+    }
+    cstr = (*env)->GetStringUTFChars(env, str, NULL);
+    if (!cstr) {
+        jthr = getPendingExceptionAndClear(env);
+        goto done;
+    }
+    newstr = strdup(cstr);
+    if (newstr == NULL) {
+        jthr = newRuntimeError(env, "classNameOfObject: out of memory");
+        goto done;
+    }
+    *name = newstr;
+    jthr = NULL;
+
+done:
+    destroyLocalReference(env, cls);
+    destroyLocalReference(env, clsClass);
+    if (str) {
+        if (cstr)
+            (*env)->ReleaseStringUTFChars(env, str, cstr);
+        (*env)->DeleteLocalRef(env, str);
+    }
+    return jthr;
+}
+
+/**
+ * Get the global JNI environemnt.
+ *
+ * We only have to create the JVM once.  After that, we can use it in
+ * every thread.  You must be holding the jvmMutex when you call this
+ * function.
+ *
+ * @return          The JNIEnv on success; error code otherwise
+ */
+static JNIEnv* getGlobalJNIEnv(void)
+{
+    const jsize vmBufLength = 1;
+    char *classpath;
+    JavaVM* vmBuf[vmBufLength]; 
+    JNIEnv *env;
+    jint rv = 0; 
+    jint noVMs = 0;
+
+    rv = GetCreatedJavaVMs(&(vmBuf[0]), vmBufLength, &noVMs);
+    if (rv != 0) {
+        fprintf(stderr, "getGlobalJNIEnv: JNI_GetCreatedJavaVMs failed "
+                "with error: %d\n", rv);
+        return NULL;
+    }
+    if (noVMs > 0) {
+        //Attach this thread to the VM
+        JavaVM* vm = vmBuf[0];
+        rv = (*vm)->AttachCurrentThread(vm, (void*)&env, 0);
+        if (rv != 0) {
+            fprintf(stderr, "getGlobalJNIEnv: Call to AttachCurrentThread "
+                    "failed with error: %d\n", rv);
+            return NULL;
+        }
+        return env;
+    }
+    //Get the environment variables for initializing the JVM
+    classpath = getenv("CLASSPATH");
+    if (!classpath) {
+        fprintf(stderr, "getGlobalJNIEnv: The CLASSPATH environment "
+                "variable was not set.");
+        return NULL;
+    } 
+    char *hadoopClassPathVMArg = "-Djava.class.path=";
+    size_t optHadoopClassPathLen = strlen(classpath) + 
+        strlen(hadoopClassPathVMArg) + 1;
+    char *optHadoopClassPath = malloc(optHadoopClassPathLen);
+    snprintf(optHadoopClassPath, optHadoopClassPathLen,
+            "%s%s", hadoopClassPathVMArg, classpath);
+
+    // Determine the # of LIBHDFS_OPTS args
+    int noArgs = 1;
+    char *hadoopJvmArgs = getenv("LIBHDFS_OPTS");
+    char jvmArgDelims[] = " ";
+    char *str, *token, *savePtr;
+    if (hadoopJvmArgs != NULL)  {
+        hadoopJvmArgs = strdup(hadoopJvmArgs);
+        for (noArgs = 1, str = hadoopJvmArgs; ; noArgs++, str = NULL) {
+            token = strtok_r(str, jvmArgDelims, &savePtr);
+            if (NULL == token) {
+                break;
+            }
+        }
+        free(hadoopJvmArgs);
+    }
+
+    // Now that we know the # args, populate the options array
+    JavaVMOption options[noArgs];
+    options[0].optionString = optHadoopClassPath;
+    hadoopJvmArgs = getenv("LIBHDFS_OPTS");
+    if (hadoopJvmArgs != NULL)  {
+        hadoopJvmArgs = strdup(hadoopJvmArgs);
+        for (noArgs = 1, str = hadoopJvmArgs; ; noArgs++, str = NULL) {
+            token = strtok_r(str, jvmArgDelims, &savePtr);
+            if (NULL == token) {
+                break;
+            }
+            options[noArgs].optionString = token;
+        }
+    }
+
+    //Create the VM
+    JavaVMInitArgs vm_args;
+    JavaVM *vm;
+    vm_args.version = JNI_VERSION_1_2;
+    vm_args.options = options;
+    vm_args.nOptions = noArgs; 
+    vm_args.ignoreUnrecognized = 1;
+
+    rv = CreateJavaVM(&vm, (void*)&env, &vm_args);
+
+    if (hadoopJvmArgs != NULL)  {
+      free(hadoopJvmArgs);
+    }
+    free(optHadoopClassPath);
+    if (rv != 0) {
+        fprintf(stderr, "Call to JNI_CreateJavaVM failed "
+                "with error: %d\n", rv);
+        return NULL;
+    }
+    return env;
+}
+
+/**
+ * getJNIEnv: A helper function to get the JNIEnv* for the given thread.
+ * If no JVM exists, then one will be created. JVM command line arguments
+ * are obtained from the LIBHDFS_OPTS environment variable.
+ *
+ * Implementation note: we rely on POSIX thread-local storage (tls).
+ * This allows us to associate a destructor function with each thread, that
+ * will detach the thread from the Java VM when the thread terminates.  If we
+ * failt to do this, it will cause a memory leak.
+ *
+ * However, POSIX TLS is not the most efficient way to do things.  It requires a
+ * key to be initialized before it can be used.  Since we don't know if this key
+ * is initialized at the start of this function, we have to lock a mutex first
+ * and check.  Luckily, most operating systems support the more efficient
+ * __thread construct, which is initialized by the linker.
+ *
+ * @param: None.
+ * @return The JNIEnv* corresponding to the thread.
+ */
+JNIEnv* getJNIEnv(void)
+{
+    JNIEnv *env;
+    struct hdfsTls *tls;
+    int ret;
+
+#ifdef HAVE_BETTER_TLS
+    static __thread struct hdfsTls *quickTls = NULL;
+    if (quickTls)
+        return quickTls->env;
+#endif
+    uv_once(&g_jni_init_once, jni_helper_init);
+    if (!g_jni_init) {
+        fprintf(stderr, "getJNIEnv: failed to initialize jni.\n");
+        return NULL;
+    }
+    tls = pthread_getspecific(g_tls_key);
+    if (tls) {
+        return tls->env;
+    }
+    env = getGlobalJNIEnv();
+    if (!env) {
+        fprintf(stderr, "getJNIEnv: getGlobalJNIEnv failed\n");
+        return NULL;
+    }
+    tls = calloc(1, sizeof(struct hdfsTls));
+    if (!tls) {
+        fprintf(stderr, "getJNIEnv: OOM allocating %zd bytes\n",
+                sizeof(struct hdfsTls));
+        return NULL;
+    }
+    tls->env = env;
+    ret = pthread_setspecific(g_tls_key, tls);
+    if (ret) {
+        fprintf(stderr, "getJNIEnv: pthread_setspecific failed with "
+            "error code %d\n", ret);
+        hdfsThreadDestructor(tls);
+        return NULL;
+    }
+#ifdef HAVE_BETTER_TLS
+    quickTls = tls;
+#endif
+    return env;
+}
+
+int javaObjectIsOfClass(JNIEnv *env, jobject obj, const char *name)
+{
+    jclass clazz;
+    int ret;
+
+    clazz = (*env)->FindClass(env, name);
+    if (!clazz) {
+        printPendingExceptionAndFree(env, PRINT_EXC_ALL,
+            "javaObjectIsOfClass(%s)", name);
+        return -1;
+    }
+    ret = (*env)->IsInstanceOf(env, obj, clazz);
+    (*env)->DeleteLocalRef(env, clazz);
+    return ret == JNI_TRUE ? 1 : 0;
+}
+
+jthrowable hadoopConfSetStr(JNIEnv *env, jobject jConfiguration,
+        const char *key, const char *value)
+{
+    jthrowable jthr;
+    jstring jkey = NULL, jvalue = NULL;
+
+    jthr = newJavaStr(env, key, &jkey);
+    if (jthr)
+        goto done;
+    jthr = newJavaStr(env, value, &jvalue);
+    if (jthr)
+        goto done;
+    jthr = invokeMethod(env, NULL, INSTANCE, jConfiguration,
+            "org/apache/hadoop/conf/Configuration", "set", 
+            "(Ljava/lang/String;Ljava/lang/String;)V",
+            jkey, jvalue);
+    if (jthr)
+        goto done;
+done:
+    (*env)->DeleteLocalRef(env, jkey);
+    (*env)->DeleteLocalRef(env, jvalue);
+    return jthr;
+}
+
+jthrowable fetchEnumInstance(JNIEnv *env, const char *className,
+                         const char *valueName, jobject *out)
+{
+    jclass clazz;
+    jfieldID fieldId;
+    jobject jEnum;
+    char prettyClass[256];
+
+    clazz = (*env)->FindClass(env, className);
+    if (!clazz) {
+        return newRuntimeError(env, "fetchEnum(%s, %s): failed to find class.",
+                className, valueName);
+    }
+    if ((size_t)snprintf(prettyClass, sizeof(prettyClass), "L%s;", className)
+          >= sizeof(prettyClass)) {
+        return newRuntimeError(env, "fetchEnum(%s, %s): class name too long.",
+                className, valueName);
+    }
+    fieldId = (*env)->GetStaticFieldID(env, clazz, valueName, prettyClass);
+    if (!fieldId) {
+        return getPendingExceptionAndClear(env);
+    }
+    jEnum = (*env)->GetStaticObjectField(env, clazz, fieldId);
+    if (!jEnum) {
+        return getPendingExceptionAndClear(env);
+    }
+    *out = jEnum;
+    return NULL;
+}
+
+// vim: ts=4:sw=4:tw=79:et

hadoop-native-core/src/main/native/jni/jni_helper.h

@@ -0,0 +1,163 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFS_JNI_HELPER_H
+#define LIBHDFS_JNI_HELPER_H
+
+#include <jni.h>
+#include <stdio.h>
+
+#include <stdlib.h>
+#include <stdarg.h>
+#include <search.h>
+#include <pthread.h>
+#include <errno.h>
+
+#define PATH_SEPARATOR ':'
+
+
+/** Denote the method we want to invoke as STATIC or INSTANCE */
+typedef enum {
+    STATIC,
+    INSTANCE
+} MethType;
+
+/**
+ * Create a new malloc'ed C string from a Java string.
+ *
+ * @param env       The JNI environment
+ * @param jstr      The Java string
+ * @param out       (out param) the malloc'ed C string
+ *
+ * @return          NULL on success; the exception otherwise
+ */
+jthrowable newCStr(JNIEnv *env, jstring jstr, char **out);
+
+/**
+ * Create a new Java string from a C string.
+ *
+ * @param env       The JNI environment
+ * @param str       The C string
+ * @param out       (out param) the java string
+ *
+ * @return          NULL on success; the exception otherwise
+ */
+jthrowable newJavaStr(JNIEnv *env, const char *str, jstring *out);
+
+/**
+ * Helper function to destroy a local reference of java.lang.Object
+ * @param env: The JNIEnv pointer. 
+ * @param jFile: The local reference of java.lang.Object object
+ * @return None.
+ */
+void destroyLocalReference(JNIEnv *env, jobject jObject);
+
+/** invokeMethod: Invoke a Static or Instance method.
+ * className: Name of the class where the method can be found
+ * methName: Name of the method
+ * methSignature: the signature of the method "(arg-types)ret-type"
+ * methType: The type of the method (STATIC or INSTANCE)
+ * instObj: Required if the methType is INSTANCE. The object to invoke
+   the method on.
+ * env: The JNIEnv pointer
+ * retval: The pointer to a union type which will contain the result of the
+   method invocation, e.g. if the method returns an Object, retval will be
+   set to that, if the method returns boolean, retval will be set to the
+   value (JNI_TRUE or JNI_FALSE), etc.
+ * exc: If the methods throws any exception, this will contain the reference
+ * Arguments (the method arguments) must be passed after methSignature
+ * RETURNS: -1 on error and 0 on success. If -1 is returned, exc will have 
+   a valid exception reference, and the result stored at retval is undefined.
+ */
+jthrowable invokeMethod(JNIEnv *env, jvalue *retval, MethType methType,
+                 jobject instObj, const char *className, const char *methName, 
+                 const char *methSignature, ...);
+
+jthrowable constructNewObjectOfClass(JNIEnv *env, jobject *out, const char *className, 
+                                  const char *ctorSignature, ...);
+
+jthrowable methodIdFromClass(const char *className, const char *methName, 
+                            const char *methSignature, MethType methType, 
+                            JNIEnv *env, jmethodID *out);
+
+jthrowable globalClassReference(const char *className, JNIEnv *env, jclass *out);
+
+/** classNameOfObject: Get an object's class name.
+ * @param jobj: The object.
+ * @param env: The JNIEnv pointer.
+ * @param name: (out param) On success, will contain a string containing the
+ * class name. This string must be freed by the caller.
+ * @return NULL on success, or the exception
+ */
+jthrowable classNameOfObject(jobject jobj, JNIEnv *env, char **name);
+
+/** getJNIEnv: A helper function to get the JNIEnv* for the given thread.
+ * If no JVM exists, then one will be created. JVM command line arguments
+ * are obtained from the LIBHDFS_OPTS environment variable.
+ * @param: None.
+ * @return The JNIEnv* corresponding to the thread.
+ * */
+JNIEnv* getJNIEnv(void);
+
+/**
+ * Figure out if a Java object is an instance of a particular class.
+ *
+ * @param env  The Java environment.
+ * @param obj  The object to check.
+ * @param name The class name to check.
+ *
+ * @return     -1 if we failed to find the referenced class name.
+ *             0 if the object is not of the given class.
+ *             1 if the object is of the given class.
+ */
+int javaObjectIsOfClass(JNIEnv *env, jobject obj, const char *name);
+
+/**
+ * Set a value in a configuration object.
+ *
+ * @param env               The JNI environment
+ * @param jConfiguration    The configuration object to modify
+ * @param key               The key to modify
+ * @param value             The value to set the key to
+ *
+ * @return                  NULL on success; exception otherwise
+ */
+jthrowable hadoopConfSetStr(JNIEnv *env, jobject jConfiguration,
+        const char *key, const char *value);
+
+/**
+ * Fetch an instance of an Enum.
+ *
+ * @param env               The JNI environment.
+ * @param className         The enum class name.
+ * @param valueName         The name of the enum value
+ * @param out               (out param) on success, a local reference to an
+ *                          instance of the enum object.  (Since Java enums are
+ *                          singletones, this is also the only instance.)
+ *
+ * @return                  NULL on success; exception otherwise
+ */
+jthrowable fetchEnumInstance(JNIEnv *env, const char *className,
+                             const char *valueName, jobject *out);
+
+#endif /*LIBHDFS_JNI_HELPER_H*/
+
+/**
+ * vim: ts=4: sw=4: et:
+ */
+

hadoop-native-core/src/main/native/jni/jnifs.c

@@ -0,0 +1,2874 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "common/hadoop_err.h"
+#include "fs/common.h"
+#include "fs/fs.h"
+#include "jni/exception.h"
+#include "jni/jni_helper.h"
+
+#include <inttypes.h>
+#include <stdio.h>
+#include <string.h>
+
+/* Some frequently used Java paths */
+#define HADOOP_CONF     "org/apache/hadoop/conf/Configuration"
+#define HADOOP_PATH     "org/apache/hadoop/fs/Path"
+#define HADOOP_LOCALFS  "org/apache/hadoop/fs/LocalFileSystem"
+#define HADOOP_FS       "org/apache/hadoop/fs/FileSystem"
+#define HADOOP_FSSTATUS "org/apache/hadoop/fs/FsStatus"
+#define HADOOP_BLK_LOC  "org/apache/hadoop/fs/BlockLocation"
+#define HADOOP_DFS      "org/apache/hadoop/hdfs/DistributedFileSystem"
+#define HADOOP_ISTRM    "org/apache/hadoop/fs/FSDataInputStream"
+#define HADOOP_OSTRM    "org/apache/hadoop/fs/FSDataOutputStream"
+#define HADOOP_STAT     "org/apache/hadoop/fs/FileStatus"
+#define HADOOP_FSPERM   "org/apache/hadoop/fs/permission/FsPermission"
+#define JAVA_NET_ISA    "java/net/InetSocketAddress"
+#define JAVA_NET_URI    "java/net/URI"
+#define JAVA_STRING     "java/lang/String"
+#define READ_OPTION     "org/apache/hadoop/fs/ReadOption"
+
+#define JAVA_VOID       "V"
+
+/* Macros for constructing method signatures */
+#define JPARAM(X)           "L" X ";"
+#define JARRPARAM(X)        "[L" X ";"
+#define JMETHOD1(X, R)      "(" X ")" R
+#define JMETHOD2(X, Y, R)   "(" X Y ")" R
+#define JMETHOD3(X, Y, Z, R)   "(" X Y Z")" R
+
+#define KERBEROS_TICKET_CACHE_PATH "hadoop.security.kerberos.ticket.cache.path"
+
+// Bit fields for hdfsFile_internal flags
+#define HDFS_FILE_SUPPORTS_DIRECT_READ (1<<0)
+
+/**
+ * The C equivalent of org.apache.org.hadoop.FSData(Input|Output)Stream .
+ */
+enum hdfsStreamType {
+    UNINITIALIZED = 0,
+    INPUT = 1,
+    OUTPUT = 2,
+};
+
+struct jni_fs {
+    struct hadoop_fs_base base;
+    jobject obj;
+};
+
+/**
+ * The 'file-handle' to a file in hdfs.
+ */
+struct jni_file {
+    struct hadoop_file_base base;
+    jobject stream;
+    enum hdfsStreamType type;
+    int flags;
+};
+
+/**
+ * Zero-copy options cache.
+ *
+ * We cache the EnumSet of ReadOptions which has to be passed into every
+ * readZero call, to avoid reconstructing it each time.  This cache is cleared
+ * whenever an element changes.
+ */
+struct jni_rz_options_cache {
+    char *pool_name;
+    int skip_checksums;
+    jobject pool_obj;
+    jobject enum_set;
+};
+
+struct jni_rz_buffer {
+    struct hadoop_rz_buffer_base base;
+    jobject byte_buffer;
+    int direct;
+};
+
+static tSize jni_read_direct(JNIEnv *env, struct jni_file *file,
+            void* buffer, tSize length);
+static jthrowable getFileInfoFromStat(JNIEnv *env, jobject jStat,
+            hdfsFileInfo *fileInfo);
+
+static int jni_file_is_open_for_read(hdfsFile bfile)
+{
+    struct jni_file *file = (struct jni_file*)bfile;
+    return (file->type == INPUT);
+}
+
+static int jni_file_is_open_for_write(hdfsFile bfile)
+{
+    struct jni_file *file = (struct jni_file*)bfile;
+    return (file->type == OUTPUT);
+}
+
+static int jni_file_get_read_statistics(hdfsFile bfile,
+            struct hdfsReadStatistics **stats)
+{
+    jthrowable jthr;
+    jobject readStats = NULL;
+    jvalue jVal;
+    struct hdfsReadStatistics *s = NULL;
+    int ret;
+    JNIEnv* env = getJNIEnv();
+    struct jni_file *file = (struct jni_file*)bfile;
+
+    if (env == NULL) {
+        errno = EINTERNAL;
+        return -1;
+    }
+    if (file->type != INPUT) {
+        ret = EINVAL;
+        goto done;
+    }
+    jthr = invokeMethod(env, &jVal, INSTANCE, file->stream, 
+                  "org/apache/hadoop/hdfs/client/HdfsDataInputStream",
+                  "getReadStatistics",
+                  "()Lorg/apache/hadoop/hdfs/DFSInputStream$ReadStatistics;");
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "hdfsFileGetReadStatistics: getReadStatistics failed");
+        goto done;
+    }
+    readStats = jVal.l;
+    s = malloc(sizeof(struct hdfsReadStatistics));
+    if (!s) {
+        ret = ENOMEM;
+        goto done;
+    }
+    jthr = invokeMethod(env, &jVal, INSTANCE, readStats,
+                  "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics",
+                  "getTotalBytesRead", "()J");
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "hdfsFileGetReadStatistics: getTotalBytesRead failed");
+        goto done;
+    }
+    s->totalBytesRead = jVal.j;
+
+    jthr = invokeMethod(env, &jVal, INSTANCE, readStats,
+                  "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics",
+                  "getTotalLocalBytesRead", "()J");
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "hdfsFileGetReadStatistics: getTotalLocalBytesRead failed");
+        goto done;
+    }
+    s->totalLocalBytesRead = jVal.j;
+
+    jthr = invokeMethod(env, &jVal, INSTANCE, readStats,
+                  "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics",
+                  "getTotalShortCircuitBytesRead", "()J");
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "hdfsFileGetReadStatistics: getTotalShortCircuitBytesRead failed");
+        goto done;
+    }
+    s->totalShortCircuitBytesRead = jVal.j;
+    jthr = invokeMethod(env, &jVal, INSTANCE, readStats,
+                  "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics",
+                  "getTotalZeroCopyBytesRead", "()J");
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "hdfsFileGetReadStatistics: getTotalZeroCopyBytesRead failed");
+        goto done;
+    }
+    s->totalZeroCopyBytesRead = jVal.j;
+    *stats = s;
+    s = NULL;
+    ret = 0;
+
+done:
+    destroyLocalReference(env, readStats);
+    free(s);
+    if (ret) {
+      errno = ret;
+      return -1;
+    }
+    return 0;
+}
+
+/**
+ * hdfsJniEnv: A wrapper struct to be used as 'value'
+ * while saving thread -> JNIEnv* mappings
+ */
+typedef struct
+{
+    JNIEnv* env;
+} hdfsJniEnv;
+
+/**
+ * Helper function to create a org.apache.hadoop.fs.Path object.
+ * @param env: The JNIEnv pointer. 
+ * @param path: The file-path for which to construct org.apache.hadoop.fs.Path
+ * object.
+ * @return Returns a jobject on success and NULL on error.
+ */
+static jthrowable constructNewObjectOfPath(JNIEnv *env, const char *path,
+                                           jobject *out)
+{
+    jthrowable jthr;
+    jstring jPathString;
+    jobject jPath;
+
+    //Construct a java.lang.String object
+    jthr = newJavaStr(env, path, &jPathString);
+    if (jthr)
+        return jthr;
+    //Construct the org.apache.hadoop.fs.Path object
+    jthr = constructNewObjectOfClass(env, &jPath, "org/apache/hadoop/fs/Path",
+                                     "(Ljava/lang/String;)V", jPathString);
+    destroyLocalReference(env, jPathString);
+    if (jthr)
+        return jthr;
+    *out = jPath;
+    return NULL;
+}
+
+static jthrowable get_str_from_conf(JNIEnv *env, jobject jConfiguration,
+        const char *key, char **val)
+{
+    jthrowable jthr;
+    jvalue jVal;
+    jstring jkey = NULL, jRet = NULL;
+
+    jthr = newJavaStr(env, key, &jkey);
+    if (jthr)
+        goto done;
+    jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration,
+            HADOOP_CONF, "get", JMETHOD1(JPARAM(JAVA_STRING),
+                                         JPARAM(JAVA_STRING)), jkey);
+    if (jthr)
+        goto done;
+    jRet = jVal.l;
+    jthr = newCStr(env, jRet, val);
+done:
+    destroyLocalReference(env, jkey);
+    destroyLocalReference(env, jRet);
+    return jthr;
+}
+
+int jni_conf_get_str(const char *key, char **val)
+{
+    JNIEnv *env;
+    int ret;
+    jthrowable jthr;
+    jobject jConfiguration = NULL;
+
+    env = getJNIEnv();
+    if (env == NULL) {
+        ret = EINTERNAL;
+        goto done;
+    }
+    jthr = constructNewObjectOfClass(env, &jConfiguration, HADOOP_CONF, "()V");
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_conf_get_str(%s): new Configuration", key);
+        goto done;
+    }
+    jthr = get_str_from_conf(env, jConfiguration, key, val);
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_conf_get_str(%s): hadoopConfGetStr", key);
+        goto done;
+    }
+    ret = 0;
+done:
+    destroyLocalReference(env, jConfiguration);
+    if (ret)
+        errno = ret;
+    return ret;
+}
+
+/**
+ * Calculate the effective URI to use, given a builder configuration.
+ *
+ * If there is not already a URI scheme, we prepend 'hdfs://'.
+ *
+ * If there is not already a port specified, and a port was given to the
+ * builder, we suffix that port.  If there is a port specified but also one in
+ * the URI, that is an error.
+ *
+ * @param bld       The hdfs builder object
+ * @param uri       (out param) dynamically allocated string representing the
+ *                  effective URI
+ *
+ * @return          0 on success; error code otherwise
+ */
+static int calcEffectiveURI(struct hdfsBuilder *bld, char ** uri)
+{
+    const char *scheme;
+    char suffix[64];
+    const char *lastColon;
+    char *u;
+    size_t uriLen;
+
+    if (!bld->nn)
+        return EINVAL;
+    scheme = (strstr(bld->nn, "://")) ? "" : "hdfs://";
+    if (bld->port == 0) {
+        suffix[0] = '\0';
+    } else {
+        lastColon = rindex(bld->nn, ':');
+        if (lastColon && (strspn(lastColon + 1, "0123456789") ==
+                          strlen(lastColon + 1))) {
+            fprintf(stderr, "port %d was given, but URI '%s' already "
+                "contains a port!\n", bld->port, bld->nn);
+            return EINVAL;
+        }
+        snprintf(suffix, sizeof(suffix), ":%d", bld->port);
+    }
+
+    uriLen = strlen(scheme) + strlen(bld->nn) + strlen(suffix);
+    u = malloc((uriLen + 1) * (sizeof(char)));
+    if (!u) {
+        fprintf(stderr, "calcEffectiveURI: out of memory");
+        return ENOMEM;
+    }
+    snprintf(u, uriLen + 1, "%s%s%s", scheme, bld->nn, suffix);
+    *uri = u;
+    return 0;
+}
+
+static const char *maybeNull(const char *str)
+{
+    return str ? str : "(NULL)";
+}
+
+static const char *hdfsBuilderToStr(const struct hdfsBuilder *bld,
+                                    char *buf, size_t bufLen)
+{
+    snprintf(buf, bufLen, "nn=%s, port=%d, "
+             "kerbTicketCachePath=%s, userName=%s",
+             maybeNull(bld->nn), bld->port,
+             maybeNull(bld->kerbTicketCachePath), maybeNull(bld->userName));
+    return buf;
+}
+
+struct hadoop_err *jni_connect(struct hdfsBuilder *bld,
+                                  struct hdfs_internal **out)
+{
+    struct jni_fs *fs = NULL;
+    JNIEnv *env = 0;
+    jobject jConfiguration = NULL, fsObj = NULL, jURI = NULL, jCachePath = NULL;
+    jstring jURIString = NULL, jUserString = NULL;
+    jvalue  jVal;
+    jthrowable jthr = NULL;
+    char *cURI = 0, buf[512];
+    int ret;
+    struct hdfsBuilderConfOpt *opt;
+    int forceNewInstance = 1;
+
+    //Get the JNIEnv* corresponding to current thread
+    env = getJNIEnv();
+    if (env == NULL) {
+        ret = EINTERNAL;
+        goto done;
+    }
+    fs = calloc(1, sizeof(*fs));
+    if (!fs) {
+        ret = ENOMEM;
+        goto done;
+    }
+    fs->base.ty = HADOOP_FS_TY_JNI;
+
+    //  jConfiguration = new Configuration();
+    jthr = constructNewObjectOfClass(env, &jConfiguration, HADOOP_CONF, "()V");
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf, sizeof(buf)));
+        goto done;
+    }
+    // set configuration values
+    for (opt = bld->opts; opt; opt = opt->next) {
+        jthr = hadoopConfSetStr(env, jConfiguration, opt->key, opt->val);
+        if (jthr) {
+            ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                "hdfsBuilderConnect(%s): error setting conf '%s' to '%s'",
+                hdfsBuilderToStr(bld, buf, sizeof(buf)), opt->key, opt->val);
+            goto done;
+        }
+    }
+ 
+    //Check what type of FileSystem the caller wants...
+    if (bld->nn == NULL) {
+        // Get a local filesystem.
+        if (forceNewInstance) {
+            // fs = FileSytem#newInstanceLocal(conf);
+            jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS,
+                    "newInstanceLocal", JMETHOD1(JPARAM(HADOOP_CONF),
+                    JPARAM(HADOOP_LOCALFS)), jConfiguration);
+            if (jthr) {
+                ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                    "hdfsBuilderConnect(%s)",
+                    hdfsBuilderToStr(bld, buf, sizeof(buf)));
+                goto done;
+            }
+            fsObj = jVal.l;
+        } else {
+            // fs = FileSytem#getLocal(conf);
+            jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, "getLocal",
+                             JMETHOD1(JPARAM(HADOOP_CONF),
+                                      JPARAM(HADOOP_LOCALFS)),
+                             jConfiguration);
+            if (jthr) {
+                ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                    "hdfsBuilderConnect(%s)",
+                    hdfsBuilderToStr(bld, buf, sizeof(buf)));
+                goto done;
+            }
+            fsObj = jVal.l;
+        }
+    } else {
+        if (!strcmp(bld->nn, "default")) {
+            // jURI = FileSystem.getDefaultUri(conf)
+            jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS,
+                          "getDefaultUri",
+                          "(Lorg/apache/hadoop/conf/Configuration;)Ljava/net/URI;",
+                          jConfiguration);
+            if (jthr) {
+                ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                    "hdfsBuilderConnect(%s)",
+                    hdfsBuilderToStr(bld, buf, sizeof(buf)));
+                goto done;
+            }
+            jURI = jVal.l;
+        } else {
+            // fs = FileSystem#get(URI, conf, ugi);
+            ret = calcEffectiveURI(bld, &cURI);
+            if (ret)
+                goto done;
+            jthr = newJavaStr(env, cURI, &jURIString);
+            if (jthr) {
+                ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                    "hdfsBuilderConnect(%s)",
+                    hdfsBuilderToStr(bld, buf, sizeof(buf)));
+                goto done;
+            }
+            jthr = invokeMethod(env, &jVal, STATIC, NULL, JAVA_NET_URI,
+                             "create", "(Ljava/lang/String;)Ljava/net/URI;",
+                             jURIString);
+            if (jthr) {
+                ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                    "hdfsBuilderConnect(%s)",
+                    hdfsBuilderToStr(bld, buf, sizeof(buf)));
+                goto done;
+            }
+            jURI = jVal.l;
+        }
+
+        if (bld->kerbTicketCachePath) {
+            jthr = hadoopConfSetStr(env, jConfiguration,
+                KERBEROS_TICKET_CACHE_PATH, bld->kerbTicketCachePath);
+            if (jthr) {
+                ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                    "hdfsBuilderConnect(%s)",
+                    hdfsBuilderToStr(bld, buf, sizeof(buf)));
+                goto done;
+            }
+        }
+        jthr = newJavaStr(env, bld->userName, &jUserString);
+        if (jthr) {
+            ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                "hdfsBuilderConnect(%s)",
+                hdfsBuilderToStr(bld, buf, sizeof(buf)));
+            goto done;
+        }
+        if (forceNewInstance) {
+            jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS,
+                    "newInstance", JMETHOD3(JPARAM(JAVA_NET_URI), 
+                        JPARAM(HADOOP_CONF), JPARAM(JAVA_STRING),
+                        JPARAM(HADOOP_FS)),
+                    jURI, jConfiguration, jUserString);
+            if (jthr) {
+                ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                    "hdfsBuilderConnect(%s)",
+                    hdfsBuilderToStr(bld, buf, sizeof(buf)));
+                goto done;
+            }
+            fsObj = jVal.l;
+        } else {
+            jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, "get",
+                    JMETHOD3(JPARAM(JAVA_NET_URI), JPARAM(HADOOP_CONF),
+                        JPARAM(JAVA_STRING), JPARAM(HADOOP_FS)),
+                        jURI, jConfiguration, jUserString);
+            if (jthr) {
+                ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                    "hdfsBuilderConnect(%s)",
+                    hdfsBuilderToStr(bld, buf, sizeof(buf)));
+                goto done;
+            }
+            fsObj = jVal.l;
+        }
+    }
+    fs->obj = (*env)->NewGlobalRef(env, fsObj);
+    if (!fs->obj) {
+        ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL,
+                    "hdfsBuilderConnect(%s)",
+                    hdfsBuilderToStr(bld, buf, sizeof(buf)));
+        goto done;
+    }
+    ret = 0;
+
+done:
+    // Release unnecessary local references
+    destroyLocalReference(env, jConfiguration);
+    destroyLocalReference(env, fsObj);
+    destroyLocalReference(env, jURI);
+    destroyLocalReference(env, jCachePath);
+    destroyLocalReference(env, jURIString);
+    destroyLocalReference(env, jUserString);
+    free(cURI);
+
+    if (ret) {
+        free(fs);
+        return hadoop_lerr_alloc(ret, "jni_connect: failed to connect: "
+                                 "error %d", ret);
+    }
+    *out = (struct hdfs_internal *)fs;
+    return NULL;
+}
+
+static int jni_disconnect(hdfsFS bfs)
+{
+    struct jni_fs *fs = (struct jni_fs*)bfs;
+    JNIEnv* env = getJNIEnv();
+    jthrowable jthr;
+    int ret;
+
+    if (env == NULL) {
+      errno = EINTERNAL;
+      return -1;
+    }
+    if (fs == NULL) {
+        errno = EBADF;
+        return -1;
+    }
+    jthr = invokeMethod(env, NULL, INSTANCE, fs->obj, HADOOP_FS,
+                     "close", "()V");
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_disconnect: FileSystem#close");
+    } else {
+        ret = 0;
+    }
+    (*env)->DeleteGlobalRef(env, fs->obj);
+    free(fs);
+    if (ret) {
+        errno = ret;
+        return -1;
+    }
+    return 0;
+}
+
+/**
+ * Get the default block size of a FileSystem object.
+ *
+ * @param env       The Java env
+ * @param jFS       The FileSystem object
+ * @param jPath     The path to find the default blocksize at
+ * @param out       (out param) the default block size
+ *
+ * @return          NULL on success; or the exception
+ */
+static jthrowable getDefaultBlockSize(JNIEnv *env, jobject jFS,
+                                      jobject jPath, jlong *out)
+{
+    jthrowable jthr;
+    jvalue jVal;
+
+    jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS,
+                 "getDefaultBlockSize", JMETHOD1(JPARAM(HADOOP_PATH), "J"), jPath);
+    if (jthr)
+        return jthr;
+    *out = jVal.j;
+    return NULL;
+}
+
+static hdfsFile jni_open_file(hdfsFS bfs, const char* path, int flags, 
+                      int bufferSize, short replication, tSize blockSize)
+{
+    struct jni_fs *fs = (struct jni_fs*)bfs;
+    jstring jStrBufferSize = NULL, jStrReplication = NULL;
+    jobject jConfiguration = NULL, jPath = NULL, jFile = NULL;
+    jthrowable jthr;
+    jvalue jVal;
+    struct jni_file *file = NULL;
+    int ret;
+
+    /*
+      JAVA EQUIVALENT:
+       File f = new File(path);
+       FSData{Input|Output}Stream f{is|os} = fs.create(f);
+       return f{is|os};
+    */
+    /* Get the JNIEnv* corresponding to current thread */
+    JNIEnv* env = getJNIEnv();
+    int accmode = flags & O_ACCMODE;
+
+    if (!env) {
+        errno = EINTERNAL;
+        return NULL;
+    }
+
+    if (accmode == O_RDONLY || accmode == O_WRONLY) {
+	/* yay */
+    } else if (accmode == O_RDWR) {
+      fprintf(stderr, "ERROR: cannot open a hadoop file in O_RDWR mode\n");
+      errno = ENOTSUP;
+      return NULL;
+    } else {
+      fprintf(stderr, "ERROR: cannot open a hadoop file in mode 0x%x\n", accmode);
+      errno = EINVAL;
+      return NULL;
+    }
+
+    if ((flags & O_CREAT) && (flags & O_EXCL)) {
+      fprintf(stderr, "WARN: hadoop does not truly support O_CREATE && O_EXCL\n");
+    }
+
+    /* The hadoop java api/signature */
+    const char* method = NULL;
+    const char* signature = NULL;
+
+    if (accmode == O_RDONLY) {
+	method = "open";
+        signature = JMETHOD2(JPARAM(HADOOP_PATH), "I", JPARAM(HADOOP_ISTRM));
+    } else if (flags & O_APPEND) {
+	method = "append";
+	signature = JMETHOD1(JPARAM(HADOOP_PATH), JPARAM(HADOOP_OSTRM));
+    } else {
+	method = "create";
+	signature = JMETHOD2(JPARAM(HADOOP_PATH), "ZISJ", JPARAM(HADOOP_OSTRM));
+    }
+
+    /* Create an object of org.apache.hadoop.fs.Path */
+    jthr = constructNewObjectOfPath(env, path, &jPath);
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_open_file(%s): constructNewObjectOfPath", path);
+        goto done;
+    }
+
+    /* Get the Configuration object from the FileSystem object */
+    jthr = invokeMethod(env, &jVal, INSTANCE, fs->obj, HADOOP_FS,
+                     "getConf", JMETHOD1("", JPARAM(HADOOP_CONF)));
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_open_file(%s): FileSystem#getConf", path);
+        goto done;
+    }
+    jConfiguration = jVal.l;
+
+    jint jBufferSize = bufferSize;
+    jshort jReplication = replication;
+    jStrBufferSize = (*env)->NewStringUTF(env, "io.file.buffer.size"); 
+    if (!jStrBufferSize) {
+        ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, "OOM");
+        goto done;
+    }
+    jStrReplication = (*env)->NewStringUTF(env, "dfs.replication");
+    if (!jStrReplication) {
+        ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, "OOM");
+        goto done;
+    }
+
+    if (!bufferSize) {
+        jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, 
+                         HADOOP_CONF, "getInt", "(Ljava/lang/String;I)I",
+                         jStrBufferSize, 4096);
+        if (jthr) {
+            ret = printExceptionAndFree(env, jthr, NOPRINT_EXC_FILE_NOT_FOUND |
+                NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_UNRESOLVED_LINK,
+                "jni_open_file(%s): Configuration#getInt(io.file.buffer.size)",
+                path);
+            goto done;
+        }
+        jBufferSize = jVal.i;
+    }
+
+    if ((accmode == O_WRONLY) && (flags & O_APPEND) == 0) {
+        if (!replication) {
+            jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, 
+                             HADOOP_CONF, "getInt", "(Ljava/lang/String;I)I",
+                             jStrReplication, 1);
+            if (jthr) {
+                ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                    "jni_open_file(%s): Configuration#getInt(dfs.replication)",
+                    path);
+                goto done;
+            }
+            jReplication = jVal.i;
+        }
+    }
+ 
+    /* Create and return either the FSDataInputStream or
+       FSDataOutputStream references jobject jStream */
+
+    // READ?
+    if (accmode == O_RDONLY) {
+        jthr = invokeMethod(env, &jVal, INSTANCE, fs->obj, HADOOP_FS,
+                       method, signature, jPath, jBufferSize);
+    }  else if ((accmode == O_WRONLY) && (flags & O_APPEND)) {
+        // WRITE/APPEND?
+       jthr = invokeMethod(env, &jVal, INSTANCE, fs->obj, HADOOP_FS,
+                       method, signature, jPath);
+    } else {
+        // WRITE/CREATE
+        jboolean jOverWrite = 1;
+        jlong jBlockSize = blockSize;
+
+        if (jBlockSize == 0) {
+            jthr = getDefaultBlockSize(env, fs->obj, jPath, &jBlockSize);
+            if (jthr) {
+                ret = EIO;
+                goto done;
+            }
+        }
+        jthr = invokeMethod(env, &jVal, INSTANCE, fs->obj, HADOOP_FS,
+                         method, signature, jPath, jOverWrite,
+                         jBufferSize, jReplication, jBlockSize);
+    }
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_open_file(%s): FileSystem#%s(%s)", path, method, signature);
+        goto done;
+    }
+    jFile = jVal.l;
+
+    file = calloc(1, sizeof(*file));
+    if (!file) {
+        fprintf(stderr, "jni_open_file(%s): OOM\n", path);
+        ret = ENOMEM;
+        goto done;
+    }
+    file->stream = (*env)->NewGlobalRef(env, jFile);
+    if (!file->stream) {
+        ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL,
+            "jni_open_file(%s): NewGlobalRef", path); 
+        goto done;
+    }
+    file->type = (((flags & O_WRONLY) == 0) ? INPUT : OUTPUT);
+    file->flags = 0;
+
+    if ((flags & O_WRONLY) == 0) {
+        // Try a test read to see if we can do direct reads
+        char buf;
+        if (jni_read_direct(env, file, &buf, 0) == 0) {
+            // Success - 0-byte read should return 0
+            file->flags |= HDFS_FILE_SUPPORTS_DIRECT_READ;
+        } else if (errno != ENOTSUP) {
+            // Unexpected error. Clear it, don't set the direct flag.
+            fprintf(stderr,
+                  "jni_open_file(%s): WARN: Unexpected error %d when testing "
+                  "for direct read compatibility\n", path, errno);
+        }
+    }
+    ret = 0;
+
+done:
+    destroyLocalReference(env, jStrBufferSize);
+    destroyLocalReference(env, jStrReplication);
+    destroyLocalReference(env, jConfiguration); 
+    destroyLocalReference(env, jPath); 
+    destroyLocalReference(env, jFile); 
+    if (ret) {
+        if (file) {
+            if (file->stream) {
+                (*env)->DeleteGlobalRef(env, file->stream);
+            }
+            free(file);
+        }
+        errno = ret;
+        return NULL;
+    }
+    return (hdfsFile)file;
+}
+
+static int jni_close_file(hdfsFS fs __attribute__((unused)), hdfsFile bfile)
+{
+    struct jni_file *file = (struct jni_file*)bfile;
+    jthrowable jthr;
+    const char* interface;
+    int ret;
+    JNIEnv* env = getJNIEnv();
+
+    if (env == NULL) {
+        errno = EINTERNAL;
+        return -1;
+    }
+    if (file->type == UNINITIALIZED) {
+        errno = EBADF;
+        return -1;
+    }
+    interface = (file->type == INPUT) ?  HADOOP_ISTRM : HADOOP_OSTRM;
+    jthr = invokeMethod(env, NULL, INSTANCE, file->stream, interface,
+                     "close", "()V");
+    if (jthr) {
+        const char *interfaceShortName = (file->type == INPUT) ? 
+            "FSDataInputStream" : "FSDataOutputStream";
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                "%s#close", interfaceShortName);
+    } else {
+        ret = 0;
+    }
+
+    //De-allocate memory
+    (*env)->DeleteGlobalRef(env, file->stream);
+    free(file);
+    if (ret) {
+        errno = ret;
+        return -1;
+    }
+    return 0;
+}
+
+static int jni_file_exists(hdfsFS bfs, const char *path)
+{
+    struct jni_fs *fs = (struct jni_fs*)bfs;
+    jobject jPath;
+    jvalue  jVal;
+    jthrowable jthr;
+    JNIEnv *env = getJNIEnv();
+
+    if (env == NULL) {
+        errno = EINTERNAL;
+        return -1;
+    }
+    if (path == NULL) {
+        errno = EINVAL;
+        return -1;
+    }
+    jthr = constructNewObjectOfPath(env, path, &jPath);
+    if (jthr) {
+        errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_file_exists: constructNewObjectOfPath");
+        return -1;
+    }
+    jthr = invokeMethod(env, &jVal, INSTANCE, fs->obj, HADOOP_FS,
+            "exists", JMETHOD1(JPARAM(HADOOP_PATH), "Z"), jPath);
+    destroyLocalReference(env, jPath);
+    if (jthr) {
+        errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_file_exists: invokeMethod(%s)",
+            JMETHOD1(JPARAM(HADOOP_PATH), "Z"));
+        return -1;
+    }
+    if (jVal.z) {
+        return 0;
+    } else {
+        errno = ENOENT;
+        return -1;
+    }
+}
+
+// Reads using the read(ByteBuffer) API, which does fewer copies
+static tSize jni_read_direct(JNIEnv *env, struct jni_file *file,
+                             void* buffer, tSize length)
+{
+    // JAVA EQUIVALENT:
+    //  ByteBuffer bbuffer = ByteBuffer.allocateDirect(length) // wraps C buffer
+    //  fis.read(bbuffer);
+    jobject bb;
+    jvalue jVal;
+    jthrowable jthr;
+
+    bb = (*env)->NewDirectByteBuffer(env, buffer, length);
+    if (bb == NULL) {
+        errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL,
+            "jni_read_direct: NewDirectByteBuffer");
+        return -1;
+    }
+    jthr = invokeMethod(env, &jVal, INSTANCE, file->stream,
+        HADOOP_ISTRM, "read", "(Ljava/nio/ByteBuffer;)I", bb);
+    destroyLocalReference(env, bb);
+    if (jthr) {
+        errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_read_direct: FSDataInputStream#read");
+        return -1;
+    }
+    return (jVal.i < 0) ? 0 : jVal.i;
+}
+
+static tSize jni_read(hdfsFS bfs __attribute__((unused)), hdfsFile bfile,
+                      void* buffer, tSize length)
+{
+    // JAVA EQUIVALENT:
+    //  byte [] bR = new byte[length];
+    //  fis.read(bR);
+    struct jni_file *file = (struct jni_file*)bfile;
+    JNIEnv* env = getJNIEnv();
+    jbyteArray jbRarray;
+    jint noReadBytes = length;
+    jvalue jVal;
+    jthrowable jthr;
+
+    if (env == NULL) {
+      errno = EINTERNAL;
+      return -1;
+    }
+    if (length == 0) {
+        return 0;
+    } else if (length < 0) {
+        errno = EINVAL;
+        return -1;
+    } else if (file->type != INPUT) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (file->flags & HDFS_FILE_SUPPORTS_DIRECT_READ) {
+        return jni_read_direct(env, file, buffer, length);
+    }
+    jbRarray = (*env)->NewByteArray(env, length);
+    if (!jbRarray) {
+        errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL,
+            "jni_read: NewByteArray");
+        return -1;
+    }
+    jthr = invokeMethod(env, &jVal, INSTANCE, file->stream, HADOOP_ISTRM,
+                               "read", "([B)I", jbRarray);
+    if (jthr) {
+        destroyLocalReference(env, jbRarray);
+        errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_read: FSDataInputStream#read");
+        return -1;
+    }
+    if (jVal.i < 0) {
+        // EOF
+        destroyLocalReference(env, jbRarray);
+        return 0;
+    } else if (jVal.i == 0) {
+        destroyLocalReference(env, jbRarray);
+        errno = EINTR;
+        return -1;
+    }
+    (*env)->GetByteArrayRegion(env, jbRarray, 0, noReadBytes, buffer);
+    destroyLocalReference(env, jbRarray);
+    if ((*env)->ExceptionCheck(env)) {
+        errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL,
+            "hdfsRead: GetByteArrayRegion");
+        return -1;
+    }
+    return jVal.i;
+}
+
+static tSize jni_pread(hdfsFS bfs __attribute__((unused)), hdfsFile bfile,
+            tOffset position, void* buffer, tSize length)
+{
+    JNIEnv* env;
+    jbyteArray jbRarray;
+    jvalue jVal;
+    jthrowable jthr;
+    struct jni_file *file = (struct jni_file*)bfile;
+
+    if (length == 0) {
+        return 0;
+    } else if (length < 0) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (file->type == UNINITIALIZED) {
+        errno = EBADF;
+        return -1;
+    }
+
+    env = getJNIEnv();
+    if (env == NULL) {
+      errno = EINTERNAL;
+      return -1;
+    }
+
+    //Error checking... make sure that this file is 'readable'
+    if (file->type != INPUT) {
+        fprintf(stderr, "Cannot read from a non-InputStream object!\n");
+        errno = EINVAL;
+        return -1;
+    }
+
+    // JAVA EQUIVALENT:
+    //  byte [] bR = new byte[length];
+    //  fis.read(pos, bR, 0, length);
+    jbRarray = (*env)->NewByteArray(env, length);
+    if (!jbRarray) {
+        errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL,
+            "jni_pread: NewByteArray");
+        return -1;
+    }
+    jthr = invokeMethod(env, &jVal, INSTANCE, file->stream, HADOOP_ISTRM,
+                     "read", "(J[BII)I", position, jbRarray, 0, length);
+    if (jthr) {
+        destroyLocalReference(env, jbRarray);
+        errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_pread: FSDataInputStream#read");
+        return -1;
+    }
+    if (jVal.i < 0) {
+        // EOF
+        destroyLocalReference(env, jbRarray);
+        return 0;
+    } else if (jVal.i == 0) {
+        destroyLocalReference(env, jbRarray);
+        errno = EINTR;
+        return -1;
+    }
+    (*env)->GetByteArrayRegion(env, jbRarray, 0, jVal.i, buffer);
+    destroyLocalReference(env, jbRarray);
+    if ((*env)->ExceptionCheck(env)) {
+        errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL,
+            "jni_pread: GetByteArrayRegion");
+        return -1;
+    }
+    return jVal.i;
+}
+
+static tSize jni_write(hdfsFS bfs __attribute__((unused)), hdfsFile bfile,
+            const void* buffer, tSize length)
+{
+    // JAVA EQUIVALENT
+    // byte b[] = str.getBytes();
+    // fso.write(b);
+
+    //Get the JNIEnv* corresponding to current thread
+    JNIEnv* env = getJNIEnv();
+    struct jni_file *file = (struct jni_file*)bfile;
+    jbyteArray jbWarray;
+    jthrowable jthr;
+    if (env == NULL) {
+      errno = EINTERNAL;
+      return -1;
+    }
+
+    //Sanity check
+    if (file->type == UNINITIALIZED) {
+        errno = EBADF;
+        return -1;
+    }
+    if (length < 0) {
+    	errno = EINVAL;
+    	return -1;
+    }
+    //Error checking... make sure that this file is 'writable'
+    if (file->type != OUTPUT) {
+        fprintf(stderr, "Cannot write into a non-OutputStream object!\n");
+        errno = EINVAL;
+        return -1;
+    }
+
+    if (length < 0) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (length == 0) {
+        return 0;
+    }
+    //Write the requisite bytes into the file
+    jbWarray = (*env)->NewByteArray(env, length);
+    if (!jbWarray) {
+        errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL,
+            "jni_write: NewByteArray");
+        return -1;
+    }
+    (*env)->SetByteArrayRegion(env, jbWarray, 0, length, buffer);
+    if ((*env)->ExceptionCheck(env)) {
+        destroyLocalReference(env, jbWarray);
+        errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL,
+            "jni_write(length = %d): SetByteArrayRegion", length);
+        return -1;
+    }
+    jthr = invokeMethod(env, NULL, INSTANCE, file->stream,
+            HADOOP_OSTRM, "write", "([B)V", jbWarray);
+    destroyLocalReference(env, jbWarray);
+    if (jthr) {
+        errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_write: FSDataOutputStream#write");
+        return -1;
+    }
+    // Unlike most Java streams, FSDataOutputStream never does partial writes.
+    // If we succeeded, all the data was written.
+    return length;
+}
+
+static int jni_seek(hdfsFS bfs __attribute__((unused)), hdfsFile bfile,
+                    tOffset desiredPos) 
+{
+    // JAVA EQUIVALENT
+    //  fis.seek(pos);
+    struct jni_file *file = (struct jni_file*)bfile;
+    jthrowable jthr;
+    JNIEnv* env = getJNIEnv();
+    if (env == NULL) {
+      errno = EINTERNAL;
+      return -1;
+    }
+    if (file->type != INPUT) {
+        errno = EBADF;
+        return -1;
+    }
+    jthr = invokeMethod(env, NULL, INSTANCE, file->stream,
+            HADOOP_ISTRM, "seek", "(J)V", desiredPos);
+    if (jthr) {
+        errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_seek(desiredPos=%" PRId64 ")"
+            ": FSDataInputStream#seek", desiredPos);
+        return -1;
+    }
+    return 0;
+}
+
+static tOffset jni_tell(hdfsFS bfs __attribute__((unused)), hdfsFile bfile)
+{
+    // JAVA EQUIVALENT
+    //  pos = f.getPos();
+
+    //Get the JNIEnv* corresponding to current thread
+    struct jni_file *file = (struct jni_file*)bfile;
+    JNIEnv* env = getJNIEnv();
+    if (env == NULL) {
+      errno = EINTERNAL;
+      return -1;
+    }
+
+    //Sanity check
+    if (file->type == UNINITIALIZED) {
+        errno = EBADF;
+        return -1;
+    }
+
+    //Parameters
+    const char* interface = (file->type == INPUT) ?
+        HADOOP_ISTRM : HADOOP_OSTRM;
+    jvalue jVal;
+    jthrowable jthr = invokeMethod(env, &jVal, INSTANCE, file->stream,
+                     interface, "getPos", "()J");
+    if (jthr) {
+        errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_tell: %s#getPos",
+            ((file->type == INPUT) ? "FSDataInputStream" :
+                                 "FSDataOutputStream"));
+        return -1;
+    }
+    return jVal.j;
+}
+
+static int jni_flush(hdfsFS bfs __attribute__((unused)), hdfsFile bfile)
+{
+    //Get the JNIEnv* corresponding to current thread
+    jthrowable jthr;
+    struct jni_file *file = (struct jni_file*)bfile;
+    JNIEnv* env = getJNIEnv();
+    if (env == NULL) {
+      errno = EINTERNAL;
+      return -1;
+    }
+    if (file->type != OUTPUT) {
+        errno = EBADF;
+        return -1;
+    }
+    jthr = invokeMethod(env, NULL, INSTANCE, file->stream,
+                     HADOOP_OSTRM, "flush", "()V");
+    if (jthr) {
+        errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_flush: FSDataOutputStream#flush");
+        return -1;
+    }
+    return 0;
+}
+
+static int jni_hflush(hdfsFS bfs __attribute__((unused)), hdfsFile bfile)
+{
+    //Get the JNIEnv* corresponding to current thread
+    jthrowable jthr;
+    struct jni_file *file = (struct jni_file*)bfile;
+    JNIEnv* env = getJNIEnv();
+    if (env == NULL) {
+      errno = EINTERNAL;
+      return -1;
+    }
+    if (file->type != OUTPUT) {
+        errno = EBADF;
+        return -1;
+    }
+    jthr = invokeMethod(env, NULL, INSTANCE, file->stream,
+                     HADOOP_OSTRM, "hflush", "()V");
+    if (jthr) {
+        errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_hflush: FSDataOutputStream#hflush");
+        return -1;
+    }
+    return 0;
+}
+
+static int jni_hsync(hdfsFS bfs __attribute__((unused)), hdfsFile bfile)
+{
+    //Get the JNIEnv* corresponding to current thread
+    jthrowable jthr;
+    struct jni_file *file = (struct jni_file*)bfile;
+    JNIEnv* env = getJNIEnv();
+    if (env == NULL) {
+      errno = EINTERNAL;
+      return -1;
+    }
+    if (file->type != OUTPUT) {
+        errno = EBADF;
+        return -1;
+    }
+    jthr = invokeMethod(env, NULL, INSTANCE, file->stream,
+                     HADOOP_OSTRM, "hsync", "()V");
+    if (jthr) {
+        errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_hsync: FSDataOutputStream#hsync");
+        return -1;
+    }
+    return 0;
+}
+
+static int jni_available(hdfsFS bfs __attribute__((unused)), hdfsFile bfile)
+{
+    // JAVA EQUIVALENT
+    //  fis.available();
+    jvalue jVal;
+    jthrowable jthr = NULL;
+    struct jni_file *file = (struct jni_file*)bfile;
+    JNIEnv* env = getJNIEnv();
+    if (env == NULL) {
+      errno = EINTERNAL;
+      return -1;
+    }
+    if (file->type != INPUT) {
+        errno = EBADF;
+        return -1;
+    }
+    jthr = invokeMethod(env, &jVal, INSTANCE, file->stream,
+                     HADOOP_ISTRM, "available", "()I");
+    if (jthr) {
+        errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_available: FSDataInputStream#available");
+        return -1;
+    }
+    return jVal.i;
+}
+
+static int jni_copy_impl(struct jni_fs *srcFS, const char* src,
+        struct jni_fs *dstFs, const char* dst, jboolean deleteSource)
+{
+    //JAVA EQUIVALENT
+    //  FileUtil#copy(srcFS, srcPath, dstFS, dstPath,
+    //                 deleteSource = false, conf)
+    jobject jConfiguration = NULL, jSrcPath = NULL, jDstPath = NULL;
+    jthrowable jthr;
+    jvalue jVal;
+    int ret;
+    JNIEnv* env = getJNIEnv();
+    if (env == NULL) {
+      errno = EINTERNAL;
+      return -1;
+    }
+    jthr = constructNewObjectOfPath(env, src, &jSrcPath);
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_copy_impl(src=%s): constructNewObjectOfPath", src);
+        goto done;
+    }
+    jthr = constructNewObjectOfPath(env, dst, &jDstPath);
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_copy_impl(dst=%s): constructNewObjectOfPath", dst);
+        goto done;
+    }
+    //Create the org.apache.hadoop.conf.Configuration object
+    jthr = constructNewObjectOfClass(env, &jConfiguration,
+                                     HADOOP_CONF, "()V");
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_copy_impl: Configuration constructor");
+        goto done;
+    }
+
+    //FileUtil#copy
+    jthr = invokeMethod(env, &jVal, STATIC,
+            NULL, "org/apache/hadoop/fs/FileUtil", "copy",
+            "(Lorg/apache/hadoop/fs/FileSystem;Lorg/apache/hadoop/fs/Path;"
+            "Lorg/apache/hadoop/fs/FileSystem;Lorg/apache/hadoop/fs/Path;"
+            "ZLorg/apache/hadoop/conf/Configuration;)Z",
+            srcFS->obj, jSrcPath, dstFs->obj, jDstPath, deleteSource, 
+            jConfiguration);
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_copy_impl(src=%s, dst=%s, deleteSource=%d): "
+            "FileUtil#copy", src, dst, deleteSource);
+        goto done;
+    }
+    if (!jVal.z) {
+        ret = EIO;
+        goto done;
+    }
+    ret = 0;
+
+done:
+    destroyLocalReference(env, jConfiguration);
+    destroyLocalReference(env, jSrcPath);
+    destroyLocalReference(env, jDstPath);
+  
+    if (ret) {
+        errno = ret;
+        return -1;
+    }
+    return 0;
+}
+
+static int jni_copy(hdfsFS srcFS, const char* src, hdfsFS dstFS,
+                    const char* dst)
+{
+    return jni_copy_impl((struct jni_fs*)srcFS, src,
+                         (struct jni_fs*)dstFS, dst, 0);
+}
+
+static int jni_move(hdfsFS srcFS, const char* src, hdfsFS dstFS,
+                    const char* dst)
+{
+    return jni_copy_impl((struct jni_fs*)srcFS, src,
+                         (struct jni_fs*)dstFS, dst, 1);
+}
+
+static int jni_unlink(hdfsFS bfs, const char *path, int recursive)
+{
+    struct jni_fs *fs = (struct jni_fs*)bfs;
+    jthrowable jthr;
+    jobject jPath;
+    jvalue jVal;
+
+    // JAVA EQUIVALENT:
+    //  Path p = new Path(path);
+    //  bool retval = fs.delete(p, recursive);
+
+    //Get the JNIEnv* corresponding to current thread
+    JNIEnv* env = getJNIEnv();
+    if (env == NULL) {
+      errno = EINTERNAL;
+      return -1;
+    }
+
+    jthr = constructNewObjectOfPath(env, path, &jPath);
+    if (jthr) {
+        errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_unlink(path=%s): constructNewObjectOfPath", path);
+        return -1;
+    }
+    jboolean jRecursive = recursive ? JNI_TRUE : JNI_FALSE;
+    jthr = invokeMethod(env, &jVal, INSTANCE, fs->obj, HADOOP_FS,
+                     "delete", "(Lorg/apache/hadoop/fs/Path;Z)Z",
+                     jPath, jRecursive);
+    destroyLocalReference(env, jPath);
+    if (jthr) {
+        errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_unlink(path=%s, recursive=%d): "
+            "FileSystem#delete", path, recursive);
+        return -1;
+    }
+    if (!jVal.z) {
+        errno = EIO;
+        return -1;
+    }
+    return 0;
+}
+
+static int jni_rename(hdfsFS bfs, const char* oldPath, const char* newPath)
+{
+    struct jni_fs *fs = (struct jni_fs*)bfs;
+
+    // JAVA EQUIVALENT:
+    //  Path old = new Path(oldPath);
+    //  Path new = new Path(newPath);
+    //  fs.rename(old, new);
+
+    //Get the JNIEnv* corresponding to current thread
+    JNIEnv* env = getJNIEnv();
+    if (env == NULL) {
+      errno = EINTERNAL;
+      return -1;
+    }
+
+    jthrowable jthr;
+    jobject jOldPath = NULL, jNewPath = NULL;
+    int ret = -1;
+    jvalue jVal;
+
+    jthr = constructNewObjectOfPath(env, oldPath, &jOldPath );
+    if (jthr) {
+        errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_rename: constructNewObjectOfPath(%s)", oldPath);
+        goto done;
+    }
+    jthr = constructNewObjectOfPath(env, newPath, &jNewPath);
+    if (jthr) {
+        errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_rename: constructNewObjectOfPath(%s)", newPath);
+        goto done;
+    }
+
+    // Rename the file
+    // TODO: use rename2 here?  (See HDFS-3592)
+    jthr = invokeMethod(env, &jVal, INSTANCE, fs->obj, HADOOP_FS, "rename",
+                     JMETHOD2(JPARAM(HADOOP_PATH), JPARAM(HADOOP_PATH), "Z"),
+                     jOldPath, jNewPath);
+    if (jthr) {
+        errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_rename(oldPath=%s, newPath=%s): FileSystem#rename",
+            oldPath, newPath);
+        goto done;
+    }
+    if (!jVal.z) {
+        errno = EIO;
+        goto done;
+    }
+    ret = 0;
+
+done:
+    destroyLocalReference(env, jOldPath);
+    destroyLocalReference(env, jNewPath);
+    return ret;
+}
+
+static char* jni_get_working_directory(hdfsFS bfs, char *buffer,
+                                       size_t bufferSize)
+{
+    struct jni_fs *fs = (struct jni_fs*)bfs;
+    // JAVA EQUIVALENT:
+    //  Path p = fs.getWorkingDirectory(); 
+    //  return p.toString()
+
+    //Get the JNIEnv* corresponding to current thread
+    JNIEnv* env = getJNIEnv();
+    if (env == NULL) {
+      errno = EINTERNAL;
+      return NULL;
+    }
+
+    jobject jPath = NULL;
+    jstring jPathString = NULL;
+    jvalue jVal;
+    jthrowable jthr;
+    int ret;
+    const char *jPathChars = NULL;
+
+    //FileSystem#getWorkingDirectory()
+    jthr = invokeMethod(env, &jVal, INSTANCE, fs->obj,
+                     HADOOP_FS, "getWorkingDirectory",
+                     "()Lorg/apache/hadoop/fs/Path;");
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_get_working_directory: FileSystem#getWorkingDirectory");
+        goto done;
+    }
+    jPath = jVal.l;
+    if (!jPath) {
+        fprintf(stderr, "jni_get_working_directory: "
+            "FileSystem#getWorkingDirectory returned NULL");
+        ret = -EIO;
+        goto done;
+    }
+
+    //Path#toString()
+    jthr = invokeMethod(env, &jVal, INSTANCE, jPath, 
+                     "org/apache/hadoop/fs/Path", "toString",
+                     "()Ljava/lang/String;");
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_get_working_directory: Path#toString");
+        goto done;
+    }
+    jPathString = jVal.l;
+    jPathChars = (*env)->GetStringUTFChars(env, jPathString, NULL);
+    if (!jPathChars) {
+        ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL,
+            "jni_get_working_directory: GetStringUTFChars");
+        goto done;
+    }
+
+    //Copy to user-provided buffer
+    ret = snprintf(buffer, bufferSize, "%s", jPathChars);
+    if ((size_t)ret >= bufferSize) {
+        ret = ENAMETOOLONG;
+        goto done;
+    }
+    ret = 0;
+
+done:
+    if (jPathChars) {
+        (*env)->ReleaseStringUTFChars(env, jPathString, jPathChars);
+    }
+    destroyLocalReference(env, jPath);
+    destroyLocalReference(env, jPathString);
+
+    if (ret) {
+        errno = ret;
+        return NULL;
+    }
+    return buffer;
+}
+
+static int jni_set_working_directory(hdfsFS bfs, const char* path)
+{
+    struct jni_fs *fs = (struct jni_fs*)bfs;
+
+    // JAVA EQUIVALENT:
+    //  fs.setWorkingDirectory(Path(path)); 
+
+    //Get the JNIEnv* corresponding to current thread
+    JNIEnv* env = getJNIEnv();
+    if (env == NULL) {
+      errno = EINTERNAL;
+      return -1;
+    }
+
+    jthrowable jthr;
+    jobject jPath;
+
+    //Create an object of org.apache.hadoop.fs.Path
+    jthr = constructNewObjectOfPath(env, path, &jPath);
+    if (jthr) {
+        errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_set_working_directory(%s): constructNewObjectOfPath",
+            path);
+        return -1;
+    }
+
+    //FileSystem#setWorkingDirectory()
+    jthr = invokeMethod(env, NULL, INSTANCE, fs->obj, HADOOP_FS,
+                     "setWorkingDirectory", 
+                     "(Lorg/apache/hadoop/fs/Path;)V", jPath);
+    destroyLocalReference(env, jPath);
+    if (jthr) {
+        errno = printExceptionAndFree(env, jthr, NOPRINT_EXC_ILLEGAL_ARGUMENT,
+            "jni_set_working_directory(%s): FileSystem#setWorkingDirectory",
+            path);
+        return -1;
+    }
+    return 0;
+}
+
+static int jni_mkdir(hdfsFS bfs, const char* path)
+{
+    struct jni_fs *fs = (struct jni_fs*)bfs;
+    // JAVA EQUIVALENT:
+    //  fs.mkdirs(new Path(path));
+
+    //Get the JNIEnv* corresponding to current thread
+    JNIEnv* env = getJNIEnv();
+    if (env == NULL) {
+      errno = EINTERNAL;
+      return -1;
+    }
+
+    jobject jPath;
+    jthrowable jthr;
+
+    //Create an object of org.apache.hadoop.fs.Path
+    jthr = constructNewObjectOfPath(env, path, &jPath);
+    if (jthr) {
+        errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_mkdir(%s): constructNewObjectOfPath", path);
+        return -1;
+    }
+
+    //Create the directory
+    jvalue jVal;
+    jVal.z = 0;
+    jthr = invokeMethod(env, &jVal, INSTANCE, fs->obj, HADOOP_FS,
+                     "mkdirs", "(Lorg/apache/hadoop/fs/Path;)Z",
+                     jPath);
+    destroyLocalReference(env, jPath);
+    if (jthr) {
+        errno = printExceptionAndFree(env, jthr,
+            NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND |
+            NOPRINT_EXC_UNRESOLVED_LINK | NOPRINT_EXC_PARENT_NOT_DIRECTORY,
+            "jni_mkdir(%s): FileSystem#mkdirs", path);
+        return -1;
+    }
+    if (!jVal.z) {
+        // It's unclear under exactly which conditions FileSystem#mkdirs
+        // is supposed to return false (as opposed to throwing an exception.)
+        // It seems like the current code never actually returns false.
+        // So we're going to translate this to EIO, since there seems to be
+        // nothing more specific we can do with it.
+        errno = EIO;
+        return -1;
+    }
+    return 0;
+}
+
+static int jni_set_replication(hdfsFS bfs, const char* path,
+                               int16_t replication)
+{
+    struct jni_fs *fs = (struct jni_fs*)bfs;
+    // JAVA EQUIVALENT:
+    //  fs.setReplication(new Path(path), replication);
+
+    //Get the JNIEnv* corresponding to current thread
+    JNIEnv* env = getJNIEnv();
+    if (env == NULL) {
+      errno = EINTERNAL;
+      return -1;
+    }
+
+    jthrowable jthr;
+
+    //Create an object of org.apache.hadoop.fs.Path
+    jobject jPath;
+    jthr = constructNewObjectOfPath(env, path, &jPath);
+    if (jthr) {
+        errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_set_replication(path=%s): constructNewObjectOfPath", path);
+        return -1;
+    }
+
+    //Create the directory
+    jvalue jVal;
+    jthr = invokeMethod(env, &jVal, INSTANCE, fs->obj, HADOOP_FS,
+                     "setReplication", "(Lorg/apache/hadoop/fs/Path;S)Z",
+                     jPath, replication);
+    destroyLocalReference(env, jPath);
+    if (jthr) {
+        errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_set_replication(path=%s, replication=%d): "
+            "FileSystem#setReplication", path, replication);
+        return -1;
+    }
+    if (!jVal.z) {
+        // setReplication returns false "if file does not exist or is a
+        // directory."  So the nearest translation to that is ENOENT.
+        errno = ENOENT;
+        return -1;
+    }
+
+    return 0;
+}
+
+static hdfsFileInfo* jni_list_directory(hdfsFS bfs, const char* path,
+                                        int *numEntries)
+{
+    struct jni_fs *fs = (struct jni_fs*)bfs;
+    // JAVA EQUIVALENT:
+    //  Path p(path);
+    //  Path []pathList = fs.listPaths(p)
+    //  foreach path in pathList 
+    //    getFileInfo(path)
+    jthrowable jthr;
+    jobject jPath = NULL;
+    hdfsFileInfo *pathList = NULL; 
+    jobjectArray jPathList = NULL;
+    jvalue jVal;
+    jsize jPathListSize = 0;
+    int ret;
+
+    //Get the JNIEnv* corresponding to current thread
+    JNIEnv* env = getJNIEnv();
+    if (env == NULL) {
+      errno = EINTERNAL;
+      return NULL;
+    }
+
+    //Create an object of org.apache.hadoop.fs.Path
+    jthr = constructNewObjectOfPath(env, path, &jPath);
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_list_directory(%s): constructNewObjectOfPath", path);
+        goto done;
+    }
+
+    jthr = invokeMethod(env, &jVal, INSTANCE, fs->obj, HADOOP_DFS, "listStatus",
+                     JMETHOD1(JPARAM(HADOOP_PATH), JARRPARAM(HADOOP_STAT)),
+                     jPath);
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr,
+            NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND |
+            NOPRINT_EXC_UNRESOLVED_LINK,
+            "jni_list_directory(%s): FileSystem#listStatus", path);
+        goto done;
+    }
+    jPathList = jVal.l;
+
+    //Figure out the number of entries in that directory
+    jPathListSize = (*env)->GetArrayLength(env, jPathList);
+    if (jPathListSize == 0) {
+        ret = 0;
+        goto done;
+    }
+
+    //Allocate memory
+    pathList = calloc(jPathListSize, sizeof(hdfsFileInfo));
+    if (pathList == NULL) {
+        ret = ENOMEM;
+        goto done;
+    }
+
+    //Save path information in pathList
+    jsize i;
+    jobject tmpStat;
+    for (i=0; i < jPathListSize; ++i) {
+        tmpStat = (*env)->GetObjectArrayElement(env, jPathList, i);
+        if (!tmpStat) {
+            ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL,
+                "jni_list_directory(%s): GetObjectArrayElement(%d out of %d)",
+                path, i, jPathListSize);
+            goto done;
+        }
+        jthr = getFileInfoFromStat(env, tmpStat, &pathList[i]);
+        destroyLocalReference(env, tmpStat);
+        if (jthr) {
+            ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                "jni_list_directory(%s): getFileInfoFromStat(%d out of %d)",
+                path, i, jPathListSize);
+            goto done;
+        }
+    }
+    ret = 0;
+
+done:
+    destroyLocalReference(env, jPath);
+    destroyLocalReference(env, jPathList);
+
+    if (ret) {
+        hdfsFreeFileInfo(pathList, jPathListSize);
+        errno = ret;
+        return NULL;
+    }
+    *numEntries = jPathListSize;
+    return pathList;
+}
+
+static void jni_rz_options_cache_free(void *vcache)
+{
+    struct jni_rz_options_cache *cache = vcache;
+    JNIEnv *env;
+
+    env = getJNIEnv();
+    if (!env) {
+        fprintf(stderr, "jni_rz_options_cache_free: failed to get JNI env\n");
+        return;
+    }
+    free(cache->pool_name);
+    if (cache->pool_obj) {
+        (*env)->DeleteGlobalRef(env, cache->pool_obj);
+    }
+    if (cache->enum_set) {
+        (*env)->DeleteGlobalRef(env, cache->enum_set);
+    }
+    free(cache);
+}
+
+static jthrowable jni_rz_setup_cache(JNIEnv *env, struct hadoopRzOptions *opts)
+{
+    if (opts->cache) {
+        if (opts->cache_teardown_cb != jni_rz_options_cache_free) {
+            return newRuntimeError(env, "jni_rz_options_get_enumset: this "
+                    "hadoopRzOptions structure is already associated with "
+                    "another filesystem.  Please create a new one.");
+        }
+        return NULL;
+    }
+    opts->cache = calloc(1, sizeof(struct jni_rz_options_cache));
+    if (!opts->cache) {
+        return newRuntimeError(env, "rz_get_or_create_cache: OOM "
+                "allocating jni_rz_options_cache");
+    }
+    opts->cache_teardown_cb = jni_rz_options_cache_free;
+    return NULL;
+}
+
+static jthrowable jni_rz_setup_enumset(JNIEnv *env,
+        struct hadoopRzOptions *opts)
+{
+    jthrowable jthr = NULL;
+    jobject enumInst = NULL, enumSetObj = NULL;
+    jvalue jVal;
+    struct jni_rz_options_cache *cache = opts->cache;
+
+    fprintf(stderr, "WATERMELON 4.1\n");
+    if (cache->enum_set) {
+    fprintf(stderr, "WATERMELON 4.2\n");
+        if (cache->skip_checksums == opts->skip_checksums) {
+    fprintf(stderr, "WATERMELON 4.3\n");
+            // If we cached the value, return it now.
+            goto done;
+        }
+    fprintf(stderr, "WATERMELON 4.4\n");
+        (*env)->DeleteGlobalRef(env, cache->enum_set);
+        cache->enum_set = NULL;
+    }
+    fprintf(stderr, "WATERMELON 4.5\n");
+    if (opts->skip_checksums) {
+    fprintf(stderr, "WATERMELON 4.6\n");
+        jthr = fetchEnumInstance(env, READ_OPTION,
+                  "SKIP_CHECKSUMS", &enumInst);
+        if (jthr) {
+            goto done;
+        }
+    fprintf(stderr, "WATERMELON 4.7\n");
+        jthr = invokeMethod(env, &jVal, STATIC, NULL,
+                "java/util/EnumSet", "of",
+                "(Ljava/lang/Enum;)Ljava/util/EnumSet;", enumInst);
+        if (jthr) {
+            goto done;
+        }
+        enumSetObj = jVal.l;
+    } else {
+    fprintf(stderr, "WATERMELON 4.8\n");
+        jclass clazz = (*env)->FindClass(env, READ_OPTION);
+        if (!clazz) {
+            jthr = newRuntimeError(env, "failed "
+                    "to find class for %s", READ_OPTION);
+            goto done;
+        }
+    fprintf(stderr, "WATERMELON 4.9\n");
+        jthr = invokeMethod(env, &jVal, STATIC, NULL,
+                "java/util/EnumSet", "noneOf",
+                "(Ljava/lang/Class;)Ljava/util/EnumSet;", clazz);
+        enumSetObj = jVal.l;
+    }
+    fprintf(stderr, "WATERMELON 4.95\n");
+    // create global ref
+    cache->enum_set = (*env)->NewGlobalRef(env, enumSetObj);
+    if (!cache->enum_set) {
+        jthr = getPendingExceptionAndClear(env);
+        goto done;
+    }
+    cache->skip_checksums = opts->skip_checksums;
+    jthr = NULL;
+done:
+    (*env)->DeleteLocalRef(env, enumInst);
+    (*env)->DeleteLocalRef(env, enumSetObj);
+    return jthr;
+}
+
+static jthrowable jni_rz_setup_bb_pool(JNIEnv *env,
+        struct hadoopRzOptions *opts)
+{
+    jthrowable jthr = NULL;
+    jobject pool_obj = NULL;
+    struct jni_rz_options_cache *cache = opts->cache;
+
+    if (!opts->pool_name) {
+        if (cache->pool_obj) {
+            (*env)->DeleteGlobalRef(env, cache->pool_obj);
+            cache->pool_obj = NULL;
+        }
+        free(cache->pool_name);
+        cache->pool_name = NULL;
+        goto done;
+    }
+    if (cache->pool_obj) {
+        if (!strcmp(cache->pool_name, opts->pool_name)) {
+            // If we cached the value, we can just use the existing value.
+            goto done;
+        }
+        (*env)->DeleteGlobalRef(env, cache->pool_obj);
+        cache->pool_obj = NULL;
+        free(cache->pool_name);
+    }
+    cache->pool_name = strdup(opts->pool_name);
+    if (!cache->pool_name) {
+        jthr = newRuntimeError(env, "jni_rz_setup_bb_pool: memory "
+                               "allocation failed.");
+        goto done;
+    }
+    jthr = constructNewObjectOfClass(env, &pool_obj, opts->pool_name, "()V");
+    if (jthr) {
+        fprintf(stderr, "jni_rz_setup_bb_pool(pool_name=%s): failed "
+            "to construct ByteBufferPool class.", opts->pool_name);
+        goto done;
+    }
+    cache->pool_obj = (*env)->NewGlobalRef(env, pool_obj);
+    if (!cache->pool_obj) {
+        jthr = getPendingExceptionAndClear(env);
+        goto done;
+    }
+    jthr = NULL;
+done:
+    (*env)->DeleteLocalRef(env, pool_obj);
+    if (jthr) {
+        free(cache->pool_name);
+        cache->pool_name = NULL;
+    }
+    return jthr;
+}
+
+static int jni_rz_extract_buffer(JNIEnv *env,
+        const struct hadoopRzOptions *opts, struct jni_rz_buffer *buf)
+{
+    int ret;
+    jthrowable jthr;
+    jvalue jVal;
+    uint8_t *direct_start;
+    void *malloc_buf = NULL;
+    jint position;
+    jarray array = NULL;
+
+    jthr = invokeMethod(env, &jVal, INSTANCE, buf->byte_buffer,
+                     "java/nio/ByteBuffer", "remaining", "()I");
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                "jni_rz_extract_buffer: ByteBuffer#remaining failed: ");
+        goto done;
+    }
+    buf->base.length = jVal.i;
+    jthr = invokeMethod(env, &jVal, INSTANCE, buf->byte_buffer,
+                     "java/nio/ByteBuffer", "position", "()I");
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                "jni_rz_extract_buffer: ByteBuffer#position failed: ");
+        goto done;
+    }
+    position = jVal.i;
+    direct_start = (*env)->GetDirectBufferAddress(env, buf->byte_buffer);
+    if (direct_start) {
+        // Handle direct buffers.
+        buf->base.ptr = direct_start + position;
+        buf->direct = 1;
+        ret = 0;
+        goto done;
+    }
+    // Handle indirect buffers.
+    // The JNI docs don't say that GetDirectBufferAddress throws any exceptions
+    // when it fails.  However, they also don't clearly say that it doesn't.  It
+    // seems safest to clear any pending exceptions here, to prevent problems on
+    // various JVMs.
+    (*env)->ExceptionClear(env);
+    if (!opts->pool_name) {
+        fputs("jni_rz_extract_buffer: we read through the "
+                "zero-copy path, but failed to get the address of the buffer via "
+                "GetDirectBufferAddress.  Please make sure your JVM supports "
+                "GetDirectBufferAddress.\n", stderr);
+        ret = ENOTSUP;
+        goto done;
+    }
+    // Get the backing array object of this buffer.
+    jthr = invokeMethod(env, &jVal, INSTANCE, buf->byte_buffer,
+                     "java/nio/ByteBuffer", "array", "()[B");
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                "jni_rz_extract_buffer: ByteBuffer#array failed: ");
+        goto done;
+    }
+    array = jVal.l;
+    if (!array) {
+        fputs("jni_rz_extract_buffer: ByteBuffer#array returned NULL.",
+              stderr);
+        ret = EIO;
+        goto done;
+    }
+    malloc_buf = malloc(buf->base.length);
+    if (!malloc_buf) {
+        fprintf(stderr, "jni_rz_extract_buffer: failed to allocate %d "
+                "bytes of memory\n", buf->base.length);
+        ret = ENOMEM;
+        goto done;
+    }
+    (*env)->GetByteArrayRegion(env, array, position,
+            buf->base.length, malloc_buf);
+    jthr = (*env)->ExceptionOccurred(env);
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                "jni_rz_extract_buffer: GetByteArrayRegion failed: ");
+        goto done;
+    }
+    buf->base.ptr = malloc_buf;
+    buf->direct = 0;
+    ret = 0;
+
+done:
+    free(malloc_buf);
+    (*env)->DeleteLocalRef(env, array);
+    return ret;
+}
+
+static int translate_zcr_exception(JNIEnv *env, jthrowable exc)
+{
+    int ret;
+    char *className = NULL;
+    jthrowable jthr = classNameOfObject(exc, env, &className);
+
+    if (jthr) {
+        fputs("jni_read_zero: failed to get class name of "
+                "exception from read().\n", stderr);
+        destroyLocalReference(env, exc);
+        destroyLocalReference(env, jthr);
+        ret = EIO;
+        goto done;
+    }
+    if (!strcmp(className, "java.lang.UnsupportedOperationException")) {
+        ret = EPROTONOSUPPORT;
+        goto done;
+    }
+    ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_read_zero failed");
+done:
+    free(className);
+    return ret;
+}
+
+static struct hadoopRzBuffer* jni_read_zero(hdfsFile bfile,
+            struct hadoopRzOptions *opts, int32_t maxLength)
+{
+    JNIEnv *env;
+    struct jni_file *file = (struct jni_file*)bfile;
+    jthrowable jthr = NULL;
+    jvalue jVal;
+    jobject byteBuffer = NULL;
+    struct jni_rz_buffer *buf = NULL;
+    int ret;
+    struct jni_rz_options_cache *cache;
+
+    env = getJNIEnv();
+    if (!env) {
+        errno = EINTERNAL;
+        return NULL;
+    }
+    if (file->type != INPUT) {
+        fputs("Cannot read from a non-InputStream object!\n", stderr);
+        ret = EINVAL;
+        goto done;
+    }
+    buf = calloc(1, sizeof(*buf));
+    if (!buf) {
+        ret = ENOMEM;
+        goto done;
+    }
+    fprintf(stderr, "WATERMELON 2\n");
+    jthr = jni_rz_setup_cache(env, opts);
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                "jni_read_zero: jni_rz_setup_cache failed: ");
+        goto done;
+    }
+    fprintf(stderr, "WATERMELON 3\n");
+    jthr = jni_rz_setup_enumset(env, opts);
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                "jni_read_zero: jni_rz_setup_enumset failed: ");
+        goto done;
+    }
+    fprintf(stderr, "WATERMELON 5\n");
+    jthr = jni_rz_setup_bb_pool(env, opts);
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                "jni_read_zero: jni_rz_setup_enumset failed: ");
+        goto done;
+    }
+    fprintf(stderr, "WATERMELON 5.1\n");
+    cache = opts->cache;
+    jthr = invokeMethod(env, &jVal, INSTANCE, file->stream, HADOOP_ISTRM,
+        "read", "(Lorg/apache/hadoop/io/ByteBufferPool;ILjava/util/EnumSet;)"
+        "Ljava/nio/ByteBuffer;", cache->pool_obj, maxLength, cache->enum_set);
+    if (jthr) {
+        ret = translate_zcr_exception(env, jthr);
+        goto done;
+    }
+    byteBuffer = jVal.l;
+    if (!byteBuffer) {
+        buf->base.ptr = NULL;
+        buf->base.length = 0;
+        buf->byte_buffer = NULL;
+        buf->direct = 0;
+    } else {
+        buf->byte_buffer = (*env)->NewGlobalRef(env, byteBuffer);
+        if (!buf->byte_buffer) {
+            ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL,
+                "jni_read_zero: failed to create global ref to ByteBuffer");
+            goto done;
+        }
+        ret = jni_rz_extract_buffer(env, opts, buf);
+        if (ret) {
+            goto done;
+        }
+    }
+    ret = 0;
+done:
+    (*env)->DeleteLocalRef(env, byteBuffer);
+    if (ret) {
+        if (buf) {
+            if (buf->byte_buffer) {
+                (*env)->DeleteGlobalRef(env, buf->byte_buffer);
+            }
+            free(buf);
+        }
+        errno = ret;
+        return NULL;
+    } else {
+        errno = 0;
+    }
+    return (struct hadoopRzBuffer*)buf;
+}
+
+static void jni_rz_buffer_free(hdfsFile bfile, struct hadoopRzBuffer *buffer)
+{
+    struct jni_file *file = (struct jni_file *)bfile;
+    struct jni_rz_buffer *buf = (struct jni_rz_buffer *)buffer;
+    jvalue jVal;
+    jthrowable jthr;
+    JNIEnv* env = getJNIEnv();
+    if (!env) {
+        fprintf(stderr, "jni_rz_buffer_free: failed to get a "
+                "thread-local JNI env");
+        return;
+    }
+    if (buf->byte_buffer) {
+        jthr = invokeMethod(env, &jVal, INSTANCE, file->stream,
+                    HADOOP_ISTRM, "releaseBuffer",
+                    "(Ljava/nio/ByteBuffer;)V", buf->byte_buffer);
+        if (jthr) {
+            printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                    "hadoopRzBufferFree: releaseBuffer failed: ");
+            // even on error, we have to delete the reference.
+        }
+        (*env)->DeleteGlobalRef(env, buf->byte_buffer);
+    }
+    if (!buf->direct) {
+        free(buf->base.ptr);
+    }
+    memset(buf, 0, sizeof(*buf));
+    free(buf);
+}
+
+char***
+jni_get_hosts(hdfsFS bfs, const char* path, tOffset start, tOffset length)
+{
+    struct jni_fs *fs = (struct jni_fs*)bfs;
+
+    // JAVA EQUIVALENT:
+    //  fs.getFileBlockLoctions(new Path(path), start, length);
+    jthrowable jthr;
+    jobject jPath = NULL;
+    jobject jFileStatus = NULL;
+    jvalue jFSVal, jVal;
+    jobjectArray jBlockLocations = NULL, jFileBlockHosts = NULL;
+    jstring jHost = NULL;
+    char*** blockHosts = NULL;
+    int i, j, ret;
+    jsize jNumFileBlocks = 0;
+
+    //Get the JNIEnv* corresponding to current thread
+    JNIEnv* env = getJNIEnv();
+    if (env == NULL) {
+      errno = EINTERNAL;
+      return NULL;
+    }
+
+    //Create an object of org.apache.hadoop.fs.Path
+    jthr = constructNewObjectOfPath(env, path, &jPath);
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_get_hosts(path=%s): constructNewObjectOfPath", path);
+        goto done;
+    }
+    jthr = invokeMethod(env, &jFSVal, INSTANCE, fs->obj,
+            HADOOP_FS, "getFileStatus", "(Lorg/apache/hadoop/fs/Path;)"
+            "Lorg/apache/hadoop/fs/FileStatus;", jPath);
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, NOPRINT_EXC_FILE_NOT_FOUND,
+                "jni_get_hosts(path=%s, start=%"PRId64", length=%"PRId64"):"
+                "FileSystem#getFileStatus", path, start, length);
+        destroyLocalReference(env, jPath);
+        goto done;
+    }
+    jFileStatus = jFSVal.l;
+
+    //org.apache.hadoop.fs.FileSystem#getFileBlockLocations
+    jthr = invokeMethod(env, &jVal, INSTANCE, fs->obj,
+                     HADOOP_FS, "getFileBlockLocations", 
+                     "(Lorg/apache/hadoop/fs/FileStatus;JJ)"
+                     "[Lorg/apache/hadoop/fs/BlockLocation;",
+                     jFileStatus, start, length);
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                "jni_get_hosts(path=%s, start=%"PRId64", length=%"PRId64"):"
+                "FileSystem#getFileBlockLocations", path, start, length);
+        goto done;
+    }
+    jBlockLocations = jVal.l;
+
+    //Figure out no of entries in jBlockLocations
+    //Allocate memory and add NULL at the end
+    jNumFileBlocks = (*env)->GetArrayLength(env, jBlockLocations);
+
+    blockHosts = calloc(jNumFileBlocks + 1, sizeof(char**));
+    if (blockHosts == NULL) {
+        ret = ENOMEM;
+        goto done;
+    }
+    if (jNumFileBlocks == 0) {
+        ret = 0;
+        goto done;
+    }
+
+    //Now parse each block to get hostnames
+    for (i = 0; i < jNumFileBlocks; ++i) {
+        jobject jFileBlock =
+            (*env)->GetObjectArrayElement(env, jBlockLocations, i);
+        if (!jFileBlock) {
+            ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL,
+                "jni_get_hosts(path=%s, start=%"PRId64", length=%"PRId64"):"
+                "GetObjectArrayElement(%d)", path, start, length, i);
+            goto done;
+        }
+        
+        jthr = invokeMethod(env, &jVal, INSTANCE, jFileBlock, HADOOP_BLK_LOC,
+                         "getHosts", "()[Ljava/lang/String;");
+        if (jthr) {
+            ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                "jni_get_hosts(path=%s, start=%"PRId64", length=%"PRId64"):"
+                "BlockLocation#getHosts", path, start, length);
+            goto done;
+        }
+        jFileBlockHosts = jVal.l;
+        if (!jFileBlockHosts) {
+            fprintf(stderr,
+                "jni_get_hosts(path=%s, start=%"PRId64", length=%"PRId64"):"
+                "BlockLocation#getHosts returned NULL", path, start, length);
+            ret = EINTERNAL;
+            goto done;
+        }
+        //Figure out no of hosts in jFileBlockHosts, and allocate the memory
+        jsize jNumBlockHosts = (*env)->GetArrayLength(env, jFileBlockHosts);
+        blockHosts[i] = calloc(jNumBlockHosts + 1, sizeof(char*));
+        if (!blockHosts[i]) {
+            ret = ENOMEM;
+            goto done;
+        }
+
+        //Now parse each hostname
+        const char *hostName;
+        for (j = 0; j < jNumBlockHosts; ++j) {
+            jHost = (*env)->GetObjectArrayElement(env, jFileBlockHosts, j);
+            if (!jHost) {
+                ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL,
+                    "jni_get_hosts(path=%s, start=%"PRId64", length=%"PRId64"): "
+                    "NewByteArray", path, start, length);
+                goto done;
+            }
+            hostName =
+                (const char*)((*env)->GetStringUTFChars(env, jHost, NULL));
+            if (!hostName) {
+                ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL,
+                    "jni_get_hosts(path=%s, start=%"PRId64", length=%"PRId64", "
+                    "j=%d out of %d): GetStringUTFChars",
+                    path, start, length, j, jNumBlockHosts);
+                goto done;
+            }
+            blockHosts[i][j] = strdup(hostName);
+            (*env)->ReleaseStringUTFChars(env, jHost, hostName);
+            if (!blockHosts[i][j]) {
+                ret = ENOMEM;
+                goto done;
+            }
+            destroyLocalReference(env, jHost);
+            jHost = NULL;
+        }
+
+        destroyLocalReference(env, jFileBlockHosts);
+        jFileBlockHosts = NULL;
+    }
+    ret = 0;
+
+done:
+    destroyLocalReference(env, jPath);
+    destroyLocalReference(env, jFileStatus);
+    destroyLocalReference(env, jBlockLocations);
+    destroyLocalReference(env, jFileBlockHosts);
+    destroyLocalReference(env, jHost);
+    if (ret) {
+        if (blockHosts) {
+            hdfsFreeHosts(blockHosts);
+        }
+        return NULL;
+    }
+
+    return blockHosts;
+}
+
+static tOffset jni_get_default_block_size(hdfsFS bfs)
+{
+    // JAVA EQUIVALENT:
+    //  fs.getDefaultBlockSize();
+    jvalue jVal;
+    jthrowable jthr;
+    struct jni_fs *fs = (struct jni_fs*)bfs;
+    JNIEnv* env = getJNIEnv();
+    if (env == NULL) {
+      errno = EINTERNAL;
+      return -1;
+    }
+    jthr = invokeMethod(env, &jVal, INSTANCE, fs->obj, HADOOP_FS,
+                     "getDefaultBlockSize", "()J");
+    if (jthr) {
+        errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_get_default_block_size: FileSystem#getDefaultBlockSize");
+        return -1;
+    }
+    return jVal.j;
+}
+
+static tOffset jni_get_default_block_size_at_path(hdfsFS bfs, const char *path)
+{
+    struct jni_fs *fs = (struct jni_fs*)bfs;
+    // JAVA EQUIVALENT:
+    //  fs.getDefaultBlockSize(path);
+
+    jthrowable jthr;
+    jobject jPath;
+    tOffset blockSize;
+    JNIEnv* env = getJNIEnv();
+
+    if (env == NULL) {
+        errno = EINTERNAL;
+        return -1;
+    }
+    jthr = constructNewObjectOfPath(env, path, &jPath);
+    if (jthr) {
+        errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_get_default_block_size_at_path(path=%s): "
+            "constructNewObjectOfPath", path);
+        return -1;
+    }
+    jthr = getDefaultBlockSize(env, fs->obj, jPath, &blockSize);
+    (*env)->DeleteLocalRef(env, jPath);
+    if (jthr) {
+        errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_get_default_block_size_at_path(path=%s): "
+            "FileSystem#getDefaultBlockSize", path);
+        return -1;
+    }
+    return blockSize;
+}
+
+static tOffset jni_get_capacity(hdfsFS bfs)
+{
+    struct jni_fs *fs = (struct jni_fs*)bfs;
+    // JAVA EQUIVALENT:
+    //  FsStatus fss = fs.getStatus();
+    //  return Fss.getCapacity();
+
+    //Get the JNIEnv* corresponding to current thread
+    JNIEnv* env = getJNIEnv();
+    if (env == NULL) {
+      errno = EINTERNAL;
+      return -1;
+    }
+
+    //FileSystem#getStatus
+    jvalue  jVal;
+    jthrowable jthr;
+    jthr = invokeMethod(env, &jVal, INSTANCE, fs->obj, HADOOP_FS,
+                     "getStatus", "()Lorg/apache/hadoop/fs/FsStatus;");
+    if (jthr) {
+        errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_get_capacity: FileSystem#getStatus");
+        return -1;
+    }
+    jobject fss = (jobject)jVal.l;
+    jthr = invokeMethod(env, &jVal, INSTANCE, fss, HADOOP_FSSTATUS,
+                     "getCapacity", "()J");
+    destroyLocalReference(env, fss);
+    if (jthr) {
+        errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_get_capacity: FsStatus#getCapacity");
+        return -1;
+    }
+    return jVal.j;
+}
+
+static jthrowable
+getFileInfoFromStat(JNIEnv *env, jobject jStat, hdfsFileInfo *fileInfo)
+{
+    jvalue jVal;
+    jthrowable jthr;
+    jobject jPath = NULL;
+    jstring jPathName = NULL;
+    jstring jUserName = NULL;
+    jstring jGroupName = NULL;
+    jobject jPermission = NULL;
+
+    jthr = invokeMethod(env, &jVal, INSTANCE, jStat,
+                     HADOOP_STAT, "isDir", "()Z");
+    if (jthr)
+        goto done;
+    fileInfo->mKind = jVal.z ? kObjectKindDirectory : kObjectKindFile;
+
+    jthr = invokeMethod(env, &jVal, INSTANCE, jStat,
+                     HADOOP_STAT, "getReplication", "()S");
+    if (jthr)
+        goto done;
+    fileInfo->mReplication = jVal.s;
+
+    jthr = invokeMethod(env, &jVal, INSTANCE, jStat,
+                     HADOOP_STAT, "getBlockSize", "()J");
+    if (jthr)
+        goto done;
+    fileInfo->mBlockSize = jVal.j;
+
+    jthr = invokeMethod(env, &jVal, INSTANCE, jStat,
+                     HADOOP_STAT, "getModificationTime", "()J");
+    if (jthr)
+        goto done;
+    fileInfo->mLastMod = jVal.j / 1000;
+
+    jthr = invokeMethod(env, &jVal, INSTANCE, jStat,
+                     HADOOP_STAT, "getAccessTime", "()J");
+    if (jthr)
+        goto done;
+    fileInfo->mLastAccess = (tTime) (jVal.j / 1000);
+
+    if (fileInfo->mKind == kObjectKindFile) {
+        jthr = invokeMethod(env, &jVal, INSTANCE, jStat,
+                         HADOOP_STAT, "getLen", "()J");
+        if (jthr)
+            goto done;
+        fileInfo->mSize = jVal.j;
+    }
+
+    jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT,
+                     "getPath", "()Lorg/apache/hadoop/fs/Path;");
+    if (jthr)
+        goto done;
+    jPath = jVal.l;
+    if (jPath == NULL) {
+        jthr = newRuntimeError(env, "org.apache.hadoop.fs.FileStatus#"
+            "getPath returned NULL!");
+        goto done;
+    }
+
+    jthr = invokeMethod(env, &jVal, INSTANCE, jPath, HADOOP_PATH,
+                     "toString", "()Ljava/lang/String;");
+    if (jthr)
+        goto done;
+    jPathName = jVal.l;
+    const char *cPathName = 
+        (const char*) ((*env)->GetStringUTFChars(env, jPathName, NULL));
+    if (!cPathName) {
+        jthr = getPendingExceptionAndClear(env);
+        goto done;
+    }
+    fileInfo->mName = strdup(cPathName);
+    (*env)->ReleaseStringUTFChars(env, jPathName, cPathName);
+    jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT,
+                    "getOwner", "()Ljava/lang/String;");
+    if (jthr)
+        goto done;
+    jUserName = jVal.l;
+    const char* cUserName = 
+        (const char*) ((*env)->GetStringUTFChars(env, jUserName, NULL));
+    if (!cUserName) {
+        jthr = getPendingExceptionAndClear(env);
+        goto done;
+    }
+    fileInfo->mOwner = strdup(cUserName);
+    (*env)->ReleaseStringUTFChars(env, jUserName, cUserName);
+
+    const char* cGroupName;
+    jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT,
+                    "getGroup", "()Ljava/lang/String;");
+    if (jthr)
+        goto done;
+    jGroupName = jVal.l;
+    cGroupName = (const char*) ((*env)->GetStringUTFChars(env, jGroupName, NULL));
+    if (!cGroupName) {
+        jthr = getPendingExceptionAndClear(env);
+        goto done;
+    }
+    fileInfo->mGroup = strdup(cGroupName);
+    (*env)->ReleaseStringUTFChars(env, jGroupName, cGroupName);
+
+    jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT,
+            "getPermission",
+            "()Lorg/apache/hadoop/fs/permission/FsPermission;");
+    if (jthr)
+        goto done;
+    if (jVal.l == NULL) {
+        jthr = newRuntimeError(env, "%s#getPermission returned NULL!",
+            HADOOP_STAT);
+        goto done;
+    }
+    jPermission = jVal.l;
+    jthr = invokeMethod(env, &jVal, INSTANCE, jPermission, HADOOP_FSPERM,
+                         "toShort", "()S");
+    if (jthr)
+        goto done;
+    fileInfo->mPermissions = jVal.s;
+    jthr = NULL;
+
+done:
+    if (jthr)
+        release_file_info_entry(fileInfo);
+    destroyLocalReference(env, jPath);
+    destroyLocalReference(env, jPathName);
+    destroyLocalReference(env, jUserName);
+    destroyLocalReference(env, jGroupName);
+    destroyLocalReference(env, jPermission);
+    destroyLocalReference(env, jPath);
+    return jthr;
+}
+
+static jthrowable
+getFileInfo(JNIEnv *env, jobject jFS, jobject jPath, hdfsFileInfo **fileInfo)
+{
+    // JAVA EQUIVALENT:
+    //  fs.isDirectory(f)
+    //  fs.getModificationTime()
+    //  fs.getAccessTime()
+    //  fs.getLength(f)
+    //  f.getPath()
+    //  f.getOwner()
+    //  f.getGroup()
+    //  f.getPermission().toShort()
+    jobject jStat;
+    jvalue  jVal;
+    jthrowable jthr;
+
+    jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS,
+                     "exists", JMETHOD1(JPARAM(HADOOP_PATH), "Z"),
+                     jPath);
+    if (jthr)
+        return jthr;
+    if (jVal.z == 0) {
+        *fileInfo = NULL;
+        return NULL;
+    }
+    jthr = invokeMethod(env, &jVal, INSTANCE, jFS,
+            HADOOP_FS, "getFileStatus",
+            JMETHOD1(JPARAM(HADOOP_PATH), JPARAM(HADOOP_STAT)), jPath);
+    if (jthr)
+        return jthr;
+    jStat = jVal.l;
+    *fileInfo = calloc(1, sizeof(hdfsFileInfo));
+    if (!*fileInfo) {
+        destroyLocalReference(env, jStat);
+        return newRuntimeError(env, "getFileInfo: OOM allocating hdfsFileInfo");
+    }
+    jthr = getFileInfoFromStat(env, jStat, *fileInfo); 
+    destroyLocalReference(env, jStat);
+    return jthr;
+}
+
+static hdfsFileInfo *jni_get_path_info(hdfsFS bfs, const char* path)
+{
+    struct jni_fs *fs = (struct jni_fs*)bfs;
+    // JAVA EQUIVALENT:
+    //  File f(path);
+    //  fs.isDirectory(f)
+    //  fs.lastModified() ??
+    //  fs.getLength(f)
+    //  f.getPath()
+
+    //Get the JNIEnv* corresponding to current thread
+    JNIEnv* env = getJNIEnv();
+    if (env == NULL) {
+      errno = EINTERNAL;
+      return NULL;
+    }
+
+    //Create an object of org.apache.hadoop.fs.Path
+    jobject jPath;
+    jthrowable jthr = constructNewObjectOfPath(env, path, &jPath);
+    if (jthr) {
+        errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_get_path_info(%s): constructNewObjectOfPath", path);
+        return NULL;
+    }
+    hdfsFileInfo *fileInfo;
+    jthr = getFileInfo(env, fs->obj, jPath, &fileInfo);
+    destroyLocalReference(env, jPath);
+    if (jthr) {
+        errno = printExceptionAndFree(env, jthr,
+            NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND |
+            NOPRINT_EXC_UNRESOLVED_LINK,
+            "jni_get_path_info(%s): getFileInfo", path);
+        return NULL;
+    }
+    if (!fileInfo) {
+        errno = ENOENT;
+        return NULL;
+    }
+    return fileInfo;
+}
+
+static tOffset jni_get_used(hdfsFS bfs)
+{
+    struct jni_fs *fs = (struct jni_fs*)bfs;
+    // JAVA EQUIVALENT:
+    //  FsStatus fss = fs.getStatus();
+    //  return Fss.getUsed();
+
+    //Get the JNIEnv* corresponding to current thread
+    JNIEnv* env = getJNIEnv();
+    if (env == NULL) {
+      errno = EINTERNAL;
+      return -1;
+    }
+
+    //FileSystem#getStatus
+    jvalue  jVal;
+    jthrowable jthr;
+    jthr = invokeMethod(env, &jVal, INSTANCE, fs->obj, HADOOP_FS,
+                     "getStatus", "()Lorg/apache/hadoop/fs/FsStatus;");
+    if (jthr) {
+        errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_get_used: FileSystem#getStatus");
+        return -1;
+    }
+    jobject fss = (jobject)jVal.l;
+    jthr = invokeMethod(env, &jVal, INSTANCE, fss, HADOOP_FSSTATUS,
+                     "getUsed", "()J");
+    destroyLocalReference(env, fss);
+    if (jthr) {
+        errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_get_used: FsStatus#getUsed");
+        return -1;
+    }
+    return jVal.j;
+}
+ 
+int jni_chown(hdfsFS bfs, const char* path, const char *owner, const char *group)
+{
+    struct jni_fs *fs = (struct jni_fs*)bfs;
+    // JAVA EQUIVALENT:
+    //  fs.setOwner(path, owner, group)
+
+    //Get the JNIEnv* corresponding to current thread
+    JNIEnv* env = getJNIEnv();
+    if (env == NULL) {
+      errno = EINTERNAL;
+      return -1;
+    }
+
+    if (owner == NULL && group == NULL) {
+      return 0;
+    }
+
+    jobject jPath = NULL;
+    jstring jOwner = NULL, jGroup = NULL;
+    jthrowable jthr;
+    int ret;
+
+    jthr = constructNewObjectOfPath(env, path, &jPath);
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_chown(path=%s): constructNewObjectOfPath", path);
+        goto done;
+    }
+
+    jthr = newJavaStr(env, owner, &jOwner); 
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_chown(path=%s): newJavaStr(%s)", path, owner);
+        goto done;
+    }
+    jthr = newJavaStr(env, group, &jGroup);
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_chown(path=%s): newJavaStr(%s)", path, group);
+        goto done;
+    }
+
+    //Create the directory
+    jthr = invokeMethod(env, NULL, INSTANCE, fs->obj, HADOOP_FS,
+            "setOwner", JMETHOD3(JPARAM(HADOOP_PATH), 
+                    JPARAM(JAVA_STRING), JPARAM(JAVA_STRING), JAVA_VOID),
+            jPath, jOwner, jGroup);
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr,
+            NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND |
+            NOPRINT_EXC_UNRESOLVED_LINK,
+            "jni_chown(path=%s, owner=%s, group=%s): "
+            "FileSystem#setOwner", path, owner, group);
+        goto done;
+    }
+    ret = 0;
+
+done:
+    destroyLocalReference(env, jPath);
+    destroyLocalReference(env, jOwner);
+    destroyLocalReference(env, jGroup);
+
+    if (ret) {
+        errno = ret;
+        return -1;
+    }
+    return 0;
+}
+
+int jni_chmod(hdfsFS bfs, const char* path, short mode)
+{
+    struct jni_fs *fs = (struct jni_fs*)bfs;
+    int ret;
+    // JAVA EQUIVALENT:
+    //  fs.setPermission(path, FsPermission)
+
+    //Get the JNIEnv* corresponding to current thread
+    JNIEnv* env = getJNIEnv();
+    if (env == NULL) {
+      errno = EINTERNAL;
+      return -1;
+    }
+
+    jthrowable jthr;
+    jobject jPath = NULL, jPermObj = NULL;
+
+    // construct jPerm = FsPermission.createImmutable(short mode);
+    jshort jmode = mode;
+    jthr = constructNewObjectOfClass(env, &jPermObj,
+                HADOOP_FSPERM,"(S)V",jmode);
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "constructNewObjectOfClass(%s)", HADOOP_FSPERM);
+        return -1;
+    }
+
+    //Create an object of org.apache.hadoop.fs.Path
+    jthr = constructNewObjectOfPath(env, path, &jPath);
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_chmod(%s): constructNewObjectOfPath", path);
+        goto done;
+    }
+
+    //Create the directory
+    jthr = invokeMethod(env, NULL, INSTANCE, fs->obj, HADOOP_FS,
+            "setPermission",
+            JMETHOD2(JPARAM(HADOOP_PATH), JPARAM(HADOOP_FSPERM), JAVA_VOID),
+            jPath, jPermObj);
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr,
+            NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND |
+            NOPRINT_EXC_UNRESOLVED_LINK,
+            "jni_chmod(%s): FileSystem#setPermission", path);
+        goto done;
+    }
+    ret = 0;
+
+done:
+    destroyLocalReference(env, jPath);
+    destroyLocalReference(env, jPermObj);
+
+    if (ret) {
+        errno = ret;
+        return -1;
+    }
+    return 0;
+}
+
+static int jni_utime(hdfsFS bfs, const char* path, tTime mtime, tTime atime)
+{
+    struct jni_fs *fs = (struct jni_fs*)bfs;
+
+    // JAVA EQUIVALENT:
+    //  fs.setTimes(src, mtime, atime)
+    jthrowable jthr;
+
+    //Get the JNIEnv* corresponding to current thread
+    JNIEnv* env = getJNIEnv();
+    if (env == NULL) {
+      errno = EINTERNAL;
+      return -1;
+    }
+
+    //Create an object of org.apache.hadoop.fs.Path
+    jobject jPath;
+    jthr = constructNewObjectOfPath(env, path, &jPath);
+    if (jthr) {
+        errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "jni_utime(path=%s): constructNewObjectOfPath", path);
+        return -1;
+    }
+
+    const tTime NO_CHANGE = -1;
+    jlong jmtime = (mtime == NO_CHANGE) ? -1 : (mtime * (jlong)1000);
+    jlong jatime = (atime == NO_CHANGE) ? -1 : (atime * (jlong)1000);
+
+    jthr = invokeMethod(env, NULL, INSTANCE, fs->obj, HADOOP_FS,
+            "setTimes", JMETHOD3(JPARAM(HADOOP_PATH), "J", "J", JAVA_VOID),
+            jPath, jmtime, jatime);
+    destroyLocalReference(env, jPath);
+    if (jthr) {
+        errno = printExceptionAndFree(env, jthr,
+            NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND |
+            NOPRINT_EXC_UNRESOLVED_LINK,
+            "jni_utime(path=%s): FileSystem#setTimes", path);
+        return -1;
+    }
+    return 0;
+}
+
+int jni_file_uses_direct_read(hdfsFile bfile)
+{
+    struct jni_file *file = (struct jni_file*)bfile;
+    return !!(file->flags & HDFS_FILE_SUPPORTS_DIRECT_READ);
+}
+
+void jni_file_disable_direct_read(hdfsFile bfile)
+{
+    struct jni_file *file = (struct jni_file*)bfile;
+    file->flags &= ~HDFS_FILE_SUPPORTS_DIRECT_READ;
+}
+
+const struct hadoop_fs_ops g_jni_ops = {
+    .name = "jnifs",
+    .file_is_open_for_read = jni_file_is_open_for_read,
+    .file_is_open_for_write = jni_file_is_open_for_write,
+    .get_read_statistics = jni_file_get_read_statistics,
+    .connect = jni_connect,
+    .disconnect = jni_disconnect,
+    .open = jni_open_file,
+    .close = jni_close_file,
+    .exists = jni_file_exists,
+    .seek = jni_seek,
+    .tell = jni_tell,
+    .read = jni_read,
+    .pread = jni_pread,
+    .write = jni_write,
+    .flush = jni_flush,
+    .hflush = jni_hflush,
+    .hsync = jni_hsync,
+    .available = jni_available,
+    .copy = jni_copy,
+    .move = jni_move,
+    .unlink = jni_unlink,
+    .rename = jni_rename,
+    .get_working_directory = jni_get_working_directory,
+    .set_working_directory = jni_set_working_directory,
+    .mkdir = jni_mkdir,
+    .set_replication = jni_set_replication,
+    .list_directory = jni_list_directory,
+    .get_path_info = jni_get_path_info,
+    .get_hosts = jni_get_hosts,
+    .get_default_block_size = jni_get_default_block_size,
+    .get_default_block_size_at_path = jni_get_default_block_size_at_path,
+    .get_capacity = jni_get_capacity,
+    .get_used = jni_get_used,
+    .chown = jni_chown,
+    .chmod = jni_chmod,
+    .utime = jni_utime,
+    .read_zero = jni_read_zero,
+    .rz_buffer_free = jni_rz_buffer_free,
+
+    // test
+    .file_uses_direct_read = jni_file_uses_direct_read,
+    .file_disable_direct_read = jni_file_disable_direct_read,
+};
+
+// vim: ts=4:sw=4:et

hadoop-native-core/src/main/native/ndfs/namenode-rpc-unit.c

@@ -0,0 +1,137 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "common/hadoop_err.h"
+#include "common/user.h"
+#include "protobuf/ClientNamenodeProtocol.call.h"
+#include "rpc/messenger.h"
+#include "rpc/proxy.h"
+#include "test/test.h"
+
+#include <netinet/in.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <strings.h>
+#include <uv.h>
+
+struct options {
+    struct sockaddr_in remote;
+    char *username;
+};
+
+static void options_from_env(struct options *opts)
+{
+    const char *ip_str;
+    const char *port_str;
+    const char *username;
+    int res, port;
+
+    ip_str = getenv("HDFS_IP");
+    if (!ip_str) {
+        fprintf(stderr, "You must set an ip via the HDFS_IP "
+                "environment variable.\n");
+        exit(EXIT_FAILURE);
+    }
+    port_str = getenv("HDFS_PORT");
+    if (!port_str) {
+        fprintf(stderr, "You must set a port via the HDFS_PORT "
+                "environment variable.\n");
+        exit(EXIT_FAILURE);
+    }
+    port = atoi(port_str);
+    res = uv_ip4_addr(ip_str, port, &opts->remote);
+    if (res) {
+        fprintf(stderr, "Invalid IP and port %s and %d: error %s\n",
+                ip_str, port, uv_strerror(res));
+        exit(EXIT_FAILURE);
+    }
+    username = getenv("HDFS_USERNAME");
+    if (username) {
+        opts->username = strdup(username);
+        if (!opts->username)
+            abort();
+        fprintf(stderr, "using HDFS username %s\n", username);
+    } else {
+        res = geteuid_string(&opts->username);
+        if (res) {
+            fprintf(stderr, "geteuid_string failed with error %d\n", res);
+            abort();
+        }
+    }
+}
+
+void set_replication_cb(SetReplicationResponseProto *resp,
+                        struct hadoop_err *err, void *cb_data)
+{
+    uv_sem_t *sem = cb_data;
+
+    if (err) {
+        fprintf(stderr, "set_replication_cb: got an error.  %s\n",
+                hadoop_err_msg(err));
+    } else {
+        fprintf(stderr, "set_replication_cb: resp->result = %d\n",
+                !!resp->result);
+    }
+
+    uv_sem_post(sem);
+    if (err) {
+        hadoop_err_free(err);
+    }
+    if (resp) {
+        set_replication_response_proto__free_unpacked(resp, NULL);
+    }
+}
+
+
+
+int main(void)
+{
+    struct hrpc_messenger_builder *msgr_bld;
+    struct hrpc_messenger *msgr;
+    struct hrpc_proxy proxy;
+    struct options opts;
+    uv_sem_t sem;
+
+    memset(&opts, 0, sizeof(opts));
+    options_from_env(&opts);
+    msgr_bld = hrpc_messenger_builder_alloc();
+    EXPECT_NONNULL(msgr_bld);
+    EXPECT_NO_HADOOP_ERR(hrpc_messenger_create(msgr_bld, &msgr));
+
+    hrpc_proxy_init(&proxy, msgr, &opts.remote,
+            "org.apache.hadoop.hdfs.protocol.ClientProtocol",
+            opts.username);
+    EXPECT_INT_ZERO(uv_sem_init(&sem, 0));
+    {
+        SetReplicationRequestProto req = SET_REPLICATION_REQUEST_PROTO__INIT;
+        req.src = "/foo2";
+        req.replication = 2;
+        cnn_async_set_replication(&proxy, &req, set_replication_cb, &sem);
+    }
+    uv_sem_wait(&sem);
+
+    hrpc_messenger_shutdown(msgr);
+    hrpc_messenger_free(msgr);
+    uv_sem_destroy(&sem);
+
+    free(opts.username);
+    return EXIT_SUCCESS;
+}
+
+// vim: ts=4:sw=4:tw=79:et

hadoop-native-core/src/main/native/ndfs/ndfs.c

@@ -0,0 +1,1081 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "common/hadoop_err.h"
+#include "common/hconf.h"
+#include "common/net.h"
+#include "common/string.h"
+#include "common/uri.h"
+#include "fs/common.h"
+#include "fs/fs.h"
+#include "protobuf/ClientNamenodeProtocol.call.h"
+#include "protobuf/hdfs.pb-c.h.s"
+#include "rpc/messenger.h"
+#include "rpc/proxy.h"
+
+#include <netinet/in.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <strings.h>
+#include <uriparser/Uri.h>
+#include <uv.h>
+
+#define DEFAULT_NN_PORT 8020
+
+#define CLIENT_NN_PROTOCOL "org.apache.hadoop.hdfs.protocol.ClientProtocol"
+
+struct native_fs {
+    /** Fields common to all filesystems. */
+    struct hadoop_fs_base base;
+
+    /**
+     * Address of the namenode.
+     * TODO: implement HA failover
+     * TODO: implement IPv6
+     */
+    struct sockaddr_in nn_addr;
+
+    /** The messenger used to perform RPCs. */
+    struct hrpc_messenger *msgr;
+
+    /** The default block size obtained from getServerDefaults. */
+    int64_t default_block_size;
+
+    /** User name to use for RPCs.  Immutable. */
+    char *user_name;
+
+    /**
+     * A dynamically allocated working directory which will be prepended to
+     * all relative paths.
+     */
+    UriUriA *working_uri; 
+
+    /** Lock which protects the working_uri. */
+    uv_mutex_t working_uri_lock;
+};
+
+/**
+ * Set if the file is read-only... otherwise, the file is assumed to be
+ * write-only.
+ */
+#define NDFS_FILE_FLAG_RO                       0x1
+
+/** This flag is for compatibility with some old test harnesses. */
+#define NDFS_FILE_FLAG_DISABLE_DIRECT_READ      0x2
+
+/** Base class for both read-only and write-only files. */
+struct native_file_base {
+    /** Fields common to all filesystems. */
+    struct hadoop_file_base base;
+
+    /** NDFS file flags. */
+    int flags;
+};
+
+/** A read-only file. */
+struct native_ro_file {
+    struct native_file_base base;
+    uint64_t bytes_read;
+};
+
+/** A write-only file. */
+struct native_wo_file {
+    struct native_file_base base;
+};
+
+/** Whole-filesystem stats sent back from the NameNode. */
+struct hadoop_vfs_stats {
+    int64_t capacity;
+    int64_t used;
+    int64_t remaining;
+    int64_t under_replicated;
+    int64_t corrupt_blocks;
+    int64_t missing_blocks;
+};
+
+/** Server defaults sent back from the NameNode. */
+struct ndfs_server_defaults {
+    uint64_t blocksize;
+};
+
+static hdfsFileInfo *ndfs_get_path_info(hdfsFS bfs, const char* uri);
+
+static void ndfs_nn_proxy_init(struct native_fs *fs, struct hrpc_proxy *proxy)
+{
+    hrpc_proxy_init(proxy, fs->msgr, &fs->nn_addr, CLIENT_NN_PROTOCOL,
+                    fs->user_name);
+}
+
+/**
+ * Construct a canonical path from a URI.
+ *
+ * @param fs        The filesystem.
+ * @param uri       The URI.
+ * @param out       (out param) the canonical path.
+ *
+ * @return          NULL on success; the error otherwise.
+ */
+static struct hadoop_err *build_path(struct native_fs *fs, const char *uri_str,
+                                     char **out)
+{
+    char *path = NULL;
+    struct hadoop_err *err = NULL;
+    UriParserStateA uri_state;
+    UriUriA uri;
+
+    memset(&uri_state, 0, sizeof(uri_state));
+
+    uv_mutex_lock(&fs->working_uri_lock);
+    err = uri_parse(uri_str, &uri_state, &uri, fs->working_uri);
+    if (err)
+        goto done;
+    // TODO: check URI scheme and user against saved values?
+    err = uri_get_path(&uri, &path);
+    if (err)
+        goto done;
+    // As a special case, when the URI given has an empty path, we assume that
+    // we want the current working directory.  This is to allow things like
+    // hdfs://mynamenode to map to the current working directory, as they do in
+    // Hadoop.  Note that this is different than hdfs://mynamenode/ (note the
+    // trailing slash) which maps to the root directory.
+    if (!path[0]) {
+        free(path);
+        path = NULL;
+        err = uri_get_path(fs->working_uri, &path);
+        if (err) {
+            goto done;
+        }
+    }
+    err = NULL;
+
+done:
+    uv_mutex_unlock(&fs->working_uri_lock);
+    if (uri_state.uri) {
+        uriFreeUriMembersA(&uri);
+    }
+    if (err) {
+        free(path);
+        return err;
+    }
+    *out = path;
+    return NULL;
+}
+
+static int ndfs_file_is_open_for_read(hdfsFile bfile)
+{
+    struct native_file_base *file = (struct native_file_base *)bfile;
+    return !!(file->flags & NDFS_FILE_FLAG_RO);
+}
+
+static int ndfs_file_is_open_for_write(hdfsFile bfile)
+{
+    struct native_file_base *file = (struct native_file_base *)bfile;
+    return !(file->flags & NDFS_FILE_FLAG_RO);
+}
+
+static int ndfs_file_get_read_statistics(hdfsFile bfile,
+            struct hdfsReadStatistics **out)
+{
+    struct hdfsReadStatistics *stats;
+    struct native_ro_file *file = (struct native_ro_file *)bfile;
+
+    if (!(file->base.flags & NDFS_FILE_FLAG_RO)) {
+        errno = EINVAL;
+        return -1;
+    }
+    stats = calloc(1, sizeof(*stats));
+    if (!stats) {
+        errno = ENOMEM; 
+        return -1;
+    }
+    stats->totalBytesRead = file->bytes_read;
+    *out = stats;
+    return 0;
+}
+
+static struct hadoop_err *ndfs_get_server_defaults(struct native_fs *fs,
+            struct ndfs_server_defaults *defaults)
+{
+    struct hadoop_err *err = NULL;
+    GetServerDefaultsRequestProto req =
+        GET_SERVER_DEFAULTS_REQUEST_PROTO__INIT;
+    GetServerDefaultsResponseProto *resp = NULL;
+    struct hrpc_proxy proxy;
+
+    ndfs_nn_proxy_init(fs, &proxy);
+    err = cnn_get_server_defaults(&proxy, &req, &resp);
+    if (err) {
+        goto done;
+    }
+    defaults->blocksize = resp->serverdefaults->blocksize;
+
+done:
+    if (resp) {
+        get_server_defaults_response_proto__free_unpacked(resp, NULL);
+    }
+    return err;
+}
+
+/**
+ * Parse an address in the form <hostname> or <hostname>:<port>.
+ *
+ * @param host          The hostname
+ * @param addr          (out param) The sockaddr.
+ * @param default_port  The default port to use, if one is not found in the
+ *                          string.
+ *
+ * @return              NULL on success; the error otherwise.
+ */
+static struct hadoop_err *parse_rpc_addr(const char *input,
+            struct sockaddr_in *out, int default_port)
+{
+    struct hadoop_err *err = NULL;
+    char *host, *colon;
+    uint32_t addr;
+    int port;
+
+    fprintf(stderr, "parse_rpc_addr(input=%s, default_port=%d)\n",
+            input, default_port);
+
+    // If the URI doesn't contain a port, we use a default.
+    // This may come either from the hdfsBuilder, or from the
+    // 'default default' for HDFS.
+    // It's kind of silly that hdfsBuilder even includes this field, since this
+    // information should just be included in the URI, but this is here for
+    // compatibility.
+    port = (default_port <= 0) ? DEFAULT_NN_PORT : default_port;
+    host = strdup(input);
+    if (!host) {
+        err = hadoop_lerr_alloc(ENOMEM, "parse_rpc_addr: OOM");
+        goto done;
+    }
+    colon = index(host, ':');
+    if (colon) {
+        // If the URI has a colon, we parse the next part as a port.
+        char *port_str = colon + 1;
+        *colon = '\0';
+        port = atoi(colon);
+        if ((port <= 0) || (port >= 65536)) {
+            err = hadoop_lerr_alloc(EINVAL, "parse_rpc_addr: invalid port "
+                                    "string %s", port_str);
+            goto done;
+        }
+    }
+    err = get_first_ipv4_addr(host, &addr);
+    if (err)
+        goto done;
+    out->sin_family = AF_INET;
+    out->sin_port = htons(port);
+    out->sin_addr.s_addr = htonl(addr);
+done:
+    free(host);
+    return err;
+}
+
+static struct hadoop_err *get_namenode_addr(const struct hdfsBuilder *hdfs_bld,
+            struct sockaddr_in *nn_addr)
+{
+    const char *nameservice_id;
+    const char *rpc_addr;
+
+    nameservice_id = hconf_get(hdfs_bld->hconf, "dfs.nameservice.id");
+    if (nameservice_id) {
+        return hadoop_lerr_alloc(ENOTSUP, "get_namenode_addr: we "
+                "don't yet support HA or federated configurations.");
+    }
+    rpc_addr = hconf_get(hdfs_bld->hconf, "dfs.namenode.rpc-address");
+    if (rpc_addr) {
+        return parse_rpc_addr(rpc_addr, nn_addr, hdfs_bld->port);
+    }
+    return parse_rpc_addr(hdfs_bld->uri_authority, nn_addr, hdfs_bld->port);
+}
+
+struct hadoop_err *ndfs_connect(struct hdfsBuilder *hdfs_bld,
+                                struct hdfs_internal **out)
+{
+    struct hadoop_err *err = NULL;
+    struct native_fs *fs = NULL;
+    struct hrpc_messenger_builder *msgr_bld;
+    struct ndfs_server_defaults defaults;
+    int working_dir_lock_created = 0;
+    char *working_dir = NULL;
+    UriParserStateA uri_state;
+
+    fs = calloc(1, sizeof(*fs));
+    if (!fs) {
+        err = hadoop_lerr_alloc(ENOMEM, "failed to allocate space "
+                                "for a native_fs structure.");
+        goto done;
+    }
+    fs->base.ty = HADOOP_FS_TY_NDFS;
+    fs->user_name = strdup(hdfs_bld->uri_user_info); 
+    if (!fs->user_name) {
+        err = hadoop_lerr_alloc(ENOMEM, "failed to allocate space "
+                                "for the user name.");
+        goto done;
+    }
+    msgr_bld = hrpc_messenger_builder_alloc();
+    if (!msgr_bld) {
+        err = hadoop_lerr_alloc(ENOMEM, "failed to allocate space "
+                                "for a messenger builder.");
+        goto done;
+    }
+    err = get_namenode_addr(hdfs_bld, &fs->nn_addr);
+    if (err)
+        goto done;
+    err = hrpc_messenger_create(msgr_bld, &fs->msgr);
+    if (err)
+        goto done;
+    // Get the default working directory
+    if (asprintf(&working_dir, "%s:///user/%s/",
+                 hdfs_bld->uri_scheme, hdfs_bld->uri_user_info) < 0) {
+        working_dir = NULL;
+        err = hadoop_lerr_alloc(ENOMEM, "ndfs_connect: OOM allocating "
+                                "working_dir");
+        goto done;
+    }
+    fs->working_uri = calloc(1, sizeof(*(fs->working_uri)));
+    if (!fs->working_uri) {
+        err = hadoop_lerr_alloc(ENOMEM, "ndfs_connect: OOM allocating "
+                                "fs->working_uri");
+        goto done;
+    }
+    err = uri_parse_abs(working_dir, &uri_state, fs->working_uri,
+                        hdfs_bld->uri_scheme);
+    if (err) {
+        free(fs->working_uri);
+        fs->working_uri = NULL;
+        goto done;
+    }
+    if (uv_mutex_init(&fs->working_uri_lock) < 0) {
+        err = hadoop_lerr_alloc(ENOMEM, "failed to create a mutex.");
+        goto done;
+    }
+    working_dir_lock_created = 1;
+
+    // Ask the NameNode about our server defaults.  We'll use this information
+    // later in ndfs_get_default_block_size, and when writing new files.  Just
+    // as important, tghis validates that we can talk to the NameNode with our
+    // current configuration.
+    memset(&defaults, 0, sizeof(defaults));
+    err = ndfs_get_server_defaults(fs, &defaults);
+    if (err)
+        goto done;
+    fs->default_block_size = defaults.blocksize;
+    err = NULL;
+
+done:
+    free(working_dir);
+    if (err) {
+        if (fs) {
+            free(fs->user_name);
+            if (fs->working_uri) {
+                uriFreeUriMembersA(fs->working_uri);
+                free(fs->working_uri);
+            }
+            if (working_dir_lock_created) {
+                uv_mutex_destroy(&fs->working_uri_lock);
+            }
+            free(fs);
+        }
+        return err; 
+    }
+    *out = (struct hdfs_internal *)fs;
+    return NULL; 
+}
+
+static int ndfs_disconnect(hdfsFS bfs)
+{
+    struct native_fs *fs = (struct native_fs*)bfs;
+
+    hrpc_messenger_shutdown(fs->msgr);
+    hrpc_messenger_free(fs->msgr);
+    free(fs->user_name);
+    uriFreeUriMembersA(fs->working_uri);
+    free(fs->working_uri);
+    uv_mutex_destroy(&fs->working_uri_lock);
+    free(fs);
+    return 0;
+}
+
+static struct hadoop_err *ndfs_open_file_for_read(
+        struct native_ro_file **out __attribute__((unused)),
+        struct native_fs *fs __attribute__((unused)),
+        const char *uri __attribute__((unused)))
+{
+    errno = ENOTSUP;
+    return NULL;
+}
+
+static struct hadoop_err *ndfs_open_file_for_write(
+        struct native_ro_file **out __attribute__((unused)),
+        struct native_fs *fs __attribute__((unused)),
+        const char *uri __attribute__((unused)),
+        int buffer_size __attribute__((unused)),
+        short replication __attribute__((unused)),
+        tSize block_size __attribute__((unused)))
+{
+    errno = ENOTSUP;
+    return NULL;
+}
+
+static hdfsFile ndfs_open_file(hdfsFS bfs, const char* uri, int flags, 
+                      int buffer_size, short replication, tSize block_size)
+{
+    struct native_fs *fs = (struct native_fs *)bfs;
+    struct native_ro_file *file = NULL;
+    struct hadoop_err *err;
+    int accmode;
+    char *path = NULL;
+
+    err = build_path(fs, uri, &path);
+    if (err) {
+        goto done;
+    }
+    accmode = flags & O_ACCMODE;
+    if (accmode == O_RDONLY) {
+        err = ndfs_open_file_for_read(&file, fs, path);
+    } else if (accmode == O_WRONLY) {
+        err = ndfs_open_file_for_write(&file, fs, path,
+                        buffer_size, replication, block_size);
+    } else {
+        err = hadoop_lerr_alloc(EINVAL, "cannot open a hadoop file in "
+                "mode 0x%x\n", accmode);
+    }
+done:
+    free(path);
+    return hadoopfs_errno_and_retptr(err, file);
+}
+
+static int ndfs_close_file(hdfsFS fs __attribute__((unused)),
+                           hdfsFile bfile __attribute__((unused)))
+{
+    errno = ENOTSUP;
+    return -1;
+}
+
+static int ndfs_file_exists(hdfsFS bfs, const char *uri)
+{
+    static hdfsFileInfo *info;
+
+    info = ndfs_get_path_info(bfs, uri);
+    if (!info) {
+        // errno will be set
+        return -1;
+    }
+    hdfsFreeFileInfo(info, 1);
+    return 0;
+}
+
+static int ndfs_seek(hdfsFS bfs __attribute__((unused)),
+                     hdfsFile bfile __attribute__((unused)),
+                     tOffset desiredPos __attribute__((unused)))
+{
+    errno = ENOTSUP;
+    return -1;
+}
+
+static tOffset ndfs_tell(hdfsFS bfs __attribute__((unused)),
+                         hdfsFile bfile __attribute__((unused)))
+{
+    errno = ENOTSUP;
+    return -1;
+}
+
+static tSize ndfs_read(hdfsFS bfs __attribute__((unused)),
+                       hdfsFile bfile __attribute__((unused)),
+                       void *buffer __attribute__((unused)),
+                       tSize length __attribute__((unused)))
+{
+    errno = ENOTSUP;
+    return -1;
+}
+
+static tSize ndfs_pread(hdfsFS bfs __attribute__((unused)),
+            hdfsFile bfile __attribute__((unused)),
+            tOffset position __attribute__((unused)),
+            void* buffer __attribute__((unused)),
+            tSize length __attribute__((unused)))
+{
+    errno = ENOTSUP;
+    return -1;
+}
+
+static tSize ndfs_write(hdfsFS bfs __attribute__((unused)),
+            hdfsFile bfile __attribute__((unused)),
+            const void* buffer __attribute__((unused)),
+            tSize length __attribute__((unused)))
+{
+    errno = ENOTSUP;
+    return -1;
+}
+
+static int ndfs_flush(hdfsFS bfs __attribute__((unused)),
+                      hdfsFile bfile __attribute__((unused)))
+{
+    errno = ENOTSUP;
+    return -1;
+}
+
+static int ndfs_hflush(hdfsFS bfs __attribute__((unused)),
+                      hdfsFile bfile __attribute__((unused)))
+{
+    errno = ENOTSUP;
+    return -1;
+}
+
+static int ndfs_hsync(hdfsFS bfs __attribute__((unused)),
+                      hdfsFile bfile __attribute__((unused)))
+{
+    errno = ENOTSUP;
+    return -1;
+}
+
+static int ndfs_available(hdfsFS bfs __attribute__((unused)),
+                          hdfsFile bfile __attribute__((unused)))
+{
+    errno = ENOTSUP;
+    return -1;
+}
+
+static int ndfs_copy(hdfsFS srcFS __attribute__((unused)),
+                     const char* src __attribute__((unused)),
+                     hdfsFS dstFS __attribute__((unused)),
+                     const char* dst __attribute__((unused)))
+{
+    errno = ENOTSUP;
+    return -1;
+}
+
+static int ndfs_move(hdfsFS srcFS __attribute__((unused)),
+                     const char* src __attribute__((unused)),
+                     hdfsFS dstFS __attribute__((unused)),
+                     const char* dst __attribute__((unused)))
+{
+    errno = ENOTSUP;
+    return -1;
+}
+
+static int ndfs_unlink(struct hdfs_internal *bfs,
+                const char *uri, int recursive)
+{
+    struct native_fs *fs = (struct native_fs*)bfs;
+    struct hadoop_err *err = NULL;
+    DeleteRequestProto req = DELETE_REQUEST_PROTO__INIT;
+    struct hrpc_proxy proxy;
+    DeleteResponseProto *resp = NULL;
+    char *path = NULL;
+
+    ndfs_nn_proxy_init(fs, &proxy);
+    err = build_path(fs, uri, &path);
+    if (err) {
+        goto done;
+    }
+    req.src = path;
+    req.recursive = !!recursive;
+    err = cnn_delete(&proxy, &req, &resp);
+    if (err) {
+        goto done;
+    }
+
+done:
+    free(path);
+    if (resp) {
+        delete_response_proto__free_unpacked(resp, NULL);
+    }
+    return hadoopfs_errno_and_retcode(err);
+}
+
+static int ndfs_rename(hdfsFS bfs, const char *src_uri, const char *dst_uri)
+{
+    struct native_fs *fs = (struct native_fs*)bfs;
+    struct hadoop_err *err = NULL;
+    Rename2RequestProto req = RENAME2_REQUEST_PROTO__INIT;
+    Rename2ResponseProto *resp = NULL;
+    struct hrpc_proxy proxy;
+    char *src_path = NULL, *dst_path = NULL;
+
+    ndfs_nn_proxy_init(fs, &proxy);
+    err = build_path(fs, src_uri, &src_path);
+    if (err) {
+        goto done;
+    }
+    err = build_path(fs, dst_uri, &dst_path);
+    if (err) {
+        goto done;
+    }
+    req.src = src_path;
+    req.dst = dst_path;
+    req.overwritedest = 0; // TODO: support overwrite
+    err = cnn_rename2(&proxy, &req, &resp);
+    if (err) {
+        goto done;
+    }
+
+done:
+    free(src_path);
+    free(dst_path);
+    if (resp) {
+        rename2_response_proto__free_unpacked(resp, NULL);
+    }
+    return hadoopfs_errno_and_retcode(err);
+}
+
+static char* ndfs_get_working_directory(hdfsFS bfs, char *buffer,
+                                       size_t bufferSize)
+{
+    size_t len;
+    struct native_fs *fs = (struct native_fs *)bfs;
+    struct hadoop_err *err = NULL;
+    char *working_path = NULL;
+
+    uv_mutex_lock(&fs->working_uri_lock);
+    err = uri_get_path(fs->working_uri, &working_path);
+    if (err) {
+        err = hadoop_err_prepend(err, 0, "ndfs_get_working_directory: failed "
+                                 "to get the path of the working_uri.");
+        goto done;
+    }
+    len = strlen(working_path);
+    if (len + 1 > bufferSize) {
+        err = hadoop_lerr_alloc(ENAMETOOLONG, "ndfs_get_working_directory: "
+                "the buffer supplied was only %zd bytes, but we would need "
+                "%zd bytes to hold the working directory.",
+                bufferSize, len + 1);
+        goto done;
+    }
+    strcpy(buffer, working_path);
+done:
+    uv_mutex_unlock(&fs->working_uri_lock);
+    free(working_path);
+    return hadoopfs_errno_and_retptr(err, buffer);
+}
+
+static int ndfs_set_working_directory(hdfsFS bfs, const char* uri_str)
+{
+    struct native_fs *fs = (struct native_fs *)bfs;
+    char *path = NULL;
+    char *scheme = NULL;
+    struct hadoop_err *err = NULL;
+    UriParserStateA uri_state;
+    UriUriA *uri = NULL;
+
+    uv_mutex_lock(&fs->working_uri_lock);
+    uri = calloc(1, sizeof(*uri));
+    if (!uri) {
+        err = hadoop_lerr_alloc(ENOMEM, "ndfs_set_working_directory: OOM");
+        goto done;
+    }
+    err = uri_get_scheme(fs->working_uri, &scheme);
+    if (err) {
+        err = hadoop_err_prepend(err, ENOMEM, "ndfs_set_working_directory: "
+                            "failed to get scheme of current working_uri");
+        goto done;
+    }
+    err = build_path(fs, uri_str, &path);
+    if (err)
+        goto done;
+    err = uri_parse_abs(path, &uri_state, uri, scheme);
+    if (err)
+        goto done;
+    uriFreeUriMembersA(fs->working_uri);
+    free(fs->working_uri);
+    fs->working_uri = uri;
+    err = NULL;
+
+done:
+    if (err) {
+        free(uri);
+    }
+    uv_mutex_unlock(&fs->working_uri_lock);
+    free(scheme);
+    free(path);
+    return hadoopfs_errno_and_retcode(err);
+}
+
+static int ndfs_mkdir(hdfsFS bfs, const char* uri)
+{
+    struct native_fs *fs = (struct native_fs *)bfs;
+    struct hadoop_err *err = NULL;
+    MkdirsRequestProto req = MKDIRS_REQUEST_PROTO__INIT;
+    MkdirsResponseProto *resp = NULL;
+    struct hrpc_proxy proxy;
+    char *path = NULL;
+
+    ndfs_nn_proxy_init(fs, &proxy);
+    err = build_path(fs, uri, &path);
+    if (err) {
+        goto done;
+    }
+    req.src = path;
+    req.createparent = 1; // TODO: add libhdfs API for non-recursive mkdir
+    err = cnn_mkdirs(&proxy, &req, &resp);
+    if (err) {
+        goto done;
+    }
+    if (!resp->result) {
+        err = hadoop_lerr_alloc(EEXIST, "ndfs_mkdir(%s): a path "
+                "component already exists as a non-directory.", path);
+        goto done;
+    }
+    err = NULL;
+
+done:
+    free(path);
+    if (resp) {
+        mkdirs_response_proto__free_unpacked(resp, NULL);
+    }
+    return hadoopfs_errno_and_retcode(err);
+}
+
+static int ndfs_set_replication(hdfsFS bfs, const char* uri,
+                               int16_t replication)
+{
+    struct native_fs *fs = (struct native_fs *)bfs;
+    struct hadoop_err *err = NULL;
+    SetReplicationRequestProto req = SET_REPLICATION_REQUEST_PROTO__INIT;
+    SetReplicationResponseProto *resp = NULL;
+    struct hrpc_proxy proxy;
+    char *path = NULL;
+
+    ndfs_nn_proxy_init(fs, &proxy);
+    err = build_path(fs, uri, &path);
+    if (err) {
+        goto done;
+    }
+    req.src = path;
+    req.replication = replication;
+    err = cnn_set_replication(&proxy, &req, &resp);
+    if (err) {
+        goto done;
+    }
+    if (!resp->result) {
+        err = hadoop_lerr_alloc(EINVAL, "ndfs_set_replication(%s): path "
+                "does not exist or is not a regular file.", path);
+        goto done;
+    }
+
+done:
+    free(path);
+    if (resp) {
+        set_replication_response_proto__free_unpacked(resp, NULL);
+    }
+    return hadoopfs_errno_and_retcode(err);
+}
+
+static hdfsFileInfo* ndfs_list_directory(hdfsFS bfs __attribute__((unused)),
+                                         const char* uri __attribute__((unused)),
+                                        int *numEntries __attribute__((unused)))
+{
+    errno = ENOTSUP;
+    return NULL;
+}
+
+static hdfsFileInfo *ndfs_get_path_info(hdfsFS bfs __attribute__((unused)),
+                                        const char* uri __attribute__((unused)))
+{
+    errno = ENOTSUP;
+    return NULL;
+}
+
+char***
+ndfs_get_hosts(hdfsFS bfs __attribute__((unused)),
+               const char* path __attribute__((unused)),
+               tOffset start __attribute__((unused)),
+               tOffset length __attribute__((unused)))
+{
+    errno = ENOTSUP;
+    return NULL;
+}
+
+static tOffset ndfs_get_default_block_size(hdfsFS bfs)
+{
+    struct native_fs *fs = (struct native_fs *)bfs;
+    return fs->default_block_size;
+}
+
+static tOffset ndfs_get_default_block_size_at_path(hdfsFS bfs,
+                    const char *uri)
+{
+    struct native_fs *fs = (struct native_fs *)bfs;
+    struct hadoop_err *err = NULL;
+    GetPreferredBlockSizeRequestProto req =
+        GET_PREFERRED_BLOCK_SIZE_REQUEST_PROTO__INIT;
+    GetPreferredBlockSizeResponseProto *resp = NULL;
+    struct hrpc_proxy proxy;
+    tOffset ret = 0;
+    char *path = NULL;
+
+    ndfs_nn_proxy_init(fs, &proxy);
+    err = build_path(fs, uri, &path);
+    if (err) {
+        goto done;
+    }
+    req.filename = path;
+    err = cnn_get_preferred_block_size(&proxy, &req, &resp);
+    if (err) {
+        goto done;
+    }
+    ret = resp->bsize;
+    err = NULL;
+
+done:
+    free(path);
+    if (resp) {
+        get_preferred_block_size_response_proto__free_unpacked(resp, NULL);
+    }
+    if (err)
+        return hadoopfs_errno_and_retcode(err);
+    return ret;
+}
+
+static struct hadoop_err *ndfs_statvfs(struct hadoop_fs_base *hfs,
+        struct hadoop_vfs_stats *stats)
+{
+    struct native_fs *fs = (struct native_fs*)hfs;
+
+    GetFsStatusRequestProto req = GET_FS_STATUS_REQUEST_PROTO__INIT;
+    GetFsStatsResponseProto *resp = NULL;
+    struct hadoop_err *err = NULL;
+    struct hrpc_proxy proxy;
+
+    ndfs_nn_proxy_init(fs, &proxy);
+    err = cnn_get_fs_stats(&proxy, &req, &resp);
+    if (err) {
+        goto done;
+    }
+    stats->capacity = resp->capacity;
+    stats->used = resp->used;
+    stats->remaining = resp->remaining;
+    stats->under_replicated = resp->under_replicated;
+    stats->corrupt_blocks = resp->corrupt_blocks;
+    stats->missing_blocks = resp->missing_blocks;
+
+done:
+    if (resp) {
+        get_fs_stats_response_proto__free_unpacked(resp, NULL);
+    }
+    return err;
+}
+
+static tOffset ndfs_get_capacity(hdfsFS bfs)
+{
+    struct hadoop_err *err;
+    struct hadoop_vfs_stats stats;
+
+    err = ndfs_statvfs((struct hadoop_fs_base *)bfs, &stats);
+    if (err)
+        return hadoopfs_errno_and_retcode(err);
+    return stats.capacity;
+}
+
+static tOffset ndfs_get_used(hdfsFS bfs)
+{
+    struct hadoop_err *err;
+    struct hadoop_vfs_stats stats;
+
+    err = ndfs_statvfs((struct hadoop_fs_base *)bfs, &stats);
+    if (err)
+        return hadoopfs_errno_and_retcode(err);
+    return stats.used;
+}
+
+static int ndfs_chown(hdfsFS bfs, const char* uri,
+                      const char *user, const char *group)
+{
+    struct native_fs *fs = (struct native_fs *)bfs;
+    struct hadoop_err *err = NULL;
+    SetOwnerRequestProto req = SET_OWNER_REQUEST_PROTO__INIT;
+    SetOwnerResponseProto *resp = NULL;
+    struct hrpc_proxy proxy;
+    char *path = NULL;
+
+    ndfs_nn_proxy_init(fs, &proxy);
+    err = build_path(fs, uri, &path);
+    if (err) {
+        goto done;
+    }
+    req.src = path;
+    req.username = (char*)user;
+    req.groupname = (char*)group;
+    err = cnn_set_owner(&proxy, &req, &resp);
+    if (err) {
+        goto done;
+    }
+
+done:
+    free(path);
+    if (resp) {
+        set_owner_response_proto__free_unpacked(resp, NULL);
+    }
+    return hadoopfs_errno_and_retcode(err);
+}
+
+static int ndfs_chmod(hdfsFS bfs, const char* uri, short mode)
+{
+    struct native_fs *fs = (struct native_fs *)bfs;
+    FsPermissionProto perm = FS_PERMISSION_PROTO__INIT;
+    SetPermissionRequestProto req = SET_PERMISSION_REQUEST_PROTO__INIT;
+    SetPermissionResponseProto *resp = NULL;
+    struct hadoop_err *err = NULL;
+    struct hrpc_proxy proxy;
+    char *path = NULL;
+
+    ndfs_nn_proxy_init(fs, &proxy);
+    err = build_path(fs, uri, &path);
+    if (err) {
+        goto done;
+    }
+    req.src = path;
+    req.permission = &perm;
+    perm.perm = mode;
+    err = cnn_set_permission(&proxy, &req, &resp);
+    if (err) {
+        goto done;
+    }
+
+done:
+    free(path);
+    if (resp) {
+        set_permission_response_proto__free_unpacked(resp, NULL);
+    }
+    return hadoopfs_errno_and_retcode(err);
+}
+
+static int ndfs_utime(hdfsFS bfs, const char* uri,
+                      int64_t mtime, int64_t atime)
+{
+    struct native_fs *fs = (struct native_fs *)bfs;
+    SetTimesRequestProto req = SET_TIMES_REQUEST_PROTO__INIT ;
+    SetTimesResponseProto *resp = NULL;
+    struct hadoop_err *err = NULL;
+    struct hrpc_proxy proxy;
+    char *path = NULL;
+
+    ndfs_nn_proxy_init(fs, &proxy);
+    err = build_path(fs, uri, &path);
+    if (err) {
+        goto done;
+    }
+    req.src = path;
+    // If mtime or atime are -1, that means "no change."
+    // Otherwise, we need to multiply by 1000, to take into account the fact
+    // that libhdfs times are in seconds, and HDFS times are in milliseconds.
+    // It's unfortunate that libhdfs doesn't support the full millisecond
+    // precision.  We need to redo the API at some point.
+    if (mtime < 0) {
+        req.mtime = -1;
+    } else {
+        req.mtime = mtime;
+        req.mtime *= 1000;
+    }
+    if (atime < 0) {
+        req.atime = -1;
+    } else {
+        req.atime = atime;
+        req.atime *= 1000;
+    }
+    err = cnn_set_times(&proxy, &req, &resp);
+    if (err) {
+        goto done;
+    }
+
+done:
+    free(path);
+    if (resp) {
+        set_times_response_proto__free_unpacked(resp, NULL);
+    }
+    return hadoopfs_errno_and_retcode(err);
+}
+
+static struct hadoopRzBuffer* ndfs_read_zero(
+            hdfsFile bfile __attribute__((unused)),
+            struct hadoopRzOptions *opts __attribute__((unused)),
+            int32_t maxLength __attribute__((unused)))
+{
+    errno = ENOTSUP;
+    return NULL;
+}
+
+static void ndfs_rz_buffer_free(hdfsFile bfile __attribute__((unused)),
+                struct hadoopRzBuffer *buffer __attribute__((unused)))
+{
+}
+
+int ndfs_file_uses_direct_read(hdfsFile bfile)
+{
+    // Set the 'disable direct reads' flag so that old test harnesses designed
+    // to test jniFS will run against NDFS.  The flag doesn't do anything,
+    // since all reads are always direct in NDFS.
+    struct native_file_base *file = (struct native_file_base *)bfile;
+    return (!(file->flags & NDFS_FILE_FLAG_DISABLE_DIRECT_READ));
+}
+
+void ndfs_file_disable_direct_read(hdfsFile bfile __attribute__((unused)))
+{
+    struct native_file_base *file = (struct native_file_base *)bfile;
+    file->flags |= NDFS_FILE_FLAG_DISABLE_DIRECT_READ;
+}
+
+const struct hadoop_fs_ops g_ndfs_ops = {
+    .name = "ndfs",
+    .file_is_open_for_read = ndfs_file_is_open_for_read,
+    .file_is_open_for_write = ndfs_file_is_open_for_write,
+    .get_read_statistics = ndfs_file_get_read_statistics,
+    .connect = ndfs_connect,
+    .disconnect = ndfs_disconnect,
+    .open = ndfs_open_file,
+    .close = ndfs_close_file,
+    .exists = ndfs_file_exists,
+    .seek = ndfs_seek,
+    .tell = ndfs_tell,
+    .read = ndfs_read,
+    .pread = ndfs_pread,
+    .write = ndfs_write,
+    .flush = ndfs_flush,
+    .hflush = ndfs_hflush,
+    .hsync = ndfs_hsync,
+    .available = ndfs_available,
+    .copy = ndfs_copy,
+    .move = ndfs_move,
+    .unlink = ndfs_unlink,
+    .rename = ndfs_rename,
+    .get_working_directory = ndfs_get_working_directory,
+    .set_working_directory = ndfs_set_working_directory,
+    .mkdir = ndfs_mkdir,
+    .set_replication = ndfs_set_replication,
+    .list_directory = ndfs_list_directory,
+    .get_path_info = ndfs_get_path_info,
+    .get_hosts = ndfs_get_hosts,
+    .get_default_block_size = ndfs_get_default_block_size,
+    .get_default_block_size_at_path = ndfs_get_default_block_size_at_path,
+    .get_capacity = ndfs_get_capacity,
+    .get_used = ndfs_get_used,
+    .chown = ndfs_chown,
+    .chmod = ndfs_chmod,
+    .utime = ndfs_utime,
+    .read_zero = ndfs_read_zero,
+    .rz_buffer_free = ndfs_rz_buffer_free,
+
+    // test
+    .file_uses_direct_read = ndfs_file_uses_direct_read,
+    .file_disable_direct_read = ndfs_file_disable_direct_read,
+};
+
+// vim: ts=4:sw=4:tw=79:et

hadoop-native-core/src/main/native/rpc/conn.c

@@ -17,6 +17,7 @@
  */
 
 #include "common/hadoop_err.h"
+#include "common/net.h"
 #include "common/string.h"
 #include "protobuf/IpcConnectionContext.pb-c.h.s"
 #include "protobuf/ProtobufRpcEngine.pb-c.h.s"
@@ -348,8 +349,11 @@ struct hadoop_err *hrpc_conn_create_outbound(struct hrpc_reactor *reactor,
     res = uv_tcp_connect(&conn->conn_req, &conn->stream,
             (struct sockaddr*)&conn->remote, conn_connect_cb);
     if (res) {
+        char remote_str[64] = { 0 };
         err = hadoop_uverr_alloc(res,
-                "hrpc_conn_create_outbound: uv_tcp_connect failed");
+                "hrpc_conn_create_outbound: uv_tcp_connect(%s) failed",
+                net_ipv4_name_and_port(&conn->remote, remote_str,
+                                       sizeof(remote_str)));
         goto done;
     }
 

hadoop-native-core/src/main/native/rpc/protoc-gen-hrpc.cc

@@ -378,13 +378,15 @@ private:
 "        $req_ty_uscore$__get_packed_size(req),\n"
 "        (hrpc_pack_cb_t)$req_ty_uscore$__pack,\n"
 "        hrpc_proxy_sync_cb, ctx);\n"
+"    uv_sem_wait(&ctx->sem);\n"
 "    if (ctx->err) {\n"
-"        hrpc_free_sync_ctx(ctx);\n"
-"        return ctx->err;\n"
+"        err = ctx->err;\n"
+"        hrpc_release_sync_ctx(ctx);\n"
+"        return err;\n"
 "    }\n"
 "    resp = $resp_ty_uscore$__unpack(NULL, ctx->resp.pb_len,\n"
 "                                                  ctx->resp.pb_base);\n"
-"    hrpc_free_sync_ctx(ctx);\n"
+"    hrpc_release_sync_ctx(ctx);\n"
 "    if (!resp) {\n"
 "        return hadoop_lerr_alloc(EINVAL,\n"
 "           \"$sync_call$: failed to parse response from server\");\n"

hadoop-native-core/src/main/native/rpc/proxy.c

@@ -92,7 +92,7 @@ void *hrpc_proxy_alloc_userdata(struct hrpc_proxy *proxy, size_t size)
 struct hrpc_sync_ctx *hrpc_proxy_alloc_sync_ctx(struct hrpc_proxy *proxy)
 {
     struct hrpc_sync_ctx *ctx = 
-        hrpc_proxy_alloc_userdata(proxy, sizeof(struct hrpc_proxy));
+        hrpc_proxy_alloc_userdata(proxy, sizeof(*ctx));
     if (!ctx) {
         return NULL;
     }
@@ -103,7 +103,7 @@ struct hrpc_sync_ctx *hrpc_proxy_alloc_sync_ctx(struct hrpc_proxy *proxy)
     return ctx;
 }
 
-void hrpc_free_sync_ctx(struct hrpc_sync_ctx *ctx)
+void hrpc_release_sync_ctx(struct hrpc_sync_ctx *ctx)
 {
     free(ctx->resp.base);
     uv_sem_destroy(&ctx->sem);

hadoop-native-core/src/main/native/rpc/proxy.h

@@ -136,11 +136,12 @@ void *hrpc_proxy_alloc_userdata(struct hrpc_proxy *proxy, size_t size);
 struct hrpc_sync_ctx *hrpc_proxy_alloc_sync_ctx(struct hrpc_proxy *proxy);
 
 /**
- * Free a sync context allocated from a proxy.
+ * Release the memory associated with a sync context.  This doesn't free the
+ * context object itself.
  *
  * @param proxy                 The sync context.
  */
-void hrpc_free_sync_ctx(struct hrpc_sync_ctx *ctx);
+void hrpc_release_sync_ctx(struct hrpc_sync_ctx *ctx);
 
 /**
  * A callback which synchronous RPCs can use.

hadoop-native-core/src/main/native/rpc/reactor.c

@@ -104,9 +104,10 @@ static void reactor_async_start_outbound(struct hrpc_reactor *reactor,
 
     conn = reuse_idle_conn(reactor, &call->remote, call);
     if (conn) {
-        reactor_log_debug(reactor, "start_outbound(remote=%s) assigning to "
-                       "connection %p\n",
-            net_ipv4_name(&call->remote, remote_str, sizeof(remote_str)), conn);
+        reactor_log_debug(reactor, "reactor_async_start_outbound(remote=%s) "
+                          "assigning to connection %p\n",
+                net_ipv4_name(&call->remote, remote_str, sizeof(remote_str)),
+                conn);
         hrpc_conn_start_outbound(conn, call);
     } else {
         err = hrpc_conn_create_outbound(reactor, call, &conn);
@@ -118,9 +119,10 @@ static void reactor_async_start_outbound(struct hrpc_reactor *reactor,
             hrpc_call_deliver_err(call, err);
             return;
         }
-        reactor_log_debug(reactor, "start_outbound(remote=%s) created new "
-                       "connection %p\n",
-            net_ipv4_name(&call->remote, remote_str, sizeof(remote_str)), conn);
+        reactor_log_debug(reactor, "reactor_async_start_outbound("
+                "remote=%s) created new connection %p\n",
+                net_ipv4_name(&call->remote, remote_str, sizeof(remote_str)),
+                conn);
     }
     // Add or re-add the connection to the reactor's tree.
     RB_INSERT(hrpc_conns, &reactor->conns, conn);

hadoop-native-core/src/main/native/rpc/varint-unit.c

@@ -16,8 +16,8 @@
  * limitations under the License.
  */
 
-#include "common/test.h"
 #include "rpc/varint.h"
+#include "test/test.h"
 
 #include <errno.h>
 #include <stdint.h>

hadoop-native-core/src/main/native/test/common/conf/core-site.xml

@@ -0,0 +1,47 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<configuration>
+    <property>
+        <name>foo.overridden</name>
+        <value>core-site-val</value>
+    </property>
+
+    <property>
+        <name>foo.final</name>
+        <value>1</value>
+        <final>true</final>
+    </property>
+
+    <property>
+        <name>foo.bar</name>
+        <value>hdfs-default.</value>
+    </property>
+
+    <property>
+        <name>foo.baz</name>
+        <value>${foo.bar}</value>
+    </property>
+
+    <property>
+        <name>foo.empty</name>
+        <value></value>
+    </property>
+</configuration>

hadoop-native-core/src/main/native/test/common/conf/hdfs-site.xml

@@ -0,0 +1,73 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<configuration>
+
+<property>
+    <name>foo.overridden</name>
+    <value>hdfs-site-val</value>
+</property>
+
+<property>
+    <name>double.foo.bar</name>
+    <value>woo:${foo.bar}${foo.bar}</value>
+</property>
+
+<property>
+    <name>triple.foo.bar</name>
+    <value>woo:${foo.bar}${double.foo.bar}</value>
+</property>
+
+<property>
+    <name>foo.final</name>
+    <value>0</value>
+</property>
+
+<property>
+    <name>tweedledum</name>
+    <value>${tweedledee}</value>
+</property>
+
+<property>
+    <name>tweedledee</name>
+    <value>${tweedledum}</value>
+</property>
+
+<property>
+    <name>joiner</name>
+    <value>${joiner.a}${joiner.b}</value>
+</property>
+
+<property>
+    <name>joiner.a</name>
+    <value>${joiner</value>
+</property>
+
+<property>
+    <name>joiner.b</name>
+    <value>.c}</value>
+</property>
+
+<property>
+    <name>joiner.c</name>
+    <value>woo</value>
+</property>
+
+</configuration>

hadoop-native-core/src/main/native/test/common/conf/include.xml

@@ -0,0 +1,26 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<configuration xmlns:xi="http://www.w3.org/2001/XInclude">
+    <property>
+        <name>included.key</name>
+        <value>included.val</value>
+    </property>
+</configuration>

hadoop-native-core/src/main/native/test/fs/test_libhdfs_meta_ops.c

@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fs/hdfs.h"
+#include "test/native_mini_dfs.h"
+#include "test/test.h"
+
+#include <errno.h>
+#include <inttypes.h>
+#include <semaphore.h>
+#include <pthread.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+/** Test performing metadata operations via libhdfs. */
+
+int main(void)
+{
+    struct hdfsBuilder *hdfs_bld = NULL;
+    hdfsFS fs = NULL;
+    struct NativeMiniDfsCluster* dfs_cluster = NULL;
+    struct NativeMiniDfsConf dfs_conf = {
+        .doFormat = 1,
+    };
+    const char *nn_uri;
+
+    nn_uri = getenv("NAMENODE_URI");
+    if (!nn_uri) {
+        dfs_cluster = nmdCreate(&dfs_conf);
+        EXPECT_NONNULL(dfs_cluster);
+        EXPECT_INT_ZERO(nmdWaitClusterUp(dfs_cluster));
+    }
+    hdfs_bld = hdfsNewBuilder();
+    if (nn_uri) {
+        hdfsBuilderSetNameNode(hdfs_bld, nn_uri);
+        EXPECT_INT_ZERO(hdfsBuilderConfSetStr(hdfs_bld,
+                "default.native.handler", "ndfs"));
+    } else {
+        hdfsBuilderSetNameNode(hdfs_bld, "localhost");
+        hdfsBuilderSetNameNodePort(hdfs_bld, nmdGetNameNodePort(dfs_cluster));
+    }
+    EXPECT_NONNULL(hdfs_bld);
+    fs = hdfsBuilderConnect(hdfs_bld);
+    EXPECT_NONNULL(fs);
+    EXPECT_INT_ZERO(hdfsDisconnect(fs));
+    if (dfs_cluster) {
+        EXPECT_INT_ZERO(nmdShutdown(dfs_cluster));
+        nmdFree(dfs_cluster);
+    }
+    return EXIT_SUCCESS;
+}
+
+// vim: ts=4:sw=4:tw=79:et

hadoop-native-core/src/main/native/test/fs/test_libhdfs_threaded.c

@@ -0,0 +1,342 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fs/hdfs.h"
+#include "test/native_mini_dfs.h"
+#include "test/test.h"
+
+#include <errno.h>
+#include <inttypes.h>
+#include <semaphore.h>
+#include <pthread.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define TO_STR_HELPER(X) #X
+#define TO_STR(X) TO_STR_HELPER(X)
+
+#define TLH_MAX_THREADS 100
+
+#define TLH_DEFAULT_BLOCK_SIZE 134217728
+
+static sem_t tlhSem;
+
+static struct NativeMiniDfsCluster* tlhCluster;
+
+struct tlhThreadInfo {
+    /** Thread index */
+    int threadIdx;
+    /** 0 = thread was successful; error code otherwise */
+    int success;
+    /** pthread identifier */
+    pthread_t thread;
+};
+
+static int hdfsSingleNameNodeConnect(struct NativeMiniDfsCluster *cl, hdfsFS *fs,
+                                     const char *username)
+{
+    int ret, port;
+    hdfsFS hdfs;
+    struct hdfsBuilder *bld;
+    
+    port = nmdGetNameNodePort(cl);
+    if (port < 0) {
+        fprintf(stderr, "hdfsSingleNameNodeConnect: nmdGetNameNodePort "
+                "returned error %d\n", port);
+        return port;
+    }
+    bld = hdfsNewBuilder();
+    if (!bld)
+        return -ENOMEM;
+    hdfsBuilderSetForceNewInstance(bld);
+    hdfsBuilderSetNameNode(bld, "localhost");
+    hdfsBuilderSetNameNodePort(bld, port);
+    hdfsBuilderConfSetStr(bld, "dfs.block.size",
+                          TO_STR(TLH_DEFAULT_BLOCK_SIZE));
+    hdfsBuilderConfSetStr(bld, "dfs.blocksize",
+                          TO_STR(TLH_DEFAULT_BLOCK_SIZE));
+    if (username) {
+        hdfsBuilderSetUserName(bld, username);
+    }
+    hdfs = hdfsBuilderConnect(bld);
+    if (!hdfs) {
+        ret = -errno;
+        return ret;
+    }
+    *fs = hdfs;
+    return 0;
+}
+
+static int doTestGetDefaultBlockSize(hdfsFS fs, const char *path)
+{
+    int64_t blockSize;
+    int ret;
+
+    blockSize = hdfsGetDefaultBlockSize(fs);
+    if (blockSize < 0) {
+        ret = errno;
+        fprintf(stderr, "hdfsGetDefaultBlockSize failed with error %d\n", ret);
+        return ret;
+    } else if (blockSize != TLH_DEFAULT_BLOCK_SIZE) {
+        fprintf(stderr, "hdfsGetDefaultBlockSize got %"PRId64", but we "
+                "expected %d\n", blockSize, TLH_DEFAULT_BLOCK_SIZE);
+        return EIO;
+    }
+
+    blockSize = hdfsGetDefaultBlockSizeAtPath(fs, path);
+    if (blockSize < 0) {
+        ret = errno;
+        fprintf(stderr, "hdfsGetDefaultBlockSizeAtPath(%s) failed with "
+                "error %d\n", path, ret);
+        return ret;
+    } else if (blockSize != TLH_DEFAULT_BLOCK_SIZE) {
+        fprintf(stderr, "hdfsGetDefaultBlockSizeAtPath(%s) got "
+                "%"PRId64", but we expected %d\n", 
+                path, blockSize, TLH_DEFAULT_BLOCK_SIZE);
+        return EIO;
+    }
+    return 0;
+}
+
+struct tlhPaths {
+    char prefix[256];
+    char file1[256];
+    char file2[256];
+};
+
+static int setupPaths(const struct tlhThreadInfo *ti, struct tlhPaths *paths)
+{
+    memset(paths, sizeof(*paths), 0);
+    if ((size_t)snprintf(paths->prefix, sizeof(paths->prefix), "/tlhData%04d",
+                 ti->threadIdx) >= sizeof(paths->prefix)) {
+        return ENAMETOOLONG;
+    }
+    if ((size_t)snprintf(paths->file1, sizeof(paths->file1), "%s/file1",
+                 paths->prefix) >= sizeof(paths->file1)) {
+        return ENAMETOOLONG;
+    }
+    if ((size_t)snprintf(paths->file2, sizeof(paths->file2), "%s/file2",
+                 paths->prefix) >= sizeof(paths->file2)) {
+        return ENAMETOOLONG;
+    }
+    return 0;
+}
+
+static int doTestHdfsOperations(hdfsFS fs, const struct tlhPaths *paths)
+{
+    char tmp[4096];
+    hdfsFile file;
+    int ret, expected;
+    hdfsFileInfo *fileInfo;
+    struct hdfsReadStatistics *readStats = NULL;
+
+    if (hdfsExists(fs, paths->prefix) == 0) {
+        EXPECT_INT_ZERO(hdfsDelete(fs, paths->prefix, 1));
+    }
+    EXPECT_INT_ZERO(hdfsCreateDirectory(fs, paths->prefix));
+
+    EXPECT_INT_ZERO(doTestGetDefaultBlockSize(fs, paths->prefix));
+
+    /* There should not be any file to open for reading. */
+    EXPECT_NULL(hdfsOpenFile(fs, paths->file1, O_RDONLY, 0, 0, 0));
+
+    /* hdfsOpenFile should not accept mode = 3 */
+    EXPECT_NULL(hdfsOpenFile(fs, paths->file1, 3, 0, 0, 0));
+
+    file = hdfsOpenFile(fs, paths->file1, O_WRONLY, 0, 0, 0);
+    EXPECT_NONNULL(file);
+
+    /* TODO: implement writeFully and use it here */
+    expected = strlen(paths->prefix);
+    ret = hdfsWrite(fs, file, paths->prefix, expected);
+    if (ret < 0) {
+        ret = errno;
+        fprintf(stderr, "hdfsWrite failed and set errno %d\n", ret);
+        return ret;
+    }
+    if (ret != expected) {
+        fprintf(stderr, "hdfsWrite was supposed to write %d bytes, but "
+                "it wrote %d\n", ret, expected);
+        return EIO;
+    }
+    EXPECT_INT_ZERO(hdfsFlush(fs, file));
+    EXPECT_INT_ZERO(hdfsHSync(fs, file));
+    EXPECT_INT_ZERO(hdfsCloseFile(fs, file));
+
+    /* Let's re-open the file for reading */
+    file = hdfsOpenFile(fs, paths->file1, O_RDONLY, 0, 0, 0);
+    EXPECT_NONNULL(file);
+
+    EXPECT_INT_ZERO(hdfsFileGetReadStatistics(file, &readStats));
+    errno = 0;
+    EXPECT_INT_ZERO(readStats->totalBytesRead);
+    EXPECT_INT_ZERO(readStats->totalLocalBytesRead);
+    EXPECT_INT_ZERO(readStats->totalShortCircuitBytesRead);
+    hdfsFileFreeReadStatistics(readStats);
+    /* TODO: implement readFully and use it here */
+    ret = hdfsRead(fs, file, tmp, sizeof(tmp));
+    if (ret < 0) {
+        ret = errno;
+        fprintf(stderr, "hdfsRead failed and set errno %d\n", ret);
+        return ret;
+    }
+    if (ret != expected) {
+        fprintf(stderr, "hdfsRead was supposed to read %d bytes, but "
+                "it read %d\n", ret, expected);
+        return EIO;
+    }
+    EXPECT_INT_ZERO(hdfsFileGetReadStatistics(file, &readStats));
+    errno = 0;
+    EXPECT_INT_EQ(expected, readStats->totalBytesRead);
+    hdfsFileFreeReadStatistics(readStats);
+    EXPECT_INT_ZERO(memcmp(paths->prefix, tmp, expected));
+    EXPECT_INT_ZERO(hdfsCloseFile(fs, file));
+
+    // TODO: Non-recursive delete should fail?
+    //EXPECT_NONZERO(hdfsDelete(fs, prefix, 0));
+    EXPECT_INT_ZERO(hdfsCopy(fs, paths->file1, fs, paths->file2));
+
+    EXPECT_INT_ZERO(hdfsChown(fs, paths->file2, NULL, NULL));
+    EXPECT_INT_ZERO(hdfsChown(fs, paths->file2, NULL, "doop"));
+    fileInfo = hdfsGetPathInfo(fs, paths->file2);
+    EXPECT_NONNULL(fileInfo);
+    EXPECT_INT_ZERO(strcmp("doop", fileInfo->mGroup));
+    hdfsFreeFileInfo(fileInfo, 1);
+
+    EXPECT_INT_ZERO(hdfsChown(fs, paths->file2, "ha", "doop2"));
+    fileInfo = hdfsGetPathInfo(fs, paths->file2);
+    EXPECT_NONNULL(fileInfo);
+    EXPECT_INT_ZERO(strcmp("ha", fileInfo->mOwner));
+    EXPECT_INT_ZERO(strcmp("doop2", fileInfo->mGroup));
+    hdfsFreeFileInfo(fileInfo, 1);
+
+    EXPECT_INT_ZERO(hdfsChown(fs, paths->file2, "ha2", NULL));
+    fileInfo = hdfsGetPathInfo(fs, paths->file2);
+    EXPECT_NONNULL(fileInfo);
+    EXPECT_INT_ZERO(strcmp("ha2", fileInfo->mOwner));
+    EXPECT_INT_ZERO(strcmp("doop2", fileInfo->mGroup));
+    hdfsFreeFileInfo(fileInfo, 1);
+
+    snprintf(tmp, sizeof(tmp), "%s/nonexistent-file-name", paths->prefix);
+    EXPECT_NEGATIVE_ONE_WITH_ERRNO(hdfsChown(fs, tmp, "ha3", NULL), ENOENT);
+    return 0;
+}
+
+static int testHdfsOperationsImpl(struct tlhThreadInfo *ti)
+{
+    hdfsFS fs = NULL;
+    struct tlhPaths paths;
+
+    fprintf(stderr, "testHdfsOperations(threadIdx=%d): starting\n",
+        ti->threadIdx);
+    EXPECT_INT_ZERO(hdfsSingleNameNodeConnect(tlhCluster, &fs, NULL));
+    EXPECT_INT_ZERO(setupPaths(ti, &paths));
+    // test some operations
+    EXPECT_INT_ZERO(doTestHdfsOperations(fs, &paths));
+    EXPECT_INT_ZERO(hdfsDisconnect(fs));
+    // reconnect as user "foo" and verify that we get permission errors
+    EXPECT_INT_ZERO(hdfsSingleNameNodeConnect(tlhCluster, &fs, "foo"));
+    EXPECT_NEGATIVE_ONE_WITH_ERRNO(hdfsChown(fs, paths.file1, "ha3", NULL), EACCES);
+    EXPECT_INT_ZERO(hdfsDisconnect(fs));
+    // reconnect to do the final delete.
+    EXPECT_INT_ZERO(hdfsSingleNameNodeConnect(tlhCluster, &fs, NULL));
+    EXPECT_INT_ZERO(hdfsDelete(fs, paths.prefix, 1));
+    EXPECT_INT_ZERO(hdfsDisconnect(fs));
+    return 0;
+}
+
+static void *testHdfsOperations(void *v)
+{
+    struct tlhThreadInfo *ti = (struct tlhThreadInfo*)v;
+    int ret = testHdfsOperationsImpl(ti);
+    ti->success = ret;
+    return NULL;
+}
+
+static int checkFailures(struct tlhThreadInfo *ti, int tlhNumThreads)
+{
+    int i, threadsFailed = 0;
+    const char *sep = "";
+
+    for (i = 0; i < tlhNumThreads; i++) {
+        if (ti[i].success != 0) {
+            threadsFailed = 1;
+        }
+    }
+    if (!threadsFailed) {
+        fprintf(stderr, "testLibHdfs: all threads succeeded.  SUCCESS.\n");
+        return EXIT_SUCCESS;
+    }
+    fprintf(stderr, "testLibHdfs: some threads failed: [");
+    for (i = 0; i < tlhNumThreads; i++) {
+        if (ti[i].success != 0) {
+            fprintf(stderr, "%s%d", sep, i);
+            sep = ", "; 
+        }
+    }
+    fprintf(stderr, "].  FAILURE.\n");
+    return EXIT_FAILURE;
+}
+
+/**
+ * Test that we can write a file with libhdfs and then read it back
+ */
+int main(void)
+{
+    int i, tlhNumThreads;
+    const char *tlhNumThreadsStr;
+    struct tlhThreadInfo ti[TLH_MAX_THREADS];
+    struct NativeMiniDfsConf conf = {
+        .doFormat = 1,
+    };
+
+    tlhNumThreadsStr = getenv("TLH_NUM_THREADS");
+    if (!tlhNumThreadsStr) {
+        tlhNumThreadsStr = "3";
+    }
+    tlhNumThreads = atoi(tlhNumThreadsStr);
+    if ((tlhNumThreads <= 0) || (tlhNumThreads > TLH_MAX_THREADS)) {
+        fprintf(stderr, "testLibHdfs: must have a number of threads "
+                "between 1 and %d inclusive, not %d\n",
+                TLH_MAX_THREADS, tlhNumThreads);
+        return EXIT_FAILURE;
+    }
+    memset(&ti[0], 0, sizeof(ti));
+    for (i = 0; i < tlhNumThreads; i++) {
+        ti[i].threadIdx = i;
+    }
+
+    EXPECT_INT_ZERO(sem_init(&tlhSem, 0, tlhNumThreads));
+    tlhCluster = nmdCreate(&conf);
+    EXPECT_NONNULL(tlhCluster);
+    EXPECT_INT_ZERO(nmdWaitClusterUp(tlhCluster));
+
+    for (i = 0; i < tlhNumThreads; i++) {
+        EXPECT_INT_ZERO(pthread_create(&ti[i].thread, NULL,
+            testHdfsOperations, &ti[i]));
+    }
+    for (i = 0; i < tlhNumThreads; i++) {
+        EXPECT_INT_ZERO(pthread_join(ti[i].thread, NULL));
+    }
+
+    EXPECT_INT_ZERO(nmdShutdown(tlhCluster));
+    nmdFree(tlhCluster);
+    EXPECT_INT_ZERO(sem_destroy(&tlhSem));
+    return checkFailures(ti, tlhNumThreads);
+}

hadoop-native-core/src/main/native/test/fs/test_libhdfs_zerocopy.c

@@ -0,0 +1,291 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fs/hdfs.h"
+#include "test/native_mini_dfs.h"
+#include "test/test.h"
+
+#include <errno.h>
+#include <inttypes.h>
+#include <semaphore.h>
+#include <pthread.h>
+#include <unistd.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+
+#define TO_STR_HELPER(X) #X
+#define TO_STR(X) TO_STR_HELPER(X)
+
+#define TEST_FILE_NAME_LENGTH 128
+#define TEST_ZEROCOPY_FULL_BLOCK_SIZE 4096
+#define TEST_ZEROCOPY_LAST_BLOCK_SIZE 3215
+#define TEST_ZEROCOPY_NUM_BLOCKS 6
+#define SMALL_READ_LEN 16
+#define TEST_ZEROCOPY_FILE_LEN \
+  (((TEST_ZEROCOPY_NUM_BLOCKS - 1) * TEST_ZEROCOPY_FULL_BLOCK_SIZE) + \
+    TEST_ZEROCOPY_LAST_BLOCK_SIZE)
+
+#define ZC_BUF_LEN 32768
+
+static uint8_t *getZeroCopyBlockData(int blockIdx)
+{
+    uint8_t *buf = malloc(TEST_ZEROCOPY_FULL_BLOCK_SIZE);
+    int i;
+    if (!buf) {
+        fprintf(stderr, "malloc(%d) failed\n", TEST_ZEROCOPY_FULL_BLOCK_SIZE);
+        exit(1);
+    }
+    for (i = 0; i < TEST_ZEROCOPY_FULL_BLOCK_SIZE; i++) {
+      buf[i] = blockIdx + (i % 17);
+    }
+    return buf;
+}
+
+static int getZeroCopyBlockLen(int blockIdx)
+{
+    if (blockIdx >= TEST_ZEROCOPY_NUM_BLOCKS) {
+        return 0;
+    } else if (blockIdx == (TEST_ZEROCOPY_NUM_BLOCKS - 1)) {
+        return TEST_ZEROCOPY_LAST_BLOCK_SIZE;
+    } else {
+        return TEST_ZEROCOPY_FULL_BLOCK_SIZE;
+    }
+}
+
+static int expectFileStats(hdfsFile file,
+      uint64_t expectedTotalBytesRead,
+      uint64_t expectedTotalLocalBytesRead,
+      uint64_t expectedTotalShortCircuitBytesRead,
+      uint64_t expectedTotalZeroCopyBytesRead)
+{
+    struct hdfsReadStatistics *stats = NULL;
+    EXPECT_INT_ZERO(hdfsFileGetReadStatistics(file, &stats));
+    fprintf(stderr, "expectFileStats(expectedTotalBytesRead=%"PRId64", "
+            "expectedTotalLocalBytesRead=%"PRId64", "
+            "expectedTotalShortCircuitBytesRead=%"PRId64", "
+            "expectedTotalZeroCopyBytesRead=%"PRId64", "
+            "totalBytesRead=%"PRId64", "
+            "totalLocalBytesRead=%"PRId64", "
+            "totalShortCircuitBytesRead=%"PRId64", "
+            "totalZeroCopyBytesRead=%"PRId64")\n",
+            expectedTotalBytesRead,
+            expectedTotalLocalBytesRead,
+            expectedTotalShortCircuitBytesRead,
+            expectedTotalZeroCopyBytesRead,
+            stats->totalBytesRead,
+            stats->totalLocalBytesRead,
+            stats->totalShortCircuitBytesRead,
+            stats->totalZeroCopyBytesRead);
+    if (expectedTotalBytesRead != UINT64_MAX) {
+        EXPECT_INT64_EQ(expectedTotalBytesRead, stats->totalBytesRead);
+    }
+    if (expectedTotalLocalBytesRead != UINT64_MAX) {
+        EXPECT_INT64_EQ(expectedTotalLocalBytesRead,
+                      stats->totalLocalBytesRead);
+    }
+    if (expectedTotalShortCircuitBytesRead != UINT64_MAX) {
+        EXPECT_INT64_EQ(expectedTotalShortCircuitBytesRead,
+                      stats->totalShortCircuitBytesRead);
+    }
+    if (expectedTotalZeroCopyBytesRead != UINT64_MAX) {
+        EXPECT_INT64_EQ(expectedTotalZeroCopyBytesRead,
+                      stats->totalZeroCopyBytesRead);
+    }
+    hdfsFileFreeReadStatistics(stats);
+    return 0;
+}
+
+static int doTestZeroCopyReads(hdfsFS fs, const char *fileName)
+{
+    hdfsFile file = NULL;
+    struct hadoopRzOptions *opts = NULL;
+    struct hadoopRzBuffer *buffer = NULL;
+    uint8_t *block;
+
+    file = hdfsOpenFile(fs, fileName, O_RDONLY, 0, 0, 0);
+    EXPECT_NONNULL(file);
+    opts = hadoopRzOptionsAlloc();
+    EXPECT_NONNULL(opts);
+    EXPECT_INT_ZERO(hadoopRzOptionsSetSkipChecksum(opts, 1));
+    /* haven't read anything yet */
+    EXPECT_INT_ZERO(expectFileStats(file, 0LL, 0LL, 0LL, 0LL));
+    block = getZeroCopyBlockData(0);
+    EXPECT_NONNULL(block);
+    /* first read is half of a block. */
+    fprintf(stderr, "WATERMELON: 0\n");
+    buffer = hadoopReadZero(file, opts, TEST_ZEROCOPY_FULL_BLOCK_SIZE / 2);
+    fprintf(stderr, "WATERMELON: 1\n");
+    EXPECT_NONNULL(buffer);
+    EXPECT_INT_EQ(TEST_ZEROCOPY_FULL_BLOCK_SIZE / 2,
+          hadoopRzBufferLength(buffer));
+    EXPECT_INT_ZERO(memcmp(hadoopRzBufferGet(buffer), block,
+          TEST_ZEROCOPY_FULL_BLOCK_SIZE / 2));
+    hadoopRzBufferFree(file, buffer);
+    /* read the next half of the block */
+    buffer = hadoopReadZero(file, opts, TEST_ZEROCOPY_FULL_BLOCK_SIZE / 2);
+    EXPECT_NONNULL(buffer);
+    EXPECT_INT_EQ(TEST_ZEROCOPY_FULL_BLOCK_SIZE / 2,
+          hadoopRzBufferLength(buffer));
+    EXPECT_INT_ZERO(memcmp(hadoopRzBufferGet(buffer),
+          block + (TEST_ZEROCOPY_FULL_BLOCK_SIZE / 2),
+          TEST_ZEROCOPY_FULL_BLOCK_SIZE / 2));
+    hadoopRzBufferFree(file, buffer);
+    free(block);
+    EXPECT_INT_ZERO(expectFileStats(file, TEST_ZEROCOPY_FULL_BLOCK_SIZE, 
+              TEST_ZEROCOPY_FULL_BLOCK_SIZE,
+              TEST_ZEROCOPY_FULL_BLOCK_SIZE,
+              TEST_ZEROCOPY_FULL_BLOCK_SIZE));
+    /* Now let's read just a few bytes. */
+    buffer = hadoopReadZero(file, opts, SMALL_READ_LEN);
+    EXPECT_NONNULL(buffer);
+    EXPECT_INT_EQ(SMALL_READ_LEN, hadoopRzBufferLength(buffer));
+    block = getZeroCopyBlockData(1);
+    EXPECT_NONNULL(block);
+    EXPECT_INT_ZERO(memcmp(block, hadoopRzBufferGet(buffer), SMALL_READ_LEN));
+    hadoopRzBufferFree(file, buffer);
+    EXPECT_INT_EQ(TEST_ZEROCOPY_FULL_BLOCK_SIZE + SMALL_READ_LEN,
+                  hdfsTell(fs, file));
+    EXPECT_INT_ZERO(expectFileStats(file,
+          TEST_ZEROCOPY_FULL_BLOCK_SIZE + SMALL_READ_LEN,
+          TEST_ZEROCOPY_FULL_BLOCK_SIZE + SMALL_READ_LEN,
+          TEST_ZEROCOPY_FULL_BLOCK_SIZE + SMALL_READ_LEN,
+          TEST_ZEROCOPY_FULL_BLOCK_SIZE + SMALL_READ_LEN));
+
+    /* Clear 'skip checksums' and test that we can't do zero-copy reads any
+     * more.  Since there is no ByteBufferPool set, we should fail with
+     * EPROTONOSUPPORT.
+     */
+    EXPECT_INT_ZERO(hadoopRzOptionsSetSkipChecksum(opts, 0));
+    EXPECT_NULL(hadoopReadZero(file, opts, TEST_ZEROCOPY_FULL_BLOCK_SIZE));
+    EXPECT_INT_EQ(EPROTONOSUPPORT, errno);
+
+    /* Verify that setting a NULL ByteBufferPool class works. */
+    EXPECT_INT_ZERO(hadoopRzOptionsSetByteBufferPool(opts, NULL));
+    EXPECT_INT_ZERO(hadoopRzOptionsSetSkipChecksum(opts, 0));
+    EXPECT_NULL(hadoopReadZero(file, opts, TEST_ZEROCOPY_FULL_BLOCK_SIZE));
+    EXPECT_INT_EQ(EPROTONOSUPPORT, errno);
+
+    /* Now set a ByteBufferPool and try again.  It should succeed this time. */
+    EXPECT_INT_ZERO(hadoopRzOptionsSetByteBufferPool(opts,
+          ELASTIC_BYTE_BUFFER_POOL_CLASS));
+    buffer = hadoopReadZero(file, opts, TEST_ZEROCOPY_FULL_BLOCK_SIZE);
+    EXPECT_NONNULL(buffer);
+    EXPECT_INT_EQ(TEST_ZEROCOPY_FULL_BLOCK_SIZE, hadoopRzBufferLength(buffer));
+    EXPECT_INT_ZERO(expectFileStats(file,
+          (2 * TEST_ZEROCOPY_FULL_BLOCK_SIZE) + SMALL_READ_LEN,
+          (2 * TEST_ZEROCOPY_FULL_BLOCK_SIZE) + SMALL_READ_LEN,
+          (2 * TEST_ZEROCOPY_FULL_BLOCK_SIZE) + SMALL_READ_LEN,
+          TEST_ZEROCOPY_FULL_BLOCK_SIZE + SMALL_READ_LEN));
+    EXPECT_INT_ZERO(memcmp(block + SMALL_READ_LEN, hadoopRzBufferGet(buffer),
+        TEST_ZEROCOPY_FULL_BLOCK_SIZE - SMALL_READ_LEN));
+    free(block);
+    block = getZeroCopyBlockData(2);
+    EXPECT_NONNULL(block);
+    EXPECT_INT_ZERO(memcmp(block, hadoopRzBufferGet(buffer) +
+        (TEST_ZEROCOPY_FULL_BLOCK_SIZE - SMALL_READ_LEN), SMALL_READ_LEN));
+    hadoopRzBufferFree(file, buffer);
+
+    /* Check the result of a zero-length read. */
+    buffer = hadoopReadZero(file, opts, 0);
+    EXPECT_NONNULL(buffer);
+    EXPECT_NONNULL(hadoopRzBufferGet(buffer));
+    EXPECT_INT_EQ(0, hadoopRzBufferLength(buffer));
+    hadoopRzBufferFree(file, buffer);
+
+    /* Check the result of reading past EOF */
+    EXPECT_INT_EQ(0, hdfsSeek(fs, file, TEST_ZEROCOPY_FILE_LEN));
+    buffer = hadoopReadZero(file, opts, 1);
+    EXPECT_NONNULL(buffer);
+    EXPECT_NULL(hadoopRzBufferGet(buffer));
+    hadoopRzBufferFree(file, buffer);
+
+    /* Cleanup */
+    free(block);
+    hadoopRzOptionsFree(opts);
+    EXPECT_INT_ZERO(hdfsCloseFile(fs, file));
+    return 0;
+}
+
+static int createZeroCopyTestFile(hdfsFS fs, char *testFileName,
+                                  size_t testFileNameLen)
+{
+    int blockIdx, blockLen;
+    hdfsFile file;
+    uint8_t *data;
+
+    snprintf(testFileName, testFileNameLen, "/zeroCopyTestFile.%d.%d",
+             getpid(), rand());
+    file = hdfsOpenFile(fs, testFileName, O_WRONLY, 0, 1,
+                        TEST_ZEROCOPY_FULL_BLOCK_SIZE);
+    EXPECT_NONNULL(file);
+    for (blockIdx = 0; blockIdx < TEST_ZEROCOPY_NUM_BLOCKS; blockIdx++) {
+        blockLen = getZeroCopyBlockLen(blockIdx);
+        data = getZeroCopyBlockData(blockIdx);
+        EXPECT_NONNULL(data);
+        EXPECT_INT_EQ(blockLen, hdfsWrite(fs, file, data, blockLen));
+    }
+    EXPECT_INT_ZERO(hdfsCloseFile(fs, file));
+    return 0;
+}
+
+/**
+ * Test that we can write a file with libhdfs and then read it back
+ */
+int main(void)
+{
+    int port;
+    struct NativeMiniDfsConf conf = {
+        .doFormat = 1,
+        .configureShortCircuit = 1,
+    };
+    char testFileName[TEST_FILE_NAME_LENGTH];
+    hdfsFS fs;
+    struct NativeMiniDfsCluster* cl;
+    struct hdfsBuilder *bld;
+
+    cl = nmdCreate(&conf);
+    EXPECT_NONNULL(cl);
+    EXPECT_INT_ZERO(nmdWaitClusterUp(cl));
+    port = nmdGetNameNodePort(cl);
+    if (port < 0) {
+        fprintf(stderr, "TEST_ERROR: test_zerocopy: "
+                "nmdGetNameNodePort returned error %d\n", port);
+        return EXIT_FAILURE;
+    }
+    bld = hdfsNewBuilder();
+    EXPECT_NONNULL(bld);
+    EXPECT_INT_ZERO(nmdConfigureHdfsBuilder(cl, bld));
+    hdfsBuilderSetForceNewInstance(bld);
+    hdfsBuilderConfSetStr(bld, "dfs.block.size",
+                          TO_STR(TEST_ZEROCOPY_FULL_BLOCK_SIZE));
+    /* ensure that we'll always get our mmaps */
+    hdfsBuilderConfSetStr(bld, "dfs.client.read.shortcircuit.skip.checksum",
+                          "true");
+    fs = hdfsBuilderConnect(bld);
+    EXPECT_NONNULL(fs);
+    EXPECT_INT_ZERO(createZeroCopyTestFile(fs, testFileName,
+          TEST_FILE_NAME_LENGTH));
+    EXPECT_INT_ZERO(doTestZeroCopyReads(fs, testFileName));
+    EXPECT_INT_ZERO(hdfsDisconnect(fs));
+    EXPECT_INT_ZERO(nmdShutdown(cl));
+    nmdFree(cl);
+    fprintf(stderr, "TEST_SUCCESS\n"); 
+    return EXIT_SUCCESS;
+}

hadoop-native-core/src/main/native/test/native_mini_dfs.c

@@ -0,0 +1,390 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fs/hdfs.h"
+#include "fs/hdfs_test.h"
+#include "jni/exception.h"
+#include "jni/jni_helper.h"
+#include "test/native_mini_dfs.h"
+
+#include <errno.h>
+#include <jni.h>
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#define MINIDFS_CLUSTER_BUILDER "org/apache/hadoop/hdfs/MiniDFSCluster$Builder"
+#define MINIDFS_CLUSTER "org/apache/hadoop/hdfs/MiniDFSCluster"
+#define HADOOP_CONF     "org/apache/hadoop/conf/Configuration"
+#define HADOOP_NAMENODE "org/apache/hadoop/hdfs/server/namenode/NameNode"
+#define JAVA_INETSOCKETADDRESS "java/net/InetSocketAddress"
+
+#define DFS_WEBHDFS_ENABLED_KEY "dfs.webhdfs.enabled"
+
+struct NativeMiniDfsCluster {
+    /**
+     * The NativeMiniDfsCluster object
+     */
+    jobject obj;
+
+    /**
+     * Path to the domain socket, or the empty string if there is none.
+     */
+    char domainSocketPath[PATH_MAX];
+};
+
+static int hdfsDisableDomainSocketSecurity(void)
+{
+    jthrowable jthr;
+    JNIEnv* env = getJNIEnv();
+    if (env == NULL) {
+      errno = EINTERNAL;
+      return -1;
+    }
+    jthr = invokeMethod(env, NULL, STATIC, NULL,
+            "org/apache/hadoop/net/unix/DomainSocket",
+            "disableBindPathValidation", "()V");
+    if (jthr) {
+        errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "DomainSocket#disableBindPathValidation");
+        return -1;
+    }
+    return 0;
+}
+
+static jthrowable nmdConfigureShortCircuit(JNIEnv *env,
+              struct NativeMiniDfsCluster *cl, jobject cobj)
+{
+    jthrowable jthr;
+    char *tmpDir;
+
+    int ret = hdfsDisableDomainSocketSecurity();
+    if (ret) {
+        return newRuntimeError(env, "failed to disable hdfs domain "
+                               "socket security: error %d", ret);
+    }
+    jthr = hadoopConfSetStr(env, cobj, "dfs.client.read.shortcircuit", "true");
+    if (jthr) {
+        return jthr;
+    }
+    tmpDir = getenv("TMPDIR");
+    if (!tmpDir) {
+        tmpDir = "/tmp";
+    }
+    snprintf(cl->domainSocketPath, PATH_MAX, "%s/native_mini_dfs.sock.%d.%d",
+             tmpDir, getpid(), rand());
+    snprintf(cl->domainSocketPath, PATH_MAX, "%s/native_mini_dfs.sock.%d.%d",
+             tmpDir, getpid(), rand());
+    jthr = hadoopConfSetStr(env, cobj, "dfs.domain.socket.path",
+                            cl->domainSocketPath);
+    if (jthr) {
+        return jthr;
+    }
+    return NULL;
+}
+
+struct NativeMiniDfsCluster* nmdCreate(struct NativeMiniDfsConf *conf)
+{
+    struct NativeMiniDfsCluster* cl = NULL;
+    jobject bld = NULL, cobj = NULL, cluster = NULL;
+    jvalue  val;
+    JNIEnv *env = getJNIEnv();
+    jthrowable jthr;
+    jstring jconfStr = NULL;
+
+    if (!env) {
+        fprintf(stderr, "nmdCreate: unable to construct JNIEnv.\n");
+        return NULL;
+    }
+    cl = calloc(1, sizeof(struct NativeMiniDfsCluster));
+    if (!cl) {
+        fprintf(stderr, "nmdCreate: OOM");
+        goto error;
+    }
+    jthr = constructNewObjectOfClass(env, &cobj, HADOOP_CONF, "()V");
+    if (jthr) {
+        printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "nmdCreate: new Configuration");
+        goto error;
+    }
+    if (conf->webhdfsEnabled) {
+        jthr = newJavaStr(env, DFS_WEBHDFS_ENABLED_KEY, &jconfStr);
+        if (jthr) {
+            printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                                  "nmdCreate: new String");
+            goto error;
+        }
+        jthr = invokeMethod(env, NULL, INSTANCE, cobj, HADOOP_CONF,
+                            "setBoolean", "(Ljava/lang/String;Z)V",
+                            jconfStr, conf->webhdfsEnabled);
+        if (jthr) {
+            printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                                  "nmdCreate: Configuration::setBoolean");
+            goto error;
+        }
+    }
+    if (jthr) {
+        printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                              "nmdCreate: Configuration::setBoolean");
+        goto error;
+    }
+    // Disable 'minimum block size' -- it's annoying in tests.
+    (*env)->DeleteLocalRef(env, jconfStr);
+    jconfStr = NULL;
+    jthr = newJavaStr(env, "dfs.namenode.fs-limits.min-block-size", &jconfStr);
+    if (jthr) {
+        printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                              "nmdCreate: new String");
+        goto error;
+    }
+    jthr = invokeMethod(env, NULL, INSTANCE, cobj, HADOOP_CONF,
+                        "setLong", "(Ljava/lang/String;J)V", jconfStr, 0LL);
+    if (jthr) {
+        printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                              "nmdCreate: Configuration::setLong");
+        goto error;
+    }
+    // Creae MiniDFSCluster object
+    jthr = constructNewObjectOfClass(env, &bld, MINIDFS_CLUSTER_BUILDER,
+                    "(L"HADOOP_CONF";)V", cobj);
+    if (jthr) {
+        printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "nmdCreate: NativeMiniDfsCluster#Builder#Builder");
+        goto error;
+    }
+    if (conf->configureShortCircuit) {
+        jthr = nmdConfigureShortCircuit(env, cl, cobj);
+        if (jthr) {
+            printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                "nmdCreate: nmdConfigureShortCircuit error");
+            goto error;
+        }
+    }
+    jthr = invokeMethod(env, &val, INSTANCE, bld, MINIDFS_CLUSTER_BUILDER,
+            "format", "(Z)L" MINIDFS_CLUSTER_BUILDER ";", conf->doFormat);
+    if (jthr) {
+        printExceptionAndFree(env, jthr, PRINT_EXC_ALL, "nmdCreate: "
+                              "Builder::format");
+        goto error;
+    }
+    (*env)->DeleteLocalRef(env, val.l);
+    if (conf->webhdfsEnabled) {
+        jthr = invokeMethod(env, &val, INSTANCE, bld, MINIDFS_CLUSTER_BUILDER,
+                        "nameNodeHttpPort", "(I)L" MINIDFS_CLUSTER_BUILDER ";",
+                        conf->namenodeHttpPort);
+        if (jthr) {
+            printExceptionAndFree(env, jthr, PRINT_EXC_ALL, "nmdCreate: "
+                                  "Builder::nameNodeHttpPort");
+            goto error;
+        }
+        (*env)->DeleteLocalRef(env, val.l);
+    }
+    jthr = invokeMethod(env, &val, INSTANCE, bld, MINIDFS_CLUSTER_BUILDER,
+            "build", "()L" MINIDFS_CLUSTER ";");
+    if (jthr) {
+        printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                              "nmdCreate: Builder#build");
+        goto error;
+    }
+    cluster = val.l;
+	  cl->obj = (*env)->NewGlobalRef(env, val.l);
+    if (!cl->obj) {
+        printPendingExceptionAndFree(env, PRINT_EXC_ALL,
+            "nmdCreate: NewGlobalRef");
+        goto error;
+    }
+    (*env)->DeleteLocalRef(env, cluster);
+    (*env)->DeleteLocalRef(env, bld);
+    (*env)->DeleteLocalRef(env, cobj);
+    (*env)->DeleteLocalRef(env, jconfStr);
+    return cl;
+
+error:
+    (*env)->DeleteLocalRef(env, cluster);
+    (*env)->DeleteLocalRef(env, bld);
+    (*env)->DeleteLocalRef(env, cobj);
+    (*env)->DeleteLocalRef(env, jconfStr);
+    free(cl);
+    return NULL;
+}
+
+void nmdFree(struct NativeMiniDfsCluster* cl)
+{
+    JNIEnv *env = getJNIEnv();
+    if (!env) {
+        fprintf(stderr, "nmdFree: getJNIEnv failed\n");
+        free(cl);
+        return;
+    }
+    (*env)->DeleteGlobalRef(env, cl->obj);
+    free(cl);
+}
+
+int nmdShutdown(struct NativeMiniDfsCluster* cl)
+{
+    JNIEnv *env = getJNIEnv();
+    jthrowable jthr;
+
+    if (!env) {
+        fprintf(stderr, "nmdShutdown: getJNIEnv failed\n");
+        return -EIO;
+    }
+    jthr = invokeMethod(env, NULL, INSTANCE, cl->obj,
+            MINIDFS_CLUSTER, "shutdown", "()V");
+    if (jthr) {
+        printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "nmdShutdown: MiniDFSCluster#shutdown");
+        return -EIO;
+    }
+    return 0;
+}
+
+int nmdWaitClusterUp(struct NativeMiniDfsCluster *cl)
+{
+    jthrowable jthr;
+    JNIEnv *env = getJNIEnv();
+    if (!env) {
+        fprintf(stderr, "nmdWaitClusterUp: getJNIEnv failed\n");
+        return -EIO;
+    }
+    jthr = invokeMethod(env, NULL, INSTANCE, cl->obj,
+            MINIDFS_CLUSTER, "waitClusterUp", "()V");
+    if (jthr) {
+        printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "nmdWaitClusterUp: MiniDFSCluster#waitClusterUp ");
+        return -EIO;
+    }
+    return 0;
+}
+
+int nmdGetNameNodePort(const struct NativeMiniDfsCluster *cl)
+{
+    JNIEnv *env = getJNIEnv();
+    jvalue jVal;
+    jthrowable jthr;
+
+    if (!env) {
+        fprintf(stderr, "nmdHdfsConnect: getJNIEnv failed\n");
+        return -EIO;
+    }
+    // Note: this will have to be updated when HA nativeMiniDfs clusters are
+    // supported
+    jthr = invokeMethod(env, &jVal, INSTANCE, cl->obj,
+            MINIDFS_CLUSTER, "getNameNodePort", "()I");
+    if (jthr) {
+        printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "nmdHdfsConnect: MiniDFSCluster#getNameNodePort");
+        return -EIO;
+    }
+    return jVal.i;
+}
+
+int nmdGetNameNodeHttpAddress(const struct NativeMiniDfsCluster *cl,
+                               int *port, const char **hostName)
+{
+    JNIEnv *env = getJNIEnv();
+    jvalue jVal;
+    jobject jNameNode, jAddress;
+    jthrowable jthr;
+    int ret = 0;
+    const char *host;
+    
+    if (!env) {
+        fprintf(stderr, "nmdHdfsConnect: getJNIEnv failed\n");
+        return -EIO;
+    }
+    // First get the (first) NameNode of the cluster
+    jthr = invokeMethod(env, &jVal, INSTANCE, cl->obj, MINIDFS_CLUSTER,
+                        "getNameNode", "()L" HADOOP_NAMENODE ";");
+    if (jthr) {
+        printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                              "nmdGetNameNodeHttpAddress: "
+                              "MiniDFSCluster#getNameNode");
+        return -EIO;
+    }
+    jNameNode = jVal.l;
+    
+    // Then get the http address (InetSocketAddress) of the NameNode
+    jthr = invokeMethod(env, &jVal, INSTANCE, jNameNode, HADOOP_NAMENODE,
+                        "getHttpAddress", "()L" JAVA_INETSOCKETADDRESS ";");
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                                    "nmdGetNameNodeHttpAddress: "
+                                    "NameNode#getHttpAddress");
+        goto error_dlr_nn;
+    }
+    jAddress = jVal.l;
+    
+    jthr = invokeMethod(env, &jVal, INSTANCE, jAddress,
+                        JAVA_INETSOCKETADDRESS, "getPort", "()I");
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                                    "nmdGetNameNodeHttpAddress: "
+                                    "InetSocketAddress#getPort");
+        goto error_dlr_addr;
+    }
+    *port = jVal.i;
+    
+    jthr = invokeMethod(env, &jVal, INSTANCE, jAddress, JAVA_INETSOCKETADDRESS,
+                        "getHostName", "()Ljava/lang/String;");
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                                    "nmdGetNameNodeHttpAddress: "
+                                    "InetSocketAddress#getHostName");
+        goto error_dlr_addr;
+    }
+    host = (*env)->GetStringUTFChars(env, jVal.l, NULL);
+    *hostName = strdup(host);
+    (*env)->ReleaseStringUTFChars(env, jVal.l, host);
+    
+error_dlr_addr:
+    (*env)->DeleteLocalRef(env, jAddress);
+error_dlr_nn:
+    (*env)->DeleteLocalRef(env, jNameNode);
+    
+    return ret;
+}
+
+int nmdConfigureHdfsBuilder(struct NativeMiniDfsCluster *cl,
+                            struct hdfsBuilder *bld)
+{
+    int port, ret;
+
+    hdfsBuilderSetNameNode(bld, "localhost");
+    port = nmdGetNameNodePort(cl);
+    if (port < 0) {
+      fprintf(stderr, "nmdGetNameNodePort failed with error %d\n", -port);
+      return EIO;
+    }
+    hdfsBuilderSetNameNodePort(bld, port);
+    if (cl->domainSocketPath[0]) {
+      ret = hdfsBuilderConfSetStr(bld, "dfs.client.read.shortcircuit", "true");
+      if (ret) {
+          return ret;
+      }
+      ret = hdfsBuilderConfSetStr(bld, "dfs.domain.socket.path",
+                            cl->domainSocketPath);
+      if (ret) {
+          return ret;
+      }
+    }
+    return 0;
+}

hadoop-native-core/src/main/native/test/native_mini_dfs.h

@@ -0,0 +1,122 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFS_NATIVE_MINI_DFS_H
+#define LIBHDFS_NATIVE_MINI_DFS_H
+
+#include <jni.h> /* for jboolean */
+
+struct hdfsBuilder;
+struct NativeMiniDfsCluster; 
+
+/**
+ * Represents a configuration to use for creating a Native MiniDFSCluster
+ */
+struct NativeMiniDfsConf {
+    /**
+     * Nonzero if the cluster should be formatted prior to startup.
+     */
+    jboolean doFormat;
+
+    /**
+     * Whether or not to enable webhdfs in MiniDfsCluster
+     */
+    jboolean webhdfsEnabled;
+
+    /**
+     * The http port of the namenode in MiniDfsCluster
+     */
+    jint namenodeHttpPort;
+
+    /**
+     * Nonzero if we should configure short circuit.
+     */
+    jboolean configureShortCircuit;
+};
+
+/**
+ * Create a NativeMiniDfsBuilder
+ *
+ * @param conf      (inout) The cluster configuration
+ *
+ * @return      a NativeMiniDfsBuilder, or a NULL pointer on error.
+ */
+struct NativeMiniDfsCluster* nmdCreate(struct NativeMiniDfsConf *conf);
+
+/**
+ * Wait until a MiniDFSCluster comes out of safe mode.
+ *
+ * @param cl        The cluster
+ *
+ * @return          0 on success; a non-zero error code if the cluster fails to
+ *                  come out of safe mode.
+ */
+int nmdWaitClusterUp(struct NativeMiniDfsCluster *cl);
+
+/**
+ * Shut down a NativeMiniDFS cluster
+ *
+ * @param cl        The cluster
+ *
+ * @return          0 on success; a non-zero error code if an exception is
+ *                  thrown.
+ */
+int nmdShutdown(struct NativeMiniDfsCluster *cl);
+
+/**
+ * Destroy a Native MiniDFSCluster
+ *
+ * @param cl        The cluster to destroy
+ */
+void nmdFree(struct NativeMiniDfsCluster* cl);
+
+/**
+ * Get the port that's in use by the given (non-HA) nativeMiniDfs
+ *
+ * @param cl        The initialized NativeMiniDfsCluster
+ *
+ * @return          the port, or a negative error code
+ */
+int nmdGetNameNodePort(const struct NativeMiniDfsCluster *cl); 
+
+/**
+ * Get the http address that's in use by the given (non-HA) nativeMiniDfs
+ *
+ * @param cl        The initialized NativeMiniDfsCluster
+ * @param port      Used to capture the http port of the NameNode 
+ *                  of the NativeMiniDfsCluster
+ * @param hostName  Used to capture the http hostname of the NameNode
+ *                  of the NativeMiniDfsCluster
+ *
+ * @return          0 on success; a non-zero error code if failing to
+ *                  get the information.
+ */
+int nmdGetNameNodeHttpAddress(const struct NativeMiniDfsCluster *cl,
+                               int *port, const char **hostName);
+
+/**
+ * Configure the HDFS builder appropriately to connect to this cluster.
+ *
+ * @param bld       The hdfs builder
+ *
+ * @return          the port, or a negative error code
+ */
+int nmdConfigureHdfsBuilder(struct NativeMiniDfsCluster *cl,
+                            struct hdfsBuilder *bld);
+
+#endif

hadoop-native-core/src/main/native/test/posix_util.c

@@ -0,0 +1,157 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.    See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.    The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.    You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "test/posix_util.h"
+
+#include <dirent.h>
+#include <errno.h>
+#include <pthread.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <limits.h>
+
+static pthread_mutex_t gTempdirLock = PTHREAD_MUTEX_INITIALIZER;
+
+static int gTempdirNonce = 0;
+
+int recursiveDeleteContents(const char *path)
+{
+    int ret;
+    DIR *dp;
+    struct dirent *de;
+    char tmp[PATH_MAX];
+
+    dp = opendir(path);
+    if (!dp) {
+        ret = -errno;
+        fprintf(stderr, "recursiveDelete(%s) failed with error %d\n", path, ret);
+        return ret;
+    }
+    while (1) {
+        de = readdir(dp);
+        if (!de) {
+            ret = 0;
+            break;
+        }
+        if ((de->d_name[0] == '.') && (de->d_name[1] == '\0'))
+            continue;
+        if ((de->d_name[0] == '.') && (de->d_name[1] == '.') &&
+                (de->d_name[2] == '\0'))
+            continue;
+        snprintf(tmp, sizeof(tmp), "%s/%s", path, de->d_name);
+        ret = recursiveDelete(tmp);
+        if (ret)
+            break;
+    }
+    if (closedir(dp)) {
+        ret = -errno;
+        fprintf(stderr, "recursiveDelete(%s): closedir failed with "
+                        "error %d\n", path, ret);
+        return ret;
+    }
+    return ret;
+}
+
+/*
+ * Simple recursive delete implementation.
+ * It could be optimized, but there is no need at the moment.
+ * TODO: use fstat, etc.
+ */
+int recursiveDelete(const char *path)
+{
+    int ret;
+    struct stat stBuf;
+
+    ret = stat(path, &stBuf);
+    if (ret != 0) {
+        ret = -errno;
+        fprintf(stderr, "recursiveDelete(%s): stat failed with "
+                        "error %d\n", path, ret);
+        return ret;
+    }
+    if (S_ISDIR(stBuf.st_mode)) {
+        ret = recursiveDeleteContents(path);
+        if (ret)
+            return ret;
+        ret = rmdir(path);
+        if (ret) {
+            ret = errno;
+            fprintf(stderr, "recursiveDelete(%s): rmdir failed with error %d\n",
+                            path, ret);
+            return ret;
+        }
+    } else {
+        ret = unlink(path);
+        if (ret) {
+            ret = -errno;
+            fprintf(stderr, "recursiveDelete(%s): unlink failed with "
+                            "error %d\n", path, ret);
+            return ret;
+        }
+    }
+    return 0;
+}
+
+int createTempDir(char *tempDir, int nameMax, int mode)
+{
+    char tmp[PATH_MAX];
+    int pid, nonce;
+    const char *base = getenv("TMPDIR");
+    if (!base)
+        base = "/tmp";
+    if (base[0] != '/') {
+        // canonicalize non-absolute TMPDIR
+        if (realpath(base, tmp) == NULL) {
+            return -errno;
+        }
+        base = tmp;
+    }
+    pid = getpid();
+    pthread_mutex_lock(&gTempdirLock);
+    nonce = gTempdirNonce++;
+    pthread_mutex_unlock(&gTempdirLock);
+    snprintf(tempDir, nameMax, "%s/temp.%08d.%08d", base, pid, nonce);
+    if (mkdir(tempDir, mode) == -1) {
+        int ret = errno;
+        return -ret;
+    }
+    return 0;
+}
+
+void sleepNoSig(int sec)
+{
+    int ret;
+    struct timespec req, rem;
+
+    rem.tv_sec = sec;
+    rem.tv_nsec = 0;
+    do {
+        req = rem;
+        ret = nanosleep(&req, &rem);
+    } while ((ret == -1) && (errno == EINTR));
+    if (ret == -1) {
+        ret = errno;
+        fprintf(stderr, "nanosleep error %d (%s)\n", ret, strerror(ret));
+    }
+}
+
+// vim: ts=4:sw=4:tw=79:et

hadoop-native-core/src/main/native/test/posix_util.h

@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __POSIX_UTIL_H__
+#define __POSIX_UTIL_H__
+
+/**
+ * Recursively delete the contents of a directory
+ *
+ * @param path      directory whose contents we should delete
+ *
+ * @return          0 on success; error code otherwise
+ */
+int recursiveDeleteContents(const char *path);
+
+/**
+ * Recursively delete a local path, using unlink or rmdir as appropriate.
+ *
+ * @param path      path to delete
+ *
+ * @return          0 on success; error code otherwise
+ */
+int recursiveDelete(const char *path);
+
+/**
+ * Get a temporary directory 
+ *
+ * @param tempDir   (out param) path to the temporary directory
+ * @param nameMax   Length of the tempDir buffer
+ * @param mode      Mode to create with
+ *
+ * @return          0 on success; error code otherwise
+ */
+int createTempDir(char *tempDir, int nameMax, int mode);
+
+/**
+ * Sleep without using signals
+ *
+ * @param sec       Number of seconds to sleep
+ */
+void sleepNoSig(int sec);
+
+#endif

hadoop-native-core/src/main/native/test/test.c

@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "test/test.h"
+
+#include <dirent.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <stdarg.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+
+void fail(const char *fmt, ...)
+{
+    va_list ap;
+
+    va_start(ap, fmt);
+    vfprintf(stderr, fmt, ap);
+    va_end(ap);
+    exit(1);
+}
+
+static int vexpect_decode(const char *expected, const char *text, int ty,
+        const char *str)
+{
+    switch (ty) {
+    case TEST_ERROR_EQ:
+        if (strcmp(expected, str) != 0) {
+            fprintf(stderr, "error: expected '%s', but got '%s', which "
+                   "is not equal. %s\n", expected, str, text);
+            return -1;
+        }
+        break;
+    case TEST_ERROR_GE:
+        if (strcmp(expected, str) > 0) {
+            fprintf(stderr, "error: expected '%s', but got '%s'. "
+                   "Expected something greater or equal.  %s\n",
+                   expected, str, text);
+            return -1;
+        }
+        break;
+    case TEST_ERROR_GT:
+        if (strcmp(expected, str) >= 0) {
+            fprintf(stderr, "error: expected '%s', but got '%s'. "
+                   "Expected something greater.  %s\n",
+                   expected, str, text);
+            return -1;
+        }
+        break;
+    case TEST_ERROR_LE:
+        if (strcmp(expected, str) < 0) {
+            fprintf(stderr, "error: expected '%s', but got '%s'. "
+                   "Expected something less or equal.  %s\n",
+                   expected, str, text);
+            return -1;
+        }
+        break;
+    case TEST_ERROR_LT:
+        if (strcmp(expected, str) <= 0) {
+            fprintf(stderr, "error: expected '%s', but got '%s'. "
+                   "Expected something less.  %s\n",
+                   expected, str, text);
+            return -1;
+        }
+        break;
+    case TEST_ERROR_NE:
+        if (strcmp(expected, str) == 0) {
+            fprintf(stderr, "error: did not expect '%s': '%s'\n",
+                   expected, text);
+            return -1;
+        }
+        break;
+    default:
+        fprintf(stderr, "runtime error: invalid type %d passed in: '%s'\n",
+                ty, text);
+        return -1;
+    }
+    return 0;
+}
+
+int vexpect(const char *expected, const char *text, int ty,
+        const char *fmt, va_list ap)
+{
+    char *str = NULL;
+    int ret;
+
+    if (vasprintf(&str, fmt, ap) < 0) { // TODO: portability
+        fprintf(stderr, "error: vasprintf failed: %s\n", text);
+        return -1;
+    }
+    ret = vexpect_decode(expected, text, ty, str);
+    free(str);
+    return ret;
+}
+
+int expect(const char *expected, const char *text, int ty,
+        const char *fmt, ...)
+{
+    int ret;
+    va_list ap;
+
+    va_start(ap, fmt);
+    ret = vexpect(expected, text, ty, fmt, ap);
+    va_end(ap);
+    return ret;
+}
+
+void *xcalloc(size_t len)
+{
+    void *v = calloc(1, len);
+    if (!v) {
+        abort();
+    }
+    return v;
+}
+
+// vim: ts=4:sw=4:tw=79:et

hadoop-native-core/src/main/native/test/test.h

@@ -0,0 +1,170 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef HADOOP_CORE_COMMON_TEST_H
+#define HADOOP_CORE_COMMON_TEST_H
+
+#include <errno.h> /* for errno */
+#include <inttypes.h> /* for PRIdPTR */
+#include <stdarg.h> /* for va_list */
+#include <unistd.h> /* for size_t */
+
+#define TEST_ERROR_EQ 0
+#define TEST_ERROR_GE 1
+#define TEST_ERROR_GT 2
+#define TEST_ERROR_LE 3
+#define TEST_ERROR_LT 4
+#define TEST_ERROR_NE 5
+
+/**
+ * Fail with an error message.
+ *
+ * @param fmt       printf-style format string.
+ * @param ...       printf-style arguments.
+ */
+void fail(const char *fmt, ...);
+
+/**
+ * Check for a test condition.
+ *
+ * @param expected  The string which is expected.
+ * @param text      Some text that will be printed out if the test
+ *                      condition fails.
+ * @param ty        Comparison type.  See TEST_ERROR_* constants.
+ * @param fmt       printf-style format string.
+ * @param ap        printf-style arguments.
+ *
+ * @return      0 on success; 1 on failure.
+ */
+int vexpect(const char *expected, const char *text, int ty,
+        const char *fmt, va_list ap);
+
+/**
+ * Check for a test condition.
+ *
+ * @param expected  The string which is expected.
+ * @param text      Some text that will be printed out if the test
+ *                      condition fails.
+ * @param ty        Comparison type.  See TEST_ERROR_* constants.
+ * @param fmt       printf-style format string.
+ * @param ...       printf-style arguments.
+ *
+ * @return          0 on success; 1 on failure.
+ */
+int expect(const char *expected, const char *text, int ty,
+        const char *fmt, ...);
+
+/**
+ * Allocate a zero-initialized region of memory, or die.
+ *
+ * @param len       The length
+ *
+ * @return          A pointer to a zero-initialized malloc'ed region.
+ */
+void *xcalloc(size_t len);
+
+#define TEST_ERROR_GET_LINE_HELPER2(x) #x
+#define TEST_ERROR_GET_LINE_HELPER(x) TEST_ERROR_GET_LINE_HELPER2(x)
+#define TEST_ERROR_LOCATION_TEXT __FILE__ " at line " \
+    TEST_ERROR_GET_LINE_HELPER(__LINE__)
+
+#define EXPECT(...) do { if (expect(__VA_ARGS__)) return 1; } while (0);
+
+#define EXPECT_EQ(expected, fmt, ...) \
+    EXPECT(expected, TEST_ERROR_LOCATION_TEXT, TEST_ERROR_EQ, \
+           fmt, __VA_ARGS__);
+
+#define EXPECT_STR_EQ(expected, str) \
+    EXPECT_EQ(expected, "%s", str)
+
+#define EXPECT_GE(expected, fmt, ...) \
+    EXPECT(expected, TEST_ERROR_LOCATION_TEXT, TEST_ERROR_GE, \
+           fmt, __VA_ARGS__);
+
+#define EXPECT_GT(expected, fmt, ...) \
+    EXPECT(expected, TEST_ERROR_LOCATION_TEXT, TEST_ERROR_GT, \
+           fmt, __VA_ARGS__);
+
+#define EXPECT_LE(expected, fmt, ...) \
+    EXPECT(expected, TEST_ERROR_LOCATION_TEXT, TEST_ERROR_LE, \
+           fmt, __VA_ARGS__);
+
+#define EXPECT_LT(expected, fmt, ...) \
+    EXPECT(expected, TEST_ERROR_LOCATION_TEXT, TEST_ERROR_LT, \
+           fmt, __VA_ARGS__);
+
+#define EXPECT_NE(expected, fmt, ...) \
+    EXPECT(expected, TEST_ERROR_LOCATION_TEXT, TEST_ERROR_NE, \
+           fmt, __VA_ARGS__);
+
+#define COMMON_TEST__TO_STR(x) #x
+#define COMMON_TEST__TO_STR2(x) COMMON_TEST__TO_STR(x)
+
+#define EXPECT_INT_EQ(expected, x) do { \
+  char expected_buf[16] = { 0 }; \
+  snprintf(expected_buf, sizeof(expected_buf), "%d", expected); \
+  EXPECT(expected_buf, TEST_ERROR_LOCATION_TEXT, TEST_ERROR_EQ, "%d", x); \
+} while(0);
+
+#define EXPECT_INT64_EQ(expected, x) do { \
+  char expected_buf[32] = { 0 }; \
+  snprintf(expected_buf, sizeof(expected_buf), "%" PRId64, expected); \
+  EXPECT(expected_buf, TEST_ERROR_LOCATION_TEXT, TEST_ERROR_EQ, \
+         "%" PRId64, x); \
+} while(0);
+
+#define EXPECT_NO_HADOOP_ERR(expr) do { \
+  struct hadoop_err *err = expr; \
+  EXPECT("", TEST_ERROR_LOCATION_TEXT, TEST_ERROR_EQ, \
+         "%s", (err ? hadoop_err_msg(err) : "")); \
+} while(0);
+
+#define EXPECT_INT_ZERO(x) \
+    EXPECT("0", TEST_ERROR_LOCATION_TEXT, TEST_ERROR_EQ, \
+           "%d", x);
+
+#define EXPECT_TRUE(x) \
+    EXPECT("1", TEST_ERROR_LOCATION_TEXT, TEST_ERROR_EQ, \
+           "%d", (!!x));
+
+#define EXPECT_FALSE(x) \
+    EXPECT("0", TEST_ERROR_LOCATION_TEXT, TEST_ERROR_EQ, \
+           "%d", (!!x));
+
+#define EXPECT_NULL(x) \
+    EXPECT("0", TEST_ERROR_LOCATION_TEXT, TEST_ERROR_EQ, \
+           "%"PRIdPTR, x);
+
+#define EXPECT_NONNULL(x) \
+    EXPECT("0", TEST_ERROR_LOCATION_TEXT, TEST_ERROR_NE, \
+           "%"PRIdPTR, x);
+
+#define EXPECT_NEGATIVE_ONE_WITH_ERRNO(expr, e) do { \
+    char expected[80] = { 0 }; \
+    int res, err; \
+    snprintf(expected, sizeof(expected), "ret -1 with errno %d", e); \
+    errno = 0; \
+    res = expr; \
+    err = errno; \
+    EXPECT(expected, TEST_ERROR_LOCATION_TEXT, TEST_ERROR_EQ, \
+            "ret %d with errno %d", res, err); \
+} while(0);
+
+#endif
+
+// vim: ts=4:sw=4:tw=79:et