HDFS-12110: libhdfs++: Rebase HDFS-8707 on trunk, minor conflict resolution in this patch. Contributed by Deepak Majeti and Anatoli Shein.

dev-support/docker/Dockerfile

@@ -38,7 +38,6 @@ ENV DEBCONF_TERSE true
 # WARNING: DO NOT PUT JAVA APPS HERE! Otherwise they will install default
 # Ubuntu Java.  See Java section below!
 ######
-<<<<<<< 5f556eacf0fedbaec3cdb819847c7c6c3bacc048
 RUN apt-get -q update && apt-get -q install -y \
     apt-utils \
     build-essential \
@@ -56,6 +55,7 @@ RUN apt-get -q update && apt-get -q install -y \
     libfuse-dev \
     libprotobuf-dev \
     libprotoc-dev \
+    libsasl2-dev \
     libsnappy-dev \
     libssl-dev \
     libtool \
@@ -65,7 +65,6 @@ RUN apt-get -q update && apt-get -q install -y \
     pkg-config \
     python \
     python2.7 \
-    pylint \
     python-pip \
     python-pkg-resources \
     python-setuptools \
@@ -74,8 +73,8 @@ RUN apt-get -q update && apt-get -q install -y \
     software-properties-common \
     snappy \
     sudo \
-    zlib1g-dev \
-    valgrind
+    valgrind \
+    zlib1g-dev
 
 #######
 # OpenJDK 8
@@ -203,3 +202,4 @@ ENV FORREST_HOME=/opt/apache-forrest
 ADD hadoop_env_checks.sh /root/hadoop_env_checks.sh
 RUN chmod 755 /root/hadoop_env_checks.sh
 RUN echo '~/hadoop_env_checks.sh' >> /root/.bashrc
+

hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml

@@ -31,12 +31,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
 
   <properties>
     <require.fuse>false</require.fuse>
-    <hadoop.component>hdfs</hadoop.component>
     <require.libwebhdfs>false</require.libwebhdfs>
     <require.valgrind>false</require.valgrind>
     <native_ctest_args></native_ctest_args>
     <native_cmake_args></native_cmake_args>
     <native_make_args></native_make_args>
+    <hadoop.component>hdfs</hadoop.component>
   </properties>
 
   <dependencies>
@@ -168,17 +168,21 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
                 <goals><goal>run</goal></goals>
                 <configuration>
                   <skip>${skipTests}</skip>
-                    <target>
-                      <property name="compile_classpath" refid="maven.compile.classpath"/>
-                      <property name="test_classpath" refid="maven.test.classpath"/>
-                      <exec executable="ctest" failonerror="true" dir="${project.build.directory}/">
-                        <arg line="--output-on-failure"/>
-                        <arg line="${native_ctest_args}"/>
-                        <env key="CLASSPATH" value="${test_classpath}:${compile_classpath}"/>
-                        <!-- Make sure libhadoop.so is on LD_LIBRARY_PATH. -->
-                        <env key="LD_LIBRARY_PATH" value="${env.LD_LIBRARY_PATH}:${project.build.directory}/target/usr/local/lib:${hadoop.common.build.dir}/native/target/usr/local/lib"/>
-                      </exec>
-                    </target>
+                  <target>
+                    <property name="compile_classpath" refid="maven.compile.classpath"/>
+                    <property name="test_classpath" refid="maven.test.classpath"/>
+                    <exec executable="ctest" failonerror="true" dir="${project.build.directory}/native">
+                      <arg line="--output-on-failure"/>
+                      <arg line="${native_ctest_args}"/>
+                      <env key="CLASSPATH" value="${test_classpath}:${compile_classpath}"/>
+                      <!-- HADOOP_HOME required to find winutils. -->
+                      <env key="HADOOP_HOME" value="${hadoop.common.build.dir}"/>
+                      <!-- Make sure hadoop.dll and jvm.dll are on PATH. -->
+                      <env key="PATH" value="${env.PATH};${hadoop.common.build.dir}/bin;${java.home}/jre/bin/server;${java.home}/bin/server"/>
+                      <!-- Make sure libhadoop.so is on LD_LIBRARY_PATH. -->
+                      <env key="LD_LIBRARY_PATH" value="${env.LD_LIBRARY_PATH}:${project.build.directory}/native/target/usr/local/lib:${hadoop.common.build.dir}/native/target/usr/local/lib"/>
+                    </exec>
+                  </target>
                 </configuration>
               </execution>
             </executions>
@@ -197,13 +201,13 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
       <build>
         <plugins>
           <plugin>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-maven-plugins</artifactId>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-antrun-plugin</artifactId>
             <executions>
               <execution>
-                <id>cmake-compile</id>
+                <id>make</id>
                 <phase>compile</phase>
-                <goals><goal>cmake-compile</goal></goals>
+                <goals><goal>run</goal></goals>
                 <configuration>
                   <target>
                     <mkdir dir="${project.build.directory}"/>
@@ -217,12 +221,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
                   </target>
                 </configuration>
               </execution>
-            </executions>
-          </plugin>
-          <plugin>
-            <groupId>org.apache.maven.plugins</groupId>
-            <artifactId>maven-antrun-plugin</artifactId>
-            <executions>
               <execution>
                 <id>native_tests</id>
                 <phase>test</phase>
@@ -237,7 +235,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
                       <arg line="${native_ctest_args}"/>
                       <env key="CLASSPATH" value="${test_classpath}:${compile_classpath}"/>
                       <!-- Make sure libhadoop.so is on LD_LIBRARY_PATH. -->
-                      <env key="LD_LIBRARY_PATH" value="${env.LD_LIBRARY_PATH}:${project.build.directory}/target/usr/local/lib:${hadoop.common.build.dir}/native/target/usr/local/lib"/>
+                      <env key="LD_LIBRARY_PATH" value="${env.LD_LIBRARY_PATH}:${project.build.directory}/native/target/usr/local/lib:${hadoop.common.build.dir}/native/target/usr/local/lib"/>
                     </exec>
                   </target>
                 </configuration>

hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt

@@ -58,19 +58,11 @@ if(WIN32)
     # Omit unneeded headers.
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DWIN32_LEAN_AND_MEAN")
     set(OS_DIR ${CMAKE_SOURCE_DIR}/main/native/libhdfs/os/windows)
-
-    # IMPORTANT: OUT_DIR MUST be relative to maven's
-    # project.build.directory (=target) and match dist-copynativelibs
-    # in order to be in a release
-    set(OUT_DIR bin)
+    set(OUT_DIR target/bin)
 else()
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fvisibility=hidden")
     set(OS_DIR ${CMAKE_SOURCE_DIR}/main/native/libhdfs/os/posix)
-
-    # IMPORTANT: OUT_DIR MUST be relative to maven's
-    # project.build.directory (=target) and match dist-copynativelibs
-    # in order to be in a release
-    set(OUT_DIR native/target/usr/local/lib)
+    set(OUT_DIR target/usr/local/lib)
 endif()
 
 # Configure JNI.

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt

@@ -35,7 +35,10 @@ SET(BUILD_SHARED_HDFSPP TRUE CACHE STRING "BUILD_SHARED_HDFSPP defaulting to 'TR
 SET(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake" ${CMAKE_MODULE_PATH})
 
 # If there's a better way to inform FindCyrusSASL.cmake, let's make this cleaner:
-SET(CMAKE_PREFIX_PATH "${CMAKE_PREFIX_PATH};${CYRUS_SASL_DIR};${GSASL_DIR}")
+SET(CMAKE_PREFIX_PATH "${CMAKE_PREFIX_PATH};${CYRUS_SASL_DIR};${GSASL_DIR};$ENV{PROTOBUF_HOME}")
+
+# Specify PROTOBUF_HOME so that find_package picks up the correct version
+SET(CMAKE_PREFIX_PATH "${CMAKE_PREFIX_PATH};$ENV{PROTOBUF_HOME}")
 
 find_package(Doxygen)
 find_package(OpenSSL REQUIRED)

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/proto/CMakeLists.txt

@@ -29,6 +29,7 @@ protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS
   ${PROTO_HDFS_DIR}/hdfs.proto
   ${PROTO_HDFS_DIR}/inotify.proto
   ${PROTO_HDFS_DIR}/xattr.proto
+  ${PROTO_HDFS_DIR}/ReconfigurationProtocol.proto
   ${PROTO_HADOOP_DIR}/IpcConnectionContext.proto
   ${PROTO_HADOOP_DIR}/ProtobufRpcEngine.proto
   ${PROTO_HADOOP_DIR}/RpcHeader.proto

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c

@@ -470,6 +470,13 @@ void hadoopRzBufferFree(hdfsFile file, struct hadoopRzBuffer *buffer) {
   return libhdfs_hadoopRzBufferFree(file->libhdfsRep, buffer);
 }
 
+int hdfsGetHedgedReadMetrics(hdfsFS fs, struct hdfsHedgedReadMetrics **metrics) {
+  return  libhdfs_hdfsGetHedgedReadMetrics(fs->libhdfsRep, (struct libhdfs_hdfsHedgedReadMetrics **) metrics);
+}
+
+void hdfsFreeHedgedReadMetrics(struct hdfsHedgedReadMetrics *metrics) {
+  return  libhdfs_hdfsFreeHedgedReadMetrics((struct libhdfs_hdfsHedgedReadMetrics *) metrics);
+}
 
 /*************
  * hdfs_ext functions

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_defines.h

@@ -90,3 +90,8 @@
 #define kObjectKindDirectory libhdfs_kObjectKindDirectory
 #define hdfsReadStatistics libhdfs_hdfsReadStatistics
 #define hdfsFileInfo libhdfs_hdfsFileInfo
+#define hdfsHedgedReadMetrics libhdfs_hdfsHedgedReadMetrics
+#define hdfsGetHedgedReadMetrics libhdfs_hdfsGetHedgedReadMetrics
+#define hdfsFreeHedgedReadMetrics libhdfs_hdfsFreeHedgedReadMetrics
+#define hdfsStreamBuilderAlloc libhdfs_hdfsStreamBuilderAlloc
+#define hdfsStreamBuilderBuild libhdfs_hdfsStreamBuilderBuild

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_undefs.h

@@ -100,3 +100,8 @@
 #undef hdfsRenameSnapshot
 #undef hdfsAllowSnapshot
 #undef hdfsDisallowSnapshot
+#undef hdfsHedgedReadMetrics
+#undef hdfsGetHedgedReadMetrics
+#undef hdfsFreeHedgedReadMetrics
+#undef hdfsStreamBuilderAlloc
+#undef hdfsStreamBuilderBuild

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfspp_wrapper_defines.h

@@ -100,3 +100,8 @@
 #define hdfsRenameSnapshot libhdfspp_hdfsRenameSnapshot
 #define hdfsAllowSnapshot libhdfspp_hdfsAllowSnapshot
 #define hdfsDisallowSnapshot libhdfspp_hdfsDisallowSnapshot
+#define hdfsHedgedReadMetrics libhdfspp_hdfsHedgedReadMetrics
+#define hdfsGetHedgedReadMetrics libhdfspp_hdfsGetHedgedReadMetrics
+#define hdfsFreeHedgedReadMetrics libhdfspp_hdfsFreeHedgedReadMetrics
+#define hdfsStreamBuilderAlloc libhdfspp_hdfsStreamBuilderAlloc
+#define hdfsStreamBuilderBuild libhdfspp_hdfsStreamBuilderBuild