HDFS-8635. Migrate HDFS native build to new CMake framework (Alan Burlison via Colin P. McCabe)

(cherry picked from commit 29df73dc0572cf8e450568f1d0301e8d55d64b48)

hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt

@@ -352,6 +352,9 @@ Release 2.8.0 - UNRELEASED
     HDFS-8493. Consolidate truncate() related implementation in a single class.
     (Rakesh R via wheat9)
 
+    HDFS-8635. Migrate HDFS native build to new CMake framework (Alan Burlison
+    via Colin P. McCabe)
+
   OPTIMIZATIONS
 
     HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than

hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt

@@ -18,106 +18,59 @@
 
 cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
 
-# Default to release builds
-set(CMAKE_BUILD_TYPE, Release)
+list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/../../../hadoop-common-project/hadoop-common)
+include(HadoopCommon)
 
-include(../../../hadoop-common-project/hadoop-common/src/JNIFlags.cmake NO_POLICY_SCOPE)
-
-# Compile a library with both shared and static variants
-function(add_dual_library LIBNAME)
-    add_library(${LIBNAME} SHARED ${ARGN})
-    add_library(${LIBNAME}_static STATIC ${ARGN})
-    # Linux builds traditionally ship a libhdfs.a (static linking) and libhdfs.so
-    # (dynamic linking).  On Windows, we cannot use the same base name for both
-    # static and dynamic, because Windows does not use distinct file extensions
-    # for a statically linked library vs. a DLL import library.  Both use the
-    # .lib extension.  On Windows, we'll build the static library as
-    # hdfs_static.lib.
-    if (NOT WIN32)
-        set_target_properties(${LIBNAME}_static PROPERTIES OUTPUT_NAME ${LIBNAME})
-    endif (NOT WIN32)
-endfunction(add_dual_library)
-
-# Link both a static and a dynamic target against some libraries
-function(target_link_dual_libraries LIBNAME)
-    target_link_libraries(${LIBNAME} ${ARGN})
-    target_link_libraries(${LIBNAME}_static ${ARGN})
-endfunction(target_link_dual_libraries)
-
-function(output_directory TGT DIR)
-    SET_TARGET_PROPERTIES(${TGT} PROPERTIES
-        RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
-    SET_TARGET_PROPERTIES(${TGT} PROPERTIES
-        ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
-    SET_TARGET_PROPERTIES(${TGT} PROPERTIES
-        LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
-endfunction(output_directory TGT DIR)
-
-function(dual_output_directory TGT DIR)
-    output_directory(${TGT} "${DIR}")
-    output_directory(${TGT}_static "${DIR}")
-endfunction(dual_output_directory TGT DIR)
+#
+# Main configuration
+#
 
-# Flatten a list into a string.
-function(FLATTEN_LIST INPUT SEPARATOR OUTPUT)
-  string (REPLACE ";" "${SEPARATOR}" _TMPS "${INPUT}")
-  set (${OUTPUT} "${_TMPS}" PARENT_SCOPE)
-endfunction()
+# The caller must specify where the generated headers have been placed.
+if(NOT GENERATED_JAVAH)
+    message(FATAL_ERROR "You must set the CMake variable GENERATED_JAVAH")
+endif()
 
 # Check to see if our compiler and linker support the __thread attribute.
-# On Linux and some other operating systems, this is a more efficient 
-# alternative to POSIX thread local storage. 
-INCLUDE(CheckCSourceCompiles)
-CHECK_C_SOURCE_COMPILES("int main(void) { static __thread int i = 0; return 0; }" HAVE_BETTER_TLS)
+# On Linux and some other operating systems, this is a more efficient
+# alternative to POSIX thread local storage.
+include(CheckCSourceCompiles)
+check_c_source_compiles("int main(void) { static __thread int i = 0; return 0; }" HAVE_BETTER_TLS)
 
 # Check to see if we have Intel SSE intrinsics.
-CHECK_C_SOURCE_COMPILES("#include <emmintrin.h>\nint main(void) { __m128d sum0 = _mm_set_pd(0.0,0.0); return 0; }" HAVE_INTEL_SSE_INTRINSICS)
+check_c_source_compileS("#include <emmintrin.h>\nint main(void) { __m128d sum0 = _mm_set_pd(0.0,0.0); return 0; }" HAVE_INTEL_SSE_INTRINSICS)
 
 # Check if we need to link dl library to get dlopen.
 # dlopen on Linux is in separate library but on FreeBSD its in libc
-INCLUDE(CheckLibraryExists)
-CHECK_LIBRARY_EXISTS(dl dlopen "" NEED_LINK_DL)
+include(CheckLibraryExists)
+check_library_exists(dl dlopen "" NEED_LINK_DL)
 
-find_package(JNI REQUIRED)
-if (NOT GENERATED_JAVAH)
-    # Must identify where the generated headers have been placed
-    MESSAGE(FATAL_ERROR "You must set the CMake variable GENERATED_JAVAH")
-endif (NOT GENERATED_JAVAH)
-
-if (WIN32)
+if(WIN32)
+    # Set the optimizer level.
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /O2")
-
     # Set warning level 4.
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /W4")
-
     # Skip "unreferenced formal parameter".
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4100")
-
     # Skip "conditional expression is constant".
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4127")
-
     # Skip deprecated POSIX function warnings.
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_CRT_NONSTDC_NO_DEPRECATE")
-
     # Skip CRT non-secure function warnings.  If we can convert usage of
     # strerror, getenv and ctime to their secure CRT equivalents, then we can
     # re-enable the CRT non-secure function warnings.
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_CRT_SECURE_NO_WARNINGS")
-
     # Omit unneeded headers.
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DWIN32_LEAN_AND_MEAN")
-
     set(OS_DIR main/native/libhdfs/os/windows)
     set(OUT_DIR target/bin)
-else (WIN32)
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -Wall -O2")
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_REENTRANT -D_GNU_SOURCE")
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64")
+else()
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fvisibility=hidden")
     set(OS_DIR main/native/libhdfs/os/posix)
-    set(OS_LINK_LIBRARIES pthread)
     set(OUT_DIR target/usr/local/lib)
-endif (WIN32)
+endif()
+
+# Configure JNI.
+include(HadoopJNI)
 
 add_definitions(-DLIBHDFS_DLL_EXPORT)
 
@@ -132,9 +85,9 @@ include_directories(
 )
 
 set(_FUSE_DFS_VERSION 0.1.0)
-CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
+configure_file(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
 
-add_dual_library(hdfs
+hadoop_add_dual_library(hdfs
     main/native/libhdfs/exception.c
     main/native/libhdfs/jni_helper.c
     main/native/libhdfs/hdfs.c
@@ -142,17 +95,17 @@ add_dual_library(hdfs
     ${OS_DIR}/mutexes.c
     ${OS_DIR}/thread_local_storage.c
 )
-if (NEED_LINK_DL)
+if(NEED_LINK_DL)
    set(LIB_DL dl)
-endif(NEED_LINK_DL)
+endif()
 
-target_link_dual_libraries(hdfs
+hadoop_target_link_dual_libraries(hdfs
     ${JAVA_JVM_LIBRARY}
     ${LIB_DL}
     ${OS_LINK_LIBRARIES}
 )
 
-dual_output_directory(hdfs ${OUT_DIR})
+hadoop_dual_output_directory(hdfs ${OUT_DIR})
 set(LIBHDFS_VERSION "0.0.0")
 set_target_properties(hdfs PROPERTIES
     SOVERSION ${LIBHDFS_VERSION})
@@ -232,25 +185,21 @@ target_link_libraries(test_htable
 
 # Skip vecsum on Windows.  This could be made to work in the future by
 # introducing an abstraction layer over the sys/mman.h functions.
-if (NOT WIN32)
-    add_executable(test_libhdfs_vecsum
-        main/native/libhdfs/test/vecsum.c
-    )
-    if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+if(NOT WIN32)
+    add_executable(test_libhdfs_vecsum main/native/libhdfs/test/vecsum.c)
+    if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
         target_link_libraries(test_libhdfs_vecsum
             hdfs
-            pthread
-        )
-    else (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+            pthread)
+    else()
         target_link_libraries(test_libhdfs_vecsum
             hdfs
             pthread
-            rt
-        )
-    endif (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
-endif(NOT WIN32)
+            rt)
+    endif()
+endif()
 
-IF(REQUIRE_LIBWEBHDFS)
+if(REQUIRE_LIBWEBHDFS)
     add_subdirectory(contrib/libwebhdfs)
-ENDIF(REQUIRE_LIBWEBHDFS)
+endif()
 add_subdirectory(main/native/fuse-dfs)

hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/CMakeLists.txt

@@ -19,12 +19,12 @@
 find_package(CURL REQUIRED)
 
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH}
-"${CMAKE_SOURCE_DIR}/contrib/libwebhdfs/resources/")
+    "${CMAKE_SOURCE_DIR}/contrib/libwebhdfs/resources/")
 
 find_package(Jansson REQUIRED)
 include_directories(${JANSSON_INCLUDE_DIR})
 
-add_dual_library(webhdfs
+hadoop_add_dual_library(webhdfs
     src/hdfs_web.c
     src/hdfs_http_client.c
     src/hdfs_http_query.c
@@ -35,13 +35,13 @@ add_dual_library(webhdfs
     ../../${OS_DIR}/thread_local_storage.c
     ../../main/native/libhdfs/common/htable.c
 )
-target_link_dual_libraries(webhdfs
+hadoop_target_link_dual_libraries(webhdfs
     ${JAVA_JVM_LIBRARY}
     ${CURL_LIBRARY}
     ${JANSSON_LIBRARY}
     pthread
 )
-dual_output_directory(webhdfs target)
+hadoop_dual_output_directory(webhdfs target)
 set(LIBWEBHDFS_VERSION "0.0.0")
 set_target_properties(webhdfs PROPERTIES
     SOVERSION ${LIBWEBHDFS_VERSION})

hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/resources/FindJansson.cmake

@@ -5,9 +5,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -24,7 +24,8 @@
 #  JANSSON_DEFINITIONS - Compiler switches required for using Jansson
 
 find_path(JANSSON_INCLUDE_DIR jansson.h
-          /usr/incluce
+          /usr/include
+          /usr/include/jansson
           /usr/local/include )
 
 find_library(JANSSON_LIBRARY NAMES jansson
@@ -39,4 +40,4 @@ include(FindPackageHandleStandardArgs)
 find_package_handle_standard_args(Jansson  DEFAULT_MSG
                                   JANSSON_LIBRARY JANSSON_INCLUDE_DIR)
 
-mark_as_advanced(JANSSON_INCLUDE_DIR JANSSON_LIBRARY )
+mark_as_advanced(JANSSON_INCLUDE_DIR JANSSON_LIBRARY )

hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/CMakeLists.txt

@@ -18,57 +18,63 @@
 
 set(CMAKE_SKIP_RPATH TRUE)
 
+# Flatten a list into a string.
+function(flatten_list INPUT SEPARATOR OUTPUT)
+  string (REPLACE ";" "${SEPARATOR}" _TMPS "${INPUT}")
+  set (${OUTPUT} "${_TMPS}" PARENT_SCOPE)
+endfunction()
+
 # Find Linux FUSE
-IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
+if(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
     find_package(PkgConfig REQUIRED)
     pkg_check_modules(FUSE fuse)
-    IF(FUSE_FOUND)
-        FLATTEN_LIST("${FUSE_CFLAGS}" " " FUSE_CFLAGS)
-        FLATTEN_LIST("${FUSE_LDFLAGS}" " " FUSE_LDFLAGS)
+    if(FUSE_FOUND)
+        flatten_list("${FUSE_CFLAGS}" " " FUSE_CFLAGS)
+        flatten_list("${FUSE_LDFLAGS}" " " FUSE_LDFLAGS)
         set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${FUSE_CFLAGS}")
         set(CMAKE_LD_FLAGS "${CMAKE_LD_FLAGS} ${FUSE_LDFLAGS}")
-        MESSAGE(STATUS "Building Linux FUSE client.")
+        message(STATUS "Building Linux FUSE client.")
         include_directories(${FUSE_INCLUDE_DIRS})
-    ELSE(FUSE_FOUND)
-        MESSAGE(STATUS "Failed to find Linux FUSE libraries or include files.  Will not build FUSE client.")
-    ENDIF(FUSE_FOUND)
-ELSE (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
-    MESSAGE(STATUS "Non-Linux system detected.  Will not build FUSE client.")
-ENDIF (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
+    else()
+        message(STATUS "Failed to find Linux FUSE libraries or include files.  Will not build FUSE client.")
+    endif(FUSE_FOUND)
+else()
+    message(STATUS "Non-Linux system detected.  Will not build FUSE client.")
+endif()
 
-IF(FUSE_FOUND)
+if(FUSE_FOUND)
     add_library(posix_util
         ../util/posix_util.c
     )
 
     add_executable(fuse_dfs
         fuse_dfs.c
-        fuse_options.c 
-        fuse_connect.c 
-        fuse_impls_access.c 
-        fuse_impls_chmod.c  
-        fuse_impls_chown.c  
-        fuse_impls_create.c  
-        fuse_impls_flush.c 
-        fuse_impls_getattr.c  
-        fuse_impls_mkdir.c  
-        fuse_impls_mknod.c  
-        fuse_impls_open.c 
-        fuse_impls_read.c 
-        fuse_impls_readdir.c 
-        fuse_impls_release.c 
-        fuse_impls_rename.c 
-        fuse_impls_rmdir.c 
-        fuse_impls_statfs.c 
-        fuse_impls_symlink.c 
-        fuse_impls_truncate.c 
-        fuse_impls_unlink.c 
-        fuse_impls_utimens.c  
+        fuse_options.c
+        fuse_connect.c
+        fuse_impls_access.c
+        fuse_impls_chmod.c
+        fuse_impls_chown.c
+        fuse_impls_create.c
+        fuse_impls_flush.c
+        fuse_impls_getattr.c
+        fuse_impls_mkdir.c
+        fuse_impls_mknod.c
+        fuse_impls_open.c
+        fuse_impls_read.c
+        fuse_impls_readdir.c
+        fuse_impls_release.c
+        fuse_impls_rename.c
+        fuse_impls_rmdir.c
+        fuse_impls_statfs.c
+        fuse_impls_symlink.c
+        fuse_impls_truncate.c
+        fuse_impls_unlink.c
+        fuse_impls_utimens.c
         fuse_impls_write.c
-        fuse_init.c 
-        fuse_stat_struct.c 
-        fuse_trash.c 
-        fuse_users.c 
+        fuse_init.c
+        fuse_stat_struct.c
+        fuse_trash.c
+        fuse_users.c
     )
     target_link_libraries(fuse_dfs
         ${FUSE_LIBRARIES}
@@ -88,8 +94,9 @@ IF(FUSE_FOUND)
         posix_util
         pthread
     )
-ELSE(FUSE_FOUND)
-    IF(REQUIRE_FUSE)
-        MESSAGE(FATAL_ERROR "Required component fuse_dfs could not be built.")
-    ENDIF(REQUIRE_FUSE)
-ENDIF(FUSE_FOUND)
+    set(REQUIRE_FUSE ${REQUIRE_FUSE}) # Prevent unused variable warning
+else()
+    if(REQUIRE_FUSE)
+        message(FATAL_ERROR "Required component fuse_dfs could not be built.")
+    endif()
+endif()