MAPREDUCE-5985. native-task: Fix build on macosx. Contributed by Binglin Chang

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/MR-2841@1612659 13f79535-47bb-0310-9956-ffa450edef68

hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt

@@ -0,0 +1,4 @@
+Changes for Hadoop Native Map Output Collector
+------------------------------------------------
+
+MAPREDUCE-5985. native-task: Fix build on macosx. Contributed by Binglin Chang

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml

@@ -1,4 +1,4 @@
-<?xml version="1.0"?>
+<?xml version="1.0" encoding="UTF-8"?>
 <!--
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -12,7 +12,8 @@
   See the License for the specific language governing permissions and
   limitations under the License. See accompanying LICENSE file.
 -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
   xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
                       http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <parent>
@@ -29,7 +30,7 @@
   <properties>
     <!-- Needed for generating FindBugs warnings using parent pom -->
     <mr.basedir>${project.parent.basedir}/../</mr.basedir>
-  </properties>  
+  </properties>
 
 
   <dependencies>
@@ -53,7 +54,7 @@
     </dependency>
     <dependency>
       <groupId>org.mockito</groupId>
-        <artifactId>mockito-all</artifactId>
+      <artifactId>mockito-all</artifactId>
       <scope>test</scope>
     </dependency>
     <dependency>
@@ -85,7 +86,7 @@
             <include>**/compresstest/*Test.java</include>
             <include>**/nonsorttest/*Test.java</include>
             <include>**/kvtest/*Test.java</include>
-          </includes> 
+          </includes>
           <additionalClasspathElements>
             <additionalClasspathElement>${basedir}/../hadoop-mapreduce-client-common/target/classes</additionalClasspathElement>
           </additionalClasspathElements>
@@ -131,26 +132,26 @@
               </execution>
             </executions>
           </plugin>
-            <plugin>
-                <groupId>org.codehaus.mojo</groupId>
-                <artifactId>native-maven-plugin</artifactId>
-                <executions>
-                    <execution>
-                      <phase>compile</phase>
-                        <goals>
-                          <goal>javah</goal>
-                        </goals>
-                        <configuration>        
-                            <javahPath>${env.JAVA_HOME}/bin/javah</javahPath>
-                            <javahClassNames>
-                                <javaClassName>org.apache.hadoop.mapred.nativetask.NativeBatchProcessor</javaClassName>
-                                <javaClassName>org.apache.hadoop.mapred.nativetask.NativeRuntime</javaClassName>
-                            </javahClassNames>
-                            <javahOutputDirectory>${project.build.directory}/native/javah</javahOutputDirectory>
-                        </configuration>
-                  </execution>
-              </executions>
-            </plugin>
+          <plugin>
+            <groupId>org.codehaus.mojo</groupId>
+            <artifactId>native-maven-plugin</artifactId>
+            <executions>
+              <execution>
+                <phase>compile</phase>
+                <goals>
+                  <goal>javah</goal>
+                </goals>
+                <configuration>
+                  <javahPath>${env.JAVA_HOME}/bin/javah</javahPath>
+                  <javahClassNames>
+                    <javaClassName>org.apache.hadoop.mapred.nativetask.NativeBatchProcessor</javaClassName>
+                    <javaClassName>org.apache.hadoop.mapred.nativetask.NativeRuntime</javaClassName>
+                  </javahClassNames>
+                  <javahOutputDirectory>${project.build.directory}/native/javah</javahOutputDirectory>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
           <plugin>
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-antrun-plugin</artifactId>
@@ -158,23 +159,27 @@
               <execution>
                 <id>make</id>
                 <phase>compile</phase>
-                <goals><goal>run</goal></goals>
+                <goals>
+                  <goal>run</goal>
+                </goals>
                 <configuration>
                   <target>
-					<copy todir="${project.build.directory}/native/test/testData" overwrite="true">
-						<fileset dir="${basedir}/src/main/native/testData" />
-					</copy>
-					<copy file="${basedir}/src/main/native/test.sh" todir="${project.build.directory}/native/test" />
-                    <exec executable="cmake" dir="${project.build.directory}/native" failonerror="true">
-						<arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DREQUIRE_SNAPPY=${require.snappy} -DCUSTOM_SNAPPY_PREFIX=${snappy.prefix} -DCUSTOM_SNAPPY_LIB=${snappy.lib} -DCUSTOM_SNAPPY_INCLUDE=${snappy.include}"/>
+                    <copy todir="${project.build.directory}/native/test/testData"
+                      overwrite="true">
+                      <fileset dir="${basedir}/src/main/native/testData" />
+                    </copy>
+                    <copy file="${basedir}/src/main/native/test.sh" todir="${project.build.directory}/native/test" />
+                    <exec executable="cmake" dir="${project.build.directory}/native"
+                      failonerror="true">
+                      <arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DREQUIRE_SNAPPY=${require.snappy} -DCUSTOM_SNAPPY_PREFIX=${snappy.prefix} -DCUSTOM_SNAPPY_LIB=${snappy.lib} -DCUSTOM_SNAPPY_INCLUDE=${snappy.include}" />
                     </exec>
                     <exec executable="make" dir="${project.build.directory}/native" failonerror="true">
-                      <arg line="VERBOSE=1"/>
+                      <arg line="VERBOSE=1" />
                     </exec>
-                    <!-- The second make is a workaround for HADOOP-9215.  It can
-                         be removed when version 2.6 of cmake is no longer supported . -->
+                    <!-- The second make is a workaround for HADOOP-9215. It can be 
+                      removed when version 2.6 of cmake is no longer supported . -->
                     <exec executable="make" dir="${project.build.directory}/native" failonerror="true"></exec>
-                   </target>
+                  </target>
                 </configuration>
               </execution>
             </executions>

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt

@@ -52,7 +52,7 @@ endfunction(dual_output_directory TGT DIR)
 
 #
 # This macro alters the behavior of find_package and find_library.
-# It does this by setting the CMAKE_FIND_LIBRARY_SUFFIXES global variable. 
+# It does this by setting the CMAKE_FIND_LIBRARY_SUFFIXES global variable.
 # You should save that variable before calling this function and restore it
 # after you have accomplished your goal.
 #
@@ -79,8 +79,8 @@ macro(set_find_shared_library_version LVERS)
 endmacro(set_find_shared_library_version LVERS)
 
 if (NOT GENERATED_JAVAH)
-	#Must identify where the generated headers have been placed
-	  MESSAGE(FATAL_ERROR "You must set the cmake variable GENERATED_JAVAH")
+    #Must identify where the generated headers have been placed
+    MESSAGE(FATAL_ERROR "You must set the cmake variable GENERATED_JAVAH")
 endif (NOT GENERATED_JAVAH)
 find_package(JNI REQUIRED)
 
@@ -91,8 +91,8 @@ SET(CMAKE_FIND_LIBRARY_SUFFIXES STORED_CMAKE_FIND_LIBRARY_SUFFIXES)
 # primitive configs
 set(PRFLAGS "-DSIMPLE_MEMCPY")
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${PRFLAGS} -Wall")
-set(CMAKE_LD_FLAGS "${CMAKE_LD_FLAGS} -no-undefined -version-info 0:1:0 
-		-L${_JAVA_HOME}/jre/lib/amd64/server -ljvm")
+set(CMAKE_LD_FLAGS "${CMAKE_LD_FLAGS} -no-undefined -version-info 0:1:0
+    -L${_JAVA_HOME}/jre/lib/amd64/server -ljvm")
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_C_FLAGS} -g -O2 -DNDEBUG -fPIC")
 set(D main/native/)
 
@@ -125,12 +125,12 @@ CHECK_FUNCTION_EXISTS(strtoul HAVE_STRTOUL)
 
 SET(STORED_CMAKE_FIND_LIBRARY_SUFFIXES CMAKE_FIND_LIBRARY_SUFFIXES)
 set_find_shared_library_version("1")
-find_library(SNAPPY_LIBRARY 
+find_library(SNAPPY_LIBRARY
     NAMES snappy
     PATHS ${CUSTOM_SNAPPY_PREFIX} ${CUSTOM_SNAPPY_PREFIX}/lib
           ${CUSTOM_SNAPPY_PREFIX}/lib64 ${CUSTOM_SNAPPY_LIB})
 SET(CMAKE_FIND_LIBRARY_SUFFIXES STORED_CMAKE_FIND_LIBRARY_SUFFIXES)
-find_path(SNAPPY_INCLUDE_DIR 
+find_path(SNAPPY_INCLUDE_DIR
     NAMES snappy.h
     PATHS ${CUSTOM_SNAPPY_PREFIX} ${CUSTOM_SNAPPY_PREFIX}/include
           ${CUSTOM_SNAPPY_INCLUDE})
@@ -147,15 +147,15 @@ else (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
 endif (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
 
 include_directories(
-	  ${GENERATED_JAVAH}
-		${D}
+    ${GENERATED_JAVAH}
+    ${D}
     ${D}/src
-		${D}/src/util
-		${D}/src/lib
-		${D}/test
+    ${D}/src/util
+    ${D}/src/lib
+    ${D}/test
     ${CMAKE_CURRENT_SOURCE_DIR}
-		#${CMAKE_CURRENT_SOURCE_DIR}/src
-		#${CMAKE_BINARY_DIR}
+    #${CMAKE_CURRENT_SOURCE_DIR}/src
+    #${CMAKE_BINARY_DIR}
     ${JNI_INCLUDE_DIRS}
     ${SNAPPY_INCLUDE_DIR}
 )
@@ -166,6 +166,13 @@ CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
 
 SET(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
 
+if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+    # macosx does not have -lrt
+    set(NT_DEPEND_LIBRARY dl pthread z ${SNAPPY_LIBRARY} ${JAVA_JVM_LIBRARY})
+else (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+    set(NT_DEPEND_LIBRARY dl rt pthread z ${SNAPPY_LIBRARY} ${JAVA_JVM_LIBRARY})
+endif (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+
 add_dual_library(nativetask
     ${D}/lz4/lz4.c
     ${D}/cityhash/city.cc
@@ -208,60 +215,51 @@ add_dual_library(nativetask
     ${D}/src/util/Timer.cc
     ${D}/src/util/WritableUtils.cc
 )
-target_link_libraries(nativetask
-	#${LIB_DL}
-	  dl
-		rt
-		pthread
-		z
-		${SNAPPY_LIBRARY}
-		${JAVA_JVM_LIBRARY}
-)
 
-add_executable(nttest 
+target_link_libraries(nativetask ${NT_DEPEND_LIBRARY})
+
+add_executable(nttest
     ${D}/gtest/gtest-all.cc
-    ${D}/test/lib/TestByteArray.cc      
-    ${D}/test/lib/TestByteBuffer.cc      
-    ${D}/test/lib/TestComparatorForDualPivotQuickSort.cc 
-    ${D}/test/lib/TestComparatorForStdSort.cc 
-    ${D}/test/lib/TestFixSizeContainer.cc 
-    ${D}/test/lib/TestMemoryPool.cc 
-    ${D}/test/lib/TestIterator.cc 
-    ${D}/test/lib/TestKVBuffer.cc 
-    ${D}/test/lib/TestMemBlockIterator.cc 
-    ${D}/test/lib/TestMemoryBlock.cc 
-    ${D}/test/lib/TestPartitionBucket.cc 
-    ${D}/test/lib/TestReadBuffer.cc 
-    ${D}/test/lib/TestReadWriteBuffer.cc 
-    ${D}/test/lib/TestTrackingCollector.cc 
-    ${D}/test/util/TestChecksum.cc             
-    ${D}/test/util/TestHash.cc                 
-    ${D}/test/util/TestStringUtil.cc           
-    ${D}/test/util/TestSyncUtils.cc            
-    ${D}/test/util/TestWritableUtils.cc        
-    ${D}/test/TestCommand.cc                   
-    ${D}/test/TestConfig.cc                   
-    ${D}/test/TestCounter.cc                   
-    ${D}/test/TestCompressions.cc              
-    ${D}/test/TestFileSystem.cc                
-    ${D}/test/TestIFile.cc                     
-    ${D}/test/TestPrimitives.cc                
-    ${D}/test/TestSort.cc                      
-    ${D}/test/TestMain.cc                      
+    ${D}/test/lib/TestByteArray.cc
+    ${D}/test/lib/TestByteBuffer.cc
+    ${D}/test/lib/TestComparatorForDualPivotQuickSort.cc
+    ${D}/test/lib/TestComparatorForStdSort.cc
+    ${D}/test/lib/TestFixSizeContainer.cc
+    ${D}/test/lib/TestMemoryPool.cc
+    ${D}/test/lib/TestIterator.cc
+    ${D}/test/lib/TestKVBuffer.cc
+    ${D}/test/lib/TestMemBlockIterator.cc
+    ${D}/test/lib/TestMemoryBlock.cc
+    ${D}/test/lib/TestPartitionBucket.cc
+    ${D}/test/lib/TestReadBuffer.cc
+    ${D}/test/lib/TestReadWriteBuffer.cc
+    ${D}/test/lib/TestTrackingCollector.cc
+    ${D}/test/util/TestChecksum.cc
+    ${D}/test/util/TestHash.cc
+    ${D}/test/util/TestStringUtil.cc
+    ${D}/test/util/TestSyncUtils.cc
+    ${D}/test/util/TestWritableUtils.cc
+    ${D}/test/TestCommand.cc
+    ${D}/test/TestConfig.cc
+    ${D}/test/TestCounter.cc
+    ${D}/test/TestCompressions.cc
+    ${D}/test/TestFileSystem.cc
+    ${D}/test/TestIFile.cc
+    ${D}/test/TestPrimitives.cc
+    ${D}/test/TestSort.cc
+    ${D}/test/TestMain.cc
     ${D}/test/test_commons.cc)
-set(CMAKE_EXE_LINKER_FLAGS "-L${_JAVA_HOME}/jre/lib/amd64/server -ljvm")
-target_link_libraries(nttest 
-   	nativetask_static
-	  dl
-	  rt
-	  pthread
-	  z
-	  ${SNAPPY_LIBRARY}
-		#	  ${JAVA_JVM_LIBRARY}
+
+
+IF (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+    # macos clang with libc++ does not have tr1/tuple, just tuple
+    SET_TARGET_PROPERTIES(nttest PROPERTIES COMPILE_FLAGS "-DGTEST_USE_OWN_TR1_TUPLE=1")
+ENDIF()
+
+target_link_libraries(nttest
+     nativetask_static
+     ${NT_DEPEND_LIBRARY}
 )
-#if (NEED_LINK_DL)
-#   set(LIB_DL dl)
-#endif (NEED_LINK_DL)
 
 IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
     #

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/commons.h

@@ -25,6 +25,7 @@
 #include <stdlib.h>
 #include <assert.h>
 #include <string.h>
+#include <unistd.h>
 #include <stdio.h>
 #include <memory.h>
 #include <fcntl.h>

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc

@@ -166,7 +166,7 @@ TEST(Perf, IFile) {
 TEST(IFile, TestGlibCBug) {
   std::string path("./testData/testGlibCBugSpill.out");
 
-  uint32_t expect[5] = {-1538241715, -1288088794, -192294464, 563552421, 1661521654};
+  int32_t expect[5] = {-1538241715, -1288088794, -192294464, 563552421, 1661521654};
 
   LOG("TestGlibCBug %s", path.c_str());
   IFileSegment * segments = new IFileSegment [1];
@@ -182,7 +182,8 @@ TEST(IFile, TestGlibCBug) {
   reader->nextPartition();
   uint32_t index = 0;
   while(NULL != (key = reader->nextKey(length))) {
-    int realKey = bswap(*(uint32_t *)(key));
+    int32_t realKey = (int32_t)bswap(*(uint32_t *)(key));
+    ASSERT_LT(index, 5);
     ASSERT_EQ(expect[index], realKey);
     index++;
   }