MAPREDUCE-4267. mavenize pipes (tgraves via bobby)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1351818 13f79535-47bb-0310-9956-ffa450edef68

hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml

@@ -0,0 +1,67 @@
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+  <id>hadoop-tools</id>
+  <formats>
+    <format>dir</format>
+  </formats>
+  <includeBaseDirectory>false</includeBaseDirectory>
+  <fileSets>
+    <fileSet>
+      <directory>../hadoop-pipes/src/main/native/pipes/api/hadoop</directory>
+      <includes>
+        <include>*.hh</include>
+      </includes>
+      <outputDirectory>/include</outputDirectory>
+    </fileSet>
+    <fileSet>
+      <directory>../hadoop-pipes/src/main/native/utils/api/hadoop</directory>
+      <includes>
+        <include>*.hh</include>
+      </includes>
+      <outputDirectory>/include</outputDirectory>
+    </fileSet>
+    <fileSet>
+      <directory>../hadoop-pipes/target/native</directory>
+      <includes>
+        <include>*.a</include>
+      </includes>
+      <outputDirectory>lib/native</outputDirectory>
+    </fileSet>
+  </fileSets>
+  <dependencySets>
+    <dependencySet>
+      <outputDirectory>/share/hadoop/${hadoop.component}/lib</outputDirectory>
+      <unpack>false</unpack>
+      <scope>runtime</scope>
+      <useProjectArtifact>false</useProjectArtifact>
+      <!-- Exclude hadoop artifacts. They will be found via HADOOP* env -->
+      <excludes>
+        <exclude>org.apache.hadoop:hadoop-common</exclude>
+        <exclude>org.apache.hadoop:hadoop-hdfs</exclude>
+        <exclude>org.apache.hadoop:hadoop-mapreduce</exclude>
+        <!-- pipes is native stuff, this just keeps pom from being package-->
+        <exclude>org.apache.hadoop:hadoop-pipes</exclude>
+        <!-- use slf4j from common to avoid multiple binding warnings -->
+        <exclude>org.slf4j:slf4j-api</exclude>
+        <exclude>org.slf4j:slf4j-log4j12</exclude>
+      </excludes>
+    </dependencySet>
+  </dependencySets>
+</assembly>

hadoop-mapreduce-project/CHANGES.txt

@@ -394,6 +394,8 @@ Release 0.23.3 - UNRELEASED
     MAPREDUCE-4301. Dedupe some strings in MRAM for memory savings 
     (bobby via tgraves)
 
+    MAPREDUCE-4267. mavenize pipes (tgraves via bobby)
+
   OPTIMIZATIONS
 
     MAPREDUCE-3850. Avoid redundant calls for tokens in TokenCache (Daryn

hadoop-mapreduce-project/src/c++/pipes/.autom4te.cfg

@@ -1,42 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-#
-# autom4te configuration for hadoop utils library
-#
-
-begin-language: "Autoheader-preselections"
-args: --no-cache 
-end-language: "Autoheader-preselections"
-
-begin-language: "Automake-preselections"
-args: --no-cache 
-end-language: "Automake-preselections"
-
-begin-language: "Autoreconf-preselections"
-args: --no-cache 
-end-language: "Autoreconf-preselections"
-
-begin-language: "Autoconf-without-aclocal-m4"
-args: --no-cache 
-end-language: "Autoconf-without-aclocal-m4"
-
-begin-language: "Autoconf"
-args: --no-cache 
-end-language: "Autoconf"
-

hadoop-mapreduce-project/src/c++/pipes/Makefile.am

@@ -1,31 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-ACLOCAL_AMFLAGS = -I ../utils/m4
-AM_CXXFLAGS=-I$(srcdir)/api -Wall -I$(HADOOP_UTILS_PREFIX)/include
-
-# List the api header files and where they will be installed
-apidir = $(includedir)/hadoop
-api_HEADERS = \
-	api/hadoop/Pipes.hh \
-	api/hadoop/TemplateFactory.hh
-
-# Define the libaries that need to be built
-lib_LIBRARIES = libhadooppipes.a
-
-# Define the sources for lib 
-libhadooppipes_a_SOURCES = \
-	impl/HadoopPipes.cc
-

hadoop-mapreduce-project/src/c++/pipes/configure.ac

@@ -1,57 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-#                                               -*- Autoconf -*-
-# Process this file with autoconf to produce a configure script.
-
-AC_PREREQ(2.59)
-AC_INIT(hadoop-pipes, 0.13.0, omalley@apache.org)
-
-AC_CONFIG_AUX_DIR([config])
-AC_CONFIG_MACRO_DIR([../utils/m4])
-
-AM_INIT_AUTOMAKE([subdir-objects foreign no-dist])
-
-AC_CONFIG_SRCDIR([impl/HadoopPipes.cc])
-AC_CONFIG_HEADER([impl/config.h])
-AC_CONFIG_FILES([Makefile])
-
-AC_PREFIX_DEFAULT(`pwd`/../install)
-
-USE_HADOOP_UTILS
-HADOOP_PIPES_SETUP
-CHECK_INSTALL_CFLAG
-
-# Checks for programs.
-AC_PROG_CXX
-AC_PROG_LIBTOOL
-
-# Checks for libraries.
-
-# Checks for header files.
-AC_LANG(C++)
-AC_CHECK_HEADERS([unistd.h])
-
-# Checks for typedefs, structures, and compiler characteristics.
-AC_HEADER_STDBOOL
-AC_C_CONST
-AC_TYPE_OFF_T
-AC_TYPE_SIZE_T
-AC_FUNC_STRERROR_R
-
-# Checks for library functions.
-AC_CHECK_FUNCS([mkdir uname])
-AC_OUTPUT

hadoop-mapreduce-project/src/c++/utils/.autom4te.cfg

@@ -1,42 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-#
-# autom4te configuration for hadoop utils library
-#
-
-begin-language: "Autoheader-preselections"
-args: --no-cache 
-end-language: "Autoheader-preselections"
-
-begin-language: "Automake-preselections"
-args: --no-cache 
-end-language: "Automake-preselections"
-
-begin-language: "Autoreconf-preselections"
-args: --no-cache 
-end-language: "Autoreconf-preselections"
-
-begin-language: "Autoconf-without-aclocal-m4"
-args: --no-cache 
-end-language: "Autoconf-without-aclocal-m4"
-
-begin-language: "Autoconf"
-args: --no-cache 
-end-language: "Autoconf"
-

hadoop-mapreduce-project/src/c++/utils/Makefile.am

@@ -1,33 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-ACLOCAL_AMFLAGS = -I m4
-AM_CXXFLAGS=-I$(srcdir)/api -Wall
-
-# List the api header files and where they will be installed
-apidir = $(includedir)/hadoop
-api_HEADERS = \
-	api/hadoop/StringUtils.hh \
-	api/hadoop/SerialUtils.hh
-
-
-# Define the libaries that need to be built
-lib_LIBRARIES = libhadooputils.a
-
-# Define the sources for lib 
-libhadooputils_a_SOURCES = \
-	impl/StringUtils.cc \
-	impl/SerialUtils.cc
-

hadoop-mapreduce-project/src/c++/utils/configure.ac

@@ -1,56 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-#                                               -*- Autoconf -*-
-# Process this file with autoconf to produce a configure script.
-
-AC_PREREQ(2.59)
-AC_INIT(hadoop-utils, 0.13.0, omalley@apache.org)
-
-AC_CONFIG_AUX_DIR([config])
-AC_CONFIG_MACRO_DIR([m4])
-
-AM_INIT_AUTOMAKE([subdir-objects foreign no-dist])
-
-AC_CONFIG_SRCDIR([impl/SerialUtils.cc])
-AC_CONFIG_HEADER([impl/config.h])
-AC_CONFIG_FILES([Makefile])
-
-AC_PREFIX_DEFAULT(`pwd`/../install)
-
-CHECK_INSTALL_CFLAG
-HADOOP_UTILS_SETUP
-
-# Checks for programs.
-AC_PROG_CXX
-AC_PROG_LIBTOOL
-
-# Checks for libraries.
-
-# Checks for header files.
-AC_LANG(C++)
-AC_CHECK_HEADERS([unistd.h])
-
-# Checks for typedefs, structures, and compiler characteristics.
-AC_HEADER_STDBOOL
-AC_C_CONST
-AC_TYPE_OFF_T
-AC_TYPE_SIZE_T
-AC_FUNC_STRERROR_R
-
-# Checks for library functions.
-AC_CHECK_FUNCS([mkdir uname])
-AC_OUTPUT

hadoop-mapreduce-project/src/c++/utils/m4/hadoop_utils.m4

@@ -1,68 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# hadoop_utils.m4
-
-# Check to see if the install program supports -C
-# If so, use "install -C" for the headers. Otherwise, every install
-# updates the timestamps on the installed headers, which causes a recompilation
-# of any downstream libraries.
-AC_DEFUN([CHECK_INSTALL_CFLAG],[
-AC_REQUIRE([AC_PROG_INSTALL])
-touch foo
-if $INSTALL -C foo bar; then
-  INSTALL_DATA="$INSTALL_DATA -C"
-fi
-rm -f foo bar
-])
-
-# Set up the things we need for compiling hadoop utils
-AC_DEFUN([HADOOP_UTILS_SETUP],[
-AC_REQUIRE([AC_GNU_SOURCE])
-AC_REQUIRE([AC_SYS_LARGEFILE])
-])
-
-# define a macro for using hadoop utils
-AC_DEFUN([USE_HADOOP_UTILS],[
-AC_REQUIRE([HADOOP_UTILS_SETUP])
-AC_ARG_WITH([hadoop-utils],
-            AS_HELP_STRING([--with-hadoop-utils=<dir>],
-                           [directory to get hadoop_utils from]),
-            [HADOOP_UTILS_PREFIX="$withval"],
-            [HADOOP_UTILS_PREFIX="\${prefix}"])
-AC_SUBST(HADOOP_UTILS_PREFIX)
-])
-
-AC_DEFUN([HADOOP_PIPES_SETUP],[
-AC_CHECK_HEADERS([pthread.h], [], 
-  AC_MSG_ERROR(Please check if you have installed the pthread library)) 
-AC_CHECK_LIB([pthread], [pthread_create], [], 
-  AC_MSG_ERROR(Cannot find libpthread.so, please check))
-AC_CHECK_LIB([crypto], [HMAC_Init], [], 
-  AC_MSG_ERROR(Cannot find libcrypto.so, please check))
-])
-
-# define a macro for using hadoop pipes
-AC_DEFUN([USE_HADOOP_PIPES],[
-AC_REQUIRE([USE_HADOOP_UTILS])
-AC_REQUIRE([HADOOP_PIPES_SETUP])
-AC_ARG_WITH([hadoop-pipes],
-            AS_HELP_STRING([--with-hadoop-pipes=<dir>],
-                           [directory to get hadoop pipes from]),
-            [HADOOP_PIPES_PREFIX="$withval"],
-            [HADOOP_PIPES_PREFIX="\${prefix}"])
-AC_SUBST(HADOOP_PIPES_PREFIX)
-])

hadoop-mapreduce-project/src/examples/pipes/.autom4te.cfg

@@ -1,42 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-#
-# autom4te configuration for hadoop utils library
-#
-
-begin-language: "Autoheader-preselections"
-args: --no-cache 
-end-language: "Autoheader-preselections"
-
-begin-language: "Automake-preselections"
-args: --no-cache 
-end-language: "Automake-preselections"
-
-begin-language: "Autoreconf-preselections"
-args: --no-cache 
-end-language: "Autoreconf-preselections"
-
-begin-language: "Autoconf-without-aclocal-m4"
-args: --no-cache 
-end-language: "Autoconf-without-aclocal-m4"
-
-begin-language: "Autoconf"
-args: --no-cache 
-end-language: "Autoconf"
-

hadoop-mapreduce-project/src/examples/pipes/Makefile.am

@@ -1,36 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-ACLOCAL_AMFLAGS = -I ../../c++/utils/m4
-AM_CXXFLAGS=-Wall -I$(HADOOP_UTILS_PREFIX)/include \
-            -I$(HADOOP_PIPES_PREFIX)/include
-LDADD=-L$(HADOOP_UTILS_PREFIX)/lib -L$(HADOOP_PIPES_PREFIX)/lib \
-      -lhadooppipes -lhadooputils
-
-bin_PROGRAMS= wordcount-simple wordcount-part wordcount-nopipe pipes-sort
-
-# Define the sources for each program
-wordcount_simple_SOURCES = \
-	impl/wordcount-simple.cc
-
-wordcount_part_SOURCES = \
-	impl/wordcount-part.cc
-
-wordcount_nopipe_SOURCES = \
-	impl/wordcount-nopipe.cc
-
-pipes_sort_SOURCES = \
-        impl/sort.cc
-

hadoop-mapreduce-project/src/examples/pipes/configure.ac

@@ -1,58 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-#                                               -*- Autoconf -*-
-# Process this file with autoconf to produce a configure script.
-
-AC_PREREQ(2.59)
-AC_INIT(hadoop-pipes-examples, 0.13.0, omalley@apache.org)
-
-AC_CONFIG_AUX_DIR([config])
-AC_CONFIG_MACRO_DIR([../../c++/utils/m4])
-
-AM_INIT_AUTOMAKE([subdir-objects foreign no-dist])
-
-AC_CONFIG_SRCDIR([impl/wordcount-simple.cc])
-AC_CONFIG_HEADER([impl/config.h])
-AC_CONFIG_FILES([Makefile])
-
-AC_PREFIX_DEFAULT(`pwd`/../install)
-
-USE_HADOOP_PIPES
-
-# Checks for programs.
-AC_PROG_CXX
-AC_PROG_INSTALL
-AC_PROG_LIBTOOL
-
-# Checks for libraries.
-
-# Checks for header files.
-AC_LANG(C++)
-AC_CHECK_HEADERS([unistd.h])
-
-# Checks for typedefs, structures, and compiler characteristics.
-AC_HEADER_STDBOOL
-AC_C_CONST
-AC_TYPE_OFF_T
-AC_TYPE_SIZE_T
-AC_FUNC_STRERROR_R
-
-# Checks for library functions.
-AC_CHECK_FUNCS([mkdir uname])
-AC_CHECK_LIB([socket],[shutdown])
-AC_CHECK_LIB([nsl],[xdr_float])
-AC_OUTPUT

hadoop-tools/hadoop-pipes/pom.xml

@@ -0,0 +1,108 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+<project>
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.hadoop</groupId>
+    <artifactId>hadoop-project</artifactId>
+    <version>3.0.0-SNAPSHOT</version>
+    <relativePath>../../hadoop-project</relativePath>
+  </parent>
+  <groupId>org.apache.hadoop</groupId>
+  <artifactId>hadoop-pipes</artifactId>
+  <version>3.0.0-SNAPSHOT</version>
+  <description>Apache Hadoop Pipes</description>
+  <name>Apache Hadoop Pipes</name>
+  <packaging>pom</packaging>
+
+  <properties>
+    <hadoop.log.dir>${project.build.directory}/log</hadoop.log.dir>
+  </properties>
+
+  <profiles>
+    <profile>
+      <id>native</id>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-antrun-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>make</id>
+                <phase>compile</phase>
+                <goals><goal>run</goal></goals>
+                <configuration>
+                  <target>
+                    <mkdir dir="${project.build.directory}/native"/>
+                    <exec executable="cmake" dir="${project.build.directory}/native" 
+                        failonerror="true">
+                      <arg line="${basedir}/src/ -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model}"/>
+                    </exec>
+                    <exec executable="make" dir="${project.build.directory}/native" failonerror="true">
+                      <arg line="VERBOSE=1"/>
+                    </exec>
+                  </target>
+                </configuration>
+              </execution>
+              <!-- TODO wire here native testcases
+              <execution>
+                <id>test</id>
+                <phase>test</phase>
+                <goals>
+                  <goal>test</goal>
+                </goals>
+                <configuration>
+                  <destDir>${project.build.directory}/native/target</destDir>
+                </configuration>
+              </execution>
+              -->
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+  </profiles> 
+
+<!--
+  <build>
+    <plugins>
+     <plugin>
+        <artifactId>maven-antrun-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>compile</id>
+            <phase>generate-sources</phase>
+            <goals>
+              <goal>run</goal>
+            </goals>
+            <configuration>
+              <target>
+                <mkdir dir="${basedir}/../target/native"/>
+                <copy toDir="${basedir}/../target/native">
+                  <fileset dir="${basedir}/src/main/native"/>
+                </copy>
+              </target>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+-->
+</project>

hadoop-tools/hadoop-pipes/src/CMakeLists.txt

@@ -0,0 +1,99 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+ 
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
+find_package(OpenSSL REQUIRED)
+
+set(CMAKE_BUILD_TYPE, Release)
+
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -Wall -O2")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -Wall -O2")
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_REENTRANT -D_FILE_OFFSET_BITS=64")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_REENTRANT -D_FILE_OFFSET_BITS=64")
+
+if (JVM_ARCH_DATA_MODEL EQUAL 32)
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32")
+    set(CMAKE_LD_FLAGS "${CMAKE_LD_FLAGS} -m32")
+    if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64")
+        set(CMAKE_SYSTEM_PROCESSOR "i686")
+    endif ()
+endif (JVM_ARCH_DATA_MODEL EQUAL 32)
+
+function(output_directory TGT DIR)
+    SET_TARGET_PROPERTIES(${TGT} PROPERTIES
+        RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
+   SET_TARGET_PROPERTIES(${TGT} PROPERTIES
+        ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
+    SET_TARGET_PROPERTIES(${TGT} PROPERTIES
+        LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
+endfunction(output_directory TGT DIR)
+
+include_directories(
+    main/native/utils/api
+    main/native/pipes/api
+    ${CMAKE_CURRENT_SOURCE_DIR}
+    ${OPENSSL_INCLUDE_DIR}
+)
+
+# Example programs
+add_executable(wordcount-simple main/native/examples/impl/wordcount-simple.cc)
+target_link_libraries(wordcount-simple hadooppipes hadooputils)
+
+add_executable(wordcount-part main/native/examples/impl/wordcount-part.cc)
+target_link_libraries(wordcount-part hadooppipes hadooputils)
+
+add_executable(wordcount-nopipe main/native/examples/impl/wordcount-nopipe.cc)
+target_link_libraries(wordcount-nopipe hadooppipes hadooputils)
+
+add_executable(pipes-sort main/native/examples/impl/sort.cc)
+target_link_libraries(pipes-sort hadooppipes hadooputils)
+
+install(TARGETS wordcount-simple wordcount-part wordcount-nopipe pipes-sort 
+    RUNTIME DESTINATION bin
+)
+
+add_library(hadooputils STATIC
+    main/native/utils/impl/StringUtils.cc
+    main/native/utils/impl/SerialUtils.cc
+)
+
+install(FILES
+    main/native/utils/api/hadoop/SerialUtils.hh
+    main/native/utils/api/hadoop/StringUtils.hh
+    DESTINATION api/hadoop
+    COMPONENT headers
+)
+install(TARGETS hadooputils DESTINATION lib)
+
+add_library(hadooppipes STATIC
+    main/native/pipes/impl/HadoopPipes.cc
+)
+target_link_libraries(hadooppipes
+    ${JAVA_JVM_LIBRARY}
+    ${OPENSSL_LIBRARIES}
+    pthread
+)
+
+install(FILES
+    main/native/pipes/api/hadoop/Pipes.hh
+    main/native/pipes/api/hadoop/TemplateFactory.hh
+    DESTINATION api/hadoop
+    COMPONENT headers
+)
+install(TARGETS hadooppipes DESTINATION lib)

hadoop-mapreduce-project/src/examples/pipes/README.txt → hadoop-tools/hadoop-pipes/src/main/native/examples/README.txt

@@ -1,16 +1,16 @@
 To run the examples, first compile them:
 
-% ant -Dcompile.c++=yes examples
+% mvn install 
 
 and then copy the binaries to dfs:
 
-% bin/hadoop fs -put build/c++-examples/Linux-i386-32/bin /examples/bin
+% hadoop fs -put target/native/wordcount-simple /examples/bin/
 
 create an input directory with text files:
 
-% bin/hadoop fs -put my-data in-dir
+% hadoop fs -put my-data in-dir
 
 and run the word count example:
 
-% bin/hadoop pipes -conf src/examples/pipes/conf/word.xml \
+% hadoop pipes -conf src/main/native/examples/conf/word.xml \
                    -input in-dir -output out-dir

hadoop-tools/hadoop-tools-dist/pom.xml

@@ -70,6 +70,13 @@
       <artifactId>hadoop-gridmix</artifactId>
       <scope>compile</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-pipes</artifactId>
+      <scope>compile</scope>
+      <type>pom</type>
+      <version>${project.version}</version>
+    </dependency>
   </dependencies>
 
   <build>
@@ -92,4 +99,44 @@
     </plugins>
   </build>
 
+  <profiles>
+    <profile>
+      <id>dist</id>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <build>
+        <plugins>
+         <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-assembly-plugin</artifactId>
+            <dependencies>
+              <dependency>
+                <groupId>org.apache.hadoop</groupId>
+                <artifactId>hadoop-assemblies</artifactId>
+                <version>${project.version}</version>
+              </dependency>
+            </dependencies>
+            <executions>
+              <execution>
+                <id>dist</id>
+                <phase>prepare-package</phase>
+                <goals>
+                  <goal>single</goal>
+                </goals>
+                <configuration>
+                  <appendAssemblyId>false</appendAssemblyId>
+                  <attach>false</attach>
+                  <finalName>${project.artifactId}-${project.version}</finalName>
+                  <descriptorRefs>
+                    <descriptorRef>hadoop-tools</descriptorRef>
+                  </descriptorRefs>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+  </profiles>
 </project>

hadoop-tools/pom.xml

@@ -39,6 +39,7 @@
     <module>hadoop-datajoin</module>
     <module>hadoop-tools-dist</module>
     <module>hadoop-extras</module>
+    <module>hadoop-pipes</module>
   </modules>
 
   <build>