HADOOP-9008. Building hadoop tarball fails on Windows. Contributed by Chris Nauroth.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-trunk-win@1408463 13f79535-47bb-0310-9956-ffa450edef68

hadoop-common-project/hadoop-common/CHANGES.branch-trunk-win.txt

@@ -51,3 +51,5 @@ branch-trunk-win changes - unreleased
   HADOOP-9005. Merge hadoop cmd line scripts from branch-1-win. (David Lao, 
   Bikas Saha, Lauren Yang, Chuan Liu, Thejas M Nair and Ivan Mitic via suresh)
 
+  HADOOP-9008. Building hadoop tarball fails on Windows. (Chris Nauroth via
+  suresh)

hadoop-dist/pom.xml

@@ -89,50 +89,6 @@
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-antrun-plugin</artifactId>
             <executions>
-              <execution>
-                <id>dist</id>
-                <phase>prepare-package</phase>
-                <goals>
-                  <goal>run</goal>
-                </goals>
-                <configuration>
-                  <target>
-                    <echo file="${project.build.directory}/dist-layout-stitching.sh">
-                      run() {
-                        echo "\$ ${@}"
-                        "${@}"
-                        res=$?
-                        if [ $res != 0 ]; then
-                          echo
-                          echo "Failed!"
-                          echo
-                          exit $res
-                        fi
-                      }
-
-                      ROOT=`cd ${basedir}/..;pwd`
-                      echo
-                      echo "Current directory `pwd`"
-                      echo
-                      run rm -rf hadoop-${project.version}
-                      run mkdir hadoop-${project.version}
-                      run cd hadoop-${project.version}
-                      run cp -r $ROOT/hadoop-common-project/hadoop-common/target/hadoop-common-${project.version}/* .
-                      run cp -r $ROOT/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-hdfs-${project.version}/* .
-                      run cp -r $ROOT/hadoop-hdfs-project/hadoop-hdfs-httpfs/target/hadoop-hdfs-httpfs-${project.version}/* .
-                      run cp -r $ROOT/hadoop-yarn-project/target/hadoop-yarn-project-${project.version}/* .
-                      run cp -r $ROOT/hadoop-mapreduce-project/target/hadoop-mapreduce-${project.version}/* .
-                      run cp -r $ROOT/hadoop-tools/hadoop-tools-dist/target/hadoop-tools-dist-${project.version}/* .
-                      echo
-                      echo "Hadoop dist layout available at: ${project.build.directory}/hadoop-${project.version}"
-                      echo
-                    </echo>
-                    <exec executable="sh" dir="${project.build.directory}" failonerror="true">
-                      <arg line="./dist-layout-stitching.sh"/>
-                    </exec>
-                  </target>
-                </configuration>
-              </execution>
               <execution>
                 <id>tar</id>
                 <phase>package</phase>
@@ -141,26 +97,61 @@
                 </goals>
                 <configuration>
                   <target if="tar">
-                    <echo file="${project.build.directory}/dist-tar-stitching.sh">
-                      run() {
-                        echo "\$ ${@}"
-                        "${@}"
-                        res=$?
-                        if [ $res != 0 ]; then
-                          echo
-                          echo "Failed!"
-                          echo
-                          exit $res
-                        fi
-                      }
+                    <!-- This script preserves permissions and symlinks. -->
+                    <!-- Python requires resetting indentation to far left. -->
+                    <echo file="${project.build.directory}/dist-maketar.py">
+from os.path import abspath, basename, isdir, join
+import tarfile
+
+def make_file_filter(root, file_name_filter):
+  def filter_func(tar_info):
+    if tar_info.name == root:
+      # Always include root directory.  Otherwise, tarfile.add assumes you are
+      # filtering out the whole directory and produces an empty tar.
+      return tar_info
+    if tar_info.isfile() or tar_info.issym():
+      # Include files and symlinks only if they match the specified name filter.
+      if file_name_filter(basename(tar_info.name)):
+        return tar_info
+    # Otherwise, exclude.
+    return None
+  return filter_func
+
+target_dirs = [
+  abspath(r"${basedir}/../hadoop-common-project/hadoop-common/target/hadoop-common-${project.version}"),
+  abspath(r"${basedir}/../hadoop-hdfs-project/hadoop-hdfs/target/hadoop-hdfs-${project.version}"),
+  abspath(r"${basedir}/../hadoop-hdfs-project/hadoop-hdfs-httpfs/target/hadoop-hdfs-httpfs-${project.version}"),
+  abspath(r"${basedir}/../hadoop-yarn-project/target/hadoop-yarn-project-${project.version}"),
+  abspath(r"${basedir}/../hadoop-mapreduce-project/target/hadoop-mapreduce-${project.version}"),
+  abspath(r"${basedir}/../hadoop-tools/hadoop-tools-dist/target/hadoop-tools-dist-${project.version}")
+]
+base_name = "hadoop" + "-" + "${project.version}"
+dir_name = abspath(join(r"${project.build.directory}", base_name))
+tar_name = dir_name + ".tar.gz"
+
+with tarfile.open(tar_name, "w:gz") as tar:
+  for target_dir in target_dirs:
+    tar.add(target_dir, arcname=base_name)
+    native_dir = abspath(join(target_dir, "../native/target/usr/local/lib"))
+    if isdir(native_dir):
+      arc_name = base_name + "/lib/native"
+      tar.add(native_dir, arcname=arc_name,
+        filter=make_file_filter(arc_name, lambda file: file.startswith("lib")))
+    bin_dir = abspath(join(target_dir, "../bin"))
+    if isdir(bin_dir):
+      arc_name = base_name + "/bin"
+      tar.add(bin_dir, arcname=arc_name)
+  if "${bundle.snappy}" == "true":
+    arc_name = base_name + "/lib/native"
+    tar.add(r"${snappy.lib}", arcname=arc_name,
+      filter=make_file_filter(arc_name, lambda file: "snappy" in file))
 
-                      run tar czf hadoop-${project.version}.tar.gz hadoop-${project.version}
-                      echo
-                      echo "Hadoop dist tar available at: ${project.build.directory}/hadoop-${project.version}.tar.gz"
-                      echo
+print
+print "Hadoop dist tar available at: " + tar_name
+print
                     </echo>
-                    <exec executable="sh" dir="${project.build.directory}" failonerror="true">
-                      <arg line="./dist-tar-stitching.sh"/>
+                    <exec executable="python" dir="${project.build.directory}" failonerror="true">
+                      <arg value="dist-maketar.py" />
                     </exec>
                   </target>
                 </configuration>

hadoop-project-dist/pom.xml

@@ -332,43 +332,6 @@
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-antrun-plugin</artifactId>
             <executions>
-              <execution>
-                <id>pre-dist</id>
-                <phase>prepare-package</phase>
-                <goals>
-                  <goal>run</goal>
-                </goals>
-                <configuration>
-                  <target>
-                    <!-- Using Unix script to preserve symlinks -->
-                    <echo file="${project.build.directory}/dist-copynativelibs.sh">
-
-                      which cygpath 2&gt; /dev/null
-                      if [ $? = 1 ]; then
-                        BUILD_DIR="${project.build.directory}"
-                      else
-                        BUILD_DIR=`cygpath --unix '${project.build.directory}'`
-                      fi
-                      TAR='tar cf -'
-                      UNTAR='tar xfBp -'
-                      LIB_DIR="${BUILD_DIR}/native/target/usr/local/lib"
-                      if [ -d $${LIB_DIR} ] ; then
-                        TARGET_DIR="${BUILD_DIR}/${project.artifactId}-${project.version}/lib/native"
-                        mkdir -p $${TARGET_DIR}
-                        cd $${LIB_DIR}
-                        $$TAR lib* | (cd $${TARGET_DIR}/; $$UNTAR)
-                        if [ "${bundle.snappy}" = "true" ] ; then
-                          cd ${snappy.lib}
-                          $$TAR *snappy* | (cd $${TARGET_DIR}/; $$UNTAR)
-                        fi
-                      fi
-                    </echo>
-                    <exec executable="sh" dir="${project.build.directory}" failonerror="true">
-                      <arg line="./dist-copynativelibs.sh"/>
-                    </exec>
-                  </target>
-                </configuration>
-              </execution>
               <execution>
                 <id>tar</id>
                 <phase>package</phase>
@@ -377,20 +340,48 @@
                 </goals>
                 <configuration>
                   <target if="tar">
-                    <!-- Using Unix script to preserve symlinks -->
-                    <echo file="${project.build.directory}/dist-maketar.sh">
+                    <!-- This script preserves permissions and symlinks. -->
+                    <!-- Python requires resetting indentation to far left. -->
+                    <echo file="${project.build.directory}/dist-maketar.py">
+from os.path import abspath, basename, isdir, join
+import tarfile
+
+def make_file_filter(root, file_name_filter):
+  def filter_func(tar_info):
+    if tar_info.name == root:
+      # Always include root directory.  Otherwise, tarfile.add assumes you are
+      # filtering out the whole directory and produces an empty tar.
+      return tar_info
+    if tar_info.isfile() or tar_info.issym():
+      # Include files and symlinks only if they match the specified name filter.
+      if file_name_filter(basename(tar_info.name)):
+        return tar_info
+    # Otherwise, exclude.
+    return None
+  return filter_func
+
+base_name = "${project.artifactId}" + "-" + "${project.version}"
+dir_name = abspath(join(r"${project.build.directory}", base_name))
+tar_name = dir_name + ".tar.gz"
 
-                      which cygpath 2&gt; /dev/null
-                      if [ $? = 1 ]; then
-                        BUILD_DIR="${project.build.directory}"
-                      else
-                        BUILD_DIR=`cygpath --unix '${project.build.directory}'`
-                      fi
-                      cd ${BUILD_DIR}
-                      tar czf ${project.artifactId}-${project.version}.tar.gz ${project.artifactId}-${project.version}
+with tarfile.open(tar_name, "w:gz") as tar:
+  tar.add(dir_name, arcname=base_name)
+  native_dir = abspath(r"${project.build.directory}/native/target/usr/local/lib")
+  if isdir(native_dir):
+    arc_name = base_name + "/lib/native"
+    tar.add(native_dir, arcname=arc_name,
+      filter=make_file_filter(arc_name, lambda file: file.startswith("lib")))
+    if "${bundle.snappy}" == "true":
+      arc_name = base_name + "/lib/native"
+      tar.add(r"${snappy.lib}", arcname=arc_name,
+        filter=make_file_filter(arc_name, lambda file: "snappy" in file))
+  bin_dir = abspath(r"${project.build.directory}/bin")
+  if isdir(bin_dir):
+    arc_name = base_name + "/bin"
+    tar.add(bin_dir, arcname=arc_name)
                     </echo>
-                    <exec executable="sh" dir="${project.build.directory}" failonerror="true">
-                      <arg line="./dist-maketar.sh"/>
+                    <exec executable="python" dir="${project.build.directory}" failonerror="true">
+                      <arg value="dist-maketar.py" />
                     </exec>
                   </target>
                 </configuration>