HADOOP-9271. Revert Python build scripts from branch-trunk-win. Contributed by Chris Nauroth.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-trunk-win@1441812 13f79535-47bb-0310-9956-ffa450edef68

hadoop-common-project/hadoop-common/CHANGES.branch-trunk-win.txt

@@ -77,6 +77,9 @@ branch-trunk-win changes - unreleased
   HADOOP-9270. Remove a stale java comment from FileUtil. (Chris Nauroth via
   szetszwo)
 
+  HADOOP-9271. Revert Python build scripts from branch-trunk-win.
+  (Chris Nauroth via suresh)
+
 Patch equivalent to trunk committed to branch-trunk-win
 
   HADOOP-8924. Add maven plugin alternative to shell script to save

hadoop-dist/pom.xml

@@ -86,6 +86,50 @@
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-antrun-plugin</artifactId>
             <executions>
+              <execution>
+                <id>dist</id>
+                <phase>prepare-package</phase>
+                <goals>
+                  <goal>run</goal>
+                </goals>
+                <configuration>
+                  <target>
+                    <echo file="${project.build.directory}/dist-layout-stitching.sh">
+                      run() {
+                        echo "\$ ${@}"
+                        "${@}"
+                        res=$?
+                        if [ $res != 0 ]; then
+                          echo
+                          echo "Failed!"
+                          echo
+                          exit $res
+                        fi
+                      }
+
+                      ROOT=`cd ../..;pwd`
+                      echo
+                      echo "Current directory `pwd`"
+                      echo
+                      run rm -rf hadoop-${project.version}
+                      run mkdir hadoop-${project.version}
+                      run cd hadoop-${project.version}
+                      run cp -r $ROOT/hadoop-common-project/hadoop-common/target/hadoop-common-${project.version}/* .
+                      run cp -r $ROOT/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-hdfs-${project.version}/* .
+                      run cp -r $ROOT/hadoop-hdfs-project/hadoop-hdfs-httpfs/target/hadoop-hdfs-httpfs-${project.version}/* .
+                      run cp -r $ROOT/hadoop-yarn-project/target/hadoop-yarn-project-${project.version}/* .
+                      run cp -r $ROOT/hadoop-mapreduce-project/target/hadoop-mapreduce-${project.version}/* .
+                      run cp -r $ROOT/hadoop-tools/hadoop-tools-dist/target/hadoop-tools-dist-${project.version}/* .
+                      echo
+                      echo "Hadoop dist layout available at: ${project.build.directory}/hadoop-${project.version}"
+                      echo
+                    </echo>
+                    <exec executable="sh" dir="${project.build.directory}" failonerror="true">
+                      <arg line="./dist-layout-stitching.sh"/>
+                    </exec>
+                  </target>
+                </configuration>
+              </execution>
               <execution>
                 <id>tar</id>
                 <phase>package</phase>
@@ -94,61 +138,27 @@
                 </goals>
                 <configuration>
                   <target if="tar">
-                    <!-- This script preserves permissions and symlinks. -->
-                    <!-- Python requires resetting indentation to far left. -->
-                    <echo file="${project.build.directory}/dist-maketar.py">
-from os.path import abspath, basename, isdir, join
-import tarfile
-
-def make_file_filter(root, file_name_filter):
-  def filter_func(tar_info):
-    if tar_info.name == root:
-      # Always include root directory.  Otherwise, tarfile.add assumes you are
-      # filtering out the whole directory and produces an empty tar.
-      return tar_info
-    if tar_info.isfile() or tar_info.issym():
-      # Include files and symlinks only if they match the specified name filter.
-      if file_name_filter(basename(tar_info.name)):
-        return tar_info
-    # Otherwise, exclude.
-    return None
-  return filter_func
-
-target_dirs = [
-  abspath(r"${basedir}/../hadoop-common-project/hadoop-common/target/hadoop-common-${project.version}"),
-  abspath(r"${basedir}/../hadoop-hdfs-project/hadoop-hdfs/target/hadoop-hdfs-${project.version}"),
-  abspath(r"${basedir}/../hadoop-hdfs-project/hadoop-hdfs-httpfs/target/hadoop-hdfs-httpfs-${project.version}"),
-  abspath(r"${basedir}/../hadoop-yarn-project/target/hadoop-yarn-project-${project.version}"),
-  abspath(r"${basedir}/../hadoop-mapreduce-project/target/hadoop-mapreduce-${project.version}"),
-  abspath(r"${basedir}/../hadoop-tools/hadoop-tools-dist/target/hadoop-tools-dist-${project.version}")
-]
-base_name = "hadoop" + "-" + "${project.version}"
-dir_name = abspath(join(r"${project.build.directory}", base_name))
-tar_name = dir_name + ".tar.gz"
-
-with tarfile.open(tar_name, "w:gz") as tar:
-  for target_dir in target_dirs:
-    tar.add(target_dir, arcname=base_name)
-    native_dir = abspath(join(target_dir, "../native/target/usr/local/lib"))
-    if isdir(native_dir):
-      arc_name = base_name + "/lib/native"
-      tar.add(native_dir, arcname=arc_name,
-        filter=make_file_filter(arc_name, lambda file: file.startswith("lib")))
-    bin_dir = abspath(join(target_dir, "../bin"))
-    if isdir(bin_dir):
-      arc_name = base_name + "/bin"
-      tar.add(bin_dir, arcname=arc_name)
-  if "${bundle.snappy}" == "true":
-    arc_name = base_name + "/lib/native"
-    tar.add(r"${snappy.lib}", arcname=arc_name,
-      filter=make_file_filter(arc_name, lambda file: "snappy" in file))
+                    <echo file="${project.build.directory}/dist-tar-stitching.sh">
+                      run() {
+                        echo "\$ ${@}"
+                        "${@}"
+                        res=$?
+                        if [ $res != 0 ]; then
+                          echo
+                          echo "Failed!"
+                          echo
+                          exit $res
+                        fi
+                      }
 
-print
-print "Hadoop dist tar available at: " + tar_name
-print
+                      run tar cf hadoop-${project.version}.tar hadoop-${project.version}
+                      run gzip hadoop-${project.version}.tar
+                      echo
+                      echo "Hadoop dist tar available at: ${project.build.directory}/hadoop-${project.version}.tar.gz"
+                      echo
                     </echo>
-                    <exec executable="python" dir="${project.build.directory}" failonerror="true">
-                      <arg value="dist-maketar.py" />
+                    <exec executable="sh" dir="${project.build.directory}" failonerror="true">
+                      <arg line="./dist-tar-stitching.sh"/>
                     </exec>
                   </target>
                 </configuration>

hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml

@@ -537,20 +537,20 @@
                     <delete dir="${project.build.directory}/tomcat.exp"/>
                     <mkdir dir="${project.build.directory}/tomcat.exp"/>
 
-                    <!-- This script preserves permissions and symlinks. -->
-                    <!-- Python requires resetting indentation to far left. -->
-                    <echo file="${project.build.directory}/tomcat-untar.py">
-from os.path import abspath
-import tarfile
+                    <!-- Using Unix script to preserve file permissions -->
+                    <echo file="${project.build.directory}/tomcat-untar.sh">
 
-src = abspath(r"${basedir}/downloads/apache-tomcat-${tomcat.version}.tar.gz")
-dest = abspath(r"${project.build.directory}/tomcat.exp")
-
-with tarfile.open(src, "r:gz") as tar:
-  tar.extractall(path=dest)
+                      which cygpath 2&gt; /dev/null
+                      if [ $? != 0 ]; then
+                      BUILD_DIR="${project.build.directory}"
+                      else
+                      BUILD_DIR=`cygpath --unix '${project.build.directory}'`
+                      fi
+                      cd $BUILD_DIR/tomcat.exp
+                      gzip -cd ../../downloads/apache-tomcat-${tomcat.version}.tar.gz | tar xf -
                     </echo>
-                    <exec executable="python" dir="${project.build.directory}" failonerror="true">
-                      <arg line="tomcat-untar.py"/>
+                    <exec executable="sh" dir="${project.build.directory}" failonerror="true">
+                      <arg line="./tomcat-untar.sh"/>
                     </exec>
 
                     <move file="${project.build.directory}/tomcat.exp/apache-tomcat-${tomcat.version}"
@@ -580,21 +580,20 @@ with tarfile.open(src, "r:gz") as tar:
                 </goals>
                 <configuration>
                   <target if="tar">
-                    <!-- This script preserves permissions and symlinks. -->
-                    <!-- Python requires resetting indentation to far left. -->
-                    <echo file="${project.build.directory}/dist-maketar.py">
-from os.path import abspath, join
-import tarfile
-
-base_name = "${project.artifactId}" + "-" + "${project.version}"
-dir_name = abspath(join(r"${project.build.directory}", base_name))
-tar_name = dir_name + ".tar.gz"
+                    <!-- Using Unix script to preserve symlinks -->
+                    <echo file="${project.build.directory}/dist-maketar.sh">
 
-with tarfile.open(tar_name, "w:gz") as tar:
-  tar.add(dir_name, arcname=base_name)
+                      which cygpath 2&gt; /dev/null
+                      if [ $? != 0 ]; then
+                      BUILD_DIR="${project.build.directory}"
+                      else
+                      BUILD_DIR=`cygpath --unix '${project.build.directory}'`
+                      fi
+                      cd $BUILD_DIR
+                      tar cf - ${project.artifactId}-${project.version} | gzip > ${project.artifactId}-${project.version}.tar.gz
                     </echo>
-                    <exec executable="python" dir="${project.build.directory}" failonerror="true">
-                      <arg value="dist-maketar.py" />
+                    <exec executable="sh" dir="${project.build.directory}" failonerror="true">
+                      <arg line="./dist-maketar.sh"/>
                     </exec>
                   </target>
                 </configuration>

hadoop-mapreduce-project/pom.xml

@@ -178,22 +178,22 @@
               <goal>run</goal>
             </goals>
             <configuration>
+              <!-- this is identical from hadoop-project-dist, eventually they must be unified -->
               <target if="tar">
-                <!-- This script preserves permissions and symlinks. -->
-                <!-- Python requires resetting indentation to far left. -->
-                <echo file="${project.build.directory}/dist-maketar.py">
-from os.path import abspath, join
-import tarfile
+                <!-- Using Unix script to preserve symlinks -->
+                <echo file="${project.build.directory}/dist-maketar.sh">
 
-base_name = "${project.artifactId}" + "-" + "${project.version}"
-dir_name = abspath(join(r"${project.build.directory}", base_name))
-tar_name = dir_name + ".tar.gz"
-
-with tarfile.open(tar_name, "w:gz") as tar:
-  tar.add(dir_name, arcname=base_name)
+                  which cygpath 2&gt; /dev/null
+                  if [ $? != 0 ]; then
+                    BUILD_DIR="${project.build.directory}"
+                  else
+                    BUILD_DIR=`cygpath --unix '${project.build.directory}'`
+                  fi
+                  cd $BUILD_DIR
+                  tar cf - ${project.artifactId}-${project.version} | gzip > ${project.artifactId}-${project.version}.tar.gz
                 </echo>
-                <exec executable="python" dir="${project.build.directory}" failonerror="true">
-                  <arg value="dist-maketar.py" />
+                <exec executable="sh" dir="${project.build.directory}" failonerror="true">
+                  <arg line="./dist-maketar.sh"/>
                 </exec>
               </target>
             </configuration>

hadoop-project-dist/pom.xml

@@ -325,6 +325,50 @@
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-antrun-plugin</artifactId>
             <executions>
+              <execution>
+                <id>pre-dist</id>
+                <phase>prepare-package</phase>
+                <goals>
+                  <goal>run</goal>
+                </goals>
+                <configuration>
+                  <target>
+                    <!-- Using Unix script to preserve symlinks -->
+                    <echo file="${project.build.directory}/dist-copynativelibs.sh">
+
+                      which cygpath 2&gt; /dev/null
+                      if [ $? != 0 ]; then
+                        BUILD_DIR="${project.build.directory}"
+                      else
+                        BUILD_DIR=`cygpath --unix '${project.build.directory}'`
+                      fi
+                      TAR='tar cf -'
+                      UNTAR='tar xfBp -'
+                      LIB_DIR="${BUILD_DIR}/native/target/usr/local/lib"
+                      if [ -d $${LIB_DIR} ] ; then
+                        TARGET_DIR="${BUILD_DIR}/${project.artifactId}-${project.version}/lib/native"
+                        mkdir -p $${TARGET_DIR}
+                        cd $${LIB_DIR}
+                        $$TAR lib* | (cd $${TARGET_DIR}/; $$UNTAR)
+                        if [ "${bundle.snappy}" = "true" ] ; then
+                          cd ${snappy.lib}
+                          $$TAR *snappy* | (cd $${TARGET_DIR}/; $$UNTAR)
+                        fi
+                      fi
+                      BIN_DIR="${BUILD_DIR}/bin"
+                      if [ -d $${BIN_DIR} ] ; then
+                        TARGET_BIN_DIR="${BUILD_DIR}/${project.artifactId}-${project.version}/bin"
+                        mkdir -p $${TARGET_BIN_DIR}
+                        cd $${BIN_DIR}
+                        $$TAR * | (cd $${TARGET_BIN_DIR}/; $$UNTAR)
+                      fi
+                    </echo>
+                    <exec executable="sh" dir="${project.build.directory}" failonerror="true">
+                      <arg line="./dist-copynativelibs.sh"/>
+                    </exec>
+                  </target>
+                </configuration>
+              </execution>
               <execution>
                 <id>tar</id>
                 <phase>package</phase>
@@ -333,48 +377,20 @@
                 </goals>
                 <configuration>
                   <target if="tar">
-                    <!-- This script preserves permissions and symlinks. -->
-                    <!-- Python requires resetting indentation to far left. -->
-                    <echo file="${project.build.directory}/dist-maketar.py">
-from os.path import abspath, basename, isdir, join
-import tarfile
-
-def make_file_filter(root, file_name_filter):
-  def filter_func(tar_info):
-    if tar_info.name == root:
-      # Always include root directory.  Otherwise, tarfile.add assumes you are
-      # filtering out the whole directory and produces an empty tar.
-      return tar_info
-    if tar_info.isfile() or tar_info.issym():
-      # Include files and symlinks only if they match the specified name filter.
-      if file_name_filter(basename(tar_info.name)):
-        return tar_info
-    # Otherwise, exclude.
-    return None
-  return filter_func
-
-base_name = "${project.artifactId}" + "-" + "${project.version}"
-dir_name = abspath(join(r"${project.build.directory}", base_name))
-tar_name = dir_name + ".tar.gz"
+                    <!-- Using Unix script to preserve symlinks -->
+                    <echo file="${project.build.directory}/dist-maketar.sh">
 
-with tarfile.open(tar_name, "w:gz") as tar:
-  tar.add(dir_name, arcname=base_name)
-  native_dir = abspath(r"${project.build.directory}/native/target/usr/local/lib")
-  if isdir(native_dir):
-    arc_name = base_name + "/lib/native"
-    tar.add(native_dir, arcname=arc_name,
-      filter=make_file_filter(arc_name, lambda file: file.startswith("lib")))
-    if "${bundle.snappy}" == "true":
-      arc_name = base_name + "/lib/native"
-      tar.add(r"${snappy.lib}", arcname=arc_name,
-        filter=make_file_filter(arc_name, lambda file: "snappy" in file))
-  bin_dir = abspath(r"${project.build.directory}/bin")
-  if isdir(bin_dir):
-    arc_name = base_name + "/bin"
-    tar.add(bin_dir, arcname=arc_name)
+                      which cygpath 2&gt; /dev/null
+                      if [ $? != 0 ]; then
+                        BUILD_DIR="${project.build.directory}"
+                      else
+                        BUILD_DIR=`cygpath --unix '${project.build.directory}'`
+                      fi
+                      cd ${BUILD_DIR}
+                      tar cf - ${project.artifactId}-${project.version} | gzip > ${project.artifactId}-${project.version}.tar.gz
                     </echo>
-                    <exec executable="python" dir="${project.build.directory}" failonerror="true">
-                      <arg value="dist-maketar.py" />
+                    <exec executable="sh" dir="${project.build.directory}" failonerror="true">
+                      <arg line="./dist-maketar.sh"/>
                     </exec>
                   </target>
                 </configuration>

hadoop-yarn-project/pom.xml

@@ -177,22 +177,22 @@
               <goal>run</goal>
             </goals>
             <configuration>
+              <!-- this is identical from hadoop-project-dist, eventually they must be unified -->
               <target if="tar">
-                <!-- This script preserves permissions and symlinks. -->
-                <!-- Python requires resetting indentation to far left. -->
-                <echo file="${project.build.directory}/dist-maketar.py">
-from os.path import abspath, join
-import tarfile
+                <!-- Using Unix script to preserve symlinks -->
+                <echo file="${project.build.directory}/dist-maketar.sh">
 
-base_name = "${project.artifactId}" + "-" + "${project.version}"
-dir_name = abspath(join(r"${project.build.directory}", base_name))
-tar_name = dir_name + ".tar.gz"
-
-with tarfile.open(tar_name, "w:gz") as tar:
-  tar.add(dir_name, arcname=base_name)
+                  which cygpath 2&gt; /dev/null
+                  if [ $? != 0 ]; then
+                    BUILD_DIR="${project.build.directory}"
+                  else
+                    BUILD_DIR=`cygpath --unix '${project.build.directory}'`
+                  fi
+                  cd $BUILD_DIR
+                  tar cf - ${project.artifactId}-${project.version} | gzip > ${project.artifactId}-${project.version}.tar.gz
                 </echo>
-                <exec executable="python" dir="${project.build.directory}" failonerror="true">
-                  <arg value="dist-maketar.py" />
+                <exec executable="sh" dir="${project.build.directory}" failonerror="true">
+                  <arg line="./dist-maketar.sh"/>
                 </exec>
               </target>
             </configuration>