HADOOP-7525. Make arguments to test-patch optional.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1155551 13f79535-47bb-0310-9956-ffa450edef68

dev-support/test-patch.sh

@@ -21,91 +21,151 @@ ulimit -n 1024
 bindir=$(dirname $0)
 . $bindir/test-patch.properties
 
+# Defaults
 if [ -z "$MAVEN_HOME" ]; then
   MVN=mvn
 else
   MVN=$MAVEN_HOME/bin/mvn
 fi
 
+PROJECT_NAME=Hadoop
+JENKINS=false
+PATCH_DIR=/tmp
+SUPPORT_DIR=/tmp
+BASEDIR=$(pwd)
+
+PS=${PS:-ps}
 AWK=${AWK:-awk}
+WGET=${WGET:-wget}
+SVN=${SVN:-svn}
+GREP=${GREP:-grep}
+PATCH=${PATCH:-patch}
+JIRACLI=${JIRA:-jira}
+FINDBUGS_HOME=${FINDBUGS_HOME}
+FORREST_HOME=${FORREST_HOME}
+ECLIPSE_HOME=${ECLIPSE_HOME}
 
 ###############################################################################
-parseArgs() {
-  case "$1" in
-    HUDSON)
-      ### Set HUDSON to true to indicate that this script is being run by Hudson
-      HUDSON=true
-      if [[ $# != 16 ]] ; then
-        echo "ERROR: usage $0 HUDSON <PATCH_DIR> <SUPPORT_DIR> <PS_CMD> <WGET_CMD> <JIRACLI> <SVN_CMD> <GREP_CMD> <PATCH_CMD> <FINDBUGS_HOME> <FORREST_HOME> <ECLIPSE_HOME> <WORKSPACE_BASEDIR> <JIRA_PASSWD> <CURL_CMD> <DEFECT> "
-        cleanupAndExit 0
-      fi
-      PATCH_DIR=$2
-      SUPPORT_DIR=$3
-      PS=$4
-      WGET=$5
-      JIRACLI=$6
-      SVN=$7
-      GREP=$8
-      PATCH=$9
-      FINDBUGS_HOME=${10}
-      FORREST_HOME=${11}
-      ECLIPSE_HOME=${12}
-      BASEDIR=${13}
-      JIRA_PASSWD=${14}
-      CURL=${15}
-      defect=${16}
-		
-      ### Retrieve the defect number
-      if [ -z "$defect" ] ; then
-        echo "Could not determine the patch to test.  Exiting."
-        cleanupAndExit 0
-      fi
-
-      if [ ! -e "$PATCH_DIR" ] ; then
-        mkdir -p $PATCH_DIR 
-      fi
+printUsage() {
+  echo "Usage: $0 [options] patch-file | defect-number"
+  echo
+  echo "Where:"
+  echo "  patch-file is a local patch file containing the changes to test"
+  echo "  defect-number is a JIRA defect number (e.g. 'HADOOP-1234') to test (Jenkins only)"
+  echo
+  echo "Options:"
+  echo "--patch-dir=<dir>      The directory for working and output files (default '/tmp')"
+  echo "--basedir=<dir>        The directory to apply the patch to (default current directory)"
+  echo "--mvn-cmd=<cmd>        The 'mvn' command to use (default \$MAVEN_HOME/bin/mvn, or 'mvn')"
+  echo "--ps-cmd=<cmd>         The 'ps' command to use (default 'ps')"
+  echo "--awk-cmd=<cmd>        The 'awk' command to use (default 'awk')"
+  echo "--svn-cmd=<cmd>        The 'svn' command to use (default 'svn')"
+  echo "--grep-cmd=<cmd>       The 'grep' command to use (default 'grep')"
+  echo "--patch-cmd=<cmd>      The 'patch' command to use (default 'patch')"
+  echo "--findbugs-home=<path> Findbugs home directory (default FINDBUGS_HOME environment variable)"
+  echo "--forrest-home=<path>  Forrest home directory (default FORREST_HOME environment variable)"
+  echo "--dirty-workspace      Allow the local SVN workspace to have uncommitted changes"
+  echo
+  echo "Jenkins-only options:"
+  echo "--jenkins              Run by Jenkins (runs tests and posts results to JIRA)"
+  echo "--support-dir=<dir>    The directory to find support files in"
+  echo "--wget-cmd=<cmd>       The 'wget' command to use (default 'wget')"
+  echo "--jira-cmd=<cmd>       The 'jira' command to use (default 'jira')"
+  echo "--jira-password=<pw>   The password for the 'jira' command"
+  echo "--eclipse-home=<path>  Eclipse home directory (default ECLIPSE_HOME environment variable)"
+}
 
-      ECLIPSE_PROPERTY="-Declipse.home=$ECLIPSE_HOME"
+###############################################################################
+parseArgs() {
+  for i in $*
+  do
+    case $i in
+    --jenkins)
+      JENKINS=true
       ;;
-    DEVELOPER)
-      ### Set HUDSON to false to indicate that this script is being run by a developer
-      HUDSON=false
-      if [[ $# != 9 ]] ; then
-        echo "ERROR: usage $0 DEVELOPER <PATCH_FILE> <SCRATCH_DIR> <SVN_CMD> <GREP_CMD> <PATCH_CMD> <FINDBUGS_HOME> <FORREST_HOME> <WORKSPACE_BASEDIR>"
-        cleanupAndExit 0
-      fi
-      ### PATCH_FILE contains the location of the patchfile
-      PATCH_FILE=$2 
-      if [[ ! -e "$PATCH_FILE" ]] ; then
-        echo "Unable to locate the patch file $PATCH_FILE"
-        cleanupAndExit 0
-      fi
-      PATCH_DIR=$3
-      ### Check if $PATCH_DIR exists. If it does not exist, create a new directory
-      if [[ ! -e "$PATCH_DIR" ]] ; then
-	mkdir "$PATCH_DIR"
-	if [[ $? == 0 ]] ; then 
-	  echo "$PATCH_DIR has been created"
-	else
-	  echo "Unable to create $PATCH_DIR"
-	  cleanupAndExit 0
-	fi
-      fi
-      SVN=$4
-      GREP=$5
-      PATCH=$6
-      FINDBUGS_HOME=$7
-      FORREST_HOME=$8
-      BASEDIR=$9
-      PS=${PS:-ps}
-      ### Obtain the patch filename to append it to the version number
-      defect=`basename $PATCH_FILE` 
+    --patch-dir=*)
+      PATCH_DIR=${i#*=}
+      ;;
+    --support-dir=*)
+      SUPPORT_DIR=${i#*=}
+      ;;
+    --basedir=*)
+      BASEDIR=${i#*=}
+      ;;
+    --mvn-cmd=*)
+      MVN=${i#*=}
+      ;;
+    --ps-cmd=*)
+      PS=${i#*=}
+      ;;
+    --awk-cmd=*)
+      AWK=${i#*=}
+      ;;
+    --wget-cmd=*)
+      WGET=${i#*=}
+      ;;
+    --svn-cmd=*)
+      SVN=${i#*=}
+      ;;
+    --grep-cmd=*)
+      GREP=${i#*=}
+      ;;
+    --patch-cmd=*)
+      PATCH=${i#*=}
+      ;;
+    --jira-cmd=*)
+      JIRACLI=${i#*=}
+      ;;
+    --jira-password=*)
+      JIRA_PASSWD=${i#*=}
+      ;;
+    --findbugs-home=*)
+      FINDBUGS_HOME=${i#*=}
+      ;;
+    --forrest-home=*)
+      FORREST_HOME=${i#*=}
+      ;;
+    --eclipse-home=*)
+      ECLIPSE_HOME=${i#*=}
+      ;;
+    --dirty-workspace)
+      DIRTY_WORKSPACE=true
       ;;
     *)
-      echo "ERROR: usage $0 HUDSON [args] | DEVELOPER [args]"
-      cleanupAndExit 0
+      PATCH_OR_DEFECT=$i
       ;;
-  esac
+    esac
+  done
+  if [ -z "$PATCH_OR_DEFECT" ]; then
+    printUsage
+    exit 1
+  fi
+  if [[ $JENKINS == "true" ]] ; then
+    echo "Running in Jenkins mode"
+    defect=$PATCH_OR_DEFECT
+    ECLIPSE_PROPERTY="-Declipse.home=$ECLIPSE_HOME"
+  else
+    echo "Running in developer mode"
+    JENKINS=false
+    ### PATCH_FILE contains the location of the patchfile
+    PATCH_FILE=$PATCH_OR_DEFECT
+    if [[ ! -e "$PATCH_FILE" ]] ; then
+      echo "Unable to locate the patch file $PATCH_FILE"
+      cleanupAndExit 0
+    fi
+    ### Check if $PATCH_DIR exists. If it does not exist, create a new directory
+    if [[ ! -e "$PATCH_DIR" ]] ; then
+      mkdir "$PATCH_DIR"
+      if [[ $? == 0 ]] ; then 
+        echo "$PATCH_DIR has been created"
+      else
+        echo "Unable to create $PATCH_DIR"
+        cleanupAndExit 0
+      fi
+    fi
+    ### Obtain the patch filename to append it to the version number
+    defect=`basename $PATCH_FILE`
+  fi
 }
 
 ###############################################################################
@@ -120,9 +180,10 @@ checkout () {
   echo ""
   echo ""
   ### When run by a developer, if the workspace contains modifications, do not continue
+  ### unless the --dirty-workspace option was set
   status=`$SVN stat --ignore-externals | sed -e '/^X[ ]*/D'`
-  if [[ $HUDSON == "false" ]] ; then
-    if [[ "$status" != "" ]] ; then
+  if [[ $JENKINS == "false" ]] ; then
+    if [[ "$status" != "" && -z $DIRTY_WORKSPACE ]] ; then
       echo "ERROR: can't run in a workspace that contains the following modifications"
       echo "$status"
       cleanupAndExit 1
@@ -140,7 +201,7 @@ checkout () {
 ###############################################################################
 setup () {
   ### Download latest patch file (ignoring .htm and .html) when run from patch process
-  if [[ $HUDSON == "true" ]] ; then
+  if [[ $JENKINS == "true" ]] ; then
     $WGET -q -O $PATCH_DIR/jira http://issues.apache.org/jira/browse/$defect
     if [[ `$GREP -c 'Patch Available' $PATCH_DIR/jira` == 0 ]] ; then
       echo "$defect is not \"Patch Available\".  Exiting."
@@ -188,8 +249,7 @@ setup () {
   echo "======================================================================"
   echo ""
   echo ""
-#  echo "$ANT_HOME/bin/ant  -Djavac.args="-Xlint -Xmaxwarns 1000" $ECLIPSE_PROPERTY -Dforrest.home=${FORREST_HOME} -D${PROJECT_NAME}PatchProcess= clean tar > $PATCH_DIR/trunkJavacWarnings.txt 2>&1"
-# $ANT_HOME/bin/ant -Djavac.args="-Xlint -Xmaxwarns 1000" $ECLIPSE_PROPERTY -Dforrest.home=${FORREST_HOME} -D${PROJECT_NAME}PatchProcess= clean tar > $PATCH_DIR/trunkJavacWarnings.txt 2>&1
+  echo "$MVN clean compile -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch > $PATCH_DIR/trunkJavacWarnings.txt 2>&1"
   $MVN clean compile -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch > $PATCH_DIR/trunkJavacWarnings.txt 2>&1
   if [[ $? != 0 ]] ; then
     echo "Trunk compilation is broken?"
@@ -238,7 +298,7 @@ checkTests () {
   testReferences=`$GREP -c -i '/test' $PATCH_DIR/patch`
   echo "There appear to be $testReferences test files referenced in the patch."
   if [[ $testReferences == 0 ]] ; then
-    if [[ $HUDSON == "true" ]] ; then
+    if [[ $JENKINS == "true" ]] ; then
       patchIsDoc=`$GREP -c -i 'title="documentation' $PATCH_DIR/jira`
       if [[ $patchIsDoc != 0 ]] ; then
         echo "The patch appears to be a documentation patch that doesn't require tests."
@@ -306,10 +366,9 @@ checkJavadocWarnings () {
   echo "======================================================================"
   echo ""
   echo ""
-  echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= clean javadoc | tee $PATCH_DIR/patchJavadocWarnings.txt"
+  echo "$MVN clean compile javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchJavadocWarnings.txt 2>&1"
   (cd hadoop-project; mvn install)
   (cd hadoop-annotations; mvn install)
-  #$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= clean javadoc | tee $PATCH_DIR/patchJavadocWarnings.txt
   $MVN clean compile javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchJavadocWarnings.txt 2>&1
   javadocWarnings=`$GREP '\[WARNING\]' $PATCH_DIR/patchJavadocWarnings.txt | $AWK '/Javadoc Warnings/,EOF' | $GREP warning | $AWK 'BEGIN {total = 0} {total += 1} END {print total}'`
   echo ""
@@ -341,8 +400,7 @@ checkJavacWarnings () {
   echo "======================================================================"
   echo ""
   echo ""
-  #echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" -Djavac.args="-Xlint -Xmaxwarns 1000" $ECLIPSE_PROPERTY -Dforrest.home=${FORREST_HOME} -DHadoopPatchProcess= clean tar > $PATCH_DIR/patchJavacWarnings.txt 2>&1"
-  #$ANT_HOME/bin/ant -Dversion="${VERSION}" -Djavac.args="-Xlint -Xmaxwarns 1000" $ECLIPSE_PROPERTY -Dforrest.home=${FORREST_HOME} -DHadoopPatchProcess= clean tar > $PATCH_DIR/patchJavacWarnings.txt 2>&1
+  echo "$MVN clean compile -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch > $PATCH_DIR/patchJavacWarnings.txt 2>&1"
   $MVN clean compile -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch > $PATCH_DIR/patchJavacWarnings.txt 2>&1
   if [[ $? != 0 ]] ; then
     JIRA_COMMENT="$JIRA_COMMENT
@@ -382,8 +440,7 @@ checkReleaseAuditWarnings () {
   echo "======================================================================"
   echo ""
   echo ""
-  #echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" -Dforrest.home=${FORREST_HOME} -DHadoopPatchProcess= releaseaudit > $PATCH_DIR/patchReleaseAuditWarnings.txt 2>&1"
-  #$ANT_HOME/bin/ant -Dversion="${VERSION}" -Dforrest.home=${FORREST_HOME} -DHadoopPatchProcess= releaseaudit > $PATCH_DIR/patchReleaseAuditWarnings.txt 2>&1
+  echo "$MVN apache-rat:check -D${PROJECT_NAME}PatchProcess 2>&1"
   $MVN apache-rat:check -D${PROJECT_NAME}PatchProcess 2>&1
   find . -name rat.txt | xargs cat > $PATCH_DIR/patchReleaseAuditWarnings.txt
 
@@ -427,8 +484,7 @@ checkStyle () {
   echo "THIS IS NOT IMPLEMENTED YET"
   echo ""
   echo ""
-  #echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= checkstyle"
-  #$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= checkstyle
+  echo "$MVN compile checkstyle:checkstyle -D${PROJECT_NAME}PatchProcess"
   $MVN compile checkstyle:checkstyle -D${PROJECT_NAME}PatchProcess
 
   JIRA_COMMENT_FOOTER="Checkstyle results: $BUILD_URL/artifact/trunk/build/test/checkstyle-errors.html
@@ -460,8 +516,7 @@ checkFindbugsWarnings () {
   echo "======================================================================"
   echo ""
   echo ""
-  #echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" -Dfindbugs.home=$FINDBUGS_HOME -Dforrest.home=${FORREST_HOME} -DHadoopPatchProcess= findbugs"
-  #$ANT_HOME/bin/ant -Dversion="${VERSION}" -Dfindbugs.home=${FINDBUGS_HOME} -Dforrest.home=${FORREST_HOME} -DHadoopPatchProcess= findbugs
+  echo "$MVN clean compile findbugs:findbugs -D${PROJECT_NAME}PatchProcess"
   $MVN clean compile findbugs:findbugs -D${PROJECT_NAME}PatchProcess
 
   if [ $? != 0 ] ; then
@@ -510,15 +565,14 @@ runCoreTests () {
   echo ""
   
   ### Kill any rogue build processes from the last attempt
-  $PS auxwww | $GREP HadoopPatchProcess | $AWK '{print $2}' | /usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null
+  $PS auxwww | $GREP ${PROJECT_NAME}PatchProcess | $AWK '{print $2}' | /usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null
   PreTestTarget=""
   if [[ $defect == MAPREDUCE-* ]] ; then
      PreTestTarget="create-c++-configure"
   fi
 
-  #echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no -Dcompile.c++=yes -Dforrest.home=$FORREST_HOME $PreTestTarget test-core"
-  #$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no -Dcompile.c++=yes -Dforrest.home=$FORREST_HOME  $PreTestTarget test-core
-  $MVN clean test -Pnative -DHadoopPatchProcess
+  echo "$MVN clean test -Pnative -D${PROJECT_NAME}PatchProcess"
+  $MVN clean test -Pnative -D${PROJECT_NAME}PatchProcess
   if [[ $? != 0 ]] ; then
     ### Find and format names of failed tests
     failed_tests=`grep -l -E "<failure|<error" $WORKSPACE/trunk/target/hadoop-common/surefire-reports/*.xml | sed -e "s|.*target/surefire-reports/TEST-|                  |g" | sed -e "s|\.xml||g"`
@@ -553,7 +607,7 @@ runContribTests () {
   fi
 
   ### Kill any rogue build processes from the last attempt
-  $PS auxwww | $GREP HadoopPatchProcess | $AWK '{print $2}' | /usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null
+  $PS auxwww | $GREP ${PROJECT_NAME}PatchProcess | $AWK '{print $2}' | /usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null
 
   #echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" $ECLIPSE_PROPERTY -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no test-contrib"
   #$ANT_HOME/bin/ant -Dversion="${VERSION}" $ECLIPSE_PROPERTY -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no test-contrib
@@ -584,7 +638,7 @@ checkInjectSystemFaults () {
   echo ""
   
   ### Kill any rogue build processes from the last attempt
-  $PS auxwww | $GREP HadoopPatchProcess | $AWK '{print $2}' | /usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null
+  $PS auxwww | $GREP ${PROJECT_NAME}PatchProcess | $AWK '{print $2}' | /usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null
 
   #echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no -Dcompile.c++=yes -Dforrest.home=$FORREST_HOME inject-system-faults"
   #$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no -Dcompile.c++=yes -Dforrest.home=$FORREST_HOME inject-system-faults
@@ -606,7 +660,7 @@ checkInjectSystemFaults () {
 submitJiraComment () {
   local result=$1
   ### Do not output the value of JIRA_COMMENT_FOOTER when run by a developer
-  if [[  $HUDSON == "false" ]] ; then
+  if [[  $JENKINS == "false" ]] ; then
     JIRA_COMMENT_FOOTER=""
   fi
   if [[ $result == 0 ]] ; then
@@ -625,7 +679,7 @@ $JIRA_COMMENT_FOOTER"
 
 $comment"  
 
-  if [[ $HUDSON == "true" ]] ; then
+  if [[ $JENKINS == "true" ]] ; then
     echo ""
     echo ""
     echo "======================================================================"
@@ -646,7 +700,7 @@ $comment"
 ### Cleanup files
 cleanupAndExit () {
   local result=$1
-  if [[ $HUDSON == "true" ]] ; then
+  if [[ $JENKINS == "true" ]] ; then
     if [ -e "$PATCH_DIR" ] ; then
       mv $PATCH_DIR $BASEDIR
     fi
@@ -678,7 +732,7 @@ cd $BASEDIR
 
 checkout
 RESULT=$?
-if [[ $HUDSON == "true" ]] ; then
+if [[ $JENKINS == "true" ]] ; then
   if [[ $RESULT != 0 ]] ; then
     exit 100
   fi
@@ -687,7 +741,7 @@ setup
 checkAuthor
 RESULT=$?
 
-if [[ $HUDSON == "true" ]] ; then
+if [[ $JENKINS == "true" ]] ; then
   cleanUpXml
 fi
 checkTests
@@ -709,7 +763,7 @@ checkFindbugsWarnings
 checkReleaseAuditWarnings
 (( RESULT = RESULT + $? ))
 ### Do not call these when run by a developer 
-if [[ $HUDSON == "true" ]] ; then
+if [[ $JENKINS == "true" ]] ; then
   runCoreTests
   (( RESULT = RESULT + $? ))
   runContribTests

hadoop-common/CHANGES.txt

@@ -309,6 +309,8 @@ Trunk (unreleased changes)
     HADOOP-7501. Publish Hadoop Common artifacts (post HADOOP-6671) to Apache
     SNAPSHOTs repo. (Alejandro Abdelnur via tomwhite)
 
+    HADOOP-7525. Make arguments to test-patch optional. (tomwhite)
+
   OPTIMIZATIONS
   
     HADOOP-7333. Performance improvement in PureJavaCrc32. (Eric Caspole