HADOOP-12651. Replace dev-support with wrappers to Yetus (aw)

(cherry picked from commit 8cecad2d564d89d0f56e98de80a6f9b0bcb39d7a)

 Conflicts:
	dev-support/test-patch.sh
	hadoop-common-project/hadoop-common/pom.xml

BUILDING.txt

@@ -37,7 +37,7 @@ and all required tools for testing and building have been installed and configur
 
 Note that from within this docker environment you ONLY have access to the Hadoop source
 tree from where you started. So if you need to run
-    dev-support/test-patch.sh /path/to/my.patch
+    dev-support/bin/test-patch /path/to/my.patch
 then the patch must be placed inside the hadoop source tree.
 
 Known issues:

dev-support/releasedocmaker.py

@@ -1,580 +0,0 @@
-#!/usr/bin/env python
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from glob import glob
-from optparse import OptionParser
-from time import gmtime, strftime
-import pprint
-import os
-import re
-import sys
-import urllib
-import urllib2
-try:
-  import json
-except ImportError:
-  import simplejson as json
-
-releaseVersion={}
-namePattern = re.compile(r' \([0-9]+\)')
-
-asflicense='''
-<!---
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
--->
-'''
-
-def clean(str):
-  return tableclean(re.sub(namePattern, "", str))
-
-def formatComponents(str):
-  str = re.sub(namePattern, '', str).replace("'", "")
-  if str != "":
-    ret = str
-  else:
-    # some markdown parsers don't like empty tables
-    ret = "."
-  return clean(ret)
-
-# convert to utf-8
-# protect some known md metachars
-# or chars that screw up doxia
-def tableclean(str):
-  str=str.encode('utf-8')
-  str=str.replace("_","\_")
-  str=str.replace("\r","")
-  str=str.rstrip()
-  return str
-
-# same thing as tableclean,
-# except table metachars are also
-# escaped as well as more
-# things we don't want doxia to
-# screw up
-def notableclean(str):
-  str=tableclean(str)
-  str=str.replace("|","\|")
-  str=str.replace("<","\<")
-  str=str.replace(">","\>")
-  str=str.replace("*","\*")
-  str=str.rstrip()
-  return str
-
-# clean output dir
-def cleanOutputDir(dir):
-    files = os.listdir(dir)
-    for name in files:
-        os.remove(os.path.join(dir,name))
-    os.rmdir(dir)
-
-def mstr(obj):
-  if (obj is None):
-    return ""
-  return unicode(obj)
-
-def buildindex(title,license):
-  versions=reversed(sorted(glob("[0-9]*.[0-9]*.[0-9]*")))
-  with open("index.md","w") as indexfile:
-    if license is True:
-      indexfile.write(asflicense)
-    for v in versions:
-      indexfile.write("* %s v%s\n" % (title,v))
-      for k in ("Changes","Release Notes"):
-        indexfile.write("    * %s (%s/%s.%s.html)\n" \
-          % (k,v,k.upper().replace(" ",""),v))
-  indexfile.close()
-
-class GetVersions:
-  """ yo """
-  def __init__(self,versions, projects):
-    versions = versions
-    projects = projects
-    self.newversions = []
-    pp = pprint.PrettyPrinter(indent=4)
-    at=0
-    end=1
-    count=100
-    versions.sort()
-    print "Looking for %s through %s"%(versions[0],versions[-1])
-    for p in projects:
-      resp = urllib2.urlopen("https://issues.apache.org/jira/rest/api/2/project/%s/versions"%p)
-      data = json.loads(resp.read())
-      for d in data:
-        if d['name'][0].isdigit and versions[0] <= d['name'] and d['name'] <= versions[-1]:
-          print "Adding %s to the list" % d['name']
-          self.newversions.append(d['name'])
-    newlist=list(set(self.newversions))
-    self.newversions=newlist
-
-  def getlist(self):
-      pp = pprint.PrettyPrinter(indent=4)
-      return(self.newversions)
-
-class Version:
-  """Represents a version number"""
-  def __init__(self, data):
-    self.mod = False
-    self.data = data
-    found = re.match('^((\d+)(\.\d+)*).*$', data)
-    if (found):
-      self.parts = [ int(p) for p in found.group(1).split('.') ]
-    else:
-      self.parts = []
-    # backfill version with zeroes if missing parts
-    self.parts.extend((0,) * (3 - len(self.parts)))
-
-  def __str__(self):
-    if (self.mod):
-      return '.'.join([ str(p) for p in self.parts ])
-    return self.data
-
-  def __cmp__(self, other):
-    return cmp(self.parts, other.parts)
-
-class Jira:
-  """A single JIRA"""
-
-  def __init__(self, data, parent):
-    self.key = data['key']
-    self.fields = data['fields']
-    self.parent = parent
-    self.notes = None
-    self.incompat = None
-    self.reviewed = None
-
-  def getId(self):
-    return mstr(self.key)
-
-  def getDescription(self):
-    return mstr(self.fields['description'])
-
-  def getReleaseNote(self):
-    if (self.notes is None):
-      field = self.parent.fieldIdMap['Release Note']
-      if (self.fields.has_key(field)):
-        self.notes=mstr(self.fields[field])
-      else:
-        self.notes=self.getDescription()
-    return self.notes
-
-  def getPriority(self):
-    ret = ""
-    pri = self.fields['priority']
-    if(pri is not None):
-      ret = pri['name']
-    return mstr(ret)
-
-  def getAssignee(self):
-    ret = ""
-    mid = self.fields['assignee']
-    if(mid is not None):
-      ret = mid['displayName']
-    return mstr(ret)
-
-  def getComponents(self):
-    if (len(self.fields['components'])>0):
-      return ", ".join([ comp['name'] for comp in self.fields['components'] ])
-    else:
-      return ""
-
-  def getSummary(self):
-    return self.fields['summary']
-
-  def getType(self):
-    ret = ""
-    mid = self.fields['issuetype']
-    if(mid is not None):
-      ret = mid['name']
-    return mstr(ret)
-
-  def getReporter(self):
-    ret = ""
-    mid = self.fields['reporter']
-    if(mid is not None):
-      ret = mid['displayName']
-    return mstr(ret)
-
-  def getProject(self):
-    ret = ""
-    mid = self.fields['project']
-    if(mid is not None):
-      ret = mid['key']
-    return mstr(ret)
-
-  def __cmp__(self,other):
-    selfsplit=self.getId().split('-')
-    othersplit=other.getId().split('-')
-    v1=cmp(selfsplit[0],othersplit[0])
-    if (v1!=0):
-      return v1
-    else:
-      if selfsplit[1] < othersplit[1]:
-        return True
-      elif selfsplit[1] > othersplit[1]:
-        return False
-    return False
-
-  def getIncompatibleChange(self):
-    if (self.incompat is None):
-      field = self.parent.fieldIdMap['Hadoop Flags']
-      self.reviewed=False
-      self.incompat=False
-      if (self.fields.has_key(field)):
-        if self.fields[field]:
-          for hf in self.fields[field]:
-            if hf['value'] == "Incompatible change":
-              self.incompat=True
-            if hf['value'] == "Reviewed":
-              self.reviewed=True
-    return self.incompat
-
-  def checkMissingComponent(self):
-      if (len(self.fields['components'])>0):
-          return False
-      return True
-
-  def checkMissingAssignee(self):
-      if (self.fields['assignee'] is not None):
-          return False
-      return True
-
-  def checkVersionString(self):
-      field = self.parent.fieldIdMap['Fix Version/s']
-      for h in self.fields[field]:
-          found = re.match('^((\d+)(\.\d+)*).*$|^(\w+\-\d+)$', h['name'])
-          if not found:
-              return True
-      return False
-
-  def getReleaseDate(self,version):
-    for j in range(len(self.fields['fixVersions'])):
-      if self.fields['fixVersions'][j]==version:
-        return(self.fields['fixVersions'][j]['releaseDate'])
-    return None
-
-class JiraIter:
-  """An Iterator of JIRAs"""
-
-  def __init__(self, version, projects):
-    self.version = version
-    self.projects = projects
-    v=str(version).replace("-SNAPSHOT","")
-
-    resp = urllib2.urlopen("https://issues.apache.org/jira/rest/api/2/field")
-    data = json.loads(resp.read())
-
-    self.fieldIdMap = {}
-    for part in data:
-      self.fieldIdMap[part['name']] = part['id']
-
-    self.jiras = []
-    at=0
-    end=1
-    count=100
-    while (at < end):
-      params = urllib.urlencode({'jql': "project in ('"+"' , '".join(projects)+"') and fixVersion in ('"+v+"') and resolution = Fixed", 'startAt':at, 'maxResults':count})
-      resp = urllib2.urlopen("https://issues.apache.org/jira/rest/api/2/search?%s"%params)
-      data = json.loads(resp.read())
-      if (data.has_key('errorMessages')):
-        raise Exception(data['errorMessages'])
-      at = data['startAt'] + data['maxResults']
-      end = data['total']
-      self.jiras.extend(data['issues'])
-
-      needaversion=False
-      if v not in releaseVersion:
-        needaversion=True
-
-      if needaversion is True:
-        for i in range(len(data['issues'])):
-          for j in range(len(data['issues'][i]['fields']['fixVersions'])):
-            if 'releaseDate' in data['issues'][i]['fields']['fixVersions'][j]:
-              releaseVersion[data['issues'][i]['fields']['fixVersions'][j]['name']]=\
-                             data['issues'][i]['fields']['fixVersions'][j]['releaseDate']
-
-    self.iter = self.jiras.__iter__()
-
-  def __iter__(self):
-    return self
-
-  def next(self):
-    data = self.iter.next()
-    j = Jira(data, self)
-    return j
-
-class Outputs:
-  """Several different files to output to at the same time"""
-
-  def __init__(self, base_file_name, file_name_pattern, keys, params={}):
-    self.params = params
-    self.base = open(base_file_name%params, 'w')
-    self.others = {}
-    for key in keys:
-      both = dict(params)
-      both['key'] = key
-      self.others[key] = open(file_name_pattern%both, 'w')
-
-  def writeAll(self, pattern):
-    both = dict(self.params)
-    both['key'] = ''
-    self.base.write(pattern%both)
-    for key in self.others.keys():
-      both = dict(self.params)
-      both['key'] = key
-      self.others[key].write(pattern%both)
-
-  def writeKeyRaw(self, key, str):
-    self.base.write(str)
-    if (self.others.has_key(key)):
-      self.others[key].write(str)
-
-  def close(self):
-    self.base.close()
-    for fd in self.others.values():
-      fd.close()
-
-  def writeList(self, mylist):
-    for jira in sorted(mylist):
-      line = '| [%s](https://issues.apache.org/jira/browse/%s) | %s |  %s | %s | %s | %s |\n' \
-        % (notableclean(jira.getId()), notableclean(jira.getId()),
-           notableclean(jira.getSummary()),
-           notableclean(jira.getPriority()),
-           formatComponents(jira.getComponents()),
-           notableclean(jira.getReporter()),
-           notableclean(jira.getAssignee()))
-      self.writeKeyRaw(jira.getProject(), line)
-
-def main():
-  parser = OptionParser(usage="usage: %prog --project PROJECT [--project PROJECT] --version VERSION [--version VERSION2 ...]",
-		epilog=
-               "Markdown-formatted CHANGES and RELEASENOTES files will be stored in a directory"
-               " named after the highest version provided.")
-  parser.add_option("-i","--index", dest="index", action="store_true",
-             default=False, help="build an index file")
-  parser.add_option("-l","--license", dest="license", action="store_false",
-             default=True, help="Add an ASF license")
-  parser.add_option("-n","--lint", dest="lint", action="store_true",
-             help="use lint flag to exit on failures")
-  parser.add_option("-p", "--project", dest="projects",
-             action="append", type="string",
-             help="projects in JIRA to include in releasenotes", metavar="PROJECT")
-  parser.add_option("-r", "--range", dest="range", action="store_true",
-             default=False, help="Given versions are a range")
-  parser.add_option("-t", "--projecttitle", dest="title",
-             type="string",
-             help="Title to use for the project (default is Apache PROJECT)")
-  parser.add_option("-u","--usetoday", dest="usetoday", action="store_true",
-             default=False, help="use current date for unreleased versions")
-  parser.add_option("-v", "--version", dest="versions",
-             action="append", type="string",
-             help="versions in JIRA to include in releasenotes", metavar="VERSION")
-  (options, args) = parser.parse_args()
-
-  if (options.versions is None):
-    options.versions = []
-
-  if (len(args) > 2):
-    options.versions.append(args[2])
-
-  if (len(options.versions) <= 0):
-    parser.error("At least one version needs to be supplied")
-
-  proxy = urllib2.ProxyHandler()
-  opener = urllib2.build_opener(proxy)
-  urllib2.install_opener(opener)
-
-  projects = options.projects
-
-  if (options.range is True):
-    versions = [ Version(v) for v in GetVersions(options.versions, projects).getlist() ]
-  else:
-    versions = [ Version(v) for v in options.versions ]
-  versions.sort();
-
-  if (options.title is None):
-    title=projects[0]
-  else:
-    title=options.title
-
-  haderrors=False
-
-  for v in versions:
-    vstr=str(v)
-    jlist = JiraIter(vstr,projects)
-
-    if vstr in releaseVersion:
-      reldate=releaseVersion[vstr]
-    elif options.usetoday:
-      reldate=strftime("%Y-%m-%d", gmtime())
-    else:
-      reldate="Unreleased"
-
-    if not os.path.exists(vstr):
-      os.mkdir(vstr)
-
-    reloutputs = Outputs("%(ver)s/RELEASENOTES.%(ver)s.md",
-      "%(ver)s/RELEASENOTES.%(key)s.%(ver)s.md",
-      [], {"ver":v, "date":reldate, "title":title})
-    choutputs = Outputs("%(ver)s/CHANGES.%(ver)s.md",
-      "%(ver)s/CHANGES.%(key)s.%(ver)s.md",
-      [], {"ver":v, "date":reldate, "title":title})
-
-    if (options.license is True):
-      reloutputs.writeAll(asflicense)
-      choutputs.writeAll(asflicense)
-
-    relhead = '# %(title)s %(key)s %(ver)s Release Notes\n\n' \
-      'These release notes cover new developer and user-facing incompatibilities, features, and major improvements.\n\n'
-    chhead = '# %(title)s Changelog\n\n' \
-      '## Release %(ver)s - %(date)s\n'\
-      '\n'
-
-    reloutputs.writeAll(relhead)
-    choutputs.writeAll(chhead)
-    errorCount=0
-    warningCount=0
-    lintMessage=""
-    incompatlist=[]
-    buglist=[]
-    improvementlist=[]
-    newfeaturelist=[]
-    subtasklist=[]
-    tasklist=[]
-    testlist=[]
-    otherlist=[]
-
-    for jira in sorted(jlist):
-      if jira.getIncompatibleChange():
-        incompatlist.append(jira)
-      elif jira.getType() == "Bug":
-        buglist.append(jira)
-      elif jira.getType() == "Improvement":
-        improvementlist.append(jira)
-      elif jira.getType() == "New Feature":
-        newfeaturelist.append(jira)
-      elif jira.getType() == "Sub-task":
-        subtasklist.append(jira)
-      elif jira.getType() == "Task":
-       tasklist.append(jira)
-      elif jira.getType() == "Test":
-        testlist.append(jira)
-      else:
-         otherlist.append(jira)
-
-      line = '* [%s](https://issues.apache.org/jira/browse/%s) | *%s* | **%s**\n' \
-          % (notableclean(jira.getId()), notableclean(jira.getId()), notableclean(jira.getPriority()),
-             notableclean(jira.getSummary()))
-
-      if (jira.getIncompatibleChange()) and (len(jira.getReleaseNote())==0):
-        warningCount+=1
-        reloutputs.writeKeyRaw(jira.getProject(),"\n---\n\n")
-        reloutputs.writeKeyRaw(jira.getProject(), line)
-        line ='\n**WARNING: No release note provided for this incompatible change.**\n\n'
-        lintMessage += "\nWARNING: incompatible change %s lacks release notes." % (notableclean(jira.getId()))
-        reloutputs.writeKeyRaw(jira.getProject(), line)
-
-      if jira.checkVersionString():
-          warningCount+=1
-          lintMessage += "\nWARNING: Version string problem for %s " % jira.getId()
-
-      if (jira.checkMissingComponent() or jira.checkMissingAssignee()):
-          errorCount+=1
-          errorMessage=[]
-          jira.checkMissingComponent() and errorMessage.append("component")
-          jira.checkMissingAssignee() and errorMessage.append("assignee")
-          lintMessage += "\nERROR: missing %s for %s " %  (" and ".join(errorMessage) , jira.getId())
-
-      if (len(jira.getReleaseNote())>0):
-        reloutputs.writeKeyRaw(jira.getProject(),"\n---\n\n")
-        reloutputs.writeKeyRaw(jira.getProject(), line)
-        line ='\n%s\n\n' % (tableclean(jira.getReleaseNote()))
-        reloutputs.writeKeyRaw(jira.getProject(), line)
-
-    if (options.lint is True):
-        print lintMessage
-        print "======================================="
-        print "%s: Error:%d, Warning:%d \n" % (vstr, errorCount, warningCount)
-        if (errorCount>0):
-           haderrors=True
-           cleanOutputDir(vstr)
-           continue
-
-    reloutputs.writeAll("\n\n")
-    reloutputs.close()
-
-    choutputs.writeAll("### INCOMPATIBLE CHANGES:\n\n")
-    choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
-    choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
-    choutputs.writeList(incompatlist)
-
-    choutputs.writeAll("\n\n### NEW FEATURES:\n\n")
-    choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
-    choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
-    choutputs.writeList(newfeaturelist)
-
-    choutputs.writeAll("\n\n### IMPROVEMENTS:\n\n")
-    choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
-    choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
-    choutputs.writeList(improvementlist)
-
-    choutputs.writeAll("\n\n### BUG FIXES:\n\n")
-    choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
-    choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
-    choutputs.writeList(buglist)
-
-    choutputs.writeAll("\n\n### TESTS:\n\n")
-    choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
-    choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
-    choutputs.writeList(testlist)
-
-    choutputs.writeAll("\n\n### SUB-TASKS:\n\n")
-    choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
-    choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
-    choutputs.writeList(subtasklist)
-
-    choutputs.writeAll("\n\n### OTHER:\n\n")
-    choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
-    choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
-    choutputs.writeList(otherlist)
-    choutputs.writeList(tasklist)
-
-    choutputs.writeAll("\n\n")
-    choutputs.close()
-
-  if options.index:
-    buildindex(title,options.license)
-
-  if haderrors is True:
-    sys.exit(1)
-
-if __name__ == "__main__":
-  main()

dev-support/smart-apply-patch.sh

@@ -1,187 +0,0 @@
-#!/usr/bin/env bash
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-
-#
-# Determine if the git diff patch file has prefixes.
-# These files are generated via "git diff" *without* the --no-prefix option.
-#
-# We can apply these patches more easily because we know that the a/ and b/
-# prefixes in the "diff" lines stands for the project root directory.
-# So we don't have to hunt for the project root.
-# And of course, we know that the patch file was generated using git, so we
-# know git apply can handle it properly.
-#
-# Arguments: git diff file name.
-# Return: 0 if it is a git diff with prefix; 1 otherwise.
-#
-has_prefix() {
-  awk '/^diff --git / { if ($3 !~ "^a/" || $4 !~ "^b/") { exit 1 } }
-    /^\+{3}|-{3} / { if ($2 !~ "^[ab]/" && $2 !~ "^/dev/null") { exit 1 } }' "$1"
-  return $?
-}
-
-PATCH_FILE=$1
-DRY_RUN=$2
-if [ -z "$PATCH_FILE" ]; then
-  echo usage: $0 patch-file
-  exit 1
-fi
-
-TMPDIR=${TMPDIR:-/tmp}
-PATCH=${PATCH:-patch} # allow overriding patch binary
-
-# Cleanup handler for temporary files
-TOCLEAN=""
-cleanup() {
-  rm $TOCLEAN
-  exit $1
-}
-trap "cleanup 1" HUP INT QUIT TERM
-
-# Allow passing "-" for stdin patches
-if [ "$PATCH_FILE" == "-" ]; then
-  PATCH_FILE="$TMPDIR/smart-apply.in.$RANDOM"
-  cat /dev/fd/0 > $PATCH_FILE
-  TOCLEAN="$TOCLEAN $PATCH_FILE"
-fi
-
-ISSUE_RE='^(HADOOP|YARN|MAPREDUCE|HDFS)-[0-9]+$'
-if [[ ${PATCH_FILE} =~ ^http || ${PATCH_FILE} =~ ${ISSUE_RE} ]]; then
-  # Allow downloading of patches
-  PFILE="$TMPDIR/smart-apply.in.$RANDOM"
-  TOCLEAN="$TOCLEAN $PFILE"
-  if [[ ${PATCH_FILE} =~ ^http ]]; then
-    patchURL="${PATCH_FILE}"
-  else # Get URL of patch from JIRA
-    wget -q -O "${PFILE}" "http://issues.apache.org/jira/browse/${PATCH_FILE}"
-    if [[ $? != 0 ]]; then
-      echo "Unable to determine what ${PATCH_FILE} may reference." 1>&2
-      cleanup 1
-    elif [[ $(grep -c 'Patch Available' "${PFILE}") == 0 ]]; then
-      echo "${PATCH_FILE} is not \"Patch Available\".  Exiting." 1>&2
-      cleanup 1
-    fi
-    relativePatchURL=$(grep -o '"/jira/secure/attachment/[0-9]*/[^"]*' "${PFILE}" | grep -v -e 'htm[l]*$' | sort | tail -1 | grep -o '/jira/secure/attachment/[0-9]*/[^"]*')
-    patchURL="http://issues.apache.org${relativePatchURL}"
-  fi
-  if [[ -n $DRY_RUN ]]; then
-    echo "Downloading ${patchURL}"
-  fi
-  wget -q -O "${PFILE}" "${patchURL}"
-  if [[ $? != 0 ]]; then
-    echo "${PATCH_FILE} could not be downloaded." 1>&2
-    cleanup 1
-  fi
-  PATCH_FILE="${PFILE}"
-fi
-
-# Case for git-diff patches
-if grep -q "^diff --git" "${PATCH_FILE}"; then
-  GIT_FLAGS="--binary -v"
-  if has_prefix "$PATCH_FILE"; then
-    GIT_FLAGS="$GIT_FLAGS -p1"
-  else
-    GIT_FLAGS="$GIT_FLAGS -p0"
-  fi
-  if [[ -z $DRY_RUN ]]; then
-    GIT_FLAGS="$GIT_FLAGS --stat --apply"
-    echo Going to apply git patch with: git apply "${GIT_FLAGS}"
-  else
-    GIT_FLAGS="$GIT_FLAGS --check"
-  fi
-  # shellcheck disable=SC2086
-  git apply ${GIT_FLAGS} "${PATCH_FILE}"
-  if [[ $? == 0 ]]; then
-    cleanup 0
-  fi
-  echo "git apply failed. Going to apply the patch with: ${PATCH}"
-fi
-
-# Come up with a list of changed files into $TMP
-TMP="$TMPDIR/smart-apply.paths.$RANDOM"
-TOCLEAN="$TOCLEAN $TMP"
-
-if $PATCH -p0 -E --dry-run < $PATCH_FILE 2>&1 > $TMP; then
-  PLEVEL=0
-  #if the patch applied at P0 there is the possability that all we are doing
-  # is adding new files and they would apply anywhere. So try to guess the
-  # correct place to put those files.
-
-  TMP2="$TMPDIR/smart-apply.paths.2.$RANDOM"
-  TOCLEAN="$TOCLEAN $TMP2"
-
-  egrep '^patching file |^checking file ' $TMP | awk '{print $3}' | grep -v /dev/null | sort -u > $TMP2
-
-  if [ ! -s $TMP2 ]; then
-    echo "Error: Patch dryrun couldn't detect changes the patch would make. Exiting."
-    cleanup 1
-  fi
-
-  #first off check that all of the files do not exist
-  FOUND_ANY=0
-  for CHECK_FILE in $(cat $TMP2)
-  do
-    if [[ -f $CHECK_FILE ]]; then
-      FOUND_ANY=1
-    fi
-  done
-
-  if [[ "$FOUND_ANY" = "0" ]]; then
-    #all of the files are new files so we have to guess where the correct place to put it is.
-
-    # if all of the lines start with a/ or b/, then this is a git patch that
-    # was generated without --no-prefix
-    if ! grep -qv '^a/\|^b/' $TMP2 ; then
-      echo Looks like this is a git patch. Stripping a/ and b/ prefixes
-      echo and incrementing PLEVEL
-      PLEVEL=$[$PLEVEL + 1]
-      sed -i -e 's,^[ab]/,,' $TMP2
-    fi
-
-    PREFIX_DIRS_AND_FILES=$(cut -d '/' -f 1 $TMP2 | sort -u)
- 
-    # if we are at the project root then nothing more to do
-    if [[ -d hadoop-common-project ]]; then
-      echo Looks like this is being run at project root
-
-    # if all of the lines start with hadoop-common/, hadoop-hdfs/, hadoop-yarn/ or hadoop-mapreduce/, this is
-    # relative to the hadoop root instead of the subproject root, so we need
-    # to chop off another layer
-    elif [[ "$PREFIX_DIRS_AND_FILES" =~ ^(hadoop-common-project|hadoop-hdfs-project|hadoop-yarn-project|hadoop-mapreduce-project)$ ]]; then
-
-      echo Looks like this is relative to project root. Increasing PLEVEL
-      PLEVEL=$[$PLEVEL + 1]
-
-    elif ! echo "$PREFIX_DIRS_AND_FILES" | grep -vxq 'hadoop-common-project\|hadoop-hdfs-project\|hadoop-yarn-project\|hadoop-mapreduce-project' ; then
-      echo Looks like this is a cross-subproject patch. Try applying from the project root
-      cleanup 1
-    fi
-  fi
-elif $PATCH -p1 -E --dry-run < $PATCH_FILE 2>&1 > /dev/null; then
-  PLEVEL=1
-elif $PATCH -p2 -E --dry-run < $PATCH_FILE 2>&1 > /dev/null; then
-  PLEVEL=2
-else
-  echo "The patch does not appear to apply with p0 to p2";
-  cleanup 1;
-fi
-
-# If this is a dry run then exit instead of applying the patch
-if [[ -n $DRY_RUN ]]; then
-  cleanup 0;
-fi
-
-echo Going to apply patch with: $PATCH -p$PLEVEL
-$PATCH -p$PLEVEL -E < $PATCH_FILE
-
-cleanup $?

dev-support/test-patch.d/checkstyle.sh

@@ -1,205 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-add_plugin checkstyle
-
-CHECKSTYLE_TIMER=0
-
-# if it ends in an explicit .sh, then this is shell code.
-# if it doesn't have an extension, we assume it is shell code too
-function checkstyle_filefilter
-{
-  local filename=$1
-
-  if [[ ${filename} =~ \.java$ ]]; then
-    add_test checkstyle
-  fi
-}
-
-function checkstyle_mvnrunner
-{
-  local logfile=$1
-  local output=$2
-  local tmp=${PATCH_DIR}/$$.${RANDOM}
-  local j
-
-  "${MVN}" clean test checkstyle:checkstyle -DskipTests \
-    -Dcheckstyle.consoleOutput=true \
-    "-D${PROJECT_NAME}PatchProcess" 2>&1 \
-      | tee "${logfile}" \
-      | ${GREP} ^/ \
-      | ${SED} -e "s,${BASEDIR},.,g" \
-          > "${tmp}"
-
-  # the checkstyle output files are massive, so
-  # let's reduce the work by filtering out files
-  # that weren't changed.  Some modules are
-  # MASSIVE and this can cut the output down to
-  # by orders of magnitude!!
-  for j in ${CHANGED_FILES}; do
-    ${GREP} "${j}" "${tmp}" >> "${output}"
-  done
-
-  rm "${tmp}" 2>/dev/null
-}
-
-function checkstyle_preapply
-{
-  local module_suffix
-  local modules=${CHANGED_MODULES}
-  local module
-
-  verify_needed_test checkstyle
-
-  if [[ $? == 0 ]]; then
-    return 0
-  fi
-
-  big_console_header "checkstyle plugin: prepatch"
-
-  start_clock
-
-  for module in ${modules}
-  do
-    pushd "${module}" >/dev/null
-    echo "  Running checkstyle in ${module}"
-    module_suffix=$(basename "${module}")
-
-    checkstyle_mvnrunner \
-      "${PATCH_DIR}/maven-${PATCH_BRANCH}checkstyle-${module_suffix}.txt" \
-      "${PATCH_DIR}/${PATCH_BRANCH}checkstyle${module_suffix}.txt"
-
-    if [[ $? != 0 ]] ; then
-      echo "Pre-patch ${PATCH_BRANCH} checkstyle compilation is broken?"
-      add_jira_table -1 checkstyle "Pre-patch ${PATCH_BRANCH} ${module} checkstyle compilation may be broken."
-    fi
-    popd >/dev/null
-  done
-
-  # keep track of how much as elapsed for us already
-  CHECKSTYLE_TIMER=$(stop_clock)
-  return 0
-}
-
-function checkstyle_calcdiffs
-{
-  local orig=$1
-  local new=$2
-  local diffout=$3
-  local tmp=${PATCH_DIR}/cs.$$.${RANDOM}
-  local count=0
-  local j
-
-  # first, pull out just the errors
-  # shellcheck disable=SC2016
-  ${AWK} -F: '{print $NF}' "${orig}" >> "${tmp}.branch"
-
-  # shellcheck disable=SC2016
-  ${AWK} -F: '{print $NF}' "${new}" >> "${tmp}.patch"
-
-  # compare the errors, generating a string of line
-  # numbers.  Sorry portability: GNU diff makes this too easy
-  ${DIFF} --unchanged-line-format="" \
-     --old-line-format="" \
-     --new-line-format="%dn " \
-     "${tmp}.branch" \
-     "${tmp}.patch" > "${tmp}.lined"
-
-  # now, pull out those lines of the raw output
-  # shellcheck disable=SC2013
-  for j in $(cat "${tmp}.lined"); do
-    # shellcheck disable=SC2086
-    head -${j} "${new}" | tail -1 >> "${diffout}"
-  done
-
-  if [[ -f "${diffout}" ]]; then
-    # shellcheck disable=SC2016
-    count=$(wc -l "${diffout}" | ${AWK} '{print $1}' )
-  fi
-  rm "${tmp}.branch" "${tmp}.patch" "${tmp}.lined" 2>/dev/null
-  echo "${count}"
-}
-
-function checkstyle_postapply
-{
-  local rc=0
-  local module
-  local modules=${CHANGED_MODULES}
-  local module_suffix
-  local numprepatch=0
-  local numpostpatch=0
-  local diffpostpatch=0
-
-  verify_needed_test checkstyle
-
-  if [[ $? == 0 ]]; then
-    return 0
-  fi
-
-  big_console_header "checkstyle plugin: postpatch"
-
-  start_clock
-
-  # add our previous elapsed to our new timer
-  # by setting the clock back
-  offset_clock "${CHECKSTYLE_TIMER}"
-
-  for module in ${modules}
-  do
-    pushd "${module}" >/dev/null
-    echo "  Running checkstyle in ${module}"
-    module_suffix=$(basename "${module}")
-
-    checkstyle_mvnrunner \
-      "${PATCH_DIR}/maven-patchcheckstyle-${module_suffix}.txt" \
-      "${PATCH_DIR}/patchcheckstyle${module_suffix}.txt"
-
-    if [[ $? != 0 ]] ; then
-      ((rc = rc +1))
-      echo "Post-patch checkstyle compilation is broken."
-      add_jira_table -1 checkstyle "Post-patch checkstyle ${module} compilation is broken."
-      continue
-    fi
-
-    #shellcheck disable=SC2016
-    diffpostpatch=$(checkstyle_calcdiffs \
-      "${PATCH_DIR}/${PATCH_BRANCH}checkstyle${module_suffix}.txt" \
-      "${PATCH_DIR}/patchcheckstyle${module_suffix}.txt" \
-      "${PATCH_DIR}/diffcheckstyle${module_suffix}.txt" )
-
-    if [[ ${diffpostpatch} -gt 0 ]] ; then
-      ((rc = rc + 1))
-
-      # shellcheck disable=SC2016
-      numprepatch=$(wc -l "${PATCH_DIR}/${PATCH_BRANCH}checkstyle${module_suffix}.txt" | ${AWK} '{print $1}')
-      # shellcheck disable=SC2016
-      numpostpatch=$(wc -l "${PATCH_DIR}/patchcheckstyle${module_suffix}.txt" | ${AWK} '{print $1}')
-
-      add_jira_table -1 checkstyle "The applied patch generated "\
-        "${diffpostpatch} new checkstyle issues (total was ${numprepatch}, now ${numpostpatch})."
-      footer="${footer} @@BASE@@/diffcheckstyle${module_suffix}.txt"
-    fi
-
-    popd >/dev/null
-  done
-
-  if [[ ${rc} -gt 0 ]] ; then
-    add_jira_footer checkstyle "${footer}"
-    return 1
-  fi
-  add_jira_table +1 checkstyle "There were no new checkstyle issues."
-  return 0
-}

dev-support/test-patch.d/shellcheck.sh

@@ -1,178 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-add_plugin shellcheck
-
-SHELLCHECK_TIMER=0
-
-SHELLCHECK=${SHELLCHECK:-$(which shellcheck 2>/dev/null)}
-
-SHELLCHECK_SPECIFICFILES=""
-
-# if it ends in an explicit .sh, then this is shell code.
-# if it doesn't have an extension, we assume it is shell code too
-function shellcheck_filefilter
-{
-  local filename=$1
-
-  if [[ ${filename} =~ \.sh$ ]]; then
-    add_test shellcheck
-    SHELLCHECK_SPECIFICFILES="${SHELLCHECK_SPECIFICFILES} ./${filename}"
-  fi
-
-  if [[ ! ${filename} =~ \. ]]; then
-    add_test shellcheck
-  fi
-}
-
-function shellcheck_private_findbash
-{
-  local i
-  local value
-  local list
-
-  while read line; do
-    value=$(find "${line}" ! -name '*.cmd' -type f \
-      | ${GREP} -E -v '(.orig$|.rej$)')
-    list="${list} ${value}"
-  done < <(find . -type d -name bin -o -type d -name sbin -o -type d -name libexec -o -type d -name shellprofile.d)
-  # shellcheck disable=SC2086
-  echo ${list} ${SHELLCHECK_SPECIFICFILES} | tr ' ' '\n' | sort -u
-}
-
-function shellcheck_preapply
-{
-  local i
-
-  verify_needed_test shellcheck
-  if [[ $? == 0 ]]; then
-    return 0
-  fi
-
-  big_console_header "shellcheck plugin: prepatch"
-
-  if [[ ! -x "${SHELLCHECK}" ]]; then
-    hadoop_error "shellcheck is not available."
-    return 0
-  fi
-
-  start_clock
-
-  # shellcheck disable=SC2016
-  SHELLCHECK_VERSION=$(${SHELLCHECK} --version | ${GREP} version: | ${AWK} '{print $NF}')
-
-  echo "Running shellcheck against all identifiable shell scripts"
-  pushd "${BASEDIR}" >/dev/null
-  for i in $(shellcheck_private_findbash); do
-    if [[ -f ${i} ]]; then
-      ${SHELLCHECK} -f gcc "${i}" >> "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt"
-    fi
-  done
-  popd > /dev/null
-  # keep track of how much as elapsed for us already
-  SHELLCHECK_TIMER=$(stop_clock)
-  return 0
-}
-
-function shellcheck_calcdiffs
-{
-  local orig=$1
-  local new=$2
-  local diffout=$3
-  local tmp=${PATCH_DIR}/sc.$$.${RANDOM}
-  local count=0
-  local j
-
-  # first, pull out just the errors
-  # shellcheck disable=SC2016
-  ${AWK} -F: '{print $NF}' "${orig}" >> "${tmp}.branch"
-
-  # shellcheck disable=SC2016
-  ${AWK} -F: '{print $NF}' "${new}" >> "${tmp}.patch"
-
-  # compare the errors, generating a string of line
-  # numbers.  Sorry portability: GNU diff makes this too easy
-  ${DIFF} --unchanged-line-format="" \
-     --old-line-format="" \
-     --new-line-format="%dn " \
-     "${tmp}.branch" \
-     "${tmp}.patch" > "${tmp}.lined"
-
-  # now, pull out those lines of the raw output
-  # shellcheck disable=SC2013
-  for j in $(cat "${tmp}.lined"); do
-    # shellcheck disable=SC2086
-    head -${j} "${new}" | tail -1 >> "${diffout}"
-  done
-
-  if [[ -f "${diffout}" ]]; then
-    # shellcheck disable=SC2016
-    count=$(wc -l "${diffout}" | ${AWK} '{print $1}' )
-  fi
-  rm "${tmp}.branch" "${tmp}.patch" "${tmp}.lined" 2>/dev/null
-  echo "${count}"
-}
-
-function shellcheck_postapply
-{
-  local i
-
-  verify_needed_test shellcheck
-  if [[ $? == 0 ]]; then
-    return 0
-  fi
-
-  big_console_header "shellcheck plugin: postpatch"
-
-  if [[ ! -x "${SHELLCHECK}" ]]; then
-    hadoop_error "shellcheck is not available."
-    add_jira_table 0 shellcheck "Shellcheck was not available."
-    return 0
-  fi
-
-  start_clock
-
-  # add our previous elapsed to our new timer
-  # by setting the clock back
-  offset_clock "${SHELLCHECK_TIMER}"
-
-  echo "Running shellcheck against all identifiable shell scripts"
-  # we re-check this in case one has been added
-  for i in $(shellcheck_private_findbash); do
-    ${SHELLCHECK} -f gcc "${i}" >> "${PATCH_DIR}/patchshellcheck-result.txt"
-  done
-
-  # shellcheck disable=SC2016
-  numPrepatch=$(wc -l "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt" | ${AWK} '{print $1}')
-  # shellcheck disable=SC2016
-  numPostpatch=$(wc -l "${PATCH_DIR}/patchshellcheck-result.txt" | ${AWK} '{print $1}')
-
-  diffPostpatch=$(shellcheck_calcdiffs \
-    "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt" \
-    "${PATCH_DIR}/patchshellcheck-result.txt" \
-      "${PATCH_DIR}/diffpatchshellcheck.txt"
-    )
-
-  if [[ ${diffPostpatch} -gt 0 ]] ; then
-    add_jira_table -1 shellcheck "The applied patch generated "\
-      "${diffPostpatch} new shellcheck (v${SHELLCHECK_VERSION}) issues (total was ${numPrepatch}, now ${numPostpatch})."
-    add_jira_footer shellcheck "@@BASE@@/diffpatchshellcheck.txt"
-    return 1
-  fi
-
-  add_jira_table +1 shellcheck "There were no new shellcheck (v${SHELLCHECK_VERSION}) issues."
-  return 0
-}

dev-support/test-patch.d/whitespace.sh

@@ -1,46 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-add_plugin whitespace
-
-function whitespace_postapply
-{
-  local count
-  local j
-
-  big_console_header "Checking for whitespace at the end of lines"
-  start_clock
-
-  pushd "${BASEDIR}" >/dev/null
-  for j in ${CHANGED_FILES}; do
-    ${GREP} -nHE '[[:blank:]]$' "./${j}" | ${GREP} -f "${GITDIFFLINES}" >> "${PATCH_DIR}/whitespace.txt"
-  done
-
-  # shellcheck disable=SC2016
-  count=$(wc -l "${PATCH_DIR}/whitespace.txt" | ${AWK} '{print $1}')
-
-  if [[ ${count} -gt 0 ]]; then
-    add_jira_table -1 whitespace "The patch has ${count}"\
-      " line(s) that end in whitespace. Use git apply --whitespace=fix."
-    add_jira_footer whitespace "@@BASE@@/whitespace.txt"
-    popd >/dev/null
-    return 1
-  fi
-
-  popd >/dev/null
-  add_jira_table +1 whitespace "The patch has no lines that end in whitespace."
-  return 0
-}

dev-support/test-patch.sh

@@ -1,2808 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-### BUILD_URL is set by Hudson if it is run by patch process
-
-this="${BASH_SOURCE-$0}"
-BINDIR=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
-CWD=$(pwd)
-USER_PARAMS=("$@")
-GLOBALTIMER=$(date +"%s")
-
-## @description  Setup the default global variables
-## @audience     public
-## @stability    stable
-## @replaceable  no
-function setup_defaults
-{
-  if [[ -z "${MAVEN_HOME:-}" ]]; then
-    MVN=mvn
-  else
-    MVN=${MAVEN_HOME}/bin/mvn
-  fi
-  # This parameter needs to be kept as an array
-  MAVEN_ARGS=()
-
-  PROJECT_NAME=hadoop
-  HOW_TO_CONTRIBUTE="https://wiki.apache.org/hadoop/HowToContribute"
-  JENKINS=false
-  BASEDIR=$(pwd)
-  RELOCATE_PATCH_DIR=false
-
-  USER_PLUGIN_DIR=""
-  LOAD_SYSTEM_PLUGINS=true
-
-  FINDBUGS_HOME=${FINDBUGS_HOME:-}
-  FINDBUGS_WARNINGS_FAIL_PRECHECK=false
-  ECLIPSE_HOME=${ECLIPSE_HOME:-}
-  BUILD_NATIVE=${BUILD_NATIVE:-true}
-  PATCH_BRANCH=""
-  PATCH_BRANCH_DEFAULT="trunk"
-  CHANGED_MODULES=""
-  USER_MODULE_LIST=""
-  OFFLINE=false
-  CHANGED_FILES=""
-  REEXECED=false
-  RESETREPO=false
-  ISSUE=""
-  ISSUE_RE='^(HADOOP|YARN|MAPREDUCE|HDFS)-[0-9]+$'
-  TIMER=$(date +"%s")
-  PATCHURL=""
-
-  OSTYPE=$(uname -s)
-
-  # Solaris needs POSIX, not SVID
-  case ${OSTYPE} in
-    SunOS)
-      PS=${PS:-ps}
-      AWK=${AWK:-/usr/xpg4/bin/awk}
-      SED=${SED:-/usr/xpg4/bin/sed}
-      WGET=${WGET:-wget}
-      GIT=${GIT:-git}
-      EGREP=${EGREP:-/usr/xpg4/bin/egrep}
-      GREP=${GREP:-/usr/xpg4/bin/grep}
-      PATCH=${PATCH:-patch}
-      DIFF=${DIFF:-/usr/gnu/bin/diff}
-      JIRACLI=${JIRA:-jira}
-      FILE=${FILE:-file}
-    ;;
-    *)
-      PS=${PS:-ps}
-      AWK=${AWK:-awk}
-      SED=${SED:-sed}
-      WGET=${WGET:-wget}
-      GIT=${GIT:-git}
-      EGREP=${EGREP:-egrep}
-      GREP=${GREP:-grep}
-      PATCH=${PATCH:-patch}
-      DIFF=${DIFF:-diff}
-      JIRACLI=${JIRA:-jira}
-      FILE=${FILE:-file}
-    ;;
-  esac
-
-  declare -a JIRA_COMMENT_TABLE
-  declare -a JIRA_FOOTER_TABLE
-  declare -a JIRA_HEADER
-  declare -a JIRA_TEST_TABLE
-
-  JFC=0
-  JTC=0
-  JTT=0
-  RESULT=0
-}
-
-## @description  Print a message to stderr
-## @audience     public
-## @stability    stable
-## @replaceable  no
-## @param        string
-function hadoop_error
-{
-  echo "$*" 1>&2
-}
-
-## @description  Print a message to stderr if --debug is turned on
-## @audience     public
-## @stability    stable
-## @replaceable  no
-## @param        string
-function hadoop_debug
-{
-  if [[ -n "${HADOOP_SHELL_SCRIPT_DEBUG}" ]]; then
-    echo "[$(date) DEBUG]: $*" 1>&2
-  fi
-}
-
-## @description  Activate the local timer
-## @audience     public
-## @stability    stable
-## @replaceable  no
-function start_clock
-{
-  hadoop_debug "Start clock"
-  TIMER=$(date +"%s")
-}
-
-## @description  Print the elapsed time in seconds since the start of the local timer
-## @audience     public
-## @stability    stable
-## @replaceable  no
-function stop_clock
-{
-  local -r stoptime=$(date +"%s")
-  local -r elapsed=$((stoptime-TIMER))
-  hadoop_debug "Stop clock"
-
-  echo ${elapsed}
-}
-
-## @description  Print the elapsed time in seconds since the start of the global timer
-## @audience     private
-## @stability    stable
-## @replaceable  no
-function stop_global_clock
-{
-  local -r stoptime=$(date +"%s")
-  local -r elapsed=$((stoptime-GLOBALTIMER))
-  hadoop_debug "Stop global clock"
-
-  echo ${elapsed}
-}
-
-## @description  Add time to the local timer
-## @audience     public
-## @stability    stable
-## @replaceable  no
-## @param        seconds
-function offset_clock
-{
-  ((TIMER=TIMER-$1))
-}
-
-## @description  Add to the header of the display
-## @audience     public
-## @stability    stable
-## @replaceable  no
-## @param        string
-function add_jira_header
-{
-  JIRA_HEADER[${JHC}]="| $* |"
-  JHC=$(( JHC+1 ))
-}
-
-## @description  Add to the output table. If the first parameter is a number
-## @description  that is the vote for that column and calculates the elapsed time
-## @description  based upon the last start_clock().  If it the string null, then it is
-## @description  a special entry that signifies extra
-## @description  content for the final column.  The second parameter is the reporting
-## @description  subsystem (or test) that is providing the vote.  The second parameter
-## @description  is always required.  The third parameter is any extra verbage that goes
-## @description  with that subsystem.
-## @audience     public
-## @stability    stable
-## @replaceable  no
-## @param        +1/0/-1/null
-## @param        subsystem
-## @param        string
-## @return       Elapsed time display
-function add_jira_table
-{
-  local value=$1
-  local subsystem=$2
-  shift 2
-
-  local color
-  local calctime=0
-
-  local -r elapsed=$(stop_clock)
-
-  if [[ ${elapsed} -lt 0 ]]; then
-    calctime="N/A"
-  else
-    printf -v calctime "%3sm %02ss" $((elapsed/60)) $((elapsed%60))
-  fi
-
-  echo ""
-  echo "Elapsed time: ${calctime}"
-  echo ""
-
-  case ${value} in
-    1|+1)
-      value="+1"
-      color="green"
-    ;;
-    -1)
-      color="red"
-    ;;
-    0)
-      color="blue"
-    ;;
-    null)
-    ;;
-  esac
-
-  if [[ -z ${color} ]]; then
-    JIRA_COMMENT_TABLE[${JTC}]="|  | ${subsystem} | | ${*:-} |"
-    JTC=$(( JTC+1 ))
-  else
-    JIRA_COMMENT_TABLE[${JTC}]="| {color:${color}}${value}{color} | ${subsystem} | ${calctime} | $* |"
-    JTC=$(( JTC+1 ))
-  fi
-}
-
-## @description  Put the final environment information at the bottom
-## @description  of the footer table
-## @stability     stable
-## @audience     private
-## @replaceable  yes
-function close_jira_footer
-{
-  # shellcheck disable=SC2016
-  local -r javaversion=$("${JAVA_HOME}/bin/java" -version 2>&1 | head -1 | ${AWK} '{print $NF}' | tr -d \")
-  local -r unamea=$(uname -a)
-
-  add_jira_footer "Java" "${javaversion}"
-  add_jira_footer "uname" "${unamea}"
-}
-
-## @description  Put the final elapsed time at the bottom of the table.
-## @audience     private
-## @stability    stable
-## @replaceable  no
-function close_jira_table
-{
-
-  local -r elapsed=$(stop_global_clock)
-
-  if [[ ${elapsed} -lt 0 ]]; then
-    calctime="N/A"
-  else
-    printf -v calctime "%3sm %02ss" $((elapsed/60)) $((elapsed%60))
-  fi
-
-  echo ""
-  echo "Total Elapsed time: ${calctime}"
-  echo ""
-
-
-  JIRA_COMMENT_TABLE[${JTC}]="| | | ${calctime} | |"
-  JTC=$(( JTC+1 ))
-}
-
-## @description  Add to the footer of the display. @@BASE@@ will get replaced with the
-## @description  correct location for the local filesystem in dev mode or the URL for
-## @description  Jenkins mode.
-## @audience     public
-## @stability    stable
-## @replaceable  no
-## @param        subsystem
-## @param        string
-function add_jira_footer
-{
-  local subsystem=$1
-  shift 1
-
-  JIRA_FOOTER_TABLE[${JFC}]="| ${subsystem} | $* |"
-  JFC=$(( JFC+1 ))
-}
-
-## @description  Special table just for unit test failures
-## @audience     public
-## @stability    stable
-## @replaceable  no
-## @param        failurereason
-## @param        testlist
-function add_jira_test_table
-{
-  local failure=$1
-  shift 1
-
-  JIRA_TEST_TABLE[${JTT}]="| ${failure} | $* |"
-  JTT=$(( JTT+1 ))
-}
-
-## @description  Large display for the user console
-## @audience     public
-## @stability    stable
-## @replaceable  no
-## @param        string
-## @return       large chunk of text
-function big_console_header
-{
-  local text="$*"
-  local spacing=$(( (75+${#text}) /2 ))
-  printf "\n\n"
-  echo "============================================================================"
-  echo "============================================================================"
-  printf "%*s\n"  ${spacing} "${text}"
-  echo "============================================================================"
-  echo "============================================================================"
-  printf "\n\n"
-}
-
-## @description  Remove {color} tags from a string
-## @audience     public
-## @stability    stable
-## @replaceable  no
-## @param        string
-## @return       string
-function colorstripper
-{
-  local string=$1
-  shift 1
-
-  local green=""
-  local white=""
-  local red=""
-  local blue=""
-
-  echo "${string}" | \
-  ${SED} -e "s,{color:red},${red},g" \
-         -e "s,{color:green},${green},g" \
-         -e "s,{color:blue},${blue},g" \
-         -e "s,{color},${white},g"
-}
-
-## @description  Find the largest size of a column of an array
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @return       size
-function findlargest
-{
-  local column=$1
-  shift
-  local a=("$@")
-  local sizeofa=${#a[@]}
-  local i=0
-
-  until [[ ${i} -gt ${sizeofa} ]]; do
-    # shellcheck disable=SC2086
-    string=$( echo ${a[$i]} | cut -f$((column + 1)) -d\| )
-    if [[ ${#string} -gt $maxlen ]]; then
-      maxlen=${#string}
-    fi
-    i=$((i+1))
-  done
-  echo "${maxlen}"
-}
-
-## @description  Verify that ${JAVA_HOME} is defined
-## @audience     public
-## @stability    stable
-## @replaceable  no
-## @return       1 - no JAVA_HOME
-## @return       0 - JAVA_HOME defined
-function find_java_home
-{
-  start_clock
-  if [[ -z ${JAVA_HOME:-} ]]; then
-    case $(uname -s) in
-      Darwin)
-        if [[ -z "${JAVA_HOME}" ]]; then
-          if [[ -x /usr/libexec/java_home ]]; then
-            JAVA_HOME="$(/usr/libexec/java_home)"
-            export JAVA_HOME
-          else
-            export JAVA_HOME=/Library/Java/Home
-          fi
-        fi
-      ;;
-      *)
-      ;;
-    esac
-  fi
-
-  if [[ -z ${JAVA_HOME:-} ]]; then
-    echo "JAVA_HOME is not defined."
-    add_jira_table -1 pre-patch "JAVA_HOME is not defined."
-    return 1
-  fi
-  return 0
-}
-
-## @description Write the contents of a file to jenkins
-## @params filename
-## @stability stable
-## @audience public
-## @returns ${JIRACLI} exit code
-function write_to_jira
-{
-  local -r commentfile=${1}
-  shift
-
-  local retval
-
-  if [[ ${OFFLINE} == false
-     && ${JENKINS} == true ]]; then
-    export USER=hudson
-    # shellcheck disable=SC2086
-    ${JIRACLI} --comment "$(cat ${commentfile})" \
-               -s https://issues.apache.org/jira \
-               -a addcomment -u hadoopqa \
-               -p "${JIRA_PASSWD}" \
-               --issue "${ISSUE}"
-    retval=$?
-    ${JIRACLI} -s https://issues.apache.org/jira \
-               -a logout -u hadoopqa \
-               -p "${JIRA_PASSWD}"
-  fi
-  return ${retval}
-}
-
-## @description Verify that the patch directory is still in working order
-## @description since bad actors on some systems wipe it out. If not,
-## @description recreate it and then exit
-## @audience    private
-## @stability   evolving
-## @replaceable yes
-## @returns     may exit on failure
-function verify_patchdir_still_exists
-{
-  local -r commentfile=/tmp/testpatch.$$.${RANDOM}
-  local extra=""
-
-  if [[ ! -d ${PATCH_DIR} ]]; then
-      rm "${commentfile}" 2>/dev/null
-
-      echo "(!) The patch artifact directory has been removed! " > "${commentfile}"
-      echo "This is a fatal error for test-patch.sh.  Aborting. " >> "${commentfile}"
-      echo
-      cat ${commentfile}
-      echo
-      if [[ ${JENKINS} == true ]]; then
-        if [[ -n ${NODE_NAME} ]]; then
-          extra=" (node ${NODE_NAME})"
-        fi
-        echo "Jenkins${extra} information at ${BUILD_URL} may provide some hints. " >> "${commentfile}"
-
-        write_to_jira ${commentfile}
-      fi
-
-      rm "${commentfile}"
-      cleanup_and_exit ${RESULT}
-    fi
-}
-
-## @description generate a list of all files and line numbers that
-## @description that were added/changed in the source repo
-## @audience    private
-## @stability   stable
-## @params      filename
-## @replaceable no
-function compute_gitdiff
-{
-  local outfile=$1
-  local file
-  local line
-  local startline
-  local counter
-  local numlines
-  local actual
-
-  pushd "${BASEDIR}" >/dev/null
-  while read line; do
-    if [[ ${line} =~ ^\+\+\+ ]]; then
-      file="./"$(echo "${line}" | cut -f2- -d/)
-      continue
-    elif [[ ${line} =~ ^@@ ]]; then
-      startline=$(echo "${line}" | cut -f3 -d' ' | cut -f1 -d, | tr -d + )
-      numlines=$(echo "${line}" | cut -f3 -d' ' | cut -s -f2 -d, )
-      # if this is empty, then just this line
-      # if it is 0, then no lines were added and this part of the patch
-      # is strictly a delete
-      if [[ ${numlines} == 0 ]]; then
-        continue
-      elif [[ -z ${numlines} ]]; then
-        numlines=1
-      fi
-      counter=0
-      until [[ ${counter} -gt ${numlines} ]]; do
-          ((actual=counter+startline))
-          echo "${file}:${actual}:" >> "${outfile}"
-          ((counter=counter+1))
-      done
-    fi
-  done < <("${GIT}" diff --unified=0 --no-color)
-  popd >/dev/null
-}
-
-## @description  Print the command to be executing to the screen. Then
-## @description  run the command, sending stdout and stderr to the given filename
-## @description  This will also ensure that any directories in ${BASEDIR} have
-## @description  the exec bit set as a pre-exec step.
-## @audience     public
-## @stability    stable
-## @param        filename
-## @param        command
-## @param        [..]
-## @replaceable  no
-## @returns      $?
-function echo_and_redirect
-{
-  local logfile=$1
-  shift
-
-  verify_patchdir_still_exists
-
-  find "${BASEDIR}" -type d -exec chmod +x {} \;
-  echo "${*} > ${logfile} 2>&1"
-  "${@}" > "${logfile}" 2>&1
-}
-
-## @description is PATCH_DIR relative to BASEDIR?
-## @audience    public
-## @stability   stable
-## @replaceable yes
-## @returns     1 - no, PATCH_DIR
-## @returns     0 - yes, PATCH_DIR - BASEDIR
-function relative_patchdir
-{
-  local p=${PATCH_DIR#${BASEDIR}}
-
-  if [[ ${#p} -eq ${#PATCH_DIR} ]]; then
-    echo ${p}
-    return 1
-  fi
-  p=${p#/}
-  echo ${p}
-  return 0
-}
-
-
-## @description  Print the usage information
-## @audience     public
-## @stability    stable
-## @replaceable  no
-function hadoop_usage
-{
-  local -r up=$(echo ${PROJECT_NAME} | tr '[:lower:]' '[:upper:]')
-
-  echo "Usage: test-patch.sh [options] patch-file | issue-number | http"
-  echo
-  echo "Where:"
-  echo "  patch-file is a local patch file containing the changes to test"
-  echo "  issue-number is a 'Patch Available' JIRA defect number (e.g. '${up}-9902') to test"
-  echo "  http is an HTTP address to download the patch file"
-  echo
-  echo "Options:"
-  echo "--basedir=<dir>        The directory to apply the patch to (default current directory)"
-  echo "--branch=<ref>         Forcibly set the branch"
-  echo "--branch-default=<ref> If the branch isn't forced and we don't detect one in the patch name, use this branch (default 'trunk')"
-  echo "--build-native=<bool>  If true, then build native components (default 'true')"
-  echo "--contrib-guide=<url>  URL to point new users towards project conventions. (default Hadoop's wiki)"
-  echo "--debug                If set, then output some extra stuff to stderr"
-  echo "--dirty-workspace      Allow the local git workspace to have uncommitted changes"
-  echo "--findbugs-home=<path> Findbugs home directory (default FINDBUGS_HOME environment variable)"
-  echo "--findbugs-strict-precheck If there are Findbugs warnings during precheck, fail"
-  echo "--issue-re=<expr>      Bash regular expression to use when trying to find a jira ref in the patch name (default '^(HADOOP|YARN|MAPREDUCE|HDFS)-[0-9]+$')"
-  echo "--modulelist=<list>    Specify additional modules to test (comma delimited)"
-  echo "--offline              Avoid connecting to the Internet"
-  echo "--patch-dir=<dir>      The directory for working and output files (default '/tmp/${PROJECT_NAME}-test-patch/pid')"
-  echo "--plugins=<dir>        A directory of user provided plugins. see test-patch.d for examples (default empty)"
-  echo "--project=<name>       The short name for project currently using test-patch (default 'hadoop')"
-  echo "--resetrepo            Forcibly clean the repo"
-  echo "--run-tests            Run all relevant tests below the base directory"
-  echo "--skip-system-plugins  Do not load plugins from ${BINDIR}/test-patch.d"
-  echo "--testlist=<list>      Specify which subsystem tests to use (comma delimited)"
-  echo "--test-parallel=<bool> Run multiple tests in parallel (default false in developer mode, true in Jenkins mode)"
-  echo "--test-threads=<int>   Number of tests to run in parallel (default defined in ${PROJECT_NAME} build)"
-
-  echo "Shell binary overrides:"
-  echo "--awk-cmd=<cmd>        The 'awk' command to use (default 'awk')"
-  echo "--diff-cmd=<cmd>       The GNU-compatible 'diff' command to use (default 'diff')"
-  echo "--file-cmd=<cmd>       The 'file' command to use (default 'file')"
-  echo "--git-cmd=<cmd>        The 'git' command to use (default 'git')"
-  echo "--grep-cmd=<cmd>       The 'grep' command to use (default 'grep')"
-  echo "--mvn-cmd=<cmd>        The 'mvn' command to use (default \${MAVEN_HOME}/bin/mvn, or 'mvn')"
-  echo "--patch-cmd=<cmd>      The 'patch' command to use (default 'patch')"
-  echo "--ps-cmd=<cmd>         The 'ps' command to use (default 'ps')"
-  echo "--sed-cmd=<cmd>        The 'sed' command to use (default 'sed')"
-
-  echo
-  echo "Jenkins-only options:"
-  echo "--jenkins              Run by Jenkins (runs tests and posts results to JIRA)"
-  echo "--eclipse-home=<path>  Eclipse home directory (default ECLIPSE_HOME environment variable)"
-  echo "--jira-cmd=<cmd>       The 'jira' command to use (default 'jira')"
-  echo "--jira-password=<pw>   The password for the 'jira' command"
-  echo "--mv-patch-dir         Move the patch-dir into the basedir during cleanup."
-  echo "--wget-cmd=<cmd>       The 'wget' command to use (default 'wget')"
-}
-
-## @description  Interpret the command line parameters
-## @audience     private
-## @stability    stable
-## @replaceable  no
-## @params       $@
-## @return       May exit on failure
-function parse_args
-{
-  local i
-  local j
-
-  for i in "$@"; do
-    case ${i} in
-      --awk-cmd=*)
-        AWK=${i#*=}
-      ;;
-      --basedir=*)
-        BASEDIR=${i#*=}
-      ;;
-      --branch=*)
-        PATCH_BRANCH=${i#*=}
-      ;;
-      --branch-default=*)
-        PATCH_BRANCH_DEFAULT=${i#*=}
-      ;;
-      --build-native=*)
-        BUILD_NATIVE=${i#*=}
-      ;;
-      --contrib-guide=*)
-        HOW_TO_CONTRIBUTE=${i#*=}
-      ;;
-      --debug)
-        HADOOP_SHELL_SCRIPT_DEBUG=true
-      ;;
-      --diff-cmd=*)
-        DIFF=${i#*=}
-      ;;
-      --dirty-workspace)
-        DIRTY_WORKSPACE=true
-      ;;
-      --eclipse-home=*)
-        ECLIPSE_HOME=${i#*=}
-      ;;
-      --file-cmd=*)
-        FILE=${i#*=}
-      ;;
-      --findbugs-home=*)
-        FINDBUGS_HOME=${i#*=}
-      ;;
-      --findbugs-strict-precheck)
-        FINDBUGS_WARNINGS_FAIL_PRECHECK=true
-      ;;
-      --git-cmd=*)
-        GIT=${i#*=}
-      ;;
-      --grep-cmd=*)
-        GREP=${i#*=}
-      ;;
-      --help|-help|-h|help|--h|--\?|-\?|\?)
-        hadoop_usage
-        exit 0
-      ;;
-      --issue-re=*)
-        ISSUE_RE=${i#*=}
-      ;;
-      --java-home=*)
-        JAVA_HOME=${i#*=}
-      ;;
-      --jenkins)
-        JENKINS=true
-        TEST_PARALLEL=${TEST_PARALLEL:-true}
-      ;;
-      --jira-cmd=*)
-        JIRACLI=${i#*=}
-      ;;
-      --jira-password=*)
-        JIRA_PASSWD=${i#*=}
-      ;;
-      --modulelist=*)
-        USER_MODULE_LIST=${i#*=}
-        USER_MODULE_LIST=${USER_MODULE_LIST//,/ }
-        hadoop_debug "Manually forcing modules ${USER_MODULE_LIST}"
-      ;;
-      --mvn-cmd=*)
-        MVN=${i#*=}
-      ;;
-      --mv-patch-dir)
-        RELOCATE_PATCH_DIR=true;
-      ;;
-      --offline)
-        OFFLINE=true
-      ;;
-      --patch-cmd=*)
-        PATCH=${i#*=}
-      ;;
-      --patch-dir=*)
-        USER_PATCH_DIR=${i#*=}
-      ;;
-      --plugins=*)
-        USER_PLUGIN_DIR=${i#*=}
-      ;;
-      --project=*)
-        PROJECT_NAME=${i#*=}
-      ;;
-      --ps-cmd=*)
-        PS=${i#*=}
-      ;;
-      --reexec)
-        REEXECED=true
-        start_clock
-        add_jira_table 0 reexec "dev-support patch detected."
-      ;;
-      --resetrepo)
-        RESETREPO=true
-      ;;
-      --run-tests)
-        RUN_TESTS=true
-      ;;
-      --skip-system-plugins)
-        LOAD_SYSTEM_PLUGINS=false
-      ;;
-      --testlist=*)
-        testlist=${i#*=}
-        testlist=${testlist//,/ }
-        for j in ${testlist}; do
-          hadoop_debug "Manually adding patch test subsystem ${j}"
-          add_test "${j}"
-        done
-      ;;
-      --test-parallel=*)
-        TEST_PARALLEL=${i#*=}
-      ;;
-      --test-threads=*)
-        TEST_THREADS=${i#*=}
-      ;;
-      --wget-cmd=*)
-        WGET=${i#*=}
-      ;;
-      *)
-        PATCH_OR_ISSUE=${i}
-      ;;
-    esac
-  done
-
-  # if we requested offline, pass that to mvn
-  if [[ ${OFFLINE} == "true" ]] ; then
-    MAVEN_ARGS=(${MAVEN_ARGS[@]} --offline)
-  fi
-
-  # we need absolute dir for ${BASEDIR}
-  cd "${CWD}"
-  BASEDIR=$(cd -P -- "${BASEDIR}" >/dev/null && pwd -P)
-
-  if [[ ${BUILD_NATIVE} == "true" ]] ; then
-    NATIVE_PROFILE=-Pnative
-    REQUIRE_TEST_LIB_HADOOP=-Drequire.test.libhadoop
-  fi
-  if [[ -z "${PATCH_OR_ISSUE}" ]]; then
-    hadoop_usage
-    exit 1
-  fi
-  if [[ ${JENKINS} == "true" ]] ; then
-    echo "Running in Jenkins mode"
-    ISSUE=${PATCH_OR_ISSUE}
-    RESETREPO=true
-    # shellcheck disable=SC2034
-    ECLIPSE_PROPERTY="-Declipse.home=${ECLIPSE_HOME}"
-  else
-    if [[ ${RESETREPO} == "true" ]] ; then
-      echo "Running in destructive (--resetrepo) developer mode"
-    else
-      echo "Running in developer mode"
-    fi
-    JENKINS=false
-  fi
-
-  if [[ -n ${USER_PATCH_DIR} ]]; then
-    PATCH_DIR="${USER_PATCH_DIR}"
-  else
-    PATCH_DIR=/tmp/${PROJECT_NAME}-test-patch/$$
-  fi
-
-  cd "${CWD}"
-  if [[ ! -d ${PATCH_DIR} ]]; then
-    mkdir -p "${PATCH_DIR}"
-    if [[ $? == 0 ]] ; then
-      echo "${PATCH_DIR} has been created"
-    else
-      echo "Unable to create ${PATCH_DIR}"
-      cleanup_and_exit 1
-    fi
-  fi
-
-  # we need absolute dir for PATCH_DIR
-  PATCH_DIR=$(cd -P -- "${PATCH_DIR}" >/dev/null && pwd -P)
-
-  GITDIFFLINES=${PATCH_DIR}/gitdifflines.txt
-
-  if [[ ${TEST_PARALLEL} == "true" ]] ; then
-    PARALLEL_TESTS_PROFILE=-Pparallel-tests
-    if [[ -n ${TEST_THREADS:-} ]]; then
-      TESTS_THREAD_COUNT="-DtestsThreadCount=$TEST_THREADS"
-    fi
-  fi
-}
-
-## @description  Locate the pom.xml file for a given directory
-## @audience     private
-## @stability    stable
-## @replaceable  no
-## @return       directory containing the pom.xml
-function find_pom_dir
-{
-  local dir
-
-  dir=$(dirname "$1")
-
-  hadoop_debug "Find pom dir for: ${dir}"
-
-  while builtin true; do
-    if [[ -f "${dir}/pom.xml" ]];then
-      echo "${dir}"
-      hadoop_debug "Found: ${dir}"
-      return
-    else
-      dir=$(dirname "${dir}")
-    fi
-  done
-}
-
-## @description  List of files that ${PATCH_DIR}/patch modifies
-## @audience     private
-## @stability    stable
-## @replaceable  no
-## @return       None; sets ${CHANGED_FILES}
-function find_changed_files
-{
-  # get a list of all of the files that have been changed,
-  # except for /dev/null (which would be present for new files).
-  # Additionally, remove any a/ b/ patterns at the front
-  # of the patch filenames and any revision info at the end
-  # shellcheck disable=SC2016
-  CHANGED_FILES=$(${GREP} -E '^(\+\+\+|---) ' "${PATCH_DIR}/patch" \
-    | ${SED} \
-      -e 's,^....,,' \
-      -e 's,^[ab]/,,' \
-    | ${GREP} -v /dev/null \
-    | ${AWK} '{print $1}' \
-    | sort -u)
-}
-
-## @description  Find the modules of the maven build that ${PATCH_DIR}/patch modifies
-## @audience     private
-## @stability    stable
-## @replaceable  no
-## @return       None; sets ${CHANGED_MODULES}
-function find_changed_modules
-{
-  # Come up with a list of changed files into ${TMP}
-  local pomdirs
-  local module
-  local pommods
-
-  # Now find all the modules that were changed
-  for file in ${CHANGED_FILES}; do
-    #shellcheck disable=SC2086
-    pomdirs="${pomdirs} $(find_pom_dir ${file})"
-  done
-
-  # Filter out modules without code
-  for module in ${pomdirs}; do
-    ${GREP} "<packaging>pom</packaging>" "${module}/pom.xml" > /dev/null
-    if [[ "$?" != 0 ]]; then
-      pommods="${pommods} ${module}"
-    fi
-  done
-
-  #shellcheck disable=SC2086
-  CHANGED_MODULES=$(echo ${pommods} ${USER_MODULE_LIST} | tr ' ' '\n' | sort -u)
-}
-
-## @description  git checkout the appropriate branch to test.  Additionally, this calls
-## @description  'determine_issue' and 'determine_branch' based upon the context provided
-## @description  in ${PATCH_DIR} and in git after checkout.
-## @audience     private
-## @stability    stable
-## @replaceable  no
-## @return       0 on success.  May exit on failure.
-function git_checkout
-{
-  local currentbranch
-  local exemptdir
-
-  big_console_header "Confirming git environment"
-
-  cd "${BASEDIR}"
-  if [[ ! -d .git ]]; then
-    hadoop_error "ERROR: ${BASEDIR} is not a git repo."
-    cleanup_and_exit 1
-  fi
-
-  if [[ ${RESETREPO} == "true" ]] ; then
-    ${GIT} reset --hard
-    if [[ $? != 0 ]]; then
-      hadoop_error "ERROR: git reset is failing"
-      cleanup_and_exit 1
-    fi
-
-    # if PATCH_DIR is in BASEDIR, then we don't want
-    # git wiping it out.
-    exemptdir=$(relative_patchdir)
-    if [[ $? == 1 ]]; then
-      ${GIT} clean -xdf
-    else
-      # we do, however, want it emptied of all _files_.
-      # we need to leave _directories_ in case we are in
-      # re-exec mode (which places a directory full of stuff in it)
-      hadoop_debug "Exempting ${exemptdir} from clean"
-      rm "${PATCH_DIR}/*" 2>/dev/null
-      ${GIT} clean -xdf -e "${exemptdir}"
-    fi
-    if [[ $? != 0 ]]; then
-      hadoop_error "ERROR: git clean is failing"
-      cleanup_and_exit 1
-    fi
-
-    ${GIT} checkout --force "${PATCH_BRANCH_DEFAULT}"
-    if [[ $? != 0 ]]; then
-      hadoop_error "ERROR: git checkout --force ${PATCH_BRANCH_DEFAULT} is failing"
-      cleanup_and_exit 1
-    fi
-
-    determine_branch
-    if [[ ${PATCH_BRANCH} =~ ^git ]]; then
-      PATCH_BRANCH=$(echo "${PATCH_BRANCH}" | cut -dt -f2)
-    fi
-
-    # we need to explicitly fetch in case the
-    # git ref hasn't been brought in tree yet
-    if [[ ${OFFLINE} == false ]]; then
-      ${GIT} pull --rebase
-      if [[ $? != 0 ]]; then
-        hadoop_error "ERROR: git pull is failing"
-        cleanup_and_exit 1
-      fi
-    fi
-    # forcibly checkout this branch or git ref
-    ${GIT} checkout --force "${PATCH_BRANCH}"
-    if [[ $? != 0 ]]; then
-      hadoop_error "ERROR: git checkout ${PATCH_BRANCH} is failing"
-      cleanup_and_exit 1
-    fi
-
-    # if we've selected a feature branch that has new changes
-    # since our last build, we'll need to rebase to see those changes.
-    if [[ ${OFFLINE} == false ]]; then
-      ${GIT} pull --rebase
-      if [[ $? != 0 ]]; then
-        hadoop_error "ERROR: git pull is failing"
-        cleanup_and_exit 1
-      fi
-    fi
-
-  else
-
-    status=$(${GIT} status --porcelain)
-    if [[ "${status}" != "" && -z ${DIRTY_WORKSPACE} ]] ; then
-      hadoop_error "ERROR: --dirty-workspace option not provided."
-      hadoop_error "ERROR: can't run in a workspace that contains the following modifications"
-      hadoop_error "${status}"
-      cleanup_and_exit 1
-    fi
-
-    determine_branch
-    if [[ ${PATCH_BRANCH} =~ ^git ]]; then
-      PATCH_BRANCH=$(echo "${PATCH_BRANCH}" | cut -dt -f2)
-    fi
-
-    currentbranch=$(${GIT} rev-parse --abbrev-ref HEAD)
-    if [[ "${currentbranch}" != "${PATCH_BRANCH}" ]];then
-      echo "WARNING: Current git branch is ${currentbranch} but patch is built for ${PATCH_BRANCH}."
-      echo "WARNING: Continuing anyway..."
-      PATCH_BRANCH=${currentbranch}
-    fi
-  fi
-
-  determine_issue
-
-  GIT_REVISION=$(${GIT} rev-parse --verify --short HEAD)
-  # shellcheck disable=SC2034
-  VERSION=${GIT_REVISION}_${ISSUE}_PATCH-${patchNum}
-
-  if [[ "${ISSUE}" == 'Unknown' ]]; then
-    echo "Testing patch on ${PATCH_BRANCH}."
-  else
-    echo "Testing ${ISSUE} patch on ${PATCH_BRANCH}."
-  fi
-
-  add_jira_footer "git revision" "${PATCH_BRANCH} / ${GIT_REVISION}"
-
-  if [[ ! -f ${BASEDIR}/pom.xml ]]; then
-    hadoop_error "ERROR: This verison of test-patch.sh only supports Maven-based builds. Aborting."
-    add_jira_table -1 pre-patch "Unsupported build system."
-    output_to_jira 1
-    cleanup_and_exit 1
-  fi
-  return 0
-}
-
-## @description  Confirm the source environment is compilable
-## @audience     private
-## @stability    stable
-## @replaceable  no
-## @return       0 on success
-## @return       1 on failure
-function precheck_without_patch
-{
-  local -r mypwd=$(pwd)
-
-  big_console_header "Pre-patch ${PATCH_BRANCH} Java verification"
-
-  start_clock
-
-  verify_needed_test javac
-
-  if [[ $? == 1 ]]; then
-    echo "Compiling ${mypwd}"
-    echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}JavacWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean test -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch
-    if [[ $? != 0 ]] ; then
-      echo "${PATCH_BRANCH} compilation is broken?"
-      add_jira_table -1 pre-patch "${PATCH_BRANCH} compilation may be broken."
-      return 1
-    fi
-  else
-    echo "Patch does not appear to need javac tests."
-  fi
-
-  verify_needed_test javadoc
-
-  if [[ $? == 1 ]]; then
-    echo "Javadoc'ing ${mypwd}"
-    echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess
-    if [[ $? != 0 ]] ; then
-      echo "Pre-patch ${PATCH_BRANCH} javadoc compilation is broken?"
-      add_jira_table -1 pre-patch "Pre-patch ${PATCH_BRANCH} JavaDoc compilation may be broken."
-      return 1
-    fi
-  else
-    echo "Patch does not appear to need javadoc tests."
-  fi
-
-  verify_needed_test site
-
-  if [[ $? == 1 ]]; then
-    echo "site creation for ${mypwd}"
-    echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}SiteWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean site site:stage -DskipTests -Dmaven.javadoc.skip=true -D${PROJECT_NAME}PatchProcess
-    if [[ $? != 0 ]] ; then
-      echo "Pre-patch ${PATCH_BRANCH} site compilation is broken?"
-      add_jira_table -1 pre-patch "Pre-patch ${PATCH_BRANCH} site compilation may be broken."
-      return 1
-    fi
-  else
-    echo "Patch does not appear to need site tests."
-  fi
-
-  precheck_findbugs
-
-  if [[ $? != 0 ]] ; then
-    return 1
-  fi
-
-  add_jira_table 0 pre-patch "Pre-patch ${PATCH_BRANCH} compilation is healthy."
-  return 0
-}
-
-## @description  Confirm the given branch is a member of the list of space
-## @description  delimited branches or a git ref
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @param        branch
-## @param        branchlist
-## @return       0 on success
-## @return       1 on failure
-function verify_valid_branch
-{
-  local branches=$1
-  local check=$2
-  local i
-
-  # shortcut some common
-  # non-resolvable names
-  if [[ -z ${check} ]]; then
-    return 1
-  fi
-
-  if [[ ${check} == patch ]]; then
-    return 1
-  fi
-
-  if [[ ${check} =~ ^git ]]; then
-    ref=$(echo "${check}" | cut -f2 -dt)
-    count=$(echo "${ref}" | wc -c | tr -d ' ')
-
-    if [[ ${count} == 8 || ${count} == 41 ]]; then
-      return 0
-    fi
-    return 1
-  fi
-
-  for i in ${branches}; do
-    if [[ "${i}" == "${check}" ]]; then
-      return 0
-    fi
-  done
-  return 1
-}
-
-## @description  Try to guess the branch being tested using a variety of heuristics
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @return       0 on success, with PATCH_BRANCH updated appropriately
-## @return       1 on failure, with PATCH_BRANCH updated to PATCH_BRANCH_DEFAULT
-function determine_branch
-{
-  local allbranches
-  local patchnamechunk
-
-  hadoop_debug "Determine branch"
-
-  # something has already set this, so move on
-  if [[ -n ${PATCH_BRANCH} ]]; then
-    return
-  fi
-
-  pushd "${BASEDIR}" > /dev/null
-
-  # developer mode, existing checkout, whatever
-  if [[ "${DIRTY_WORKSPACE}" == true ]];then
-    PATCH_BRANCH=$(${GIT} rev-parse --abbrev-ref HEAD)
-    echo "dirty workspace mode; applying against existing branch"
-    return
-  fi
-
-  allbranches=$(${GIT} branch -r | tr -d ' ' | ${SED} -e s,origin/,,g)
-
-  for j in "${PATCHURL}" "${PATCH_OR_ISSUE}"; do
-    hadoop_debug "Determine branch: starting with ${j}"
-    # shellcheck disable=SC2016
-    patchnamechunk=$(echo "${j}" | ${AWK} -F/ '{print $NF}')
-
-    # ISSUE.branch.##.patch
-    hadoop_debug "Determine branch: ISSUE.branch.##.patch"
-    PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f2 -d. )
-    verify_valid_branch "${allbranches}" "${PATCH_BRANCH}"
-    if [[ $? == 0 ]]; then
-      return
-    fi
-
-    # ISSUE-branch-##.patch
-    hadoop_debug "Determine branch: ISSUE-branch-##.patch"
-    PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d- | cut -f1,2 -d-)
-    verify_valid_branch "${allbranches}" "${PATCH_BRANCH}"
-    if [[ $? == 0 ]]; then
-      return
-    fi
-
-    # ISSUE-##.patch.branch
-    hadoop_debug "Determine branch: ISSUE-##.patch.branch"
-    # shellcheck disable=SC2016
-    PATCH_BRANCH=$(echo "${patchnamechunk}" | ${AWK} -F. '{print $NF}')
-    verify_valid_branch "${allbranches}" "${PATCH_BRANCH}"
-    if [[ $? == 0 ]]; then
-      return
-    fi
-
-    # ISSUE-branch.##.patch
-    hadoop_debug "Determine branch: ISSUE-branch.##.patch"
-    # shellcheck disable=SC2016
-    PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d- | ${AWK} -F. '{print $(NF-2)}' 2>/dev/null)
-    verify_valid_branch "${allbranches}" "${PATCH_BRANCH}"
-    if [[ $? == 0 ]]; then
-      return
-    fi
-  done
-
-  PATCH_BRANCH="${PATCH_BRANCH_DEFAULT}"
-
-  popd >/dev/null
-}
-
-## @description  Try to guess the issue being tested using a variety of heuristics
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @return       0 on success, with ISSUE updated appropriately
-## @return       1 on failure, with ISSUE updated to "Unknown"
-function determine_issue
-{
-  local patchnamechunk
-  local maybeissue
-
-  hadoop_debug "Determine issue"
-
-  # we can shortcut jenkins
-  if [[ ${JENKINS} == true ]]; then
-    ISSUE=${PATCH_OR_ISSUE}
-    return 0
-  fi
-
-  # shellcheck disable=SC2016
-  patchnamechunk=$(echo "${PATCH_OR_ISSUE}" | ${AWK} -F/ '{print $NF}')
-
-  maybeissue=$(echo "${patchnamechunk}" | cut -f1,2 -d-)
-
-  if [[ ${maybeissue} =~ ${ISSUE_RE} ]]; then
-    ISSUE=${maybeissue}
-    return 0
-  fi
-
-  ISSUE="Unknown"
-  return 1
-}
-
-## @description  Add the given test type
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        test
-function add_test
-{
-  local testname=$1
-
-  hadoop_debug "Testing against ${testname}"
-
-  if [[ -z ${NEEDED_TESTS} ]]; then
-    hadoop_debug "Setting tests to ${testname}"
-    NEEDED_TESTS=${testname}
-  elif [[ ! ${NEEDED_TESTS} =~ ${testname} ]] ; then
-    hadoop_debug "Adding ${testname}"
-    NEEDED_TESTS="${NEEDED_TESTS} ${testname}"
-  fi
-}
-
-## @description  Verify if a given test was requested
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        test
-## @return       1 = yes
-## @return       0 = no
-function verify_needed_test
-{
-  local i=$1
-
-  if [[ ${NEEDED_TESTS} =~ $i ]]; then
-    return 1
-  fi
-  return 0
-}
-
-## @description  Use some heuristics to determine which long running
-## @description  tests to run
-## @audience     private
-## @stability    stable
-## @replaceable  no
-function determine_needed_tests
-{
-  local i
-
-  for i in ${CHANGED_FILES}; do
-    if [[ ${i} =~ src/main/webapp ]]; then
-      hadoop_debug "tests/webapp: ${i}"
-    elif [[ ${i} =~ \.sh
-         || ${i} =~ \.cmd
-         ]]; then
-      hadoop_debug "tests/shell: ${i}"
-    elif [[ ${i} =~ \.md$
-         || ${i} =~ \.md\.vm$
-         || ${i} =~ src/site
-         || ${i} =~ src/main/docs
-         ]]; then
-      hadoop_debug "tests/site: ${i}"
-      add_test site
-    elif [[ ${i} =~ \.c$
-         || ${i} =~ \.cc$
-         || ${i} =~ \.h$
-         || ${i} =~ \.hh$
-         || ${i} =~ \.proto$
-         || ${i} =~ src/test
-         || ${i} =~ \.cmake$
-         || ${i} =~ CMakeLists.txt
-         ]]; then
-      hadoop_debug "tests/units: ${i}"
-      add_test javac
-      add_test unit
-    elif [[ ${i} =~ pom.xml$
-         || ${i} =~ \.java$
-         || ${i} =~ src/main
-         ]]; then
-      hadoop_debug "tests/javadoc+units: ${i}"
-      add_test javadoc
-      add_test javac
-      add_test unit
-    fi
-
-    if [[ ${i} =~ \.java$ ]]; then
-      add_test findbugs
-    fi
-
-    for plugin in ${PLUGINS}; do
-      if declare -f ${plugin}_filefilter >/dev/null 2>&1; then
-        "${plugin}_filefilter" "${i}"
-      fi
-    done
-  done
-
-  add_jira_footer "Optional Tests" "${NEEDED_TESTS}"
-}
-
-## @description  Given ${PATCH_ISSUE}, determine what type of patch file is in use, and do the
-## @description  necessary work to place it into ${PATCH_DIR}/patch.
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @return       0 on success
-## @return       1 on failure, may exit
-function locate_patch
-{
-  local notSureIfPatch=false
-  hadoop_debug "locate patch"
-
-  if [[ -f ${PATCH_OR_ISSUE} ]]; then
-    PATCH_FILE="${PATCH_OR_ISSUE}"
-  else
-    if [[ ${PATCH_OR_ISSUE} =~ ^http ]]; then
-      echo "Patch is being downloaded at $(date) from"
-      PATCHURL="${PATCH_OR_ISSUE}"
-    else
-      ${WGET} -q -O "${PATCH_DIR}/jira" "http://issues.apache.org/jira/browse/${PATCH_OR_ISSUE}"
-
-      if [[ $? != 0 ]];then
-        hadoop_error "ERROR: Unable to determine what ${PATCH_OR_ISSUE} may reference."
-        cleanup_and_exit 1
-      fi
-
-      if [[ $(${GREP} -c 'Patch Available' "${PATCH_DIR}/jira") == 0 ]] ; then
-        if [[ ${JENKINS} == true ]]; then
-          hadoop_error "ERROR: ${PATCH_OR_ISSUE} is not \"Patch Available\"."
-          cleanup_and_exit 1
-        else
-          hadoop_error "WARNING: ${PATCH_OR_ISSUE} is not \"Patch Available\"."
-        fi
-      fi
-
-      relativePatchURL=$(${GREP} -o '"/jira/secure/attachment/[0-9]*/[^"]*' "${PATCH_DIR}/jira" | ${GREP} -v -e 'htm[l]*$' | sort | tail -1 | ${GREP} -o '/jira/secure/attachment/[0-9]*/[^"]*')
-      PATCHURL="http://issues.apache.org${relativePatchURL}"
-      if [[ ! ${PATCHURL} =~ \.patch$ ]]; then
-        notSureIfPatch=true
-      fi
-      patchNum=$(echo "${PATCHURL}" | ${GREP} -o '[0-9]*/' | ${GREP} -o '[0-9]*')
-      echo "${ISSUE} patch is being downloaded at $(date) from"
-    fi
-    echo "${PATCHURL}"
-    add_jira_footer "Patch URL" "${PATCHURL}"
-    ${WGET} -q -O "${PATCH_DIR}/patch" "${PATCHURL}"
-    if [[ $? != 0 ]];then
-      hadoop_error "ERROR: ${PATCH_OR_ISSUE} could not be downloaded."
-      cleanup_and_exit 1
-    fi
-    PATCH_FILE="${PATCH_DIR}/patch"
-  fi
-
-  if [[ ! -f "${PATCH_DIR}/patch" ]]; then
-    cp "${PATCH_FILE}" "${PATCH_DIR}/patch"
-    if [[ $? == 0 ]] ; then
-      echo "Patch file ${PATCH_FILE} copied to ${PATCH_DIR}"
-    else
-      hadoop_error "ERROR: Could not copy ${PATCH_FILE} to ${PATCH_DIR}"
-      cleanup_and_exit 1
-    fi
-  fi
-  if [[ ${notSureIfPatch} == "true" ]]; then
-    guess_patch_file "${PATCH_DIR}/patch"
-    if [[ $? != 0 ]]; then
-      hadoop_error "ERROR: ${PATCHURL} is not a patch file."
-      cleanup_and_exit 1
-    else
-      hadoop_debug "The patch ${PATCHURL} was not named properly, but it looks like a patch file. proceeding, but issue/branch matching might go awry."
-      add_jira_table 0 patch "The patch file was not named according to ${PROJECT_NAME}'s naming conventions. Please see ${HOW_TO_CONTRIBUTE} for instructions."
-    fi
-  fi
-}
-
-## @description Given a possible patch file, guess if it's a patch file without using smart-apply-patch
-## @audience private
-## @stability evolving
-## @param path to patch file to test
-## @return 0 we think it's a patch file
-## @return 1 we think it's not a patch file
-function guess_patch_file
-{
-  local patch=$1
-  local fileOutput
-
-  hadoop_debug "Trying to guess is ${patch} is a patch file."
-  fileOutput=$("${FILE}" "${patch}")
-  if [[ $fileOutput =~ \ diff\  ]]; then
-    hadoop_debug "file magic says it's a diff."
-    return 0
-  fi
-  fileOutput=$(head -n 1 "${patch}" | "${EGREP}" "^(From [a-z0-9]* Mon Sep 17 00:00:00 2001)|(diff .*)|(Index: .*)$")
-  if [[ $? == 0 ]]; then
-    hadoop_debug "first line looks like a patch file."
-    return 0
-  fi
-  return 1
-}
-
-## @description  Given ${PATCH_DIR}/patch, verify the patch is good using ${BINDIR}/smart-apply-patch.sh
-## @description  in dryrun mode.
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @return       0 on success
-## @return       1 on failure
-function verify_patch_file
-{
-  # Before building, check to make sure that the patch is valid
-  export PATCH
-
-  "${BINDIR}/smart-apply-patch.sh" "${PATCH_DIR}/patch" dryrun
-  if [[ $? != 0 ]] ; then
-    echo "PATCH APPLICATION FAILED"
-    add_jira_table -1 patch "The patch command could not apply the patch during dryrun."
-    return 1
-  else
-    return 0
-  fi
-}
-
-## @description  Given ${PATCH_DIR}/patch, apply the patch using ${BINDIR}/smart-apply-patch.sh
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @return       0 on success
-## @return       exit on failure
-function apply_patch_file
-{
-  big_console_header "Applying patch"
-
-  export PATCH
-  "${BINDIR}/smart-apply-patch.sh" "${PATCH_DIR}/patch"
-  if [[ $? != 0 ]] ; then
-    echo "PATCH APPLICATION FAILED"
-    ((RESULT = RESULT + 1))
-    add_jira_table -1 patch "The patch command could not apply the patch."
-    output_to_console 1
-    output_to_jira 1
-    cleanup_and_exit 1
-  fi
-  return 0
-}
-
-
-## @description  If this actually patches the files used for the QA process
-## @description  under dev-support and its subdirectories, then
-## @description  run with the patched version for the test.
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @return       none; otherwise relaunches
-function check_reexec
-{
-  local commentfile=${PATCH_DIR}/tp.${RANDOM}
-
-  if [[ ${REEXECED} == true ]]; then
-    big_console_header "Re-exec mode detected. Continuing."
-    return
-  fi
-
-  if [[ ! ${CHANGED_FILES} =~ dev-support/test-patch
-     && ! ${CHANGED_FILES} =~ dev-support/smart-apply ]] ; then
-    return
-  fi
-
-  big_console_header "dev-support patch detected"
-
-  if [[ ${RESETREPO} == false ]]; then
-    ((RESULT = RESULT + 1))
-    hadoop_debug "can't destructively change the working directory. run with '--resetrepo' please. :("
-    add_jira_table -1 dev-support "Couldn't test dev-support changes because we aren't configured to destructively change the working directory."
-    return
-  fi
-
-  printf "\n\nRe-executing against patched versions to test.\n\n"
-
-  apply_patch_file
-
-  if [[ ${JENKINS} == true ]]; then
-
-    rm "${commentfile}" 2>/dev/null
-
-    echo "(!) A patch to the files used for the QA process has been detected. " > "${commentfile}"
-    echo "Re-executing against the patched versions to perform further tests. " >> "${commentfile}"
-    echo "The console is at ${BUILD_URL}console in case of problems." >> "${commentfile}"
-
-    write_to_jira "${commentfile}"
-    rm "${commentfile}"
-  fi
-
-  cd "${CWD}"
-  mkdir -p "${PATCH_DIR}/dev-support-test"
-  cp -pr "${BASEDIR}"/dev-support/test-patch* "${PATCH_DIR}/dev-support-test"
-  cp -pr "${BASEDIR}"/dev-support/smart-apply* "${PATCH_DIR}/dev-support-test"
-
-  big_console_header "exec'ing test-patch.sh now..."
-
-  exec "${PATCH_DIR}/dev-support-test/test-patch.sh" \
-    --reexec \
-    --branch="${PATCH_BRANCH}" \
-    --patch-dir="${PATCH_DIR}" \
-      "${USER_PARAMS[@]}"
-}
-
-## @description  Check the current directory for @author tags
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @return       0 on success
-## @return       1 on failure
-function check_author
-{
-  local authorTags
-
-  big_console_header "Checking there are no @author tags in the patch."
-
-  start_clock
-
-  if [[ ${CHANGED_FILES} =~ dev-support/test-patch ]]; then
-    add_jira_table 0 @author "Skipping @author checks as test-patch has been patched."
-    return 0
-  fi
-
-  authorTags=$("${GREP}" -c -i '^[^-].*@author' "${PATCH_DIR}/patch")
-  echo "There appear to be ${authorTags} @author tags in the patch."
-  if [[ ${authorTags} != 0 ]] ; then
-    add_jira_table -1 @author \
-      "The patch appears to contain ${authorTags} @author tags which the Hadoop" \
-      " community has agreed to not allow in code contributions."
-    return 1
-  fi
-  add_jira_table +1 @author "The patch does not contain any @author tags."
-  return 0
-}
-
-## @description  Check the patch file for changed/new tests
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @return       0 on success
-## @return       1 on failure
-function check_modified_unittests
-{
-  local testReferences=0
-  local i
-
-  verify_needed_test unit
-
-  if [[ $? == 0 ]]; then
-    return 0
-  fi
-
-  big_console_header "Checking there are new or changed tests in the patch."
-
-  start_clock
-
-  for i in ${CHANGED_FILES}; do
-    if [[ ${i} =~ /test/ ]]; then
-      ((testReferences=testReferences + 1))
-    fi
-  done
-
-  echo "There appear to be ${testReferences} test file(s) referenced in the patch."
-  if [[ ${testReferences} == 0 ]] ; then
-    add_jira_table -1 "tests included" \
-      "The patch doesn't appear to include any new or modified tests. " \
-      "Please justify why no new tests are needed for this patch." \
-      "Also please list what manual steps were performed to verify this patch."
-    return 1
-  fi
-  add_jira_table +1 "tests included" \
-    "The patch appears to include ${testReferences} new or modified test files."
-  return 0
-}
-
-## @description  Helper for check_javadoc
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @return       0 on success
-## @return       1 on failure
-function count_javadoc_warns
-{
-  local warningfile=$1
-
-  #shellcheck disable=SC2016,SC2046
-  return $(${EGREP} "^[0-9]+ warnings$" "${warningfile}" | ${AWK} '{sum+=$1} END {print sum}')
-}
-
-## @description  Count and compare the number of JavaDoc warnings pre- and post- patch
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @return       0 on success
-## @return       1 on failure
-function check_javadoc
-{
-  local numBranchJavadocWarnings
-  local numPatchJavadocWarnings
-
-  verify_needed_test javadoc
-
-  if [[ $? == 0 ]]; then
-    echo "This patch does not appear to need javadoc checks."
-    return 0
-  fi
-
-  big_console_header "Determining number of patched javadoc warnings"
-
-  start_clock
-
-  if [[ -d hadoop-project ]]; then
-    (cd hadoop-project; "${MVN}" "${MAVEN_ARGS[@]}" install > /dev/null 2>&1)
-  fi
-  if [[ -d hadoop-common-project/hadoop-annotations ]]; then
-    (cd hadoop-common-project/hadoop-annotations; "${MVN}" "${MAVEN_ARGS[@]}" install > /dev/null 2>&1)
-  fi
-  echo_and_redirect "${PATCH_DIR}/patchJavadocWarnings.txt"  "${MVN}" "${MAVEN_ARGS[@]}" clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess
-  count_javadoc_warns "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarnings.txt"
-  numBranchJavadocWarnings=$?
-  count_javadoc_warns "${PATCH_DIR}/patchJavadocWarnings.txt"
-  numPatchJavadocWarnings=$?
-
-  echo "There appear to be ${numBranchJavadocWarnings} javadoc warnings before the patch and ${numPatchJavadocWarnings} javadoc warnings after applying the patch."
-  if [[ ${numBranchJavadocWarnings} != "" && ${numPatchJavadocWarnings} != "" ]] ; then
-    if [[ ${numPatchJavadocWarnings} -gt ${numBranchJavadocWarnings} ]] ; then
-
-      ${GREP} -i warning "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarnings.txt" > "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarningsFiltered.txt"
-      ${GREP} -i warning "${PATCH_DIR}/patchJavadocWarnings.txt" > "${PATCH_DIR}/patchJavadocWarningsFiltered.txt"
-      ${DIFF} -u "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarningsFiltered.txt" \
-        "${PATCH_DIR}/patchJavadocWarningsFiltered.txt" \
-        > "${PATCH_DIR}/diffJavadocWarnings.txt"
-      rm -f "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarningsFiltered.txt" "${PATCH_DIR}/patchJavadocWarningsFiltered.txt"
-
-      add_jira_table -1 javadoc "The applied patch generated "\
-      "$((numPatchJavadocWarnings-numBranchJavadocWarnings))" \
-      " additional warning messages."
-      add_jira_footer javadoc "@@BASE@@/diffJavadocWarnings.txt"
-      return 1
-    fi
-  fi
-  add_jira_table +1 javadoc "There were no new javadoc warning messages."
-  return 0
-}
-
-## @description  Make sure site still compiles
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @return       0 on success
-## @return       1 on failure
-function check_site
-{
-  local -r mypwd=$(pwd)
-
-  verify_needed_test site
-
-  if [[ $? == 0 ]]; then
-    echo "This patch does not appear to need site checks."
-    return 0
-  fi
-
-  big_console_header "Determining if patched site still builds"
-
-  start_clock
-
-  echo "site creation for ${mypwd}"
-  echo_and_redirect "${PATCH_DIR}/patchSiteWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean site site:stage -DskipTests -Dmaven.javadoc.skip=true -D${PROJECT_NAME}PatchProcess
-  if [[ $? != 0 ]] ; then
-    echo "Site compilation is broken"
-    add_jira_table -1 site "Site compilation is broken."
-    add_jira_footer site "@@BASE@@/patchSiteWarnings.txt"
-    return 1
-  fi
-  add_jira_table +1 site "Site still builds."
-  return 0
-}
-
-## @description  Helper for check_javac
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @return       0 on success
-## @return       1 on failure
-function count_javac_warns
-{
-  local warningfile=$1
-  #shellcheck disable=SC2016,SC2046
-  return $(${AWK} 'BEGIN {total = 0} {total += 1} END {print total}' "${warningfile}")
-}
-
-## @description  Count and compare the number of javac warnings pre- and post- patch
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @return       0 on success
-## @return       1 on failure
-function check_javac
-{
-  local branchJavacWarnings
-  local patchJavacWarnings
-
-  verify_needed_test javac
-
-  if [[ $? == 0 ]]; then
-    echo "This patch does not appear to need javac checks."
-    return 0
-  fi
-
-  big_console_header "Determining number of patched javac warnings."
-
-  start_clock
-
-  echo_and_redirect "${PATCH_DIR}/patchJavacWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean test -DskipTests -D${PROJECT_NAME}PatchProcess ${NATIVE_PROFILE} -Ptest-patch
-  if [[ $? != 0 ]] ; then
-    add_jira_table -1 javac "The patch appears to cause the build to fail."
-    return 2
-  fi
-  ### Compare ${PATCH_BRANCH} and patch javac warning numbers
-  if [[ -f ${PATCH_DIR}/patchJavacWarnings.txt ]] ; then
-    ${GREP} '\[WARNING\]' "${PATCH_DIR}/${PATCH_BRANCH}JavacWarnings.txt" > "${PATCH_DIR}/filtered${PATCH_BRANCH}JavacWarnings.txt"
-    ${GREP} '\[WARNING\]' "${PATCH_DIR}/patchJavacWarnings.txt" > "${PATCH_DIR}/filteredPatchJavacWarnings.txt"
-
-    count_javac_warns "${PATCH_DIR}/filtered${PATCH_BRANCH}JavacWarnings.txt"
-    branchJavacWarnings=$?
-    count_javac_warns "${PATCH_DIR}/filteredPatchJavacWarnings.txt"
-    patchJavacWarnings=$?
-
-    echo "There appear to be ${branchJavacWarnings} javac compiler warnings before the patch and ${patchJavacWarnings} javac compiler warnings after applying the patch."
-    if [[ ${patchJavacWarnings} != "" && ${branchJavacWarnings} != "" ]] ; then
-      if [[ ${patchJavacWarnings} -gt ${branchJavacWarnings} ]] ; then
-
-        ${DIFF} "${PATCH_DIR}/filtered${PATCH_BRANCH}JavacWarnings.txt" \
-        "${PATCH_DIR}/filteredPatchJavacWarnings.txt" \
-        > "${PATCH_DIR}/diffJavacWarnings.txt"
-
-        add_jira_table -1 javac "The applied patch generated "\
-        "$((patchJavacWarnings-branchJavacWarnings))" \
-        " additional warning messages."
-
-        add_jira_footer javac "@@BASE@@/diffJavacWarnings.txt"
-
-        return 1
-      fi
-    fi
-  fi
-
-  add_jira_table +1 javac "There were no new javac warning messages."
-  return 0
-}
-
-## @description  Verify all files have an Apache License
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @return       0 on success
-## @return       1 on failure
-function check_apachelicense
-{
-  big_console_header "Determining number of patched release audit warnings."
-
-  start_clock
-
-  echo_and_redirect "${PATCH_DIR}/patchReleaseAuditOutput.txt" "${MVN}" "${MAVEN_ARGS[@]}" apache-rat:check -D${PROJECT_NAME}PatchProcess
-  #shellcheck disable=SC2038
-  find "${BASEDIR}" -name rat.txt | xargs cat > "${PATCH_DIR}/patchReleaseAuditWarnings.txt"
-
-  ### Compare ${PATCH_BRANCH} and patch release audit warning numbers
-  if [[ -f ${PATCH_DIR}/patchReleaseAuditWarnings.txt ]] ; then
-    patchReleaseAuditWarnings=$("${GREP}" -c '\!?????' "${PATCH_DIR}/patchReleaseAuditWarnings.txt")
-    echo ""
-    echo ""
-    echo "There appear to be ${patchReleaseAuditWarnings} release audit warnings after applying the patch."
-    if [[ ${patchReleaseAuditWarnings} != "" ]] ; then
-      if [[ ${patchReleaseAuditWarnings} -gt 0 ]] ; then
-        add_jira_table -1 "release audit" "The applied patch generated ${patchReleaseAuditWarnings} release audit warnings."
-
-        ${GREP} '\!?????' "${PATCH_DIR}/patchReleaseAuditWarnings.txt" \
-        >  "${PATCH_DIR}/patchReleaseAuditProblems.txt"
-
-        echo "Lines that start with ????? in the release audit "\
-            "report indicate files that do not have an Apache license header." \
-            >> "${PATCH_DIR}/patchReleaseAuditProblems.txt"
-
-        add_jira_footer "Release Audit" "@@BASE@@/patchReleaseAuditProblems.txt"
-
-        return 1
-      fi
-    fi
-  fi
-  add_jira_table 1 "release audit" "The applied patch does not increase the total number of release audit warnings."
-  return 0
-}
-
-## @description  Verify mvn install works
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @return       0 on success
-## @return       1 on failure
-function check_mvn_install
-{
-  local retval
-
-  verify_needed_test javadoc
-  retval=$?
-
-  verify_needed_test javac
-  ((retval = retval + $? ))
-
-  if [[ ${retval} == 0 ]]; then
-    echo "This patch does not appear to need mvn install checks."
-    return 0
-  fi
-
-  big_console_header "Installing all of the jars"
-
-  start_clock
-  echo_and_redirect "${PATCH_DIR}/jarinstall.txt" "${MVN}" "${MAVEN_ARGS[@]}" install -Dmaven.javadoc.skip=true -DskipTests -D${PROJECT_NAME}PatchProcess
-  retval=$?
-  if [[ ${retval} != 0 ]]; then
-    add_jira_table -1 install "The patch causes mvn install to fail."
-  else
-    add_jira_table +1 install "mvn install still works."
-  fi
-  return ${retval}
-}
-
-## @description  are the needed bits for findbugs present?
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @return       0 findbugs will work for our use
-## @return       1 findbugs is missing some component
-function findbugs_is_installed
-{
-  if [[ ! -e "${FINDBUGS_HOME}/bin/findbugs" ]]; then
-    printf "\n\n%s is not executable.\n\n" "${FINDBUGS_HOME}/bin/findbugs"
-    add_jira_table -1 findbugs "Findbugs is not installed."
-    return 1
-  fi
-  return 0
-}
-
-## @description  Run the maven findbugs plugin and record found issues in a bug database
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @return       0 on success
-## @return       1 on failure
-function findbugs_mvnrunner
-{
-  local name=$1
-  local logfile=$2
-  local warnings_file=$3
-
-  echo_and_redirect "${logfile}" "${MVN}" "${MAVEN_ARGS[@]}" clean test findbugs:findbugs -DskipTests \
-    "-D${PROJECT_NAME}PatchProcess" < /dev/null
-  if [[ $? != 0 ]]; then
-    return 1
-  fi
-  cp target/findbugsXml.xml "${warnings_file}.xml"
-
-  "${FINDBUGS_HOME}/bin/setBugDatabaseInfo" -name "${name}" \
-      "${warnings_file}.xml" "${warnings_file}.xml"
-  if [[ $? != 0 ]]; then
-    return 1
-  fi
-
-  "${FINDBUGS_HOME}/bin/convertXmlToText" -html "${warnings_file}.xml" \
-      "${warnings_file}.html"
-  if [[ $? != 0 ]]; then
-    return 1
-  fi
-
-  return 0
-}
-
-## @description  Track pre-existing findbugs warnings
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @return       0 on success
-## @return       1 on failure
-function precheck_findbugs
-{
-  local -r mypwd=$(pwd)
-  local module_suffix
-  local modules=${CHANGED_MODULES}
-  local module
-  local findbugs_version
-  local rc=0
-  local module_findbugs_warnings
-  local findbugs_warnings=0
-
-  verify_needed_test findbugs
-
-  if [[ $? == 0 ]]; then
-    echo "Patch does not appear to need findbugs tests."
-    return 0
-  fi
-
-  echo "findbugs baseline for ${mypwd}"
-
-  findbugs_is_installed
-  if [[ $? != 0 ]]; then
-    return 1
-  fi
-
-  for module in ${modules}
-  do
-    pushd "${module}" >/dev/null
-    echo "  Running findbugs in ${module}"
-    module_suffix=$(basename "${module}")
-    findbugs_mvnrunner "${PATCH_BRANCH}" \
-        "${PATCH_DIR}/${PATCH_BRANCH}FindBugsOutput${module_suffix}.txt" \
-        "${PATCH_DIR}/${PATCH_BRANCH}FindbugsWarnings${module_suffix}"
-    (( rc = rc + $? ))
-
-    if [[ "${FINDBUGS_WARNINGS_FAIL_PRECHECK}" == "true" ]]; then
-      #shellcheck disable=SC2016
-      module_findbugs_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -first \
-          "${PATCH_BRANCH}" \
-          "${PATCH_DIR}/${PATCH_BRANCH}FindbugsWarnings${module_suffix}".xml \
-          "${PATCH_DIR}/${PATCH_BRANCH}FindbugsWarnings${module_suffix}".xml \
-          | ${AWK} '{print $1}')
-      if [[ $? != 0 ]]; then
-        popd >/dev/null
-        return 1
-      fi
-
-      findbugs_warnings=$((findbugs_warnings+module_findbugs_warnings))
-
-      if [[ ${module_findbugs_warnings} -gt 0 ]] ; then
-        add_jira_footer "Pre-patch Findbugs warnings" "@@BASE@@/${PATCH_BRANCH}FindbugsWarnings${module_suffix}.html"
-      fi
-    fi
-    popd >/dev/null
-  done
-
-  #shellcheck disable=SC2016
-  findbugs_version=$(${AWK} 'match($0, /findbugs-maven-plugin:[^:]*:findbugs/) { print substr($0, RSTART + 22, RLENGTH - 31); exit }' "${PATCH_DIR}/${PATCH_BRANCH}FindBugsOutput${module_suffix}.txt")
-
-  if [[ ${rc} -ne 0 ]]; then
-    echo "Pre-patch ${PATCH_BRANCH} findbugs is broken?"
-    add_jira_table -1 pre-patch "Findbugs (version ${findbugs_version}) appears to be broken on ${PATCH_BRANCH}."
-    return 1
-  fi
-
-  if [[ "${FINDBUGS_WARNINGS_FAIL_PRECHECK}" == "true" && \
-        ${findbugs_warnings} -gt 0 ]] ; then
-    echo "Pre-patch ${PATCH_BRANCH} findbugs has ${findbugs_warnings} warnings."
-    add_jira_table -1 pre-patch "Pre-patch ${PATCH_BRANCH} has ${findbugs_warnings} extant Findbugs (version ${findbugs_version}) warnings."
-    return 1
-  fi
-
-  return 0
-}
-
-## @description  Verify patch does not trigger any findbugs warnings
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @return       0 on success
-## @return       1 on failure
-function check_findbugs
-{
-  local rc=0
-  local module
-  local modules=${CHANGED_MODULES}
-  local module_suffix
-  local combined_xml
-  local newBugs
-  local new_findbugs_warnings
-  local new_findbugs_fixed_warnings
-  local findbugs_warnings=0
-  local findbugs_fixed_warnings=0
-  local line
-  local firstpart
-  local secondpart
-  local findbugs_version
-
-  verify_needed_test findbugs
-
-  if [[ $? == 0 ]]; then
-    return 0
-  fi
-
-  big_console_header "Determining number of patched Findbugs warnings."
-
-  start_clock
-
-  findbugs_is_installed
-  if [[ $? != 0 ]]; then
-    return 1
-  fi
-
-  for module in ${modules}
-  do
-    pushd "${module}" >/dev/null
-    echo "  Running findbugs in ${module}"
-    module_suffix=$(basename "${module}")
-
-    findbugs_mvnrunner patch \
-        "${PATCH_DIR}/patchFindBugsOutput${module_suffix}.txt" \
-        "${PATCH_DIR}/patchFindbugsWarnings${module_suffix}"
-
-    if [[ $? != 0 ]] ; then
-      ((rc = rc +1))
-      echo "Post-patch findbugs compilation is broken."
-      add_jira_table -1 findbugs "Post-patch findbugs ${module} compilation is broken."
-      continue
-    fi
-
-    combined_xml="$PATCH_DIR/combinedFindbugsWarnings${module_suffix}.xml"
-    newBugs="${PATCH_DIR}/newPatchFindbugsWarnings${module_suffix}"
-    "${FINDBUGS_HOME}/bin/computeBugHistory" -useAnalysisTimes -withMessages \
-        -output "${combined_xml}" \
-        "${PATCH_DIR}/${PATCH_BRANCH}FindbugsWarnings${module_suffix}.xml" \
-        "${PATCH_DIR}/patchFindbugsWarnings${module_suffix}.xml"
-    if [[ $? != 0 ]]; then
-      popd >/dev/null
-      return 1
-    fi
-
-    #shellcheck disable=SC2016
-    new_findbugs_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -first patch \
-        "${combined_xml}" "${newBugs}.xml" | ${AWK} '{print $1}')
-    if [[ $? != 0 ]]; then
-      popd >/dev/null
-      return 1
-    fi
-    #shellcheck disable=SC2016
-    new_findbugs_fixed_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -fixed patch \
-        "${combined_xml}" "${newBugs}.xml" | ${AWK} '{print $1}')
-    if [[ $? != 0 ]]; then
-      popd >/dev/null
-      return 1
-    fi
-
-    echo "Found ${new_findbugs_warnings} new Findbugs warnings and ${new_findbugs_fixed_warnings} newly fixed warnings."
-    findbugs_warnings=$((findbugs_warnings+new_findbugs_warnings))
-    findbugs_fixed_warnings=$((findbugs_fixed_warnings+new_findbugs_fixed_warnings))
-
-    "${FINDBUGS_HOME}/bin/convertXmlToText" -html "${newBugs}.xml" \
-        "${newBugs}.html"
-    if [[ $? != 0 ]]; then
-      popd >/dev/null
-      return 1
-    fi
-
-    if [[ ${new_findbugs_warnings} -gt 0 ]] ; then
-      populate_test_table FindBugs "module:${module_suffix}"
-      while read line; do
-        firstpart=$(echo "${line}" | cut -f2 -d:)
-        secondpart=$(echo "${line}" | cut -f9- -d' ')
-        add_jira_test_table "" "${firstpart}:${secondpart}"
-      done < <("${FINDBUGS_HOME}/bin/convertXmlToText" "${newBugs}.xml")
-
-      add_jira_footer "Findbugs warnings" "@@BASE@@/newPatchFindbugsWarnings${module_suffix}.html"
-    fi
-
-    popd >/dev/null
-  done
-
-  #shellcheck disable=SC2016
-  findbugs_version=$(${AWK} 'match($0, /findbugs-maven-plugin:[^:]*:findbugs/) { print substr($0, RSTART + 22, RLENGTH - 31); exit }' "${PATCH_DIR}/patchFindBugsOutput${module_suffix}.txt")
-
-  if [[ ${findbugs_warnings} -gt 0 ]] ; then
-    add_jira_table -1 findbugs "The patch appears to introduce ${findbugs_warnings} new Findbugs (version ${findbugs_version}) warnings."
-    return 1
-  fi
-
-  if [[ ${findbugs_fixed_warnings} -gt 0 ]] ; then
-    add_jira_table +1 findbugs "The patch does not introduce any new Findbugs (version ${findbugs_version}) warnings, and fixes ${findbugs_fixed_warnings} pre-existing warnings."
-  else
-    add_jira_table +1 findbugs "The patch does not introduce any new Findbugs (version ${findbugs_version}) warnings."
-  fi
-  return 0
-}
-
-## @description  Make sure Maven's eclipse generation works.
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @return       0 on success
-## @return       1 on failure
-function check_mvn_eclipse
-{
-  big_console_header "Running mvn eclipse:eclipse."
-
-  verify_needed_test javac
-  if [[ $? == 0 ]]; then
-    echo "Patch does not touch any java files. Skipping mvn eclipse:eclipse"
-    return 0
-  fi
-
-  start_clock
-
-  echo_and_redirect "${PATCH_DIR}/patchEclipseOutput.txt" "${MVN}" "${MAVEN_ARGS[@]}" eclipse:eclipse -D${PROJECT_NAME}PatchProcess
-  if [[ $? != 0 ]] ; then
-    add_jira_table -1 eclipse:eclipse "The patch failed to build with eclipse:eclipse."
-    return 1
-  fi
-  add_jira_table +1 eclipse:eclipse "The patch built with eclipse:eclipse."
-  return 0
-}
-
-## @description  Utility to push many tests into the failure list
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @param        testdesc
-## @param        testlist
-function populate_test_table
-{
-  local reason=$1
-  shift
-  local first=""
-  local i
-
-  for i in "$@"; do
-    if [[ -z "${first}" ]]; then
-      add_jira_test_table "${reason}" "${i}"
-      first="${reason}"
-    else
-      add_jira_test_table " " "${i}"
-    fi
-  done
-}
-
-## @description  Run and verify the output of the appropriate unit tests
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @return       0 on success
-## @return       1 on failure
-function check_unittests
-{
-  verify_needed_test unit
-
-  if [[ $? == 0 ]]; then
-    echo "Existing unit tests do not test patched files. Skipping."
-    return 0
-  fi
-
-  big_console_header "Running unit tests"
-
-  start_clock
-
-  local failed_tests=""
-  local modules=${CHANGED_MODULES}
-  local building_common=0
-  local hdfs_modules
-  local ordered_modules=""
-  local failed_test_builds=""
-  local test_timeouts=""
-  local test_logfile
-  local test_build_result
-  local module_test_timeouts=""
-  local result
-  local totalresult=0
-  local module_prefix
-
-  #
-  # If we are building hadoop-hdfs-project, we must build the native component
-  # of hadoop-common-project first.  In order to accomplish this, we move the
-  # hadoop-hdfs subprojects to the end of the list so that common will come
-  # first.
-  #
-  # Of course, we may not be building hadoop-common at all-- in this case, we
-  # explicitly insert a mvn compile -Pnative of common, to ensure that the
-  # native libraries show up where we need them.
-  #
-
-  for module in ${modules}; do
-    if [[ ${module} == hadoop-hdfs-project* ]]; then
-      hdfs_modules="${hdfs_modules} ${module}"
-    elif [[ ${module} == hadoop-common-project* ]]; then
-      ordered_modules="${ordered_modules} ${module}"
-      building_common=1
-    else
-      ordered_modules="${ordered_modules} ${module}"
-    fi
-  done
-
-  if [[ -n "${hdfs_modules}" ]]; then
-    ordered_modules="${ordered_modules} ${hdfs_modules}"
-    if [[ ${building_common} -eq 0 ]]; then
-      echo "  Building hadoop-common with -Pnative in order to provide libhadoop.so to the hadoop-hdfs unit tests."
-      echo_and_redirect "${PATCH_DIR}/testrun_native.txt" "${MVN}" "${MAVEN_ARGS[@]}" compile ${NATIVE_PROFILE} "-D${PROJECT_NAME}PatchProcess"
-      if [[ $? != 0 ]]; then
-        add_jira_table -1 "native" "Failed to build the native portion " \
-          "of hadoop-common prior to running the unit tests in ${ordered_modules}"
-        return 1
-      else
-        add_jira_table +1 "native" "Pre-build of native portion"
-      fi
-    fi
-  fi
-
-  for module in ${ordered_modules}; do
-    result=0
-    start_clock
-    pushd "${module}" >/dev/null
-    module_suffix=$(basename "${module}")
-    module_prefix=$(echo "${module}" | cut -f2 -d- )
-
-    test_logfile=${PATCH_DIR}/testrun_${module_suffix}.txt
-    echo "  Running tests in ${module_suffix}"
-    # shellcheck disable=2086
-    echo_and_redirect "${test_logfile}" "${MVN}" "${MAVEN_ARGS[@]}" clean install -fae ${NATIVE_PROFILE} ${REQUIRE_TEST_LIB_HADOOP} ${PARALLEL_TESTS_PROFILE} ${TESTS_THREAD_COUNT} -D${PROJECT_NAME}PatchProcess
-    test_build_result=$?
-
-    add_jira_footer "${module_suffix} test log" "@@BASE@@/testrun_${module_suffix}.txt"
-
-    # shellcheck disable=2016
-    module_test_timeouts=$(${AWK} '/^Running / { array[$NF] = 1 } /^Tests run: .* in / { delete array[$NF] } END { for (x in array) { print x } }' "${test_logfile}")
-    if [[ -n "${module_test_timeouts}" ]] ; then
-      test_timeouts="${test_timeouts} ${module_test_timeouts}"
-      result=1
-    fi
-
-    #shellcheck disable=SC2026,SC2038,SC2016
-    module_failed_tests=$(find . -name 'TEST*.xml'\
-      | xargs "${GREP}" -l -E "<failure|<error"\
-      | ${AWK} -F/ '{sub("TEST-org.apache.",""); sub(".xml",""); print $NF}')
-
-    if [[ -n "${module_failed_tests}" ]] ; then
-      failed_tests="${failed_tests} ${module_failed_tests}"
-      result=1
-    fi
-    if [[ ${test_build_result} != 0 && -z "${module_failed_tests}" && -z "${module_test_timeouts}" ]] ; then
-      failed_test_builds="${failed_test_builds} ${module_suffix}"
-      result=1
-    fi
-    popd >/dev/null
-
-    if [[ $result == 1 ]]; then
-      add_jira_table -1 "${module_prefix} tests" "Tests failed in ${module_suffix}."
-    else
-      add_jira_table +1 "${module_prefix} tests" "Tests passed in ${module_suffix}."
-    fi
-
-    ((totalresult = totalresult + result))
-  done
-
-  if [[ -n "${failed_tests}" ]] ; then
-    # shellcheck disable=SC2086
-    populate_test_table "Failed unit tests" ${failed_tests}
-  fi
-  if [[ -n "${test_timeouts}" ]] ; then
-    # shellcheck disable=SC2086
-    populate_test_table "Timed out tests" ${test_timeouts}
-  fi
-  if [[ -n "${failed_test_builds}" ]] ; then
-    # shellcheck disable=SC2086
-    populate_test_table "Failed build" ${failed_test_builds}
-  fi
-
-  if [[ ${JENKINS} == true ]]; then
-    add_jira_footer "Test Results" "${BUILD_URL}testReport/"
-  fi
-
-  if [[ ${totalresult} -gt 0 ]]; then
-    return 1
-  else
-    return 0
-  fi
-}
-
-## @description  Print out the finished details on the console
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @param        runresult
-## @return       0 on success
-## @return       1 on failure
-function output_to_console
-{
-  local result=$1
-  shift
-  local i
-  local ourstring
-  local vote
-  local subs
-  local ela
-  local comment
-  local commentfile1="${PATCH_DIR}/comment.1"
-  local commentfile2="${PATCH_DIR}/comment.2"
-  local normaltop
-  local line
-  local seccoladj=0
-  local spcfx=${PATCH_DIR}/spcl.txt
-
-  if [[ ${result} == 0 ]]; then
-    if [[ ${JENKINS} == false ]]; then
-      {
-        printf "IF9fX19fX19fX18gCjwgU3VjY2VzcyEgPgogLS0tLS0tLS0tLSAKIFwgICAg";
-        printf "IC9cICBfX18gIC9cCiAgXCAgIC8vIFwvICAgXC8gXFwKICAgICAoKCAgICBP";
-        printf "IE8gICAgKSkKICAgICAgXFwgLyAgICAgXCAvLwogICAgICAgXC8gIHwgfCAg";
-        printf "XC8gCiAgICAgICAgfCAgfCB8ICB8ICAKICAgICAgICB8ICB8IHwgIHwgIAog";
-        printf "ICAgICAgIHwgICBvICAgfCAgCiAgICAgICAgfCB8ICAgfCB8ICAKICAgICAg";
-        printf "ICB8bXwgICB8bXwgIAo"
-      } > "${spcfx}"
-    fi
-    printf "\n\n+1 overall\n\n"
-  else
-    if [[ ${JENKINS} == false ]]; then
-      {
-        printf "IF9fX19fICAgICBfIF8gICAgICAgICAgICAgICAgXyAKfCAgX19ffF8gXyhf";
-        printf "KSB8XyAgIF8gXyBfXyBfX198IHwKfCB8XyAvIF9gIHwgfCB8IHwgfCB8ICdf";
-        printf "Xy8gXyBcIHwKfCAgX3wgKF98IHwgfCB8IHxffCB8IHwgfCAgX18vX3wKfF98";
-        printf "ICBcX18sX3xffF98XF9fLF98X3wgIFxfX18oXykKICAgICAgICAgICAgICAg";
-        printf "ICAgICAgICAgICAgICAgICAK"
-      } > "${spcfx}"
-    fi
-    printf "\n\n-1 overall\n\n"
-  fi
-
-  if [[ -f ${spcfx} ]]; then
-    if which base64 >/dev/null 2>&1; then
-      base64 --decode "${spcfx}" 2>/dev/null
-    elif which openssl >/dev/null 2>&1; then
-      openssl enc -A -d -base64 -in "${spcfx}" 2>/dev/null
-    fi
-    echo
-    echo
-    rm "${spcfx}"
-  fi
-
-  seccoladj=$(findlargest 2 "${JIRA_COMMENT_TABLE[@]}")
-  if [[ ${seccoladj} -lt 10 ]]; then
-    seccoladj=10
-  fi
-
-  seccoladj=$((seccoladj + 2 ))
-  i=0
-  until [[ $i -eq ${#JIRA_HEADER[@]} ]]; do
-    printf "%s\n" "${JIRA_HEADER[${i}]}"
-    ((i=i+1))
-  done
-
-  printf "| %s | %*s |  %s   | %s\n" "Vote" ${seccoladj} Subsystem Runtime "Comment"
-  echo "============================================================================"
-  i=0
-  until [[ $i -eq ${#JIRA_COMMENT_TABLE[@]} ]]; do
-    ourstring=$(echo "${JIRA_COMMENT_TABLE[${i}]}" | tr -s ' ')
-    vote=$(echo "${ourstring}" | cut -f2 -d\|)
-    vote=$(colorstripper "${vote}")
-    subs=$(echo "${ourstring}"  | cut -f3 -d\|)
-    ela=$(echo "${ourstring}" | cut -f4 -d\|)
-    comment=$(echo "${ourstring}"  | cut -f5 -d\|)
-
-    echo "${comment}" | fold -s -w $((78-seccoladj-22)) > "${commentfile1}"
-    normaltop=$(head -1 "${commentfile1}")
-    ${SED} -e '1d' "${commentfile1}"  > "${commentfile2}"
-
-    printf "| %4s | %*s | %-10s |%-s\n" "${vote}" ${seccoladj} \
-      "${subs}" "${ela}" "${normaltop}"
-    while read line; do
-      printf "|      | %*s |            | %-s\n" ${seccoladj} " " "${line}"
-    done < "${commentfile2}"
-
-    ((i=i+1))
-    rm "${commentfile2}" "${commentfile1}" 2>/dev/null
-  done
-
-  if [[ ${#JIRA_TEST_TABLE[@]} -gt 0 ]]; then
-    seccoladj=$(findlargest 1 "${JIRA_TEST_TABLE[@]}")
-    printf "\n\n%*s | Tests\n" "${seccoladj}" "Reason"
-    i=0
-    until [[ $i -eq ${#JIRA_TEST_TABLE[@]} ]]; do
-      ourstring=$(echo "${JIRA_TEST_TABLE[${i}]}" | tr -s ' ')
-      vote=$(echo "${ourstring}" | cut -f2 -d\|)
-      subs=$(echo "${ourstring}"  | cut -f3 -d\|)
-      printf "%*s | %s\n" "${seccoladj}" "${vote}" "${subs}"
-      ((i=i+1))
-    done
-  fi
-
-  printf "\n\n|| Subsystem || Report/Notes ||\n"
-  echo "============================================================================"
-  i=0
-
-  until [[ $i -eq ${#JIRA_FOOTER_TABLE[@]} ]]; do
-    comment=$(echo "${JIRA_FOOTER_TABLE[${i}]}" |
-              ${SED} -e "s,@@BASE@@,${PATCH_DIR},g")
-    printf "%s\n" "${comment}"
-    ((i=i+1))
-  done
-}
-
-## @description  Print out the finished details to the JIRA issue
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @param        runresult
-function output_to_jira
-{
-  local result=$1
-  local i
-  local commentfile=${PATCH_DIR}/commentfile
-  local comment
-
-  rm "${commentfile}" 2>/dev/null
-
-  if [[ ${JENKINS} != "true" ]] ; then
-    return 0
-  fi
-
-  big_console_header "Adding comment to JIRA"
-
-  add_jira_footer "Console output" "${BUILD_URL}console"
-
-  if [[ ${result} == 0 ]]; then
-    add_jira_header "(/) *{color:green}+1 overall{color}*"
-  else
-    add_jira_header "(x) *{color:red}-1 overall{color}*"
-  fi
-
-
-  { echo "\\\\" ; echo "\\\\"; } >>  "${commentfile}"
-
-  i=0
-  until [[ $i -eq ${#JIRA_HEADER[@]} ]]; do
-    printf "%s\n" "${JIRA_HEADER[${i}]}" >> "${commentfile}"
-    ((i=i+1))
-  done
-
-  { echo "\\\\" ; echo "\\\\"; } >>  "${commentfile}"
-
-  echo "|| Vote || Subsystem || Runtime || Comment ||" >> "${commentfile}"
-
-  i=0
-  until [[ $i -eq ${#JIRA_COMMENT_TABLE[@]} ]]; do
-    printf "%s\n" "${JIRA_COMMENT_TABLE[${i}]}" >> "${commentfile}"
-    ((i=i+1))
-  done
-
-
-  if [[ ${#JIRA_TEST_TABLE[@]} -gt 0 ]]; then
-    { echo "\\\\" ; echo "\\\\"; } >>  "${commentfile}"
-
-    echo "|| Reason || Tests ||" >>  "${commentfile}"
-    i=0
-    until [[ $i -eq ${#JIRA_TEST_TABLE[@]} ]]; do
-      printf "%s\n" "${JIRA_TEST_TABLE[${i}]}" >> "${commentfile}"
-      ((i=i+1))
-    done
-  fi
-
-  { echo "\\\\" ; echo "\\\\"; } >>  "${commentfile}"
-
-  echo "|| Subsystem || Report/Notes ||" >> "${commentfile}"
-  i=0
-  until [[ $i -eq ${#JIRA_FOOTER_TABLE[@]} ]]; do
-    comment=$(echo "${JIRA_FOOTER_TABLE[${i}]}" |
-              ${SED} -e "s,@@BASE@@,${BUILD_URL}artifact/patchprocess,g")
-    printf "%s\n" "${comment}" >> "${commentfile}"
-    ((i=i+1))
-  done
-
-  printf "\n\nThis message was automatically generated.\n\n" >> "${commentfile}"
-
-  write_to_jira "${commentfile}"
-}
-
-## @description  Clean the filesystem as appropriate and then exit
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @param        runresult
-function cleanup_and_exit
-{
-  local result=$1
-
-  if [[ ${JENKINS} == "true" && ${RELOCATE_PATCH_DIR} == "true" && \
-      -e ${PATCH_DIR} && -d ${PATCH_DIR} ]] ; then
-    # if PATCH_DIR is already inside BASEDIR, then
-    # there is no need to move it since we assume that
-    # Jenkins or whatever already knows where it is at
-    # since it told us to put it there!
-    relative_patchdir >/dev/null
-    if [[ $? == 1 ]]; then
-      hadoop_debug "mv ${PATCH_DIR} ${BASEDIR}"
-      mv "${PATCH_DIR}" "${BASEDIR}"
-    fi
-  fi
-  big_console_header "Finished build."
-
-  # shellcheck disable=SC2086
-  exit ${result}
-}
-
-## @description  Driver to execute _postcheckout routines
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-function postcheckout
-{
-  local routine
-  local plugin
-
-  for routine in find_java_home verify_patch_file
-  do
-    verify_patchdir_still_exists
-
-    hadoop_debug "Running ${routine}"
-    ${routine}
-
-    (( RESULT = RESULT + $? ))
-    if [[ ${RESULT} != 0 ]] ; then
-      output_to_console 1
-      output_to_jira 1
-      cleanup_and_exit 1
-    fi
-  done
-
-  for plugin in ${PLUGINS}; do
-    verify_patchdir_still_exists
-
-    if declare -f ${plugin}_postcheckout >/dev/null 2>&1; then
-
-      hadoop_debug "Running ${plugin}_postcheckout"
-      #shellcheck disable=SC2086
-      ${plugin}_postcheckout
-
-      (( RESULT = RESULT + $? ))
-      if [[ ${RESULT} != 0 ]] ; then
-        output_to_console 1
-        output_to_jira 1
-        cleanup_and_exit 1
-      fi
-    fi
-  done
-}
-
-## @description  Driver to execute _preapply routines
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-function preapply
-{
-  local routine
-  local plugin
-
-  for routine in precheck_without_patch check_author \
-                 check_modified_unittests
-  do
-    verify_patchdir_still_exists
-
-    hadoop_debug "Running ${routine}"
-    ${routine}
-
-    (( RESULT = RESULT + $? ))
-  done
-
-  for plugin in ${PLUGINS}; do
-    verify_patchdir_still_exists
-
-    if declare -f ${plugin}_preapply >/dev/null 2>&1; then
-
-      hadoop_debug "Running ${plugin}_preapply"
-      #shellcheck disable=SC2086
-      ${plugin}_preapply
-
-      (( RESULT = RESULT + $? ))
-    fi
-  done
-}
-
-## @description  Driver to execute _postapply routines
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-function postapply
-{
-  local routine
-  local plugin
-  local retval
-
-  compute_gitdiff "${GITDIFFLINES}"
-
-  check_javac
-  retval=$?
-  if [[ ${retval} -gt 1 ]] ; then
-    output_to_console 1
-    output_to_jira 1
-    cleanup_and_exit 1
-  fi
-
-  ((RESULT = RESULT + retval))
-
-  for routine in check_javadoc check_apachelicense check_site
-  do
-    verify_patchdir_still_exists
-    hadoop_debug "Running ${routine}"
-    $routine
-
-    (( RESULT = RESULT + $? ))
-
-  done
-
-  for plugin in ${PLUGINS}; do
-    verify_patchdir_still_exists
-    if declare -f ${plugin}_postapply >/dev/null 2>&1; then
-      hadoop_debug "Running ${plugin}_postapply"
-      #shellcheck disable=SC2086
-      ${plugin}_postapply
-      (( RESULT = RESULT + $? ))
-    fi
-  done
-}
-
-## @description  Driver to execute _postinstall routines
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-function postinstall
-{
-  local routine
-  local plugin
-
-  for routine in check_mvn_eclipse check_findbugs
-  do
-    verify_patchdir_still_exists
-    hadoop_debug "Running ${routine}"
-    ${routine}
-    (( RESULT = RESULT + $? ))
-  done
-
-  for plugin in ${PLUGINS}; do
-    verify_patchdir_still_exists
-    if declare -f ${plugin}_postinstall >/dev/null 2>&1; then
-      hadoop_debug "Running ${plugin}_postinstall"
-      #shellcheck disable=SC2086
-      ${plugin}_postinstall
-      (( RESULT = RESULT + $? ))
-    fi
-  done
-
-}
-
-## @description  Driver to execute _tests routines
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-function runtests
-{
-  local plugin
-
-  ### Run tests for Jenkins or if explictly asked for by a developer
-  if [[ ${JENKINS} == "true" || ${RUN_TESTS} == "true" ]] ; then
-
-    verify_patchdir_still_exists
-    check_unittests
-
-    (( RESULT = RESULT + $? ))
-  fi
-
-  for plugin in ${PLUGINS}; do
-    verify_patchdir_still_exists
-    if declare -f ${plugin}_tests >/dev/null 2>&1; then
-      hadoop_debug "Running ${plugin}_tests"
-      #shellcheck disable=SC2086
-      ${plugin}_tests
-      (( RESULT = RESULT + $? ))
-    fi
-  done
-}
-
-## @description  Import content from test-patch.d and optionally
-## @description  from user provided plugin directory
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-function importplugins
-{
-  local i
-  local files=()
-
-  if [[ ${LOAD_SYSTEM_PLUGINS} == "true" ]]; then
-    if [[ -d "${BINDIR}/test-patch.d" ]]; then
-      files=(${BINDIR}/test-patch.d/*.sh)
-    fi
-  fi
-
-  if [[ -n "${USER_PLUGIN_DIR}" && -d "${USER_PLUGIN_DIR}" ]]; then
-    hadoop_debug "Loading user provided plugins from ${USER_PLUGIN_DIR}"
-    files=("${files[@]}" ${USER_PLUGIN_DIR}/*.sh)
-  fi
-
-  for i in "${files[@]}"; do
-    hadoop_debug "Importing ${i}"
-    . "${i}"
-  done
-}
-
-## @description  Register test-patch.d plugins
-## @audience     public
-## @stability    stable
-## @replaceable  no
-function add_plugin
-{
-  PLUGINS="${PLUGINS} $1"
-}
-
-###############################################################################
-###############################################################################
-###############################################################################
-
-big_console_header "Bootstrapping test harness"
-
-setup_defaults
-
-parse_args "$@"
-
-importplugins
-
-locate_patch
-
-find_changed_files
-
-determine_needed_tests
-
-# from here on out, we'll be in ${BASEDIR} for cwd
-# routines need to pushd/popd if they change.
-git_checkout
-RESULT=$?
-if [[ ${JENKINS} == "true" ]] ; then
-  if [[ ${RESULT} != 0 ]] ; then
-    exit 100
-  fi
-fi
-
-check_reexec
-
-postcheckout
-
-find_changed_modules
-
-preapply
-
-apply_patch_file
-
-postapply
-
-check_mvn_install
-
-postinstall
-
-runtests
-
-close_jira_footer
-
-close_jira_table
-
-output_to_console ${RESULT}
-output_to_jira ${RESULT}
-cleanup_and_exit ${RESULT}

hadoop-common-project/hadoop-common/pom.xml

@@ -822,15 +822,25 @@
                         <goal>exec</goal>
                     </goals>
                     <configuration>
-                        <executable>python</executable>
+                        <executable>${basedir}/../../dev-support/bin/releasedocmaker</executable>
                         <workingDirectory>src/site/markdown/release/</workingDirectory>
                         <requiresOnline>true</requiresOnline>
                         <arguments>
-                            <argument>${basedir}/../../dev-support/releasedocmaker.py</argument>
-                            <argument>--version</argument>
-                            <argument>${project.version}</argument>
                             <argument>--index</argument>
+                            <argument>--license</argument>
+                            <argument>--project</argument>
+                            <argument>HADOOP</argument>
+                            <argument>--project</argument>
+                            <argument>HDFS</argument>
+                            <argument>--project</argument>
+                            <argument>MAPREDUCE</argument>
+                            <argument>--project</argument>
+                            <argument>YARN</argument>
+                            <argument>--projecttitle</argument>
+                            <argument>"Apache Hadoop"</argument>
                             <argument>--usetoday</argument>
+                            <argument>--version</argument>
+                            <argument>${project.version}</argument>
                         </arguments>
                     </configuration>
                 </execution>