hadoop/dev-support/bin/checkcompatibility.py

#!/usr/bin/env python
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# Script which checks Java API compatibility between two revisions of the
# Java client.
#
# Originally sourced from Apache Kudu, which was based on the
# compatibility checker from the Apache HBase project, but ported to
# Python for better readability.

import logging
import os
import re
import shutil
import subprocess
import sys
import urllib2
try:
  import argparse
except ImportError:
  sys.stderr.write("Please install argparse, e.g. via `pip install argparse`.")
  sys.exit(2)

# Various relative paths
REPO_DIR = os.getcwd()

def check_output(*popenargs, **kwargs):
  r"""Run command with arguments and return its output as a byte string.
  Backported from Python 2.7 as it's implemented as pure python on stdlib.
  >>> check_output(['/usr/bin/python', '--version'])
  Python 2.6.2
  """
  process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
  output, _ = process.communicate()
  retcode = process.poll()
  if retcode:
    cmd = kwargs.get("args")
    if cmd is None:
      cmd = popenargs[0]
    error = subprocess.CalledProcessError(retcode, cmd)
    error.output = output
    raise error
  return output

def get_repo_dir():
  """ Return the path to the top of the repo. """
  dirname, _ = os.path.split(os.path.abspath(__file__))
  return os.path.join(dirname, "../..")

def get_scratch_dir():
  """ Return the path to the scratch dir that we build within. """
  scratch_dir = os.path.join(get_repo_dir(), "target", "compat-check")
  if not os.path.exists(scratch_dir):
    os.makedirs(scratch_dir)
  return scratch_dir

def get_java_acc_dir():
  """ Return the path where we check out the Java API Compliance Checker. """
  return os.path.join(get_repo_dir(), "target", "java-acc")


def clean_scratch_dir(scratch_dir):
  """ Clean up and re-create the scratch directory. """
  if os.path.exists(scratch_dir):
    logging.info("Removing scratch dir %s...", scratch_dir)
    shutil.rmtree(scratch_dir)
  logging.info("Creating empty scratch dir %s...", scratch_dir)
  os.makedirs(scratch_dir)


def checkout_java_tree(rev, path):
  """ Check out the Java source tree for the given revision into
  the given path. """
  logging.info("Checking out %s in %s", rev, path)
  os.makedirs(path)
  # Extract java source
  subprocess.check_call(["bash", '-o', 'pipefail', "-c",
                         ("git archive --format=tar %s | " +
                          "tar -C \"%s\" -xf -") % (rev, path)],
                        cwd=get_repo_dir())

def get_git_hash(revname):
  """ Convert 'revname' to its SHA-1 hash. """
  return check_output(["git", "rev-parse", revname],
                      cwd=get_repo_dir()).strip()

def get_repo_name():
  """Get the name of the repo based on the git remote."""
  remotes = check_output(["git", "remote", "-v"],
                         cwd=get_repo_dir()).strip().split("\n")
  # Example output:
  # origin	https://github.com/apache/hadoop.git (fetch)
  # origin	https://github.com/apache/hadoop.git (push)
  remote_url = remotes[0].split("\t")[1].split(" ")[0]
  remote = remote_url.split("/")[-1]
  if remote.endswith(".git"):
    remote = remote[:-4]
  return remote

def build_tree(java_path):
  """ Run the Java build within 'path'. """
  logging.info("Building in %s...", java_path)
  subprocess.check_call(["mvn", "-DskipTests", "-Dmaven.javadoc.skip=true",
                         "package"],
                        cwd=java_path)


def checkout_java_acc(force):
  """
  Check out the Java API Compliance Checker. If 'force' is true, will
  re-download even if the directory exists.
  """
  acc_dir = get_java_acc_dir()
  if os.path.exists(acc_dir):
    logging.info("Java ACC is already downloaded.")
    if not force:
      return
    logging.info("Forcing re-download.")
    shutil.rmtree(acc_dir)

  logging.info("Downloading Java ACC...")

  url = "https://github.com/lvc/japi-compliance-checker/archive/1.8.tar.gz"
  scratch_dir = get_scratch_dir()
  path = os.path.join(scratch_dir, os.path.basename(url))
  jacc = urllib2.urlopen(url)
  with open(path, 'wb') as w:
    w.write(jacc.read())

  subprocess.check_call(["tar", "xzf", path],
                        cwd=scratch_dir)

  shutil.move(os.path.join(scratch_dir, "japi-compliance-checker-1.8"),
              os.path.join(acc_dir))


def find_jars(path):
  """ Return a list of jars within 'path' to be checked for compatibility. """
  all_jars = set(check_output(["find", path, "-name", "*.jar"]).splitlines())

  return [j for j in all_jars if (
      "-tests" not in j and
      "-sources" not in j and
      "-with-dependencies" not in j)]

def write_xml_file(path, version, jars):
  """Write the XML manifest file for JACC."""
  with open(path, "wt") as f:
    f.write("<version>" + version + "</version>\n")
    f.write("<archives>")
    for j in jars:
      f.write(j + "\n")
    f.write("</archives>")

def run_java_acc(src_name, src_jars, dst_name, dst_jars, annotations):
  """ Run the compliance checker to compare 'src' and 'dst'. """
  logging.info("Will check compatibility between original jars:\n\t%s\n" +
               "and new jars:\n\t%s",
               "\n\t".join(src_jars),
               "\n\t".join(dst_jars))

  java_acc_path = os.path.join(get_java_acc_dir(), "japi-compliance-checker.pl")

  src_xml_path = os.path.join(get_scratch_dir(), "src.xml")
  dst_xml_path = os.path.join(get_scratch_dir(), "dst.xml")
  write_xml_file(src_xml_path, src_name, src_jars)
  write_xml_file(dst_xml_path, dst_name, dst_jars)

  out_path = os.path.join(get_scratch_dir(), "report.html")

  args = ["perl", java_acc_path,
          "-l", get_repo_name(),
          "-d1", src_xml_path,
          "-d2", dst_xml_path,
          "-report-path", out_path]

  if annotations is not None:
    annotations_path = os.path.join(get_scratch_dir(), "annotations.txt")
    with file(annotations_path, "w") as f:
      for ann in annotations:
        print >>f, ann
    args += ["-annotations-list", annotations_path]

  subprocess.check_call(args)

def filter_jars(jars, include_filters, exclude_filters):
  """Filter the list of JARs based on include and exclude filters."""
  filtered = []
  # Apply include filters
  for j in jars:
    found = False
    basename = os.path.basename(j)
    for f in include_filters:
      if f.match(basename):
        found = True
        break
    if found:
      filtered += [j]
    else:
      logging.debug("Ignoring JAR %s", j)
  # Apply exclude filters
  exclude_filtered = []
  for j in filtered:
    basename = os.path.basename(j)
    found = False
    for f in exclude_filters:
      if f.match(basename):
        found = True
        break
    if found:
      logging.debug("Ignoring JAR %s", j)
    else:
      exclude_filtered += [j]

  return exclude_filtered


def main():
  """Main function."""
  logging.basicConfig(level=logging.INFO)
  parser = argparse.ArgumentParser(
      description="Run Java API Compliance Checker.")
  parser.add_argument("-f", "--force-download",
                      action="store_true",
                      help="Download dependencies (i.e. Java JAVA_ACC) " +
                      "even if they are already present")
  parser.add_argument("-i", "--include-file",
                      action="append",
                      dest="include_files",
                      help="Regex filter for JAR files to be included. " +
                      "Applied before the exclude filters. " +
                      "Can be specified multiple times.")
  parser.add_argument("-e", "--exclude-file",
                      action="append",
                      dest="exclude_files",
                      help="Regex filter for JAR files to be excluded. " +
                      "Applied after the include filters. " +
                      "Can be specified multiple times.")
  parser.add_argument("-a", "--annotation",
                      action="append",
                      dest="annotations",
                      help="Fully-qualified Java annotation. " +
                      "Java ACC will only check compatibility of " +
                      "annotated classes. Can be specified multiple times.")
  parser.add_argument("--skip-clean",
                      action="store_true",
                      help="Skip cleaning the scratch directory.")
  parser.add_argument("--skip-build",
                      action="store_true",
                      help="Skip building the projects.")
  parser.add_argument("src_rev", nargs=1, help="Source revision.")
  parser.add_argument("dst_rev", nargs="?", default="HEAD",
                      help="Destination revision. " +
                      "If not specified, will use HEAD.")

  if len(sys.argv) == 1:
    parser.print_help()
    sys.exit(1)

  args = parser.parse_args()

  src_rev, dst_rev = args.src_rev[0], args.dst_rev

  logging.info("Source revision: %s", src_rev)
  logging.info("Destination revision: %s", dst_rev)

  # Construct the JAR regex patterns for filtering.
  include_filters = []
  if args.include_files is not None:
    for f in args.include_files:
      logging.info("Applying JAR filename include filter: %s", f)
      include_filters += [re.compile(f)]
  else:
    include_filters = [re.compile(".*")]

  exclude_filters = []
  if args.exclude_files is not None:
    for f in args.exclude_files:
      logging.info("Applying JAR filename exclude filter: %s", f)
      exclude_filters += [re.compile(f)]

  # Construct the annotation list
  annotations = args.annotations
  if annotations is not None:
    logging.info("Filtering classes using %d annotation(s):", len(annotations))
    for a in annotations:
      logging.info("\t%s", a)

  # Download deps.
  checkout_java_acc(args.force_download)

  # Set up the build.
  scratch_dir = get_scratch_dir()
  src_dir = os.path.join(scratch_dir, "src")
  dst_dir = os.path.join(scratch_dir, "dst")

  if args.skip_clean:
    logging.info("Skipping cleaning the scratch directory")
  else:
    clean_scratch_dir(scratch_dir)
    # Check out the src and dst source trees.
    checkout_java_tree(get_git_hash(src_rev), src_dir)
    checkout_java_tree(get_git_hash(dst_rev), dst_dir)

  # Run the build in each.
  if args.skip_build:
    logging.info("Skipping the build")
  else:
    build_tree(src_dir)
    build_tree(dst_dir)

  # Find the JARs.
  src_jars = find_jars(src_dir)
  dst_jars = find_jars(dst_dir)

  # Filter the JARs.
  src_jars = filter_jars(src_jars, include_filters, exclude_filters)
  dst_jars = filter_jars(dst_jars, include_filters, exclude_filters)

  if len(src_jars) == 0 or len(dst_jars) == 0:
    logging.error("No JARs found! Are your filters too strong?")
    sys.exit(1)

  run_java_acc(src_rev, src_jars,
               dst_rev, dst_jars, annotations)


if __name__ == "__main__":
  main()