hadoop/dev-support/releasedocmaker.py

#!/usr/bin/env python
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from glob import glob
from optparse import OptionParser
from time import gmtime, strftime
import pprint
import os
import re
import sys
import urllib
import urllib2
try:
  import json
except ImportError:
  import simplejson as json

releaseVersion={}
namePattern = re.compile(r' \([0-9]+\)')

asflicense='''
<!---
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-->
'''

def clean(str):
  return tableclean(re.sub(namePattern, "", str))

def formatComponents(str):
  str = re.sub(namePattern, '', str).replace("'", "")
  if str != "":
    ret = str
  else:
    # some markdown parsers don't like empty tables
    ret = "."
  return clean(ret)

# convert to utf-8
# protect some known md metachars
# or chars that screw up doxia
def tableclean(str):
  str=str.encode('utf-8')
  str=str.replace("_","\_")
  str=str.replace("\r","")
  str=str.rstrip()
  return str

# same thing as tableclean,
# except table metachars are also
# escaped as well as more
# things we don't want doxia to
# screw up
def notableclean(str):
  str=tableclean(str)
  str=str.replace("|","\|")
  str=str.replace("<","\<")
  str=str.replace(">","\>")
  str=str.replace("*","\*")
  str=str.rstrip()
  return str

# clean output dir
def cleanOutputDir(dir):
    files = os.listdir(dir)
    for name in files:
        os.remove(os.path.join(dir,name))
    os.rmdir(dir)

def mstr(obj):
  if (obj is None):
    return ""
  return unicode(obj)

def buildindex(title,license):
  versions=reversed(sorted(glob("[0-9]*.[0-9]*.[0-9]*")))
  with open("index.md","w") as indexfile:
    if license is True:
      indexfile.write(asflicense)
    for v in versions:
      indexfile.write("* %s v%s\n" % (title,v))
      for k in ("Changes","Release Notes"):
        indexfile.write("    * %s (%s/%s.%s.html)\n" \
          % (k,v,k.upper().replace(" ",""),v))
  indexfile.close()

class GetVersions:
  """ yo """
  def __init__(self,versions, projects):
    versions = versions
    projects = projects
    self.newversions = []
    pp = pprint.PrettyPrinter(indent=4)
    at=0
    end=1
    count=100
    versions.sort()
    print "Looking for %s through %s"%(versions[0],versions[-1])
    for p in projects:
      resp = urllib2.urlopen("https://issues.apache.org/jira/rest/api/2/project/%s/versions"%p)
      data = json.loads(resp.read())
      for d in data:
        if d['name'][0].isdigit and versions[0] <= d['name'] and d['name'] <= versions[-1]:
          print "Adding %s to the list" % d['name']
          self.newversions.append(d['name'])
    newlist=list(set(self.newversions))
    self.newversions=newlist

  def getlist(self):
      pp = pprint.PrettyPrinter(indent=4)
      return(self.newversions)

class Version:
  """Represents a version number"""
  def __init__(self, data):
    self.mod = False
    self.data = data
    found = re.match('^((\d+)(\.\d+)*).*$', data)
    if (found):
      self.parts = [ int(p) for p in found.group(1).split('.') ]
    else:
      self.parts = []
    # backfill version with zeroes if missing parts
    self.parts.extend((0,) * (3 - len(self.parts)))

  def __str__(self):
    if (self.mod):
      return '.'.join([ str(p) for p in self.parts ])
    return self.data

  def __cmp__(self, other):
    return cmp(self.parts, other.parts)

class Jira:
  """A single JIRA"""

  def __init__(self, data, parent):
    self.key = data['key']
    self.fields = data['fields']
    self.parent = parent
    self.notes = None
    self.incompat = None
    self.reviewed = None

  def getId(self):
    return mstr(self.key)

  def getDescription(self):
    return mstr(self.fields['description'])

  def getReleaseNote(self):
    if (self.notes is None):
      field = self.parent.fieldIdMap['Release Note']
      if (self.fields.has_key(field)):
        self.notes=mstr(self.fields[field])
      else:
        self.notes=self.getDescription()
    return self.notes

  def getPriority(self):
    ret = ""
    pri = self.fields['priority']
    if(pri is not None):
      ret = pri['name']
    return mstr(ret)

  def getAssignee(self):
    ret = ""
    mid = self.fields['assignee']
    if(mid is not None):
      ret = mid['displayName']
    return mstr(ret)

  def getComponents(self):
    if (len(self.fields['components'])>0):
      return ", ".join([ comp['name'] for comp in self.fields['components'] ])
    else:
      return ""

  def getSummary(self):
    return self.fields['summary']

  def getType(self):
    ret = ""
    mid = self.fields['issuetype']
    if(mid is not None):
      ret = mid['name']
    return mstr(ret)

  def getReporter(self):
    ret = ""
    mid = self.fields['reporter']
    if(mid is not None):
      ret = mid['displayName']
    return mstr(ret)

  def getProject(self):
    ret = ""
    mid = self.fields['project']
    if(mid is not None):
      ret = mid['key']
    return mstr(ret)

  def __cmp__(self,other):
    selfsplit=self.getId().split('-')
    othersplit=other.getId().split('-')
    v1=cmp(selfsplit[0],othersplit[0])
    if (v1!=0):
      return v1
    else:
      if selfsplit[1] < othersplit[1]:
        return True
      elif selfsplit[1] > othersplit[1]:
        return False
    return False

  def getIncompatibleChange(self):
    if (self.incompat is None):
      field = self.parent.fieldIdMap['Hadoop Flags']
      self.reviewed=False
      self.incompat=False
      if (self.fields.has_key(field)):
        if self.fields[field]:
          for hf in self.fields[field]:
            if hf['value'] == "Incompatible change":
              self.incompat=True
            if hf['value'] == "Reviewed":
              self.reviewed=True
    return self.incompat

  def checkMissingComponent(self):
      if (len(self.fields['components'])>0):
          return False
      return True

  def checkMissingAssignee(self):
      if (self.fields['assignee'] is not None):
          return False
      return True

  def checkVersionString(self):
      field = self.parent.fieldIdMap['Fix Version/s']
      for h in self.fields[field]:
          found = re.match('^((\d+)(\.\d+)*).*$|^(\w+\-\d+)$', h['name'])
          if not found:
              return True
      return False

  def getReleaseDate(self,version):
    for j in range(len(self.fields['fixVersions'])):
      if self.fields['fixVersions'][j]==version:
        return(self.fields['fixVersions'][j]['releaseDate'])
    return None

class JiraIter:
  """An Iterator of JIRAs"""

  def __init__(self, version, projects):
    self.version = version
    self.projects = projects
    v=str(version).replace("-SNAPSHOT","")

    resp = urllib2.urlopen("https://issues.apache.org/jira/rest/api/2/field")
    data = json.loads(resp.read())

    self.fieldIdMap = {}
    for part in data:
      self.fieldIdMap[part['name']] = part['id']

    self.jiras = []
    at=0
    end=1
    count=100
    while (at < end):
      params = urllib.urlencode({'jql': "project in ('"+"' , '".join(projects)+"') and fixVersion in ('"+v+"') and resolution = Fixed", 'startAt':at, 'maxResults':count})
      resp = urllib2.urlopen("https://issues.apache.org/jira/rest/api/2/search?%s"%params)
      data = json.loads(resp.read())
      if (data.has_key('errorMessages')):
        raise Exception(data['errorMessages'])
      at = data['startAt'] + data['maxResults']
      end = data['total']
      self.jiras.extend(data['issues'])

      needaversion=False
      if v not in releaseVersion:
        needaversion=True

      if needaversion is True:
        for i in range(len(data['issues'])):
          for j in range(len(data['issues'][i]['fields']['fixVersions'])):
            if 'releaseDate' in data['issues'][i]['fields']['fixVersions'][j]:
              releaseVersion[data['issues'][i]['fields']['fixVersions'][j]['name']]=\
                             data['issues'][i]['fields']['fixVersions'][j]['releaseDate']

    self.iter = self.jiras.__iter__()

  def __iter__(self):
    return self

  def next(self):
    data = self.iter.next()
    j = Jira(data, self)
    return j

class Outputs:
  """Several different files to output to at the same time"""

  def __init__(self, base_file_name, file_name_pattern, keys, params={}):
    self.params = params
    self.base = open(base_file_name%params, 'w')
    self.others = {}
    for key in keys:
      both = dict(params)
      both['key'] = key
      self.others[key] = open(file_name_pattern%both, 'w')

  def writeAll(self, pattern):
    both = dict(self.params)
    both['key'] = ''
    self.base.write(pattern%both)
    for key in self.others.keys():
      both = dict(self.params)
      both['key'] = key
      self.others[key].write(pattern%both)

  def writeKeyRaw(self, key, str):
    self.base.write(str)
    if (self.others.has_key(key)):
      self.others[key].write(str)

  def close(self):
    self.base.close()
    for fd in self.others.values():
      fd.close()

  def writeList(self, mylist):
    for jira in sorted(mylist):
      line = '| [%s](https://issues.apache.org/jira/browse/%s) | %s |  %s | %s | %s | %s |\n' \
        % (notableclean(jira.getId()), notableclean(jira.getId()),
           notableclean(jira.getSummary()),
           notableclean(jira.getPriority()),
           formatComponents(jira.getComponents()),
           notableclean(jira.getReporter()),
           notableclean(jira.getAssignee()))
      self.writeKeyRaw(jira.getProject(), line)

def main():
  parser = OptionParser(usage="usage: %prog --project PROJECT [--project PROJECT] --version VERSION [--version VERSION2 ...]",
		epilog=
               "Markdown-formatted CHANGES and RELEASENOTES files will be stored in a directory"
               " named after the highest version provided.")
  parser.add_option("-i","--index", dest="index", action="store_true",
             default=False, help="build an index file")
  parser.add_option("-l","--license", dest="license", action="store_false",
             default=True, help="Add an ASF license")
  parser.add_option("-n","--lint", dest="lint", action="store_true",
             help="use lint flag to exit on failures")
  parser.add_option("-p", "--project", dest="projects",
             action="append", type="string",
             help="projects in JIRA to include in releasenotes", metavar="PROJECT")
  parser.add_option("-r", "--range", dest="range", action="store_true",
             default=False, help="Given versions are a range")
  parser.add_option("-t", "--projecttitle", dest="title",
             type="string",
             help="Title to use for the project (default is Apache PROJECT)")
  parser.add_option("-u","--usetoday", dest="usetoday", action="store_true",
             default=False, help="use current date for unreleased versions")
  parser.add_option("-v", "--version", dest="versions",
             action="append", type="string",
             help="versions in JIRA to include in releasenotes", metavar="VERSION")
  (options, args) = parser.parse_args()

  if (options.versions is None):
    options.versions = []

  if (len(args) > 2):
    options.versions.append(args[2])

  if (len(options.versions) <= 0):
    parser.error("At least one version needs to be supplied")

  proxy = urllib2.ProxyHandler()
  opener = urllib2.build_opener(proxy)
  urllib2.install_opener(opener)

  projects = options.projects

  if (options.range is True):
    versions = [ Version(v) for v in GetVersions(options.versions, projects).getlist() ]
  else:
    versions = [ Version(v) for v in options.versions ]
  versions.sort();

  if (options.title is None):
    title=projects[0]
  else:
    title=options.title

  haderrors=False

  for v in versions:
    vstr=str(v)
    jlist = JiraIter(vstr,projects)

    if vstr in releaseVersion:
      reldate=releaseVersion[vstr]
    elif options.usetoday:
      reldate=strftime("%Y-%m-%d", gmtime())
    else:
      reldate="Unreleased"

    if not os.path.exists(vstr):
      os.mkdir(vstr)

    reloutputs = Outputs("%(ver)s/RELEASENOTES.%(ver)s.md",
      "%(ver)s/RELEASENOTES.%(key)s.%(ver)s.md",
      [], {"ver":v, "date":reldate, "title":title})
    choutputs = Outputs("%(ver)s/CHANGES.%(ver)s.md",
      "%(ver)s/CHANGES.%(key)s.%(ver)s.md",
      [], {"ver":v, "date":reldate, "title":title})

    if (options.license is True):
      reloutputs.writeAll(asflicense)
      choutputs.writeAll(asflicense)

    relhead = '# %(title)s %(key)s %(ver)s Release Notes\n\n' \
      'These release notes cover new developer and user-facing incompatibilities, features, and major improvements.\n\n'
    chhead = '# %(title)s Changelog\n\n' \
      '## Release %(ver)s - %(date)s\n'\
      '\n'

    reloutputs.writeAll(relhead)
    choutputs.writeAll(chhead)
    errorCount=0
    warningCount=0
    lintMessage=""
    incompatlist=[]
    buglist=[]
    improvementlist=[]
    newfeaturelist=[]
    subtasklist=[]
    tasklist=[]
    testlist=[]
    otherlist=[]

    for jira in sorted(jlist):
      if jira.getIncompatibleChange():
        incompatlist.append(jira)
      elif jira.getType() == "Bug":
        buglist.append(jira)
      elif jira.getType() == "Improvement":
        improvementlist.append(jira)
      elif jira.getType() == "New Feature":
        newfeaturelist.append(jira)
      elif jira.getType() == "Sub-task":
        subtasklist.append(jira)
      elif jira.getType() == "Task":
       tasklist.append(jira)
      elif jira.getType() == "Test":
        testlist.append(jira)
      else:
         otherlist.append(jira)

      line = '* [%s](https://issues.apache.org/jira/browse/%s) | *%s* | **%s**\n' \
          % (notableclean(jira.getId()), notableclean(jira.getId()), notableclean(jira.getPriority()),
             notableclean(jira.getSummary()))

      if (jira.getIncompatibleChange()) and (len(jira.getReleaseNote())==0):
        warningCount+=1
        reloutputs.writeKeyRaw(jira.getProject(),"\n---\n\n")
        reloutputs.writeKeyRaw(jira.getProject(), line)
        line ='\n**WARNING: No release note provided for this incompatible change.**\n\n'
        lintMessage += "\nWARNING: incompatible change %s lacks release notes." % (notableclean(jira.getId()))
        reloutputs.writeKeyRaw(jira.getProject(), line)

      if jira.checkVersionString():
          warningCount+=1
          lintMessage += "\nWARNING: Version string problem for %s " % jira.getId()

      if (jira.checkMissingComponent() or jira.checkMissingAssignee()):
          errorCount+=1
          errorMessage=[]
          jira.checkMissingComponent() and errorMessage.append("component")
          jira.checkMissingAssignee() and errorMessage.append("assignee")
          lintMessage += "\nERROR: missing %s for %s " %  (" and ".join(errorMessage) , jira.getId())

      if (len(jira.getReleaseNote())>0):
        reloutputs.writeKeyRaw(jira.getProject(),"\n---\n\n")
        reloutputs.writeKeyRaw(jira.getProject(), line)
        line ='\n%s\n\n' % (tableclean(jira.getReleaseNote()))
        reloutputs.writeKeyRaw(jira.getProject(), line)

    if (options.lint is True):
        print lintMessage
        print "======================================="
        print "%s: Error:%d, Warning:%d \n" % (vstr, errorCount, warningCount)
        if (errorCount>0):
           haderrors=True
           cleanOutputDir(vstr)
           continue

    reloutputs.writeAll("\n\n")
    reloutputs.close()

    choutputs.writeAll("### INCOMPATIBLE CHANGES:\n\n")
    choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
    choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
    choutputs.writeList(incompatlist)

    choutputs.writeAll("\n\n### NEW FEATURES:\n\n")
    choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
    choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
    choutputs.writeList(newfeaturelist)

    choutputs.writeAll("\n\n### IMPROVEMENTS:\n\n")
    choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
    choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
    choutputs.writeList(improvementlist)

    choutputs.writeAll("\n\n### BUG FIXES:\n\n")
    choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
    choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
    choutputs.writeList(buglist)

    choutputs.writeAll("\n\n### TESTS:\n\n")
    choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
    choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
    choutputs.writeList(testlist)

    choutputs.writeAll("\n\n### SUB-TASKS:\n\n")
    choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
    choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
    choutputs.writeList(subtasklist)

    choutputs.writeAll("\n\n### OTHER:\n\n")
    choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
    choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
    choutputs.writeList(otherlist)
    choutputs.writeList(tasklist)

    choutputs.writeAll("\n\n")
    choutputs.close()

  if options.index:
    buildindex(title,options.license)

  if haderrors is True:
    sys.exit(1)

if __name__ == "__main__":
  main()