HADOOP-12135. cleanup releasedocmaker

(cherry picked from commit 3fee9f8d18dd60d83da674b3cfbefe666915fad8)

dev-support/releasedocmaker.py

@@ -19,6 +19,7 @@
 from glob import glob
 from optparse import OptionParser
 from time import gmtime, strftime
+import pprint
 import os
 import re
 import sys
@@ -99,23 +100,44 @@ def mstr(obj):
     return ""
   return unicode(obj)
 
-def buildindex(master):
+def buildindex(title,license):
   versions=reversed(sorted(glob("[0-9]*.[0-9]*.[0-9]*")))
   with open("index.md","w") as indexfile:
+    if license is True:
+      indexfile.write(asflicense)
     for v in versions:
-      indexfile.write("* Apache Hadoop v%s\n" % (v))
+      indexfile.write("* %s v%s\n" % (title,v))
       for k in ("Changes","Release Notes"):
-        indexfile.write("    *  %s\n" %(k))
-        indexfile.write("        * [Combined %s](%s/%s.%s.html)\n" \
+        indexfile.write("    * %s (%s/%s.%s.html)\n" \
           % (k,v,k.upper().replace(" ",""),v))
-        if not master:
-          indexfile.write("        * [Hadoop Common %s](%s/%s.HADOOP.%s.html)\n" \
-            % (k,v,k.upper().replace(" ",""),v))
-          for p in ("HDFS","MapReduce","YARN"):
-            indexfile.write("        * [%s %s](%s/%s.%s.%s.html)\n" \
-              % (p,k,v,k.upper().replace(" ",""),p.upper(),v))
   indexfile.close()
 
+class GetVersions:
+  """ yo """
+  def __init__(self,versions, projects):
+    versions = versions
+    projects = projects
+    self.newversions = []
+    pp = pprint.PrettyPrinter(indent=4)
+    at=0
+    end=1
+    count=100
+    versions.sort()
+    print "Looking for %s through %s"%(versions[0],versions[-1])
+    for p in projects:
+      resp = urllib.urlopen("https://issues.apache.org/jira/rest/api/2/project/%s/versions"%p)
+      data = json.loads(resp.read())
+      for d in data:
+        if d['name'][0].isdigit and versions[0] <= d['name'] and d['name'] <= versions[-1]:
+          print "Adding %s to the list" % d['name']
+          self.newversions.append(d['name'])
+    newlist=list(set(self.newversions))
+    self.newversions=newlist
+
+  def getlist(self):
+      pp = pprint.PrettyPrinter(indent=4)
+      return(self.newversions)
+
 class Version:
   """Represents a version number"""
   def __init__(self, data):
@@ -261,8 +283,10 @@ class Jira:
 class JiraIter:
   """An Iterator of JIRAs"""
 
-  def __init__(self, versions):
-    self.versions = versions
+  def __init__(self, version, projects):
+    self.version = version
+    self.projects = projects
+    v=str(version).replace("-SNAPSHOT","")
 
     resp = urllib.urlopen("https://issues.apache.org/jira/rest/api/2/field")
     data = json.loads(resp.read())
@@ -276,7 +300,7 @@ class JiraIter:
     end=1
     count=100
     while (at < end):
-      params = urllib.urlencode({'jql': "project in (HADOOP,HDFS,MAPREDUCE,YARN) and fixVersion in ('"+"' , '".join([str(v).replace("-SNAPSHOT","") for v in versions])+"') and resolution = Fixed", 'startAt':at, 'maxResults':count})
+      params = urllib.urlencode({'jql': "project in ('"+"' , '".join(projects)+"') and fixVersion in ('"+v+"') and resolution = Fixed", 'startAt':at, 'maxResults':count})
       resp = urllib.urlopen("https://issues.apache.org/jira/rest/api/2/search?%s"%params)
       data = json.loads(resp.read())
       if (data.has_key('errorMessages')):
@@ -286,10 +310,8 @@ class JiraIter:
       self.jiras.extend(data['issues'])
 
       needaversion=False
-      for j in versions:
-        v=str(j).replace("-SNAPSHOT","")
-        if v not in releaseVersion:
-          needaversion=True
+      if v not in releaseVersion:
+        needaversion=True
 
       if needaversion is True:
         for i in range(len(data['issues'])):
@@ -351,21 +373,29 @@ class Outputs:
       self.writeKeyRaw(jira.getProject(), line)
 
 def main():
-  parser = OptionParser(usage="usage: %prog --version VERSION [--version VERSION2 ...]",
+  parser = OptionParser(usage="usage: %prog --project PROJECT [--project PROJECT] --version VERSION [--version VERSION2 ...]",
 		epilog=
                "Markdown-formatted CHANGES and RELEASENOTES files will be stored in a directory"
                " named after the highest version provided.")
-  parser.add_option("-v", "--version", dest="versions",
-             action="append", type="string",
-             help="versions in JIRA to include in releasenotes", metavar="VERSION")
-  parser.add_option("-m","--master", dest="master", action="store_true",
-             help="only create the master, merged project files")
   parser.add_option("-i","--index", dest="index", action="store_true",
-             help="build an index file")
-  parser.add_option("-u","--usetoday", dest="usetoday", action="store_true",
-             help="use current date for unreleased versions")
+             default=False, help="build an index file")
+  parser.add_option("-l","--license", dest="license", action="store_false",
+             default=True, help="Add an ASF license")
   parser.add_option("-n","--lint", dest="lint", action="store_true",
              help="use lint flag to exit on failures")
+  parser.add_option("-p", "--project", dest="projects",
+             action="append", type="string",
+             help="projects in JIRA to include in releasenotes", metavar="PROJECT")
+  parser.add_option("-r", "--range", dest="range", action="store_true",
+             default=False, help="Given versions are a range")
+  parser.add_option("-t", "--projecttitle", dest="title",
+             type="string",
+             help="Title to use for the project (default is Apache PROJECT)")
+  parser.add_option("-u","--usetoday", dest="usetoday", action="store_true",
+             default=False, help="use current date for unreleased versions")
+  parser.add_option("-v", "--version", dest="versions",
+             action="append", type="string",
+             help="versions in JIRA to include in releasenotes", metavar="VERSION")
   (options, args) = parser.parse_args()
 
   if (options.versions is None):
@@ -377,169 +407,169 @@ def main():
   if (len(options.versions) <= 0):
     parser.error("At least one version needs to be supplied")
 
-  versions = [ Version(v) for v in options.versions ];
+  projects = options.projects
+
+  if (options.range is True):
+    versions = [ Version(v) for v in GetVersions(options.versions, projects).getlist() ]
+  else:
+    versions = [ Version(v) for v in options.versions ]
   versions.sort();
 
-  maxVersion = str(versions[-1])
+  if (options.title is None):
+    title=projects[0]
+  else:
+    title=options.title
 
-  jlist = JiraIter(versions)
-  version = maxVersion
+  for v in versions:
+    vstr=str(v)
+    jlist = JiraIter(vstr,projects)
 
-  if version in releaseVersion:
-    reldate=releaseVersion[version]
-  elif options.usetoday:
-    reldate=strftime("%Y-%m-%d", gmtime())
-  else:
-    reldate="Unreleased"
+    if vstr in releaseVersion:
+      reldate=releaseVersion[vstr]
+    elif options.usetoday:
+      reldate=strftime("%Y-%m-%d", gmtime())
+    else:
+      reldate="Unreleased"
 
-  if not os.path.exists(version):
-    os.mkdir(version)
+    if not os.path.exists(vstr):
+      os.mkdir(vstr)
 
-  if options.master:
     reloutputs = Outputs("%(ver)s/RELEASENOTES.%(ver)s.md",
       "%(ver)s/RELEASENOTES.%(key)s.%(ver)s.md",
-      [], {"ver":maxVersion, "date":reldate})
+      [], {"ver":v, "date":reldate, "title":title})
     choutputs = Outputs("%(ver)s/CHANGES.%(ver)s.md",
       "%(ver)s/CHANGES.%(key)s.%(ver)s.md",
-      [], {"ver":maxVersion, "date":reldate})
-  else:
-    reloutputs = Outputs("%(ver)s/RELEASENOTES.%(ver)s.md",
-      "%(ver)s/RELEASENOTES.%(key)s.%(ver)s.md",
-      ["HADOOP","HDFS","MAPREDUCE","YARN"], {"ver":maxVersion, "date":reldate})
-    choutputs = Outputs("%(ver)s/CHANGES.%(ver)s.md",
-      "%(ver)s/CHANGES.%(key)s.%(ver)s.md",
-      ["HADOOP","HDFS","MAPREDUCE","YARN"], {"ver":maxVersion, "date":reldate})
-
-  reloutputs.writeAll(asflicense)
-  choutputs.writeAll(asflicense)
-
-  relhead = '# Hadoop %(key)s %(ver)s Release Notes\n\n' \
-    'These release notes cover new developer and user-facing incompatibilities, features, and major improvements.\n\n'
-
-  chhead = '# Hadoop Changelog\n\n' \
-    '## Release %(ver)s - %(date)s\n'\
-    '\n'
-
-  reloutputs.writeAll(relhead)
-  choutputs.writeAll(chhead)
-
-  errorCount=0
-  warningCount=0
-  lintMessage=""
-  incompatlist=[]
-  buglist=[]
-  improvementlist=[]
-  newfeaturelist=[]
-  subtasklist=[]
-  tasklist=[]
-  testlist=[]
-  otherlist=[]
-
-  for jira in sorted(jlist):
-    if jira.getIncompatibleChange():
-      incompatlist.append(jira)
-      if (len(jira.getReleaseNote())==0):
-          warningCount+=1
-
-    if jira.checkVersionString():
-       warningCount+=1
-
-    if jira.checkMissingComponent() or jira.checkMissingAssignee():
-      errorCount+=1
-    elif jira.getType() == "Bug":
-      buglist.append(jira)
-    elif jira.getType() == "Improvement":
-      improvementlist.append(jira)
-    elif jira.getType() == "New Feature":
-      newfeaturelist.append(jira)
-    elif jira.getType() == "Sub-task":
-      subtasklist.append(jira)
-    elif jira.getType() == "Task":
-     tasklist.append(jira)
-    elif jira.getType() == "Test":
-      testlist.append(jira)
-    else:
-       otherlist.append(jira)
-
-    line = '* [%s](https://issues.apache.org/jira/browse/%s) | *%s* | **%s**\n' \
-        % (notableclean(jira.getId()), notableclean(jira.getId()), notableclean(jira.getPriority()),
-           notableclean(jira.getSummary()))
-
-    if (jira.getIncompatibleChange()) and (len(jira.getReleaseNote())==0):
-      reloutputs.writeKeyRaw(jira.getProject(),"\n---\n\n")
-      reloutputs.writeKeyRaw(jira.getProject(), line)
-      line ='\n**WARNING: No release note provided for this incompatible change.**\n\n'
-      lintMessage += "\nWARNING: incompatible change %s lacks release notes." % (notableclean(jira.getId()))
-      reloutputs.writeKeyRaw(jira.getProject(), line)
-
-    if jira.checkVersionString():
-        lintMessage += "\nWARNING: Version string problem for %s " % jira.getId()
-
-    if (jira.checkMissingComponent() or jira.checkMissingAssignee()):
-        errorMessage=[]
-        jira.checkMissingComponent() and errorMessage.append("component")
-        jira.checkMissingAssignee() and errorMessage.append("assignee")
-        lintMessage += "\nERROR: missing %s for %s " %  (" and ".join(errorMessage) , jira.getId())
-
-    if (len(jira.getReleaseNote())>0):
-      reloutputs.writeKeyRaw(jira.getProject(),"\n---\n\n")
-      reloutputs.writeKeyRaw(jira.getProject(), line)
-      line ='\n%s\n\n' % (tableclean(jira.getReleaseNote()))
-      reloutputs.writeKeyRaw(jira.getProject(), line)
-
-  if (options.lint is True):
-      print lintMessage
-      print "======================================="
-      print "Error:%d, Warning:%d \n" % (errorCount, warningCount)
-
-      if (errorCount>0):
-          cleanOutputDir(version)
-          sys.exit(1)
-
-  reloutputs.writeAll("\n\n")
-  reloutputs.close()
-
-  choutputs.writeAll("### INCOMPATIBLE CHANGES:\n\n")
-  choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
-  choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
-  choutputs.writeList(incompatlist)
-
-  choutputs.writeAll("\n\n### NEW FEATURES:\n\n")
-  choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
-  choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
-  choutputs.writeList(newfeaturelist)
-
-  choutputs.writeAll("\n\n### IMPROVEMENTS:\n\n")
-  choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
-  choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
-  choutputs.writeList(improvementlist)
-
-  choutputs.writeAll("\n\n### BUG FIXES:\n\n")
-  choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
-  choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
-  choutputs.writeList(buglist)
-
-  choutputs.writeAll("\n\n### TESTS:\n\n")
-  choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
-  choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
-  choutputs.writeList(testlist)
-
-  choutputs.writeAll("\n\n### SUB-TASKS:\n\n")
-  choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
-  choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
-  choutputs.writeList(subtasklist)
-
-  choutputs.writeAll("\n\n### OTHER:\n\n")
-  choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
-  choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
-  choutputs.writeList(otherlist)
-  choutputs.writeList(tasklist)
-
-  choutputs.writeAll("\n\n")
-  choutputs.close()
+      [], {"ver":v, "date":reldate, "title":title})
+
+    if (options.license is True):
+      reloutputs.writeAll(asflicense)
+      choutputs.writeAll(asflicense)
+
+    relhead = '# %(title)s %(key)s %(ver)s Release Notes\n\n' \
+      'These release notes cover new developer and user-facing incompatibilities, features, and major improvements.\n\n'
+    chhead = '# %(title)s Changelog\n\n' \
+      '## Release %(ver)s - %(date)s\n'\
+      '\n'
+
+    reloutputs.writeAll(relhead)
+    choutputs.writeAll(chhead)
+    errorCount=0
+    warningCount=0
+    lintMessage=""
+    incompatlist=[]
+    buglist=[]
+    improvementlist=[]
+    newfeaturelist=[]
+    subtasklist=[]
+    tasklist=[]
+    testlist=[]
+    otherlist=[]
+
+    for jira in sorted(jlist):
+      if jira.getIncompatibleChange():
+        incompatlist.append(jira)
+        if (len(jira.getReleaseNote())==0):
+            warningCount+=1
+
+      if jira.checkVersionString():
+         warningCount+=1
+
+      if jira.checkMissingComponent() or jira.checkMissingAssignee():
+        errorCount+=1
+      elif jira.getType() == "Bug":
+        buglist.append(jira)
+      elif jira.getType() == "Improvement":
+        improvementlist.append(jira)
+      elif jira.getType() == "New Feature":
+        newfeaturelist.append(jira)
+      elif jira.getType() == "Sub-task":
+        subtasklist.append(jira)
+      elif jira.getType() == "Task":
+       tasklist.append(jira)
+      elif jira.getType() == "Test":
+        testlist.append(jira)
+      else:
+         otherlist.append(jira)
+
+      line = '* [%s](https://issues.apache.org/jira/browse/%s) | *%s* | **%s**\n' \
+          % (notableclean(jira.getId()), notableclean(jira.getId()), notableclean(jira.getPriority()),
+             notableclean(jira.getSummary()))
+
+      if (jira.getIncompatibleChange()) and (len(jira.getReleaseNote())==0):
+        reloutputs.writeKeyRaw(jira.getProject(),"\n---\n\n")
+        reloutputs.writeKeyRaw(jira.getProject(), line)
+        line ='\n**WARNING: No release note provided for this incompatible change.**\n\n'
+        lintMessage += "\nWARNING: incompatible change %s lacks release notes." % (notableclean(jira.getId()))
+        reloutputs.writeKeyRaw(jira.getProject(), line)
+
+      if jira.checkVersionString():
+          lintMessage += "\nWARNING: Version string problem for %s " % jira.getId()
+
+      if (jira.checkMissingComponent() or jira.checkMissingAssignee()):
+          errorMessage=[]
+          jira.checkMissingComponent() and errorMessage.append("component")
+          jira.checkMissingAssignee() and errorMessage.append("assignee")
+          lintMessage += "\nERROR: missing %s for %s " %  (" and ".join(errorMessage) , jira.getId())
+
+      if (len(jira.getReleaseNote())>0):
+        reloutputs.writeKeyRaw(jira.getProject(),"\n---\n\n")
+        reloutputs.writeKeyRaw(jira.getProject(), line)
+        line ='\n%s\n\n' % (tableclean(jira.getReleaseNote()))
+        reloutputs.writeKeyRaw(jira.getProject(), line)
+
+    if (options.lint is True):
+        print lintMessage
+        print "======================================="
+        print "Error:%d, Warning:%d \n" % (errorCount, warningCount)
+
+        if (errorCount>0):
+            cleanOutputDir(version)
+            sys.exit(1)
+
+    reloutputs.writeAll("\n\n")
+    reloutputs.close()
+
+    choutputs.writeAll("### INCOMPATIBLE CHANGES:\n\n")
+    choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
+    choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
+    choutputs.writeList(incompatlist)
+
+    choutputs.writeAll("\n\n### NEW FEATURES:\n\n")
+    choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
+    choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
+    choutputs.writeList(newfeaturelist)
+
+    choutputs.writeAll("\n\n### IMPROVEMENTS:\n\n")
+    choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
+    choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
+    choutputs.writeList(improvementlist)
+
+    choutputs.writeAll("\n\n### BUG FIXES:\n\n")
+    choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
+    choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
+    choutputs.writeList(buglist)
+
+    choutputs.writeAll("\n\n### TESTS:\n\n")
+    choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
+    choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
+    choutputs.writeList(testlist)
+
+    choutputs.writeAll("\n\n### SUB-TASKS:\n\n")
+    choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
+    choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
+    choutputs.writeList(subtasklist)
+
+    choutputs.writeAll("\n\n### OTHER:\n\n")
+    choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
+    choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
+    choutputs.writeList(otherlist)
+    choutputs.writeList(tasklist)
+
+    choutputs.writeAll("\n\n")
+    choutputs.close()
 
   if options.index:
-    buildindex(options.master)
+    buildindex(title,options.license)
 
 if __name__ == "__main__":
   main()