Merge pull request #9960 from Carreau/ghpro

Recommend and document GhPro for backporting.

docs/source/coredev/index.rst

@@ -1,29 +1,56 b''
 .. _core_developer_guide:
 
 =================================
 Guide for IPython core Developers
 =================================
 
 This guide documents the development of IPython itself.  Alternatively,
 developers of third party tools and libraries that use IPython should see the
 :doc:`../development/index`.
 
 
 For instruction on how to make a developer install see :ref:`devinstall`.
 
 .. toctree::
    :maxdepth: 1
 
    release_process
 
 
+Backporting Pull requests
+-------------------------
+
+All pull requests should usually be made against ``master``, if a Pull Request
+need to be backported to an earlier release; then it should be tagged with the
+correct ``milestone``. We then use `ghpro <https://pypi.python.org/pypi/ghpro>`
+to automatically list and apply the PR on other branches. For example:
+
+.. code-block:: bash
+    
+    $ backport-pr todo --milestone 5.2
+    [...snip..]
+    The following PRs have been backported
+    9848
+    9851
+    9953
+    9955
+    The following PRs should be backported:
+    9417
+    9863
+    9925
+    9947
+
+    $ backport-pr apply 5.x 9947
+    [...snip...]
+
+
 Old Documentation
 =================
 
 Out of date documentation is still available and have been kept for archival purposes.
 
 .. note::
 
   Developers documentation used to be on the IPython wiki, but are now out of
   date. The wiki is though still available for historical reasons: `Old IPython
   GitHub Wiki.  <https://github.com/ipython/ipython/wiki/Dev:-Index>`_

tools/backport_pr.py

@@ -1,189 +1,193 b''
 #!/usr/bin/env python
 """
 Backport pull requests to a particular branch.
 
 Usage: backport_pr.py [org/repository] branch [PR] [PR2]
 
 e.g.:
 
     python tools/backport_pr.py 0.13.1 123 155
 
 to backport PRs #123 and #155 onto branch 0.13.1
 
 or
 
     python tools/backport_pr.py 2.1
 
 to see what PRs are marked for backport with milestone=2.1 that have yet to be applied
 to branch 2.x
 
 or
 
     python tools/backport_pr.py jupyter/notebook 0.13.1 123 155
 
 to backport PRs #123 and #155 of the `jupyter/notebook` repo onto branch 0.13.1
 of that repo.
 
 """
 
 from __future__ import print_function
 
 import os
 import re
 import sys
 
 from subprocess import Popen, PIPE, check_call, check_output
 try:
     from urllib.request import urlopen
 except:
     from urllib import urlopen
 
 from gh_api import (
     get_issues_list,
     get_pull_request,
     get_pull_request_files,
     is_pull_request,
     get_milestone_id,
 )
 
 def find_rejects(root='.'):
     for dirname, dirs, files in os.walk(root):
         for fname in files:
             if fname.endswith('.rej'):
                 yield os.path.join(dirname, fname)
 
 def get_current_branch():
     branches = check_output(['git', 'branch'])
     for branch in branches.splitlines():
         if branch.startswith(b'*'):
             return branch[1:].strip().decode('utf-8')
 
 def backport_pr(branch, num, project='ipython/ipython'):
     current_branch = get_current_branch()
     if branch != current_branch:
         check_call(['git', 'checkout', branch])
     check_call(['git', 'pull'])
     pr = get_pull_request(project, num, auth=True)
     files = get_pull_request_files(project, num, auth=True)
     patch_url = pr['patch_url']
     title = pr['title']
     description = pr['body']
     fname = "PR%i.patch" % num
     if os.path.exists(fname):
         print("using patch from {fname}".format(**locals()))
         with open(fname, 'rb') as f:
             patch = f.read()
     else:
         req = urlopen(patch_url)
         patch = req.read()
 
     lines = description.splitlines()
     if len(lines) > 5:
         lines = lines[:5] + ['...']
         description = '\n'.join(lines)
 
     msg = "Backport PR #%i: %s" % (num, title) + '\n\n' + description
     check = Popen(['git', 'apply', '--check', '--verbose'], stdin=PIPE)
     a,b = check.communicate(patch)
 
     if check.returncode:
         print("patch did not apply, saving to {fname}".format(**locals()))
         print("edit {fname} until `cat {fname} | git apply --check` succeeds".format(**locals()))
         print("then run tools/backport_pr.py {num} again".format(**locals()))
         if not os.path.exists(fname):
             with open(fname, 'wb') as f:
                 f.write(patch)
         return 1
 
     p = Popen(['git', 'apply'], stdin=PIPE)
     a,b = p.communicate(patch)
 
     filenames = [ f['filename'] for f in files ]
 
     check_call(['git', 'add'] + filenames)
 
     check_call(['git', 'commit', '-m', msg])
 
     print("PR #%i applied, with msg:" % num)
     print()
     print(msg)
     print()
 
     if branch != current_branch:
         check_call(['git', 'checkout', current_branch])
 
     return 0
 
 backport_re = re.compile(r"(?:[Bb]ackport|[Mm]erge).*#(\d+)")
 
 def already_backported(branch, since_tag=None):
     """return set of PRs that have been backported already"""
     if since_tag is None:
         since_tag = check_output(['git','describe', branch, '--abbrev=0']).decode('utf8').strip()
     cmd = ['git', 'log', '%s..%s' % (since_tag, branch), '--oneline']
     lines = check_output(cmd).decode('utf8')
     return set(int(num) for num in backport_re.findall(lines))
 
 def should_backport(labels=None, milestone=None, project='ipython/ipython'):
     """return set of PRs marked for backport"""
     if labels is None and milestone is None:
         raise ValueError("Specify one of labels or milestone.")
     elif labels is not None and milestone is not None:
         raise ValueError("Specify only one of labels or milestone.")
     if labels is not None:
         issues = get_issues_list(project,
                 labels=labels,
                 state='closed',
                 auth=True,
         )
     else:
         milestone_id = get_milestone_id(project, milestone,
                 auth=True)
         issues = get_issues_list(project,
                 milestone=milestone_id,
                 state='closed',
                 auth=True,
         )
 
     should_backport = set()
     for issue in issues:
         if not is_pull_request(issue):
             continue
         pr = get_pull_request(project, issue['number'],
                 auth=True)
         if not pr['merged']:
             print ("Marked PR closed without merge: %i" % pr['number'])
             continue
         if pr['base']['ref'] != 'master':
             continue
         should_backport.add(pr['number'])
     return should_backport
 
 if __name__ == '__main__':
     project = 'ipython/ipython'
+
+    print("DEPRECATE: backport_pr.py is deprecated and is is now recommended"
+          "to install `ghpro` from PyPI.", file=sys.stderr)
+
     args = list(sys.argv)
     if len(args) >= 2:
         if '/' in args[1]:
             project = args[1]
             del args[1]
 
     if len(args) < 2:
         print(__doc__)
         sys.exit(1)
 
     if len(args) < 3:
         milestone = args[1]
         branch = milestone.split('.')[0] + '.x'
         already = already_backported(branch)
         should = should_backport(milestone=milestone, project=project)
         print ("The following PRs should be backported:")
         for pr in sorted(should.difference(already)):
             print (pr)
         sys.exit(0)
     
     for prno in map(int, args[2:]):
         print("Backporting PR #%i" % prno)
         rc = backport_pr(args[1], prno, project=project)
         if rc:
             print("Backporting PR #%i failed" % prno)
             sys.exit(rc)

tools/github_stats.py

@@ -1,231 +1,235 b''
 #!/usr/bin/env python
 """Simple tools to query github.com and gather stats about issues.
 
 To generate a report for IPython 2.0, run:
 
     python github_stats.py --milestone 2.0 --since-tag rel-1.0.0
 """
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from __future__ import print_function
 
 import codecs
 import sys
 
 from argparse import ArgumentParser
 from datetime import datetime, timedelta
 from subprocess import check_output
 
 from gh_api import (
     get_paged_request, make_auth_header, get_pull_request, is_pull_request,
     get_milestone_id, get_issues_list, get_authors,
 )
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 ISO8601 = "%Y-%m-%dT%H:%M:%SZ"
 PER_PAGE = 100
 
 #-----------------------------------------------------------------------------
 # Functions
 #-----------------------------------------------------------------------------
 
 def round_hour(dt):
     return dt.replace(minute=0,second=0,microsecond=0)
 
 def _parse_datetime(s):
     """Parse dates in the format returned by the Github API."""
     if s:
         return datetime.strptime(s, ISO8601)
     else:
         return datetime.fromtimestamp(0)
 
 def issues2dict(issues):
     """Convert a list of issues to a dict, keyed by issue number."""
     idict = {}
     for i in issues:
         idict[i['number']] = i
     return idict
 
 def split_pulls(all_issues, project="ipython/ipython"):
     """split a list of closed issues into non-PR Issues and Pull Requests"""
     pulls = []
     issues = []
     for i in all_issues:
         if is_pull_request(i):
             pull = get_pull_request(project, i['number'], auth=True)
             pulls.append(pull)
         else:
             issues.append(i)
     return issues, pulls
 
 
 def issues_closed_since(period=timedelta(days=365), project="ipython/ipython", pulls=False):
     """Get all issues closed since a particular point in time. period
     can either be a datetime object, or a timedelta object. In the
     latter case, it is used as a time before the present.
     """
 
     which = 'pulls' if pulls else 'issues'
 
     if isinstance(period, timedelta):
         since = round_hour(datetime.utcnow() - period)
     else:
         since = period
     url = "https://api.github.com/repos/%s/%s?state=closed&sort=updated&since=%s&per_page=%i" % (project, which, since.strftime(ISO8601), PER_PAGE)
     allclosed = get_paged_request(url, headers=make_auth_header())
     
     filtered = [ i for i in allclosed if _parse_datetime(i['closed_at']) > since ]
     if pulls:
         filtered = [ i for i in filtered if _parse_datetime(i['merged_at']) > since ]
         # filter out PRs not against master (backports)
         filtered = [ i for i in filtered if i['base']['ref'] == 'master' ]
     else:
         filtered = [ i for i in filtered if not is_pull_request(i) ]
     
     return filtered
 
 
 def sorted_by_field(issues, field='closed_at', reverse=False):
     """Return a list of issues sorted by closing date date."""
     return sorted(issues, key = lambda i:i[field], reverse=reverse)
 
 
 def report(issues, show_urls=False):
     """Summary report about a list of issues, printing number and title."""
     if show_urls:
         for i in issues:
             role = 'ghpull' if 'merged_at' in i else 'ghissue'
             print(u'* :%s:`%d`: %s' % (role, i['number'],
                                         i['title'].replace(u'`', u'``')))
     else:
         for i in issues:
             print(u'* %d: %s' % (i['number'], i['title'].replace(u'`', u'``')))
 
 #-----------------------------------------------------------------------------
 # Main script
 #-----------------------------------------------------------------------------
 
 if __name__ == "__main__":
+
+    print("DEPRECATE: backport_pr.py is deprecated and is is now recommended"
+          "to install `ghpro` from PyPI.", file=sys.stderr)
+
     # deal with unicode
     if sys.version_info < (3,):
         sys.stdout = codecs.getwriter('utf8')(sys.stdout)
     
     # Whether to add reST urls for all issues in printout.
     show_urls = True
     
     parser = ArgumentParser()
     parser.add_argument('--since-tag', type=str,
         help="The git tag to use for the starting point (typically the last major release)."
     )
     parser.add_argument('--milestone', type=str,
         help="The GitHub milestone to use for filtering issues [optional]."
     )
     parser.add_argument('--days', type=int,
         help="The number of days of data to summarize (use this or --since-tag)."
     )
     parser.add_argument('--project', type=str, default="ipython/ipython",
         help="The project to summarize."
     )
     parser.add_argument('--links', action='store_true', default=False,
         help="Include links to all closed Issues and PRs in the output."
     )
     
     opts = parser.parse_args()
     tag = opts.since_tag
     
     # set `since` from days or git tag
     if opts.days:
         since = datetime.utcnow() - timedelta(days=opts.days)
     else:
         if not tag:
             tag = check_output(['git', 'describe', '--abbrev=0']).strip().decode('utf8')
         cmd = ['git', 'log', '-1', '--format=%ai', tag]
         tagday, tz = check_output(cmd).strip().decode('utf8').rsplit(' ', 1)
         since = datetime.strptime(tagday, "%Y-%m-%d %H:%M:%S")
         h = int(tz[1:3])
         m = int(tz[3:])
         td = timedelta(hours=h, minutes=m)
         if tz[0] == '-':
             since += td
         else:
             since -= td
     
     since = round_hour(since)
     
     milestone = opts.milestone
     project = opts.project
 
     print("fetching GitHub stats since %s (tag: %s, milestone: %s)" % (since, tag, milestone), file=sys.stderr)
     if milestone:
         milestone_id = get_milestone_id(project=project, milestone=milestone,
                 auth=True)
         issues_and_pulls = get_issues_list(project=project,
                 milestone=milestone_id,
                 state='closed',
                 auth=True,
         )
         issues, pulls = split_pulls(issues_and_pulls, project=project)
     else:
         issues = issues_closed_since(since, project=project, pulls=False)
         pulls = issues_closed_since(since, project=project, pulls=True)
     
     # For regular reports, it's nice to show them in reverse chronological order
     issues = sorted_by_field(issues, reverse=True)
     pulls = sorted_by_field(pulls, reverse=True)
     
     n_issues, n_pulls = map(len, (issues, pulls))
     n_total = n_issues + n_pulls
     
     # Print summary report we can directly include into release notes.
     
     print()
     since_day = since.strftime("%Y/%m/%d")
     today = datetime.today().strftime("%Y/%m/%d")
     print("GitHub stats for %s - %s (tag: %s)" % (since_day, today, tag))
     print()
     print("These lists are automatically generated, and may be incomplete or contain duplicates.")
     print()
     
     ncommits = 0
     all_authors = []
     if tag:
         # print git info, in addition to GitHub info:
         since_tag = tag+'..'
         cmd = ['git', 'log', '--oneline', since_tag]
         ncommits += len(check_output(cmd).splitlines())
         
         author_cmd = ['git', 'log', '--use-mailmap', "--format=* %aN", since_tag]
         all_authors.extend(check_output(author_cmd).decode('utf-8', 'replace').splitlines())
     
     pr_authors = []
     for pr in pulls:
         pr_authors.extend(get_authors(pr))
     ncommits = len(pr_authors) + ncommits - len(pulls)
     author_cmd = ['git', 'check-mailmap'] + pr_authors
     with_email = check_output(author_cmd).decode('utf-8', 'replace').splitlines()
     all_authors.extend([ u'* ' + a.split(' <')[0] for a in with_email ])
     unique_authors = sorted(set(all_authors), key=lambda s: s.lower())
 
     print("We closed %d issues and merged %d pull requests." % (n_issues, n_pulls))
     if milestone:
         print("The full list can be seen `on GitHub <https://github.com/{project}/issues?q=milestone%3A{milestone}+>`__".format(project=project,milestone=milestone)
         )
     
     print()
     print("The following %i authors contributed %i commits." % (len(unique_authors), ncommits))
     print()
     print('\n'.join(unique_authors))
     
     if opts.links:
         print()
         print("GitHub issues and pull requests:")
         print()
         print('Pull Requests (%d):\n' % n_pulls)
         report(pulls, show_urls)
         print()
         print('Issues (%d):\n' % n_issues)
         report(issues, show_urls)