Allow checking for backports via milestone

tools/backport_pr.py

@@ -1,143 +1,159 b''
 #!/usr/bin/env python
 """
 Backport pull requests to a particular branch.
 
 Usage: backport_pr.py branch [PR]
 
 e.g.:
 
     python tools/backport_pr.py 0.13.1 123
 
 to backport PR #123 onto branch 0.13.1
 
 or
 
     python tools/backport_pr.py 1.x
 
 to see what PRs are marked for backport that have yet to be applied.
 
 """
 
 from __future__ import print_function
 
 import os
 import re
 import sys
 
 from subprocess import Popen, PIPE, check_call, check_output
 from urllib import urlopen
 
 from gh_api import (
     get_issues_list,
     get_pull_request,
     get_pull_request_files,
     is_pull_request,
+    get_milestone_id,
 )
 
 def find_rejects(root='.'):
     for dirname, dirs, files in os.walk(root):
         for fname in files:
             if fname.endswith('.rej'):
                 yield os.path.join(dirname, fname)
 
 def get_current_branch():
     branches = check_output(['git', 'branch'])
     for branch in branches.splitlines():
         if branch.startswith('*'):
             return branch[1:].strip()
 
 def backport_pr(branch, num, project='ipython/ipython'):
     current_branch = get_current_branch()
     if branch != current_branch:
         check_call(['git', 'checkout', branch])
     check_call(['git', 'pull'])
     pr = get_pull_request(project, num, auth=True)
     files = get_pull_request_files(project, num, auth=True)
     patch_url = pr['patch_url']
     title = pr['title']
     description = pr['body']
     fname = "PR%i.patch" % num
     if os.path.exists(fname):
         print("using patch from {fname}".format(**locals()))
         with open(fname) as f:
             patch = f.read()
     else:
         req = urlopen(patch_url)
         patch = req.read()
     
     msg = "Backport PR #%i: %s" % (num, title) + '\n\n' + description
     check = Popen(['git', 'apply', '--check', '--verbose'], stdin=PIPE)
     a,b = check.communicate(patch)
     
     if check.returncode:
         print("patch did not apply, saving to {fname}".format(**locals()))
         print("edit {fname} until `cat {fname} | git apply --check` succeeds".format(**locals()))
         print("then run tools/backport_pr.py {num} again".format(**locals()))
         if not os.path.exists(fname):
             with open(fname, 'wb') as f:
                 f.write(patch)
         return 1
     
     p = Popen(['git', 'apply'], stdin=PIPE)
     a,b = p.communicate(patch)
     
     filenames = [ f['filename'] for f in files ]
 
     check_call(['git', 'add'] + filenames)
     
     check_call(['git', 'commit', '-m', msg])
     
     print("PR #%i applied, with msg:" % num)
     print()
     print(msg)
     print()
     
     if branch != current_branch:
         check_call(['git', 'checkout', current_branch])
     
     return 0
 
 backport_re = re.compile(r"[Bb]ackport.*?(\d+)")
 
 def already_backported(branch, since_tag=None):
     """return set of PRs that have been backported already"""
     if since_tag is None:
         since_tag = check_output(['git','describe', branch, '--abbrev=0']).decode('utf8').strip()
     cmd = ['git', 'log', '%s..%s' % (since_tag, branch), '--oneline']
     lines = check_output(cmd).decode('utf8')
     return set(int(num) for num in backport_re.findall(lines))
 
-def should_backport(labels):
+def should_backport(labels=None, milestone=None):
     """return set of PRs marked for backport"""
+    if labels is None and milestone is None:
+        raise ValueError("Specify one of labels or milestone.")
+    elif labels is not None and milestone is not None:
+        raise ValueError("Specify only one of labels or milestone.")
+    if labels is not None:
     issues = get_issues_list("ipython/ipython",
             labels=labels,
             state='closed',
             auth=True,
     )
+    else:
+        milestone_id = get_milestone_id("ipython/ipython", milestone,
+                auth=True)
+        issues = get_issues_list("ipython/ipython",
+                milestone=milestone_id,
+                state='closed',
+                auth=True,
+        )
+
     should_backport = set()
     for issue in issues:
         if not is_pull_request(issue):
             continue
-        pr = get_pull_request("ipython/ipython", issue['number'], auth=True)
+        pr = get_pull_request("ipython/ipython", issue['number'],
+                auth=True)
         if not pr['merged']:
             print ("Marked PR closed without merge: %i" % pr['number'])
             continue
         should_backport.add(pr['number'])
     return should_backport
 
 if __name__ == '__main__':
     
     if len(sys.argv) < 2:
         print(__doc__)
         sys.exit(1)
     
     if len(sys.argv) < 3:
         branch = sys.argv[1]
         already = already_backported(branch)
         should = should_backport("backport-1.2")
         print ("The following PRs should be backported:")
         for pr in should.difference(already):
             print (pr)
         sys.exit(0)
     
     sys.exit(backport_pr(sys.argv[1], int(sys.argv[2])))

tools/gh_api.py

@@ -1,259 +1,276 b''
 """Functions for Github API requests."""
 from __future__ import print_function
 
 try:
     input = raw_input
 except NameError:
     pass
 
 import os
 import re
 import sys
 
 import requests
 import getpass
 import json
 
 try:
     import requests_cache
 except ImportError:
     print("no cache")
 else:
     requests_cache.install_cache("gh_api")
 
 # Keyring stores passwords by a 'username', but we're not storing a username and
 # password
 fake_username = 'ipython_tools'
 
 class Obj(dict):
     """Dictionary with attribute access to names."""
     def __getattr__(self, name):
         try:
             return self[name]
         except KeyError:
             raise AttributeError(name)
     
     def __setattr__(self, name, val):
         self[name] = val
 
 token = None
 def get_auth_token():
     global token
     
     if token is not None:
         return token
     
     import keyring
     token = keyring.get_password('github', fake_username)
     if token is not None:
         return token
     
     print("Please enter your github username and password. These are not "
            "stored, only used to get an oAuth token. You can revoke this at "
            "any time on Github.")
     user = input("Username: ")
     pw = getpass.getpass("Password: ")
     
     auth_request = {
       "scopes": [
         "public_repo",
         "gist"
       ],
       "note": "IPython tools",
       "note_url": "https://github.com/ipython/ipython/tree/master/tools",
     }
     response = requests.post('https://api.github.com/authorizations',
                             auth=(user, pw), data=json.dumps(auth_request))
     response.raise_for_status()
     token = json.loads(response.text)['token']
     keyring.set_password('github', fake_username, token)
     return token
 
 def make_auth_header():
     return {'Authorization': 'token ' + get_auth_token()}
 
 def post_issue_comment(project, num, body):
     url = 'https://api.github.com/repos/{project}/issues/{num}/comments'.format(project=project, num=num)
     payload = json.dumps({'body': body})
     requests.post(url, data=payload, headers=make_auth_header())
 
 def post_gist(content, description='', filename='file', auth=False):
     """Post some text to a Gist, and return the URL."""
     post_data = json.dumps({
       "description": description,
       "public": True,
       "files": {
         filename: {
           "content": content
         }
       }
     }).encode('utf-8')
     
     headers = make_auth_header() if auth else {}
     response = requests.post("https://api.github.com/gists", data=post_data, headers=headers)
     response.raise_for_status()
     response_data = json.loads(response.text)
     return response_data['html_url']
     
 def get_pull_request(project, num, auth=False):
     """get pull request info  by number
     """
     url = "https://api.github.com/repos/{project}/pulls/{num}".format(project=project, num=num)
     if auth:
         header = make_auth_header()
     else:
         header = None
     response = requests.get(url, headers=header)
     response.raise_for_status()
     return json.loads(response.text, object_hook=Obj)
 
 def get_pull_request_files(project, num, auth=False):
     """get list of files in a pull request"""
     url = "https://api.github.com/repos/{project}/pulls/{num}/files".format(project=project, num=num)
     if auth:
         header = make_auth_header()
     else:
         header = None
     return get_paged_request(url, headers=header)
 
 element_pat = re.compile(r'<(.+?)>')
 rel_pat = re.compile(r'rel=[\'"](\w+)[\'"]')
 
 def get_paged_request(url, headers=None, **params):
     """get a full list, handling APIv3's paging"""
     results = []
     params.setdefault("per_page", 100)
     while True:
         print("fetching %s with %s" % (url, params), file=sys.stderr)
         response = requests.get(url, headers=headers, params=params)
         response.raise_for_status()
         results.extend(response.json())
         if 'next' in response.links:
             url = response.links['next']['url']
         else:
             break
     return results
 
 def get_pulls_list(project, auth=False, **params):
     """get pull request list"""
     params.setdefault("state", "closed")
     url = "https://api.github.com/repos/{project}/pulls".format(project=project)
     if auth:
         headers = make_auth_header()
     else:
         headers = None
     pages = get_paged_request(url, headers=headers, params=params)
     return pages
 
 def get_issues_list(project, auth=False, **params):
     """get issues list"""
     params.setdefault("state", "closed")
     url = "https://api.github.com/repos/{project}/issues".format(project=project)
     if auth:
         headers = make_auth_header()
     else:
         headers = None
     pages = get_paged_request(url, headers=headers, **params)
     return pages
 
+def get_milestones(project, auth=False, **params):
+    url = "https://api.github.com/repos/{project}/milestones".format(project=project)
+    if auth:
+        headers = make_auth_header()
+    else:
+        headers = None
+    pages = get_paged_request(url, headers=headers, **params)
+    return pages
+
+def get_milestone_id(project, milestone, auth=False, **params):
+    pages = get_milestones(project, auth=auth, **params)
+    for page in pages:
+        if page['title'] == milestone:
+            return page['number']
+    else:
+        raise ValueError("milestone %s not found" % milestone)
+
 def is_pull_request(issue):
     """Return True if the given issue is a pull request."""
     return bool(issue.get('pull_request', {}).get('html_url', None))
 
 # encode_multipart_formdata is from urllib3.filepost
 # The only change is to iter_fields, to enforce S3's required key ordering
 
 def iter_fields(fields):
     fields = fields.copy()
     for key in ('key', 'acl', 'Filename', 'success_action_status', 'AWSAccessKeyId',
         'Policy', 'Signature', 'Content-Type', 'file'):
         yield (key, fields.pop(key))
     for (k,v) in fields.items():
         yield k,v
 
 def encode_multipart_formdata(fields, boundary=None):
     """
     Encode a dictionary of ``fields`` using the multipart/form-data mime format.
 
     :param fields:
         Dictionary of fields or list of (key, value) field tuples.  The key is
         treated as the field name, and the value as the body of the form-data
         bytes. If the value is a tuple of two elements, then the first element
         is treated as the filename of the form-data section.
 
         Field names and filenames must be unicode.
 
     :param boundary:
         If not specified, then a random boundary will be generated using
         :func:`mimetools.choose_boundary`.
     """
     # copy requests imports in here:
     from io import BytesIO
     from requests.packages.urllib3.filepost import (
         choose_boundary, six, writer, b, get_content_type
     )
     body = BytesIO()
     if boundary is None:
         boundary = choose_boundary()
 
     for fieldname, value in iter_fields(fields):
         body.write(b('--%s\r\n' % (boundary)))
 
         if isinstance(value, tuple):
             filename, data = value
             writer(body).write('Content-Disposition: form-data; name="%s"; '
                                'filename="%s"\r\n' % (fieldname, filename))
             body.write(b('Content-Type: %s\r\n\r\n' %
                        (get_content_type(filename))))
         else:
             data = value
             writer(body).write('Content-Disposition: form-data; name="%s"\r\n'
                                % (fieldname))
             body.write(b'Content-Type: text/plain\r\n\r\n')
 
         if isinstance(data, int):
             data = str(data)  # Backwards compatibility
         if isinstance(data, six.text_type):
             writer(body).write(data)
         else:
             body.write(data)
 
         body.write(b'\r\n')
 
     body.write(b('--%s--\r\n' % (boundary)))
 
     content_type = b('multipart/form-data; boundary=%s' % boundary)
 
     return body.getvalue(), content_type
 
 
 def post_download(project, filename, name=None, description=""):
     """Upload a file to the GitHub downloads area"""
     if name is None:
         name = os.path.basename(filename)
     with open(filename, 'rb') as f:
         filedata = f.read()
     
     url = "https://api.github.com/repos/{project}/downloads".format(project=project)
     
     payload = json.dumps(dict(name=name, size=len(filedata),
                     description=description))
     response = requests.post(url, data=payload, headers=make_auth_header())
     response.raise_for_status()
     reply = json.loads(response.content)
     s3_url = reply['s3_url']
     
     fields = dict(
         key=reply['path'],
         acl=reply['acl'],
         success_action_status=201,
         Filename=reply['name'],
         AWSAccessKeyId=reply['accesskeyid'],
         Policy=reply['policy'],
         Signature=reply['signature'],
         file=(reply['name'], filedata),
     )
     fields['Content-Type'] = reply['mime_type']
     data, content_type = encode_multipart_formdata(fields)
     s3r = requests.post(s3_url, data=data, headers={'Content-Type': content_type})
     return s3r