Merge pull request #13396 from Carreau/cleanup-tools

Cleanup tools

tools/gh_api.py

@@ -1,304 +1,163 b''
 """Functions for Github API requests."""
 
 try:
     input = raw_input
 except NameError:
     pass
 
-import os
 import re
 import sys
 
 import requests
 import getpass
 import json
-from pathlib import Path
 
 try:
     import requests_cache
 except ImportError:
     print("cache not available, install `requests_cache` for caching.", file=sys.stderr)
 else:
     requests_cache.install_cache("gh_api", expire_after=3600)
 
 # Keyring stores passwords by a 'username', but we're not storing a username and
 # password
 import socket
 fake_username = 'ipython_tools_%s' % socket.gethostname().replace('.','_').replace('-','_')
 
 class Obj(dict):
     """Dictionary with attribute access to names."""
     def __getattr__(self, name):
         try:
             return self[name]
         except KeyError as e:
             raise AttributeError(name) from e
 
     def __setattr__(self, name, val):
         self[name] = val
 
 token = None
 def get_auth_token():
     global token
 
     if token is not None:
         return token
 
     import keyring
     token = keyring.get_password('github', fake_username)
     if token is not None:
         return token
 
     print("Please enter your github username and password. These are not "
           "stored, only used to get an oAuth token. You can revoke this at "
           "any time on Github.\n"
           "Username: ", file=sys.stderr, end='')
     user = input('')
     pw = getpass.getpass("Password: ", stream=sys.stderr)
 
     auth_request = {
       "scopes": [
         "public_repo",
         "gist"
       ],
       "note": "IPython tools %s" % socket.gethostname(),
       "note_url": "https://github.com/ipython/ipython/tree/master/tools",
     }
     response = requests.post('https://api.github.com/authorizations',
                             auth=(user, pw), data=json.dumps(auth_request))
     if response.status_code == 401 and \
             'required;' in response.headers.get('X-GitHub-OTP', ''):
         print("Your login API requested a one time password", file=sys.stderr)
         otp = getpass.getpass("One Time Password: ", stream=sys.stderr)
         response = requests.post('https://api.github.com/authorizations',
                             auth=(user, pw), 
                             data=json.dumps(auth_request),
                             headers={'X-GitHub-OTP':otp})
     response.raise_for_status()
     token = json.loads(response.text)['token']
     keyring.set_password('github', fake_username, token)
     return token
 
 def make_auth_header():
     return {'Authorization': 'token ' + get_auth_token()}
 
-def post_issue_comment(project, num, body):
-    url = 'https://api.github.com/repos/{project}/issues/{num}/comments'.format(project=project, num=num)
-    payload = json.dumps({'body': body})
-    requests.post(url, data=payload, headers=make_auth_header())
-
-def post_gist(content, description='', filename='file', auth=False):
-    """Post some text to a Gist, and return the URL."""
-    post_data = json.dumps({
-      "description": description,
-      "public": True,
-      "files": {
-        filename: {
-          "content": content
-        }
-      }
-    }).encode('utf-8')
-
-    headers = make_auth_header() if auth else {}
-    response = requests.post("https://api.github.com/gists", data=post_data, headers=headers)
-    response.raise_for_status()
-    response_data = json.loads(response.text)
-    return response_data['html_url']
 
 def get_pull_request(project, num, auth=False):
     """get pull request info  by number
     """
     url = "https://api.github.com/repos/{project}/pulls/{num}".format(project=project, num=num)
     if auth:
         header = make_auth_header()
     else:
         header = None
     print("fetching %s" % url, file=sys.stderr)
     response = requests.get(url, headers=header)
     response.raise_for_status()
     return json.loads(response.text, object_hook=Obj)
 
-def get_pull_request_files(project, num, auth=False):
-    """get list of files in a pull request"""
-    url = "https://api.github.com/repos/{project}/pulls/{num}/files".format(project=project, num=num)
-    if auth:
-        header = make_auth_header()
-    else:
-        header = None
-    return get_paged_request(url, headers=header)
-
 element_pat = re.compile(r'<(.+?)>')
 rel_pat = re.compile(r'rel=[\'"](\w+)[\'"]')
 
 def get_paged_request(url, headers=None, **params):
     """get a full list, handling APIv3's paging"""
     results = []
     params.setdefault("per_page", 100)
     while True:
         if '?' in url:
             params = None
             print("fetching %s" % url, file=sys.stderr)
         else:
             print("fetching %s with %s" % (url, params), file=sys.stderr)
         response = requests.get(url, headers=headers, params=params)
         response.raise_for_status()
         results.extend(response.json())
         if 'next' in response.links:
             url = response.links['next']['url']
         else:
             break
     return results
 
-def get_pulls_list(project, auth=False, **params):
-    """get pull request list"""
-    params.setdefault("state", "closed")
-    url = "https://api.github.com/repos/{project}/pulls".format(project=project)
-    if auth:
-        headers = make_auth_header()
-    else:
-        headers = None
-    pages = get_paged_request(url, headers=headers, **params)
-    return pages
-
 def get_issues_list(project, auth=False, **params):
     """get issues list"""
     params.setdefault("state", "closed")
     url = "https://api.github.com/repos/{project}/issues".format(project=project)
     if auth:
         headers = make_auth_header()
     else:
         headers = None
     pages = get_paged_request(url, headers=headers, **params)
     return pages
 
 def get_milestones(project, auth=False, **params):
     params.setdefault('state', 'all')
     url = "https://api.github.com/repos/{project}/milestones".format(project=project)
     if auth:
         headers = make_auth_header()
     else:
         headers = None
     milestones = get_paged_request(url, headers=headers, **params)
     return milestones
 
 def get_milestone_id(project, milestone, auth=False, **params):
     milestones = get_milestones(project, auth=auth, **params)
     for mstone in milestones:
         if mstone['title'] == milestone:
             return mstone['number']
     else:
         raise ValueError("milestone %s not found" % milestone)
 
 def is_pull_request(issue):
     """Return True if the given issue is a pull request."""
     return bool(issue.get('pull_request', {}).get('html_url', None))
 
 def get_authors(pr):
     print("getting authors for #%i" % pr['number'], file=sys.stderr)
     h = make_auth_header()
     r = requests.get(pr['commits_url'], headers=h)
     r.raise_for_status()
     commits = r.json()
     authors = []
     for commit in commits:
         author = commit['commit']['author']
         authors.append("%s <%s>" % (author['name'], author['email']))
     return authors
 
-# encode_multipart_formdata is from urllib3.filepost
-# The only change is to iter_fields, to enforce S3's required key ordering
-
-def iter_fields(fields):
-    fields = fields.copy()
-    for key in ('key', 'acl', 'Filename', 'success_action_status', 'AWSAccessKeyId',
-        'Policy', 'Signature', 'Content-Type', 'file'):
-        yield (key, fields.pop(key))
-    for (k,v) in fields.items():
-        yield k,v
-
-def encode_multipart_formdata(fields, boundary=None):
-    """
-    Encode a dictionary of ``fields`` using the multipart/form-data mime format.
-
-    :param fields:
-        Dictionary of fields or list of (key, value) field tuples.  The key is
-        treated as the field name, and the value as the body of the form-data
-        bytes. If the value is a tuple of two elements, then the first element
-        is treated as the filename of the form-data section.
-
-        Field names and filenames must be unicode.
-
-    :param boundary:
-        If not specified, then a random boundary will be generated using
-        :func:`mimetools.choose_boundary`.
-    """
-    # copy requests imports in here:
-    from io import BytesIO
-    from requests.packages.urllib3.filepost import (
-        choose_boundary, six, writer, b, get_content_type
-    )
-    body = BytesIO()
-    if boundary is None:
-        boundary = choose_boundary()
-
-    for fieldname, value in iter_fields(fields):
-        body.write(b('--%s\r\n' % (boundary)))
-
-        if isinstance(value, tuple):
-            filename, data = value
-            writer(body).write('Content-Disposition: form-data; name="%s"; '
-                               'filename="%s"\r\n' % (fieldname, filename))
-            body.write(b('Content-Type: %s\r\n\r\n' %
-                       (get_content_type(filename))))
-        else:
-            data = value
-            writer(body).write('Content-Disposition: form-data; name="%s"\r\n'
-                               % (fieldname))
-            body.write(b'Content-Type: text/plain\r\n\r\n')
-
-        if isinstance(data, int):
-            data = str(data)  # Backwards compatibility
-        if isinstance(data, six.text_type):
-            writer(body).write(data)
-        else:
-            body.write(data)
-
-        body.write(b'\r\n')
-
-    body.write(b('--%s--\r\n' % (boundary)))
-
-    content_type = b('multipart/form-data; boundary=%s' % boundary)
-
-    return body.getvalue(), content_type
-
-
-def post_download(project, filename, name=None, description=""):
-    """Upload a file to the GitHub downloads area"""
-    if name is None:
-        name = Path(filename).name
-    with open(filename, 'rb') as f:
-        filedata = f.read()
-
-    url = "https://api.github.com/repos/{project}/downloads".format(project=project)
-
-    payload = json.dumps(dict(name=name, size=len(filedata),
-                    description=description))
-    response = requests.post(url, data=payload, headers=make_auth_header())
-    response.raise_for_status()
-    reply = json.loads(response.content)
-    s3_url = reply['s3_url']
-
-    fields = dict(
-        key=reply['path'],
-        acl=reply['acl'],
-        success_action_status=201,
-        Filename=reply['name'],
-        AWSAccessKeyId=reply['accesskeyid'],
-        Policy=reply['policy'],
-        Signature=reply['signature'],
-        file=(reply['name'], filedata),
-    )
-    fields['Content-Type'] = reply['mime_type']
-    data, content_type = encode_multipart_formdata(fields)
-    s3r = requests.post(s3_url, data=data, headers={'Content-Type': content_type})
-    return s3r

tools/github_stats.py

@@ -1,231 +1,230 b''
 #!/usr/bin/env python
 """Simple tools to query github.com and gather stats about issues.
 
 To generate a report for IPython 2.0, run:
 
     python github_stats.py --milestone 2.0 --since-tag rel-1.0.0
 """
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 
-import codecs
 import sys
 
 from argparse import ArgumentParser
 from datetime import datetime, timedelta
 from subprocess import check_output
 
 from gh_api import (
     get_paged_request, make_auth_header, get_pull_request, is_pull_request,
     get_milestone_id, get_issues_list, get_authors,
 )
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 ISO8601 = "%Y-%m-%dT%H:%M:%SZ"
 PER_PAGE = 100
 
 #-----------------------------------------------------------------------------
 # Functions
 #-----------------------------------------------------------------------------
 
 def round_hour(dt):
     return dt.replace(minute=0,second=0,microsecond=0)
 
 def _parse_datetime(s):
     """Parse dates in the format returned by the Github API."""
     if s:
         return datetime.strptime(s, ISO8601)
     else:
         return datetime.fromtimestamp(0)
 
 def issues2dict(issues):
     """Convert a list of issues to a dict, keyed by issue number."""
     idict = {}
     for i in issues:
         idict[i['number']] = i
     return idict
 
 def split_pulls(all_issues, project="ipython/ipython"):
     """split a list of closed issues into non-PR Issues and Pull Requests"""
     pulls = []
     issues = []
     for i in all_issues:
         if is_pull_request(i):
             pull = get_pull_request(project, i['number'], auth=True)
             pulls.append(pull)
         else:
             issues.append(i)
     return issues, pulls
 
 
 def issues_closed_since(period=timedelta(days=365), project="ipython/ipython", pulls=False):
     """Get all issues closed since a particular point in time. period
     can either be a datetime object, or a timedelta object. In the
     latter case, it is used as a time before the present.
     """
 
     which = 'pulls' if pulls else 'issues'
 
     if isinstance(period, timedelta):
         since = round_hour(datetime.utcnow() - period)
     else:
         since = period
     url = "https://api.github.com/repos/%s/%s?state=closed&sort=updated&since=%s&per_page=%i" % (project, which, since.strftime(ISO8601), PER_PAGE)
     allclosed = get_paged_request(url, headers=make_auth_header())
     
     filtered = [ i for i in allclosed if _parse_datetime(i['closed_at']) > since ]
     if pulls:
         filtered = [ i for i in filtered if _parse_datetime(i['merged_at']) > since ]
         # filter out PRs not against master (backports)
         filtered = [ i for i in filtered if i['base']['ref'] == 'master' ]
     else:
         filtered = [ i for i in filtered if not is_pull_request(i) ]
     
     return filtered
 
 
 def sorted_by_field(issues, field='closed_at', reverse=False):
     """Return a list of issues sorted by closing date date."""
     return sorted(issues, key = lambda i:i[field], reverse=reverse)
 
 
 def report(issues, show_urls=False):
     """Summary report about a list of issues, printing number and title."""
     if show_urls:
         for i in issues:
             role = 'ghpull' if 'merged_at' in i else 'ghissue'
             print(u'* :%s:`%d`: %s' % (role, i['number'],
                                         i['title'].replace(u'`', u'``')))
     else:
         for i in issues:
             print(u'* %d: %s' % (i['number'], i['title'].replace(u'`', u'``')))
 
 #-----------------------------------------------------------------------------
 # Main script
 #-----------------------------------------------------------------------------
 
 if __name__ == "__main__":
 
     print("DEPRECATE: backport_pr.py is deprecated and it is now recommended"
           "to install `ghpro` from PyPI.", file=sys.stderr)
 
     
     # Whether to add reST urls for all issues in printout.
     show_urls = True
     
     parser = ArgumentParser()
     parser.add_argument('--since-tag', type=str,
         help="The git tag to use for the starting point (typically the last major release)."
     )
     parser.add_argument('--milestone', type=str,
         help="The GitHub milestone to use for filtering issues [optional]."
     )
     parser.add_argument('--days', type=int,
         help="The number of days of data to summarize (use this or --since-tag)."
     )
     parser.add_argument('--project', type=str, default="ipython/ipython",
         help="The project to summarize."
     )
     parser.add_argument('--links', action='store_true', default=False,
         help="Include links to all closed Issues and PRs in the output."
     )
     
     opts = parser.parse_args()
     tag = opts.since_tag
     
     # set `since` from days or git tag
     if opts.days:
         since = datetime.utcnow() - timedelta(days=opts.days)
     else:
         if not tag:
             tag = check_output(['git', 'describe', '--abbrev=0']).strip().decode('utf8')
         cmd = ['git', 'log', '-1', '--format=%ai', tag]
         tagday, tz = check_output(cmd).strip().decode('utf8').rsplit(' ', 1)
         since = datetime.strptime(tagday, "%Y-%m-%d %H:%M:%S")
         h = int(tz[1:3])
         m = int(tz[3:])
         td = timedelta(hours=h, minutes=m)
         if tz[0] == '-':
             since += td
         else:
             since -= td
     
     since = round_hour(since)
     
     milestone = opts.milestone
     project = opts.project
 
     print("fetching GitHub stats since %s (tag: %s, milestone: %s)" % (since, tag, milestone), file=sys.stderr)
     if milestone:
         milestone_id = get_milestone_id(project=project, milestone=milestone,
                 auth=True)
         issues_and_pulls = get_issues_list(project=project,
                 milestone=milestone_id,
                 state='closed',
                 auth=True,
         )
         issues, pulls = split_pulls(issues_and_pulls, project=project)
     else:
         issues = issues_closed_since(since, project=project, pulls=False)
         pulls = issues_closed_since(since, project=project, pulls=True)
     
     # For regular reports, it's nice to show them in reverse chronological order
     issues = sorted_by_field(issues, reverse=True)
     pulls = sorted_by_field(pulls, reverse=True)
     
     n_issues, n_pulls = map(len, (issues, pulls))
     n_total = n_issues + n_pulls
     
     # Print summary report we can directly include into release notes.
     
     print()
     since_day = since.strftime("%Y/%m/%d")
     today = datetime.today().strftime("%Y/%m/%d")
     print("GitHub stats for %s - %s (tag: %s)" % (since_day, today, tag))
     print()
     print("These lists are automatically generated, and may be incomplete or contain duplicates.")
     print()
     
     ncommits = 0
     all_authors = []
     if tag:
         # print git info, in addition to GitHub info:
         since_tag = tag+'..'
         cmd = ['git', 'log', '--oneline', since_tag]
         ncommits += len(check_output(cmd).splitlines())
         
         author_cmd = ['git', 'log', '--use-mailmap', "--format=* %aN", since_tag]
         all_authors.extend(check_output(author_cmd).decode('utf-8', 'replace').splitlines())
     
     pr_authors = []
     for pr in pulls:
         pr_authors.extend(get_authors(pr))
     ncommits = len(pr_authors) + ncommits - len(pulls)
     author_cmd = ['git', 'check-mailmap'] + pr_authors
     with_email = check_output(author_cmd).decode('utf-8', 'replace').splitlines()
     all_authors.extend([ u'* ' + a.split(' <')[0] for a in with_email ])
     unique_authors = sorted(set(all_authors), key=lambda s: s.lower())
 
     print("We closed %d issues and merged %d pull requests." % (n_issues, n_pulls))
     if milestone:
         print("The full list can be seen `on GitHub <https://github.com/{project}/issues?q=milestone%3A{milestone}>`__".format(project=project,milestone=milestone)
         )
     
     print()
     print("The following %i authors contributed %i commits." % (len(unique_authors), ncommits))
     print()
     print('\n'.join(unique_authors))
     
     if opts.links:
         print()
         print("GitHub issues and pull requests:")
         print()
         print('Pull Requests (%d):\n' % n_pulls)
         report(pulls, show_urls)
         print()
         print('Issues (%d):\n' % n_issues)
         report(issues, show_urls)

tools/release

@@ -1,92 +1,85 b''
 #!/usr/bin/env python3
 """IPython release script.
 
 This should ONLY be run at real release time.
 """
 from __future__ import print_function
 
 import os
 from glob import glob
 from subprocess import call
 import sys
 
 from toollib import (get_ipdir, pjoin, cd, execfile, sh, archive,
          archive_user, archive_dir)
-from gh_api import post_download
 
 # Get main ipython dir, this will raise if it doesn't pass some checks
 ipdir = get_ipdir()
 tooldir = pjoin(ipdir, 'tools')
 distdir = pjoin(ipdir, 'dist')
 
 # Where I keep static backups of each release
 ipbackupdir = os.path.expanduser('~/ipython/backup')
 if not os.path.exists(ipbackupdir):
     os.makedirs(ipbackupdir)
 
 # Start in main IPython dir
 cd(ipdir)
 
 # Load release info
 version = None
 execfile(pjoin('IPython','core','release.py'), globals())
 
 # Build site addresses for file uploads
 release_site = '%s/release/%s' % (archive, version)
 backup_site =  '%s/backup/' % archive
 
 # Start actual release process
 print()
 print('Releasing IPython')
 print('=================')
 print()
 print('Version:', version)
 print()
 print('Source IPython directory:', ipdir)
 print()
 
 # Perform local backup, go to tools dir to run it.
 cd(tooldir)
 
 if 'upload' in sys.argv:
     cd(distdir)
 
     # do not upload OS specific files like .DS_Store
     to_upload = glob('*.whl')+glob('*.tar.gz')
-    for fname in to_upload:
-        # TODO: update to GitHub releases API
-        continue
-        print('uploading %s to GitHub' % fname)
-        desc = "IPython %s source distribution" % version
-        post_download("ipython/ipython", fname, description=desc)
 
     # Make target dir if it doesn't exist
     print('1. Uploading IPython to archive.ipython.org')
     sh('ssh %s "mkdir -p %s/release/%s" ' % (archive_user, archive_dir, version))
     sh('scp *.tar.gz *.tar.xz *.whl %s' % release_site)
 
     print('2. Uploading backup files...')
     cd(ipbackupdir)
     sh('scp `ls -1tr *tgz | tail -1` %s' % backup_site)
 
     print('3. Uploading to PyPI using twine')
     cd(distdir)
     call(['twine', 'upload', '--verbose'] + to_upload)
 
 else:
     # Build, but don't upload
 
     # Make backup tarball
     sh('./make_tarball.py')
     sh('mv ipython-*.tgz %s' % ipbackupdir)
 
     # Build release files
     sh('./build_release')
 
     cd(ipdir)
 
     print("`./release upload` to upload source distribution on PyPI and ipython archive")
     sys.exit(0)