updates to release scripts

- unicode tweaks in github_stats
- remove unused Windows and egg bits
- update Python 3 builder to 3.4

tools/build_release

@@ -1,49 +1,27 b''
 #!/usr/bin/env python
 """IPython release build script.
 """
 
 import os
 
 from toollib import *
 
-# The windows builds are fairly painful to set up on a posix system via wine,
-# so by default we've disabled them and we just build the windows installers
-# separately in a true Windows VM.
-do_windows = False
-
-# Egg building is also disabled by default.  They serve no real purpose in
-# python2, and getting a setupegg.py file that has valid python2/3 syntax is a
-# pain in the ass.  Since the python devs were too stubborn to leave execfile()
-# in place in python3, then we just don't build eggs.
-do_eggs = False
-
 # Get main ipython dir, this will raise if it doesn't pass some checks
 ipdir = get_ipdir()
 cd(ipdir)
 
 # Load release info
 execfile(pjoin('IPython', 'core', 'release.py'))
 
 # Check that everything compiles
 compile_tree()
 
 # Cleanup
 for d in ['build', 'dist', pjoin('docs', 'build'), pjoin('docs', 'dist'),
           pjoin('docs', 'source', 'api', 'generated')]:
     if os.path.isdir(d):
         remove_tree(d)
 
 # Build source and binary distros
 sh(sdists)
-
-# Build eggs
-if do_eggs:
-    sh(eggs)
-
-if do_windows:
-    map(sh, win_builds)
-    # Change name so retarded Vista runs the installer correctly
-    sh("rename 's/linux-i686/win32/' dist/*.exe")
-    sh("rename 's/linux-x86_64/win32/' dist/*.exe")
-    # exe files aren't really executable under *nix.
-    sh("chmod -x dist/*.exe")
+sh(wheels)

tools/github_stats.py

@@ -1,224 +1,224 b''
 #!/usr/bin/env python
 """Simple tools to query github.com and gather stats about issues.
 
 To generate a report for IPython 2.0, run:
 
     python github_stats.py --milestone 2.0 --since-tag rel-1.0.0
 """
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from __future__ import print_function
 
 import codecs
 import sys
 
 from argparse import ArgumentParser
 from datetime import datetime, timedelta
 from subprocess import check_output
 
 from gh_api import (
     get_paged_request, make_auth_header, get_pull_request, is_pull_request,
     get_milestone_id, get_issues_list, get_authors,
 )
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 ISO8601 = "%Y-%m-%dT%H:%M:%SZ"
 PER_PAGE = 100
 
 #-----------------------------------------------------------------------------
 # Functions
 #-----------------------------------------------------------------------------
 
 def round_hour(dt):
     return dt.replace(minute=0,second=0,microsecond=0)
 
 def _parse_datetime(s):
     """Parse dates in the format returned by the Github API."""
     if s:
         return datetime.strptime(s, ISO8601)
     else:
         return datetime.fromtimestamp(0)
 
 def issues2dict(issues):
     """Convert a list of issues to a dict, keyed by issue number."""
     idict = {}
     for i in issues:
         idict[i['number']] = i
     return idict
 
 def split_pulls(all_issues, project="ipython/ipython"):
     """split a list of closed issues into non-PR Issues and Pull Requests"""
     pulls = []
     issues = []
     for i in all_issues:
         if is_pull_request(i):
             pull = get_pull_request(project, i['number'], auth=True)
             pulls.append(pull)
         else:
             issues.append(i)
     return issues, pulls
 
 
 def issues_closed_since(period=timedelta(days=365), project="ipython/ipython", pulls=False):
     """Get all issues closed since a particular point in time. period
     can either be a datetime object, or a timedelta object. In the
     latter case, it is used as a time before the present.
     """
 
     which = 'pulls' if pulls else 'issues'
 
     if isinstance(period, timedelta):
         since = round_hour(datetime.utcnow() - period)
     else:
         since = period
     url = "https://api.github.com/repos/%s/%s?state=closed&sort=updated&since=%s&per_page=%i" % (project, which, since.strftime(ISO8601), PER_PAGE)
     allclosed = get_paged_request(url, headers=make_auth_header())
     
     filtered = [ i for i in allclosed if _parse_datetime(i['closed_at']) > since ]
     if pulls:
         filtered = [ i for i in filtered if _parse_datetime(i['merged_at']) > since ]
         # filter out PRs not against master (backports)
         filtered = [ i for i in filtered if i['base']['ref'] == 'master' ]
     else:
         filtered = [ i for i in filtered if not is_pull_request(i) ]
     
     return filtered
 
 
 def sorted_by_field(issues, field='closed_at', reverse=False):
     """Return a list of issues sorted by closing date date."""
     return sorted(issues, key = lambda i:i[field], reverse=reverse)
 
 
 def report(issues, show_urls=False):
     """Summary report about a list of issues, printing number and title.
     """
     # titles may have unicode in them, so we must encode everything below
     if show_urls:
         for i in issues:
             role = 'ghpull' if 'merged_at' in i else 'ghissue'
-            print('* :%s:`%d`: %s' % (role, i['number'],
-                                        i['title'].replace('`', '``').encode('utf-8')))
+            print(u'* :%s:`%d`: %s' % (role, i['number'],
+                                        i['title'].replace(u'`', u'``')))
     else:
         for i in issues:
-            print('* %d: %s' % (i['number'], i['title'].replace('`', '``').encode('utf-8')))
+            print(u'* %d: %s' % (i['number'], i['title'].replace(u'`', u'``')))
 
 #-----------------------------------------------------------------------------
 # Main script
 #-----------------------------------------------------------------------------
 
 if __name__ == "__main__":
     # deal with unicode
     sys.stdout = codecs.getwriter('utf8')(sys.stdout)
     
     # Whether to add reST urls for all issues in printout.
     show_urls = True
     
     parser = ArgumentParser()
     parser.add_argument('--since-tag', type=str,
         help="The git tag to use for the starting point (typically the last major release)."
     )
     parser.add_argument('--milestone', type=str,
         help="The GitHub milestone to use for filtering issues [optional]."
     )
     parser.add_argument('--days', type=int,
         help="The number of days of data to summarize (use this or --since-tag)."
     )
     parser.add_argument('--project', type=str, default="ipython/ipython",
         help="The project to summarize."
     )
     
     opts = parser.parse_args()
     tag = opts.since_tag
     
     # set `since` from days or git tag
     if opts.days:
         since = datetime.utcnow() - timedelta(days=opts.days)
     else:
         if not tag:
             tag = check_output(['git', 'describe', '--abbrev=0']).strip()
         cmd = ['git', 'log', '-1', '--format=%ai', tag]
         tagday, tz = check_output(cmd).strip().rsplit(' ', 1)
         since = datetime.strptime(tagday, "%Y-%m-%d %H:%M:%S")
         h = int(tz[1:3])
         m = int(tz[3:])
         td = timedelta(hours=h, minutes=m)
         if tz[0] == '-':
             since += td
         else:
             since -= td
     
     since = round_hour(since)
     
     milestone = opts.milestone
     project = opts.project
 
     print("fetching GitHub stats since %s (tag: %s, milestone: %s)" % (since, tag, milestone), file=sys.stderr)
     if milestone:
         milestone_id = get_milestone_id(project=project, milestone=milestone,
                 auth=True)
         issues_and_pulls = get_issues_list(project=project,
                 milestone=milestone_id,
                 state='closed',
                 auth=True,
         )
         issues, pulls = split_pulls(issues_and_pulls)
     else:
         issues = issues_closed_since(since, project=project, pulls=False)
         pulls = issues_closed_since(since, project=project, pulls=True)
     
     # For regular reports, it's nice to show them in reverse chronological order
     issues = sorted_by_field(issues, reverse=True)
     pulls = sorted_by_field(pulls, reverse=True)
     
     n_issues, n_pulls = map(len, (issues, pulls))
     n_total = n_issues + n_pulls
     
     # Print summary report we can directly include into release notes.
     
     print()
     since_day = since.strftime("%Y/%m/%d")
     today = datetime.today().strftime("%Y/%m/%d")
     print("GitHub stats for %s - %s (tag: %s)" % (since_day, today, tag))
     print()
     print("These lists are automatically generated, and may be incomplete or contain duplicates.")
     print()
     
     ncommits = 0
     all_authors = []
     if tag:
         # print git info, in addition to GitHub info:
         since_tag = tag+'..'
         cmd = ['git', 'log', '--oneline', since_tag]
         ncommits += len(check_output(cmd).splitlines())
         
         author_cmd = ['git', 'log', '--use-mailmap', "--format=* %aN", since_tag]
         all_authors.extend(check_output(author_cmd).decode('utf-8', 'replace').splitlines())
     
     pr_authors = []
     for pr in pulls:
         pr_authors.extend(get_authors(pr))
     ncommits = len(pr_authors) + ncommits - len(pulls)
     author_cmd = ['git', 'check-mailmap'] + pr_authors
     with_email = check_output(author_cmd).decode('utf-8', 'replace').splitlines()
     all_authors.extend([ u'* ' + a.split(' <')[0] for a in with_email ])
     unique_authors = sorted(set(all_authors), key=lambda s: s.lower())
 
     print("The following %i authors contributed %i commits." % (len(unique_authors), ncommits))
     print()
     print('\n'.join(unique_authors))
     
     print()
     print("We closed %d issues and merged %d pull requests;\n"
           "this is the full list (generated with the script \n"
           ":file:`tools/github_stats.py`):" % (n_pulls, n_issues))
     print()
     print('Pull Requests (%d):\n' % n_pulls)
     report(pulls, show_urls)
     print()
     print('Issues (%d):\n' % n_issues)
     report(issues, show_urls)

tools/release

@@ -1,77 +1,79 b''
 #!/usr/bin/env python
 """IPython release script.
 
 This should ONLY be run at real release time.
 """
 from __future__ import print_function
 
 from toollib import *
 from gh_api import post_download
 
 # Get main ipython dir, this will raise if it doesn't pass some checks
 ipdir = get_ipdir()
 tooldir = pjoin(ipdir, 'tools')
 distdir = pjoin(ipdir, 'dist')
 
 # Where I keep static backups of each release
 ipbackupdir = os.path.expanduser('~/ipython/backup')
+if not os.path.exists(ipbackupdir):
+    os.makedirs(ipbackupdir)
 
 # Start in main IPython dir
 cd(ipdir)
 
 # Load release info
 execfile(pjoin('IPython','core','release.py'))
 # ensure js version is in sync
 sh('./setup.py jsversion')
 
 # Build site addresses for file uploads
 release_site = '%s/release/%s' % (archive, version)
 backup_site =  '%s/backup/' % archive
 
 # Start actual release process
 print()
 print('Releasing IPython')
 print('=================')
 print()
 print('Version:', version)
 print()
 print('Source IPython directory:', ipdir)
 print()
 
 # Perform local backup, go to tools dir to run it.
 cd(tooldir)
 sh('./make_tarball.py')
 sh('mv ipython-*.tgz %s' % ipbackupdir)
 
 # Build release files
 sh('./build_release %s' % ipdir)
 
 # Register with the Python Package Index (PyPI)
 print( 'Registering with PyPI...')
 cd(ipdir)
 sh('./setup.py register')
 
 # Upload all files
 sh(sdists + ' upload')
-for py in ('2.7', '3.3'):
+for py in ('2.7', '3.4'):
     sh('python%s setupegg.py bdist_wheel' % py)
 
 cd(distdir)
 print( 'Uploading distribution files...')
 
 for fname in os.listdir('.'):
-    # GitHub doesn't have an API for uploads at the moment
+    # TODO: update to GitHub releases API
     continue
     print('uploading %s to GitHub' % fname)
     desc = "IPython %s source distribution" % version
     post_download("ipython/ipython", fname, description=desc)
 
 # Make target dir if it doesn't exist
 sh('ssh %s "mkdir -p %s/release/%s" ' % (archive_user, archive_dir, version))
 sh('scp * %s' % release_site)
 
 print( 'Uploading backup files...')
 cd(ipbackupdir)
 sh('scp `ls -1tr *tgz | tail -1` %s' % backup_site)
 
 print('Done!')

tools/toollib.py

@@ -1,77 +1,61 b''
 """Various utilities common to IPython release and maintenance tools.
 """
 from __future__ import print_function
 
 # Library imports
 import os
 import sys
 
 # Useful shorthands
 pjoin = os.path.join
 cd = os.chdir
 
 # Constants
 
 # SSH root address of the archive site
 archive_user = '[e-mail]'
 archive_dir = 'archive.ipython.org'
 archive = '%s:%s' % (archive_user, archive_dir)
 
 # Build commands
 # Source dists
 sdists = './setup.py sdist --formats=gztar,zip'
-# Eggs
-eggs = './setupegg.py bdist_egg'
-
-# Windows builds.
-# We do them separately, so that the extra Windows scripts don't get pulled
-# into Unix builds (setup.py has code which checks for bdist_wininst).  Note
-# that the install scripts args are added to the main distutils call in
-# setup.py, so they don't need to be passed here.
-#
-# The Windows 64-bit installer can't be built by a Linux/Mac Python because ofa
-# bug in distutils:  http://bugs.python.org/issue6792.
-# So we have to build it with a wine-installed native Windows Python...
-win_builds = ["python setup.py bdist_wininst "
-              "--install-script=ipython_win_post_install.py",
-              r"%s/.wine/dosdevices/c\:/Python32/python.exe setup.py build "
-              "--plat-name=win-amd64 bdist_wininst "
-              "--install-script=ipython_win_post_install.py" %
-              os.environ['HOME'] ]
+# Binary dists
+wheels = './setupegg.py bdist_wheel'
 
 # Utility functions
 def sh(cmd):
     """Run system command in shell, raise SystemExit if it returns an error."""
     print("$", cmd)
     stat = os.system(cmd)
     #stat = 0  # Uncomment this and comment previous to run in debug mode
     if stat:
         raise SystemExit("Command %s failed with code: %s" % (cmd, stat))
 
 # Backwards compatibility
 c = sh
 
 def get_ipdir():
     """Get IPython directory from command line, or assume it's the one above."""
 
     # Initialize arguments and check location
     try:
         ipdir = sys.argv[1]
     except IndexError:
         ipdir = '..'
 
     ipdir = os.path.abspath(ipdir)
 
     cd(ipdir)
     if not os.path.isdir('IPython') and os.path.isfile('setup.py'):
         raise SystemExit('Invalid ipython directory: %s' % ipdir)
     return ipdir
 
 
 def compile_tree():
     """Compile all Python files below current directory."""
     stat = os.system('python -m compileall .')
     if stat:
         msg = '*** ERROR: Some Python files in tree do NOT compile! ***\n'
         msg += 'See messages above for the actual file that produced it.\n'
         raise SystemExit(msg)