##// END OF EJS Templates
update tools/github_stats.py to use GitHub API v3
MinRK -
Show More
@@ -1,109 +1,149 b''
1 1 #!/usr/bin/env python
2 2 """Simple tools to query github.com and gather stats about issues.
3 3 """
4 4 #-----------------------------------------------------------------------------
5 5 # Imports
6 6 #-----------------------------------------------------------------------------
7 7
8 8 from __future__ import print_function
9 9
10 10 import json
11 import re
11 12 import sys
12 13
13 14 from datetime import datetime, timedelta
14 15 from urllib import urlopen
15 16
16 17 #-----------------------------------------------------------------------------
18 # Globals
19 #-----------------------------------------------------------------------------
20
21 ISO8601 = "%Y-%m-%dT%H:%M:%SZ"
22 PER_PAGE = 100
23
24 element_pat = re.compile(r'<(.+?)>')
25 rel_pat = re.compile(r'rel=[\'"](\w+)[\'"]')
26
27 #-----------------------------------------------------------------------------
17 28 # Functions
18 29 #-----------------------------------------------------------------------------
19 30
20 def get_issues(project="ipython/ipython/", state="open"):
31 def parse_link_header(headers):
32 link_s = headers.get('link', '')
33 urls = element_pat.findall(link_s)
34 rels = rel_pat.findall(link_s)
35 d = {}
36 for rel,url in zip(rels, urls):
37 d[rel] = url
38 return d
39
40 def get_paged_request(url):
41 """get a full list, handling APIv3's paging"""
42 results = []
43 while url:
44 print("fetching %s" % url, file=sys.stderr)
45 f = urlopen(url)
46 results.extend(json.load(f))
47 links = parse_link_header(f.headers)
48 url = links.get('next')
49 return results
50
51 def get_issues(project="ipython/ipython", state="closed", pulls=False):
21 52 """Get a list of the issues from the Github API."""
22 f = urlopen("http://github.com/api/v2/json/issues/list/%s%s" % (project,
23 state))
24 return json.load(f)['issues']
53 which = 'pulls' if pulls else 'issues'
54 url = "https://api.github.com/repos/%s/%s?state=%s&per_page=%i" % (project, which, state, PER_PAGE)
55 return get_paged_request(url)
25 56
26 57
27 58 def _parse_datetime(s):
28 59 """Parse dates in the format returned by the Github API."""
29 return datetime.strptime(s.rpartition(" ")[0], "%Y/%m/%d %H:%M:%S")
60 if s:
61 return datetime.strptime(s, ISO8601)
62 else:
63 return datetime.fromtimestamp(0)
30 64
31 65
32 66 def issues2dict(issues):
33 67 """Convert a list of issues to a dict, keyed by issue number."""
34 68 idict = {}
35 69 for i in issues:
36 70 idict[i['number']] = i
37 71 return idict
38 72
39 73
40 74 def is_pull_request(issue):
41 75 """Return True if the given issue is a pull request."""
42 76 return 'pull_request_url' in issue
43 77
44 78
45 def issues_closed_since(period=timedelta(days=365), project="ipython/ipython/"):
79 def issues_closed_since(period=timedelta(days=365), project="ipython/ipython", pulls=False):
46 80 """Get all issues closed since a particular point in time. period
47 81 can either be a datetime object, or a timedelta object. In the
48 82 latter case, it is used as a time before the present."""
49 allclosed = get_issues(project=project, state='closed')
83
84 which = 'pulls' if pulls else 'issues'
85
50 86 if isinstance(period, timedelta):
51 87 period = datetime.now() - period
52 return [i for i in allclosed if _parse_datetime(i['closed_at']) > period]
88 url = "https://api.github.com/repos/%s/%s?state=closed&sort=updated&since=%s&per_page=%i" % (project, which, period.strftime(ISO8601), PER_PAGE)
89 allclosed = get_paged_request(url)
90 # allclosed = get_issues(project=project, state='closed', pulls=pulls, since=period)
91 filtered = [i for i in allclosed if _parse_datetime(i['closed_at']) > period]
92 return filtered
53 93
54 94
55 95 def sorted_by_field(issues, field='closed_at', reverse=False):
56 96 """Return a list of issues sorted by closing date date."""
57 97 return sorted(issues, key = lambda i:i[field], reverse=reverse)
58 98
59 99
60 100 def report(issues, show_urls=False):
61 101 """Summary report about a list of issues, printing number and title.
62 102 """
63 103 # titles may have unicode in them, so we must encode everything below
64 104 if show_urls:
65 105 for i in issues:
66 print('* `%d <%s>`_: %s' % (i['number'],
67 i['html_url'].encode('utf-8'),
106 role = 'ghpull' if 'merged' in i else 'ghissue'
107 print('* :%s:`%d`: %s' % (role, i['number'],
68 108 i['title'].encode('utf-8')))
69 109 else:
70 110 for i in issues:
71 111 print('* %d: %s' % (i['number'], i['title'].encode('utf-8')))
72 112
73 113 #-----------------------------------------------------------------------------
74 114 # Main script
75 115 #-----------------------------------------------------------------------------
76 116
77 117 if __name__ == "__main__":
78 118 # Whether to add reST urls for all issues in printout.
79 119 show_urls = True
80 120
81 121 # By default, search one month back
82 122 if len(sys.argv) > 1:
83 123 days = int(sys.argv[1])
84 124 else:
85 125 days = 30
86 126
87 127 # turn off to play interactively without redownloading, use %run -i
88 128 if 1:
89 issues = issues_closed_since(timedelta(days=days))
129 issues = issues_closed_since(timedelta(days=days), pulls=False)
130 pulls = issues_closed_since(timedelta(days=days), pulls=True)
90 131
91 132 # For regular reports, it's nice to show them in reverse chronological order
92 133 issues = sorted_by_field(issues, reverse=True)
93
94 # Break up into pull requests and regular issues
95 pulls = filter(is_pull_request, issues)
96 regular = filter(lambda i: not is_pull_request(i), issues)
97 n_issues, n_pulls, n_regular = map(len, (issues, pulls, regular))
134 pulls = sorted_by_field(pulls, reverse=True)
135
136 n_issues, n_pulls = map(len, (issues, pulls))
137 n_total = n_issues + n_pulls
98 138
99 139 # Print summary report we can directly include into release notes.
100 print("Github stats for the last %d days." % days)
140 print("GitHub stats for the last %d days." % days)
101 141 print("We closed a total of %d issues, %d pull requests and %d regular \n"
102 142 "issues; this is the full list (generated with the script \n"
103 "`tools/github_stats.py`):" % (n_issues, n_pulls, n_regular))
143 "`tools/github_stats.py`):" % (n_total, n_pulls, n_issues))
104 144 print()
105 print('Pull requests (%d):\n' % n_pulls)
145 print('Pull Requests (%d):\n' % n_pulls)
106 146 report(pulls, show_urls)
107 147 print()
108 print('Regular issues (%d):\n' % n_regular)
109 report(regular, show_urls)
148 print('Issues (%d):\n' % n_issues)
149 report(issues, show_urls)
General Comments 0
You need to be logged in to leave comments. Login now