Show More
@@ -1,200 +1,200 b'' | |||||
1 | #!/usr/bin/env python |
|
1 | #!/usr/bin/env python | |
2 | """Simple tools to query github.com and gather stats about issues. |
|
2 | """Simple tools to query github.com and gather stats about issues. | |
3 | """ |
|
3 | """ | |
4 | #----------------------------------------------------------------------------- |
|
4 | #----------------------------------------------------------------------------- | |
5 | # Imports |
|
5 | # Imports | |
6 | #----------------------------------------------------------------------------- |
|
6 | #----------------------------------------------------------------------------- | |
7 |
|
7 | |||
8 | from __future__ import print_function |
|
8 | from __future__ import print_function | |
9 |
|
9 | |||
10 | import json |
|
10 | import json | |
11 | import re |
|
11 | import re | |
12 | import sys |
|
12 | import sys | |
13 |
|
13 | |||
14 | from datetime import datetime, timedelta |
|
14 | from datetime import datetime, timedelta | |
15 | from subprocess import check_output |
|
15 | from subprocess import check_output | |
16 | from gh_api import get_paged_request, make_auth_header, get_pull_request |
|
16 | from gh_api import get_paged_request, make_auth_header, get_pull_request | |
17 |
|
17 | |||
18 | #----------------------------------------------------------------------------- |
|
18 | #----------------------------------------------------------------------------- | |
19 | # Globals |
|
19 | # Globals | |
20 | #----------------------------------------------------------------------------- |
|
20 | #----------------------------------------------------------------------------- | |
21 |
|
21 | |||
22 | ISO8601 = "%Y-%m-%dT%H:%M:%SZ" |
|
22 | ISO8601 = "%Y-%m-%dT%H:%M:%SZ" | |
23 | PER_PAGE = 100 |
|
23 | PER_PAGE = 100 | |
24 |
|
24 | |||
25 | #----------------------------------------------------------------------------- |
|
25 | #----------------------------------------------------------------------------- | |
26 | # Functions |
|
26 | # Functions | |
27 | #----------------------------------------------------------------------------- |
|
27 | #----------------------------------------------------------------------------- | |
28 |
|
28 | |||
29 | def get_issues(project="ipython/ipython", state="closed", pulls=False): |
|
29 | def get_issues(project="ipython/ipython", state="closed", pulls=False): | |
30 | """Get a list of the issues from the Github API.""" |
|
30 | """Get a list of the issues from the Github API.""" | |
31 | which = 'pulls' if pulls else 'issues' |
|
31 | which = 'pulls' if pulls else 'issues' | |
32 | url = "https://api.github.com/repos/%s/%s?state=%s&per_page=%i" % (project, which, state, PER_PAGE) |
|
32 | url = "https://api.github.com/repos/%s/%s?state=%s&per_page=%i" % (project, which, state, PER_PAGE) | |
33 | return get_paged_request(url, headers=make_auth_header()) |
|
33 | return get_paged_request(url, headers=make_auth_header()) | |
34 |
|
34 | |||
35 | def round_hour(dt): |
|
35 | def round_hour(dt): | |
36 | return dt.replace(minute=0,second=0,microsecond=0) |
|
36 | return dt.replace(minute=0,second=0,microsecond=0) | |
37 |
|
37 | |||
38 | def _parse_datetime(s): |
|
38 | def _parse_datetime(s): | |
39 | """Parse dates in the format returned by the Github API.""" |
|
39 | """Parse dates in the format returned by the Github API.""" | |
40 | if s: |
|
40 | if s: | |
41 | return datetime.strptime(s, ISO8601) |
|
41 | return datetime.strptime(s, ISO8601) | |
42 | else: |
|
42 | else: | |
43 | return datetime.fromtimestamp(0) |
|
43 | return datetime.fromtimestamp(0) | |
44 |
|
44 | |||
45 |
|
45 | |||
46 | def issues2dict(issues): |
|
46 | def issues2dict(issues): | |
47 | """Convert a list of issues to a dict, keyed by issue number.""" |
|
47 | """Convert a list of issues to a dict, keyed by issue number.""" | |
48 | idict = {} |
|
48 | idict = {} | |
49 | for i in issues: |
|
49 | for i in issues: | |
50 | idict[i['number']] = i |
|
50 | idict[i['number']] = i | |
51 | return idict |
|
51 | return idict | |
52 |
|
52 | |||
53 |
|
53 | |||
54 | def is_pull_request(issue): |
|
54 | def is_pull_request(issue): | |
55 | """Return True if the given issue is a pull request.""" |
|
55 | """Return True if the given issue is a pull request.""" | |
56 | return bool(issue.get('pull_request', {}).get('html_url', None)) |
|
56 | return bool(issue.get('pull_request', {}).get('html_url', None)) | |
57 |
|
57 | |||
58 |
|
58 | |||
59 | def split_pulls(all_issues, project="ipython/ipython"): |
|
59 | def split_pulls(all_issues, project="ipython/ipython"): | |
60 | """split a list of closed issues into non-PR Issues and Pull Requests""" |
|
60 | """split a list of closed issues into non-PR Issues and Pull Requests""" | |
61 | pulls = [] |
|
61 | pulls = [] | |
62 | issues = [] |
|
62 | issues = [] | |
63 | for i in all_issues: |
|
63 | for i in all_issues: | |
64 | if is_pull_request(i): |
|
64 | if is_pull_request(i): | |
65 | pull = get_pull_request(project, i['number'], auth=True) |
|
65 | pull = get_pull_request(project, i['number'], auth=True) | |
66 | pulls.append(pull) |
|
66 | pulls.append(pull) | |
67 | else: |
|
67 | else: | |
68 | issues.append(i) |
|
68 | issues.append(i) | |
69 | return issues, pulls |
|
69 | return issues, pulls | |
70 |
|
70 | |||
71 |
|
71 | |||
72 |
|
72 | |||
73 | def issues_closed_since(period=timedelta(days=365), project="ipython/ipython", pulls=False): |
|
73 | def issues_closed_since(period=timedelta(days=365), project="ipython/ipython", pulls=False): | |
74 | """Get all issues closed since a particular point in time. period |
|
74 | """Get all issues closed since a particular point in time. period | |
75 | can either be a datetime object, or a timedelta object. In the |
|
75 | can either be a datetime object, or a timedelta object. In the | |
76 | latter case, it is used as a time before the present. |
|
76 | latter case, it is used as a time before the present. | |
77 | """ |
|
77 | """ | |
78 |
|
78 | |||
79 | which = 'pulls' if pulls else 'issues' |
|
79 | which = 'pulls' if pulls else 'issues' | |
80 |
|
80 | |||
81 | if isinstance(period, timedelta): |
|
81 | if isinstance(period, timedelta): | |
82 | since = round_hour(datetime.utcnow() - period) |
|
82 | since = round_hour(datetime.utcnow() - period) | |
83 | else: |
|
83 | else: | |
84 | since = period |
|
84 | since = period | |
85 | url = "https://api.github.com/repos/%s/%s?state=closed&sort=updated&since=%s&per_page=%i" % (project, which, since.strftime(ISO8601), PER_PAGE) |
|
85 | url = "https://api.github.com/repos/%s/%s?state=closed&sort=updated&since=%s&per_page=%i" % (project, which, since.strftime(ISO8601), PER_PAGE) | |
86 | allclosed = get_paged_request(url, headers=make_auth_header()) |
|
86 | allclosed = get_paged_request(url, headers=make_auth_header()) | |
87 |
|
87 | |||
88 | filtered = [ i for i in allclosed if _parse_datetime(i['closed_at']) > since ] |
|
88 | filtered = [ i for i in allclosed if _parse_datetime(i['closed_at']) > since ] | |
89 | if pulls: |
|
89 | if pulls: | |
90 | filtered = [ i for i in filtered if _parse_datetime(i['merged_at']) > since ] |
|
90 | filtered = [ i for i in filtered if _parse_datetime(i['merged_at']) > since ] | |
91 | # filter out PRs not against master (backports) |
|
91 | # filter out PRs not against master (backports) | |
92 | filtered = [ i for i in filtered if i['base']['ref'] == 'master' ] |
|
92 | filtered = [ i for i in filtered if i['base']['ref'] == 'master' ] | |
93 | else: |
|
93 | else: | |
94 | filtered = [ i for i in filtered if not is_pull_request(i) ] |
|
94 | filtered = [ i for i in filtered if not is_pull_request(i) ] | |
95 |
|
95 | |||
96 | return filtered |
|
96 | return filtered | |
97 |
|
97 | |||
98 |
|
98 | |||
99 | def sorted_by_field(issues, field='closed_at', reverse=False): |
|
99 | def sorted_by_field(issues, field='closed_at', reverse=False): | |
100 | """Return a list of issues sorted by closing date date.""" |
|
100 | """Return a list of issues sorted by closing date date.""" | |
101 | return sorted(issues, key = lambda i:i[field], reverse=reverse) |
|
101 | return sorted(issues, key = lambda i:i[field], reverse=reverse) | |
102 |
|
102 | |||
103 |
|
103 | |||
104 | def report(issues, show_urls=False): |
|
104 | def report(issues, show_urls=False): | |
105 | """Summary report about a list of issues, printing number and title. |
|
105 | """Summary report about a list of issues, printing number and title. | |
106 | """ |
|
106 | """ | |
107 | # titles may have unicode in them, so we must encode everything below |
|
107 | # titles may have unicode in them, so we must encode everything below | |
108 | if show_urls: |
|
108 | if show_urls: | |
109 | for i in issues: |
|
109 | for i in issues: | |
110 | role = 'ghpull' if 'merged_at' in i else 'ghissue' |
|
110 | role = 'ghpull' if 'merged_at' in i else 'ghissue' | |
111 | print('* :%s:`%d`: %s' % (role, i['number'], |
|
111 | print('* :%s:`%d`: %s' % (role, i['number'], | |
112 | i['title'].encode('utf-8'))) |
|
112 | i['title'].encode('utf-8'))) | |
113 | else: |
|
113 | else: | |
114 | for i in issues: |
|
114 | for i in issues: | |
115 | print('* %d: %s' % (i['number'], i['title'].encode('utf-8'))) |
|
115 | print('* %d: %s' % (i['number'], i['title'].encode('utf-8'))) | |
116 |
|
116 | |||
117 | #----------------------------------------------------------------------------- |
|
117 | #----------------------------------------------------------------------------- | |
118 | # Main script |
|
118 | # Main script | |
119 | #----------------------------------------------------------------------------- |
|
119 | #----------------------------------------------------------------------------- | |
120 |
|
120 | |||
121 | if __name__ == "__main__": |
|
121 | if __name__ == "__main__": | |
122 | # deal with unicode |
|
122 | # deal with unicode | |
123 | import codecs |
|
123 | import codecs | |
124 | sys.stdout = codecs.getwriter('utf8')(sys.stdout) |
|
124 | sys.stdout = codecs.getwriter('utf8')(sys.stdout) | |
125 |
|
125 | |||
126 | # Whether to add reST urls for all issues in printout. |
|
126 | # Whether to add reST urls for all issues in printout. | |
127 | show_urls = True |
|
127 | show_urls = True | |
128 |
|
128 | |||
129 | # By default, search one month back |
|
129 | # By default, search one month back | |
130 | tag = None |
|
130 | tag = None | |
131 | if len(sys.argv) > 1: |
|
131 | if len(sys.argv) > 1: | |
132 | try: |
|
132 | try: | |
133 | days = int(sys.argv[1]) |
|
133 | days = int(sys.argv[1]) | |
134 | except: |
|
134 | except: | |
135 | tag = sys.argv[1] |
|
135 | tag = sys.argv[1] | |
136 | else: |
|
136 | else: | |
137 | tag = check_output(['git', 'describe', '--abbrev=0']).strip() |
|
137 | tag = check_output(['git', 'describe', '--abbrev=0']).strip() | |
138 |
|
138 | |||
139 | if tag: |
|
139 | if tag: | |
140 | cmd = ['git', 'log', '-1', '--format=%ai', tag] |
|
140 | cmd = ['git', 'log', '-1', '--format=%ai', tag] | |
141 | tagday, tz = check_output(cmd).strip().rsplit(' ', 1) |
|
141 | tagday, tz = check_output(cmd).strip().rsplit(' ', 1) | |
142 | since = datetime.strptime(tagday, "%Y-%m-%d %H:%M:%S") |
|
142 | since = datetime.strptime(tagday, "%Y-%m-%d %H:%M:%S") | |
143 | h = int(tz[1:3]) |
|
143 | h = int(tz[1:3]) | |
144 | m = int(tz[3:]) |
|
144 | m = int(tz[3:]) | |
145 | td = timedelta(hours=h, minutes=m) |
|
145 | td = timedelta(hours=h, minutes=m) | |
146 | if tz[0] == '-': |
|
146 | if tz[0] == '-': | |
147 | since += td |
|
147 | since += td | |
148 | else: |
|
148 | else: | |
149 | since -= td |
|
149 | since -= td | |
150 | else: |
|
150 | else: | |
151 | since = datetime.utcnow() - timedelta(days=days) |
|
151 | since = datetime.utcnow() - timedelta(days=days) | |
152 |
|
152 | |||
153 | since = round_hour(since) |
|
153 | since = round_hour(since) | |
154 |
|
154 | |||
155 | print("fetching GitHub stats since %s (tag: %s)" % (since, tag), file=sys.stderr) |
|
155 | print("fetching GitHub stats since %s (tag: %s)" % (since, tag), file=sys.stderr) | |
156 | # turn off to play interactively without redownloading, use %run -i |
|
156 | # turn off to play interactively without redownloading, use %run -i | |
157 | if 1: |
|
157 | if 1: | |
158 | issues = issues_closed_since(since, pulls=False) |
|
158 | issues = issues_closed_since(since, pulls=False) | |
159 | pulls = issues_closed_since(since, pulls=True) |
|
159 | pulls = issues_closed_since(since, pulls=True) | |
160 |
|
160 | |||
161 | # For regular reports, it's nice to show them in reverse chronological order |
|
161 | # For regular reports, it's nice to show them in reverse chronological order | |
162 | issues = sorted_by_field(issues, reverse=True) |
|
162 | issues = sorted_by_field(issues, reverse=True) | |
163 | pulls = sorted_by_field(pulls, reverse=True) |
|
163 | pulls = sorted_by_field(pulls, reverse=True) | |
164 |
|
164 | |||
165 | n_issues, n_pulls = map(len, (issues, pulls)) |
|
165 | n_issues, n_pulls = map(len, (issues, pulls)) | |
166 | n_total = n_issues + n_pulls |
|
166 | n_total = n_issues + n_pulls | |
167 |
|
167 | |||
168 | # Print summary report we can directly include into release notes. |
|
168 | # Print summary report we can directly include into release notes. | |
169 |
|
169 | |||
170 | print() |
|
170 | print() | |
171 | since_day = since.strftime("%Y/%m/%d") |
|
171 | since_day = since.strftime("%Y/%m/%d") | |
172 | today = datetime.today().strftime("%Y/%m/%d") |
|
172 | today = datetime.today().strftime("%Y/%m/%d") | |
173 | print("GitHub stats for %s - %s (tag: %s)" % (since_day, today, tag)) |
|
173 | print("GitHub stats for %s - %s (tag: %s)" % (since_day, today, tag)) | |
174 | print() |
|
174 | print() | |
175 | print("These lists are automatically generated, and may be incomplete or contain duplicates.") |
|
175 | print("These lists are automatically generated, and may be incomplete or contain duplicates.") | |
176 | print() |
|
176 | print() | |
177 | if tag: |
|
177 | if tag: | |
178 | # print git info, in addition to GitHub info: |
|
178 | # print git info, in addition to GitHub info: | |
179 | since_tag = tag+'..' |
|
179 | since_tag = tag+'..' | |
180 | cmd = ['git', 'log', '--oneline', since_tag] |
|
180 | cmd = ['git', 'log', '--oneline', since_tag] | |
181 | ncommits = len(check_output(cmd).splitlines()) |
|
181 | ncommits = len(check_output(cmd).splitlines()) | |
182 |
|
182 | |||
183 |
author_cmd = ['git', 'log', '--use-mailmap', "--format= |
|
183 | author_cmd = ['git', 'log', '--use-mailmap', "--format=* %aN", since_tag] | |
184 | all_authors = check_output(author_cmd).decode('utf-8', 'replace').splitlines() |
|
184 | all_authors = check_output(author_cmd).decode('utf-8', 'replace').splitlines() | |
185 | unique_authors = sorted(set(all_authors), key=lambda s: s.lower()) |
|
185 | unique_authors = sorted(set(all_authors), key=lambda s: s.lower()) | |
186 | print("The following %i authors contributed %i commits." % (len(unique_authors), ncommits)) |
|
186 | print("The following %i authors contributed %i commits." % (len(unique_authors), ncommits)) | |
187 | print() |
|
187 | print() | |
188 | print('\n'.join(unique_authors)) |
|
188 | print('\n'.join(unique_authors)) | |
189 | print() |
|
189 | print() | |
190 |
|
190 | |||
191 | print() |
|
191 | print() | |
192 | print("We closed a total of %d issues, %d pull requests and %d regular issues;\n" |
|
192 | print("We closed a total of %d issues, %d pull requests and %d regular issues;\n" | |
193 | "this is the full list (generated with the script \n" |
|
193 | "this is the full list (generated with the script \n" | |
194 | ":file:`tools/github_stats.py`):" % (n_total, n_pulls, n_issues)) |
|
194 | ":file:`tools/github_stats.py`):" % (n_total, n_pulls, n_issues)) | |
195 | print() |
|
195 | print() | |
196 | print('Pull Requests (%d):\n' % n_pulls) |
|
196 | print('Pull Requests (%d):\n' % n_pulls) | |
197 | report(pulls, show_urls) |
|
197 | report(pulls, show_urls) | |
198 | print() |
|
198 | print() | |
199 | print('Issues (%d):\n' % n_issues) |
|
199 | print('Issues (%d):\n' % n_issues) | |
200 | report(issues, show_urls) |
|
200 | report(issues, show_urls) |
General Comments 0
You need to be logged in to leave comments.
Login now