##// END OF EJS Templates
fix milestone link in github_stats
Min RK -
Show More
@@ -1,232 +1,232 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 """Simple tools to query github.com and gather stats about issues.
2 """Simple tools to query github.com and gather stats about issues.
3
3
4 To generate a report for IPython 2.0, run:
4 To generate a report for IPython 2.0, run:
5
5
6 python github_stats.py --milestone 2.0 --since-tag rel-1.0.0
6 python github_stats.py --milestone 2.0 --since-tag rel-1.0.0
7 """
7 """
8 #-----------------------------------------------------------------------------
8 #-----------------------------------------------------------------------------
9 # Imports
9 # Imports
10 #-----------------------------------------------------------------------------
10 #-----------------------------------------------------------------------------
11
11
12 from __future__ import print_function
12 from __future__ import print_function
13
13
14 import codecs
14 import codecs
15 import sys
15 import sys
16
16
17 from argparse import ArgumentParser
17 from argparse import ArgumentParser
18 from datetime import datetime, timedelta
18 from datetime import datetime, timedelta
19 from subprocess import check_output
19 from subprocess import check_output
20
20
21 from gh_api import (
21 from gh_api import (
22 get_paged_request, make_auth_header, get_pull_request, is_pull_request,
22 get_paged_request, make_auth_header, get_pull_request, is_pull_request,
23 get_milestone_id, get_issues_list, get_authors,
23 get_milestone_id, get_issues_list, get_authors,
24 )
24 )
25 #-----------------------------------------------------------------------------
25 #-----------------------------------------------------------------------------
26 # Globals
26 # Globals
27 #-----------------------------------------------------------------------------
27 #-----------------------------------------------------------------------------
28
28
29 ISO8601 = "%Y-%m-%dT%H:%M:%SZ"
29 ISO8601 = "%Y-%m-%dT%H:%M:%SZ"
30 PER_PAGE = 100
30 PER_PAGE = 100
31
31
32 #-----------------------------------------------------------------------------
32 #-----------------------------------------------------------------------------
33 # Functions
33 # Functions
34 #-----------------------------------------------------------------------------
34 #-----------------------------------------------------------------------------
35
35
36 def round_hour(dt):
36 def round_hour(dt):
37 return dt.replace(minute=0,second=0,microsecond=0)
37 return dt.replace(minute=0,second=0,microsecond=0)
38
38
39 def _parse_datetime(s):
39 def _parse_datetime(s):
40 """Parse dates in the format returned by the Github API."""
40 """Parse dates in the format returned by the Github API."""
41 if s:
41 if s:
42 return datetime.strptime(s, ISO8601)
42 return datetime.strptime(s, ISO8601)
43 else:
43 else:
44 return datetime.fromtimestamp(0)
44 return datetime.fromtimestamp(0)
45
45
46 def issues2dict(issues):
46 def issues2dict(issues):
47 """Convert a list of issues to a dict, keyed by issue number."""
47 """Convert a list of issues to a dict, keyed by issue number."""
48 idict = {}
48 idict = {}
49 for i in issues:
49 for i in issues:
50 idict[i['number']] = i
50 idict[i['number']] = i
51 return idict
51 return idict
52
52
53 def split_pulls(all_issues, project="ipython/ipython"):
53 def split_pulls(all_issues, project="ipython/ipython"):
54 """split a list of closed issues into non-PR Issues and Pull Requests"""
54 """split a list of closed issues into non-PR Issues and Pull Requests"""
55 pulls = []
55 pulls = []
56 issues = []
56 issues = []
57 for i in all_issues:
57 for i in all_issues:
58 if is_pull_request(i):
58 if is_pull_request(i):
59 pull = get_pull_request(project, i['number'], auth=True)
59 pull = get_pull_request(project, i['number'], auth=True)
60 pulls.append(pull)
60 pulls.append(pull)
61 else:
61 else:
62 issues.append(i)
62 issues.append(i)
63 return issues, pulls
63 return issues, pulls
64
64
65
65
66 def issues_closed_since(period=timedelta(days=365), project="ipython/ipython", pulls=False):
66 def issues_closed_since(period=timedelta(days=365), project="ipython/ipython", pulls=False):
67 """Get all issues closed since a particular point in time. period
67 """Get all issues closed since a particular point in time. period
68 can either be a datetime object, or a timedelta object. In the
68 can either be a datetime object, or a timedelta object. In the
69 latter case, it is used as a time before the present.
69 latter case, it is used as a time before the present.
70 """
70 """
71
71
72 which = 'pulls' if pulls else 'issues'
72 which = 'pulls' if pulls else 'issues'
73
73
74 if isinstance(period, timedelta):
74 if isinstance(period, timedelta):
75 since = round_hour(datetime.utcnow() - period)
75 since = round_hour(datetime.utcnow() - period)
76 else:
76 else:
77 since = period
77 since = period
78 url = "https://api.github.com/repos/%s/%s?state=closed&sort=updated&since=%s&per_page=%i" % (project, which, since.strftime(ISO8601), PER_PAGE)
78 url = "https://api.github.com/repos/%s/%s?state=closed&sort=updated&since=%s&per_page=%i" % (project, which, since.strftime(ISO8601), PER_PAGE)
79 allclosed = get_paged_request(url, headers=make_auth_header())
79 allclosed = get_paged_request(url, headers=make_auth_header())
80
80
81 filtered = [ i for i in allclosed if _parse_datetime(i['closed_at']) > since ]
81 filtered = [ i for i in allclosed if _parse_datetime(i['closed_at']) > since ]
82 if pulls:
82 if pulls:
83 filtered = [ i for i in filtered if _parse_datetime(i['merged_at']) > since ]
83 filtered = [ i for i in filtered if _parse_datetime(i['merged_at']) > since ]
84 # filter out PRs not against master (backports)
84 # filter out PRs not against master (backports)
85 filtered = [ i for i in filtered if i['base']['ref'] == 'master' ]
85 filtered = [ i for i in filtered if i['base']['ref'] == 'master' ]
86 else:
86 else:
87 filtered = [ i for i in filtered if not is_pull_request(i) ]
87 filtered = [ i for i in filtered if not is_pull_request(i) ]
88
88
89 return filtered
89 return filtered
90
90
91
91
92 def sorted_by_field(issues, field='closed_at', reverse=False):
92 def sorted_by_field(issues, field='closed_at', reverse=False):
93 """Return a list of issues sorted by closing date date."""
93 """Return a list of issues sorted by closing date date."""
94 return sorted(issues, key = lambda i:i[field], reverse=reverse)
94 return sorted(issues, key = lambda i:i[field], reverse=reverse)
95
95
96
96
97 def report(issues, show_urls=False):
97 def report(issues, show_urls=False):
98 """Summary report about a list of issues, printing number and title."""
98 """Summary report about a list of issues, printing number and title."""
99 if show_urls:
99 if show_urls:
100 for i in issues:
100 for i in issues:
101 role = 'ghpull' if 'merged_at' in i else 'ghissue'
101 role = 'ghpull' if 'merged_at' in i else 'ghissue'
102 print(u'* :%s:`%d`: %s' % (role, i['number'],
102 print(u'* :%s:`%d`: %s' % (role, i['number'],
103 i['title'].replace(u'`', u'``')))
103 i['title'].replace(u'`', u'``')))
104 else:
104 else:
105 for i in issues:
105 for i in issues:
106 print(u'* %d: %s' % (i['number'], i['title'].replace(u'`', u'``')))
106 print(u'* %d: %s' % (i['number'], i['title'].replace(u'`', u'``')))
107
107
108 #-----------------------------------------------------------------------------
108 #-----------------------------------------------------------------------------
109 # Main script
109 # Main script
110 #-----------------------------------------------------------------------------
110 #-----------------------------------------------------------------------------
111
111
112 if __name__ == "__main__":
112 if __name__ == "__main__":
113 # deal with unicode
113 # deal with unicode
114 if sys.version_info < (3,):
114 if sys.version_info < (3,):
115 sys.stdout = codecs.getwriter('utf8')(sys.stdout)
115 sys.stdout = codecs.getwriter('utf8')(sys.stdout)
116
116
117 # Whether to add reST urls for all issues in printout.
117 # Whether to add reST urls for all issues in printout.
118 show_urls = True
118 show_urls = True
119
119
120 parser = ArgumentParser()
120 parser = ArgumentParser()
121 parser.add_argument('--since-tag', type=str,
121 parser.add_argument('--since-tag', type=str,
122 help="The git tag to use for the starting point (typically the last major release)."
122 help="The git tag to use for the starting point (typically the last major release)."
123 )
123 )
124 parser.add_argument('--milestone', type=str,
124 parser.add_argument('--milestone', type=str,
125 help="The GitHub milestone to use for filtering issues [optional]."
125 help="The GitHub milestone to use for filtering issues [optional]."
126 )
126 )
127 parser.add_argument('--days', type=int,
127 parser.add_argument('--days', type=int,
128 help="The number of days of data to summarize (use this or --since-tag)."
128 help="The number of days of data to summarize (use this or --since-tag)."
129 )
129 )
130 parser.add_argument('--project', type=str, default="ipython/ipython",
130 parser.add_argument('--project', type=str, default="ipython/ipython",
131 help="The project to summarize."
131 help="The project to summarize."
132 )
132 )
133 parser.add_argument('--links', action='store_true', default=False,
133 parser.add_argument('--links', action='store_true', default=False,
134 help="Include links to all closed Issues and PRs in the output."
134 help="Include links to all closed Issues and PRs in the output."
135 )
135 )
136
136
137 opts = parser.parse_args()
137 opts = parser.parse_args()
138 tag = opts.since_tag
138 tag = opts.since_tag
139
139
140 # set `since` from days or git tag
140 # set `since` from days or git tag
141 if opts.days:
141 if opts.days:
142 since = datetime.utcnow() - timedelta(days=opts.days)
142 since = datetime.utcnow() - timedelta(days=opts.days)
143 else:
143 else:
144 if not tag:
144 if not tag:
145 tag = check_output(['git', 'describe', '--abbrev=0']).strip().decode('utf8')
145 tag = check_output(['git', 'describe', '--abbrev=0']).strip().decode('utf8')
146 cmd = ['git', 'log', '-1', '--format=%ai', tag]
146 cmd = ['git', 'log', '-1', '--format=%ai', tag]
147 tagday, tz = check_output(cmd).strip().decode('utf8').rsplit(' ', 1)
147 tagday, tz = check_output(cmd).strip().decode('utf8').rsplit(' ', 1)
148 since = datetime.strptime(tagday, "%Y-%m-%d %H:%M:%S")
148 since = datetime.strptime(tagday, "%Y-%m-%d %H:%M:%S")
149 h = int(tz[1:3])
149 h = int(tz[1:3])
150 m = int(tz[3:])
150 m = int(tz[3:])
151 td = timedelta(hours=h, minutes=m)
151 td = timedelta(hours=h, minutes=m)
152 if tz[0] == '-':
152 if tz[0] == '-':
153 since += td
153 since += td
154 else:
154 else:
155 since -= td
155 since -= td
156
156
157 since = round_hour(since)
157 since = round_hour(since)
158
158
159 milestone = opts.milestone
159 milestone = opts.milestone
160 project = opts.project
160 project = opts.project
161
161
162 print("fetching GitHub stats since %s (tag: %s, milestone: %s)" % (since, tag, milestone), file=sys.stderr)
162 print("fetching GitHub stats since %s (tag: %s, milestone: %s)" % (since, tag, milestone), file=sys.stderr)
163 if milestone:
163 if milestone:
164 milestone_id = get_milestone_id(project=project, milestone=milestone,
164 milestone_id = get_milestone_id(project=project, milestone=milestone,
165 auth=True)
165 auth=True)
166 issues_and_pulls = get_issues_list(project=project,
166 issues_and_pulls = get_issues_list(project=project,
167 milestone=milestone_id,
167 milestone=milestone_id,
168 state='closed',
168 state='closed',
169 auth=True,
169 auth=True,
170 )
170 )
171 issues, pulls = split_pulls(issues_and_pulls)
171 issues, pulls = split_pulls(issues_and_pulls)
172 else:
172 else:
173 issues = issues_closed_since(since, project=project, pulls=False)
173 issues = issues_closed_since(since, project=project, pulls=False)
174 pulls = issues_closed_since(since, project=project, pulls=True)
174 pulls = issues_closed_since(since, project=project, pulls=True)
175
175
176 # For regular reports, it's nice to show them in reverse chronological order
176 # For regular reports, it's nice to show them in reverse chronological order
177 issues = sorted_by_field(issues, reverse=True)
177 issues = sorted_by_field(issues, reverse=True)
178 pulls = sorted_by_field(pulls, reverse=True)
178 pulls = sorted_by_field(pulls, reverse=True)
179
179
180 n_issues, n_pulls = map(len, (issues, pulls))
180 n_issues, n_pulls = map(len, (issues, pulls))
181 n_total = n_issues + n_pulls
181 n_total = n_issues + n_pulls
182
182
183 # Print summary report we can directly include into release notes.
183 # Print summary report we can directly include into release notes.
184
184
185 print()
185 print()
186 since_day = since.strftime("%Y/%m/%d")
186 since_day = since.strftime("%Y/%m/%d")
187 today = datetime.today().strftime("%Y/%m/%d")
187 today = datetime.today().strftime("%Y/%m/%d")
188 print("GitHub stats for %s - %s (tag: %s)" % (since_day, today, tag))
188 print("GitHub stats for %s - %s (tag: %s)" % (since_day, today, tag))
189 print()
189 print()
190 print("These lists are automatically generated, and may be incomplete or contain duplicates.")
190 print("These lists are automatically generated, and may be incomplete or contain duplicates.")
191 print()
191 print()
192
192
193 ncommits = 0
193 ncommits = 0
194 all_authors = []
194 all_authors = []
195 if tag:
195 if tag:
196 # print git info, in addition to GitHub info:
196 # print git info, in addition to GitHub info:
197 since_tag = tag+'..'
197 since_tag = tag+'..'
198 cmd = ['git', 'log', '--oneline', since_tag]
198 cmd = ['git', 'log', '--oneline', since_tag]
199 ncommits += len(check_output(cmd).splitlines())
199 ncommits += len(check_output(cmd).splitlines())
200
200
201 author_cmd = ['git', 'log', '--use-mailmap', "--format=* %aN", since_tag]
201 author_cmd = ['git', 'log', '--use-mailmap', "--format=* %aN", since_tag]
202 all_authors.extend(check_output(author_cmd).decode('utf-8', 'replace').splitlines())
202 all_authors.extend(check_output(author_cmd).decode('utf-8', 'replace').splitlines())
203
203
204 pr_authors = []
204 pr_authors = []
205 for pr in pulls:
205 for pr in pulls:
206 pr_authors.extend(get_authors(pr))
206 pr_authors.extend(get_authors(pr))
207 ncommits = len(pr_authors) + ncommits - len(pulls)
207 ncommits = len(pr_authors) + ncommits - len(pulls)
208 author_cmd = ['git', 'check-mailmap'] + pr_authors
208 author_cmd = ['git', 'check-mailmap'] + pr_authors
209 with_email = check_output(author_cmd).decode('utf-8', 'replace').splitlines()
209 with_email = check_output(author_cmd).decode('utf-8', 'replace').splitlines()
210 all_authors.extend([ u'* ' + a.split(' <')[0] for a in with_email ])
210 all_authors.extend([ u'* ' + a.split(' <')[0] for a in with_email ])
211 unique_authors = sorted(set(all_authors), key=lambda s: s.lower())
211 unique_authors = sorted(set(all_authors), key=lambda s: s.lower())
212
212
213 print("We closed %d issues and merged %d pull requests." % (n_issues, n_pulls))
213 print("We closed %d issues and merged %d pull requests." % (n_issues, n_pulls))
214 if milestone:
214 if milestone:
215 print("The full list can be seen `on GitHub <https://github.com/%s/milestone/%s>`"
215 print("The full list can be seen `on GitHub <https://github.com/%s/milestones/%s>`_"
216 % (project, milestone)
216 % (project, milestone)
217 )
217 )
218
218
219 print()
219 print()
220 print("The following %i authors contributed %i commits." % (len(unique_authors), ncommits))
220 print("The following %i authors contributed %i commits." % (len(unique_authors), ncommits))
221 print()
221 print()
222 print('\n'.join(unique_authors))
222 print('\n'.join(unique_authors))
223
223
224 if opts.links:
224 if opts.links:
225 print()
225 print()
226 print("GitHub issues and pull requests:")
226 print("GitHub issues and pull requests:")
227 print()
227 print()
228 print('Pull Requests (%d):\n' % n_pulls)
228 print('Pull Requests (%d):\n' % n_pulls)
229 report(pulls, show_urls)
229 report(pulls, show_urls)
230 print()
230 print()
231 print('Issues (%d):\n' % n_issues)
231 print('Issues (%d):\n' % n_issues)
232 report(issues, show_urls)
232 report(issues, show_urls)
General Comments 0
You need to be logged in to leave comments. Login now