##// END OF EJS Templates
teach github_stats about milestones...
MinRK -
Show More
@@ -1,194 +1,209 b''
1 1 #!/usr/bin/env python
2 2 """Simple tools to query github.com and gather stats about issues.
3
4 To generate a report for IPython 2.0, run:
5
6 python github_stats.py --milestone 2.0 --since-tag rel-1.0.0
3 7 """
4 8 #-----------------------------------------------------------------------------
5 9 # Imports
6 10 #-----------------------------------------------------------------------------
7 11
8 12 from __future__ import print_function
9 13
14 import codecs
10 15 import json
11 16 import re
12 17 import sys
13 18
19 from argparse import ArgumentParser
14 20 from datetime import datetime, timedelta
15 21 from subprocess import check_output
16 from gh_api import get_paged_request, make_auth_header, get_pull_request, is_pull_request
17
22 from gh_api import (
23 get_paged_request, make_auth_header, get_pull_request, is_pull_request,
24 get_milestone_id, get_issues_list,
25 )
18 26 #-----------------------------------------------------------------------------
19 27 # Globals
20 28 #-----------------------------------------------------------------------------
21 29
22 30 ISO8601 = "%Y-%m-%dT%H:%M:%SZ"
23 31 PER_PAGE = 100
24 32
25 33 #-----------------------------------------------------------------------------
26 34 # Functions
27 35 #-----------------------------------------------------------------------------
28 36
29 def get_issues(project="ipython/ipython", state="closed", pulls=False):
30 """Get a list of the issues from the Github API."""
31 which = 'pulls' if pulls else 'issues'
32 url = "https://api.github.com/repos/%s/%s?state=%s&per_page=%i" % (project, which, state, PER_PAGE)
33 return get_paged_request(url, headers=make_auth_header())
34
35 37 def round_hour(dt):
36 38 return dt.replace(minute=0,second=0,microsecond=0)
37 39
38 40 def _parse_datetime(s):
39 41 """Parse dates in the format returned by the Github API."""
40 42 if s:
41 43 return datetime.strptime(s, ISO8601)
42 44 else:
43 45 return datetime.fromtimestamp(0)
44 46
45
46 47 def issues2dict(issues):
47 48 """Convert a list of issues to a dict, keyed by issue number."""
48 49 idict = {}
49 50 for i in issues:
50 51 idict[i['number']] = i
51 52 return idict
52 53
53 54 def split_pulls(all_issues, project="ipython/ipython"):
54 55 """split a list of closed issues into non-PR Issues and Pull Requests"""
55 56 pulls = []
56 57 issues = []
57 58 for i in all_issues:
58 59 if is_pull_request(i):
59 60 pull = get_pull_request(project, i['number'], auth=True)
60 61 pulls.append(pull)
61 62 else:
62 63 issues.append(i)
63 64 return issues, pulls
64 65
65 66
66
67 67 def issues_closed_since(period=timedelta(days=365), project="ipython/ipython", pulls=False):
68 68 """Get all issues closed since a particular point in time. period
69 69 can either be a datetime object, or a timedelta object. In the
70 70 latter case, it is used as a time before the present.
71 71 """
72 72
73 73 which = 'pulls' if pulls else 'issues'
74 74
75 75 if isinstance(period, timedelta):
76 76 since = round_hour(datetime.utcnow() - period)
77 77 else:
78 78 since = period
79 79 url = "https://api.github.com/repos/%s/%s?state=closed&sort=updated&since=%s&per_page=%i" % (project, which, since.strftime(ISO8601), PER_PAGE)
80 80 allclosed = get_paged_request(url, headers=make_auth_header())
81 81
82 82 filtered = [ i for i in allclosed if _parse_datetime(i['closed_at']) > since ]
83 83 if pulls:
84 84 filtered = [ i for i in filtered if _parse_datetime(i['merged_at']) > since ]
85 85 # filter out PRs not against master (backports)
86 86 filtered = [ i for i in filtered if i['base']['ref'] == 'master' ]
87 87 else:
88 88 filtered = [ i for i in filtered if not is_pull_request(i) ]
89 89
90 90 return filtered
91 91
92 92
93 93 def sorted_by_field(issues, field='closed_at', reverse=False):
94 94 """Return a list of issues sorted by closing date date."""
95 95 return sorted(issues, key = lambda i:i[field], reverse=reverse)
96 96
97 97
98 98 def report(issues, show_urls=False):
99 99 """Summary report about a list of issues, printing number and title.
100 100 """
101 101 # titles may have unicode in them, so we must encode everything below
102 102 if show_urls:
103 103 for i in issues:
104 104 role = 'ghpull' if 'merged_at' in i else 'ghissue'
105 105 print('* :%s:`%d`: %s' % (role, i['number'],
106 106 i['title'].encode('utf-8')))
107 107 else:
108 108 for i in issues:
109 109 print('* %d: %s' % (i['number'], i['title'].encode('utf-8')))
110 110
111 111 #-----------------------------------------------------------------------------
112 112 # Main script
113 113 #-----------------------------------------------------------------------------
114 114
115 115 if __name__ == "__main__":
116 116 # deal with unicode
117 import codecs
118 117 sys.stdout = codecs.getwriter('utf8')(sys.stdout)
119 118
120 119 # Whether to add reST urls for all issues in printout.
121 120 show_urls = True
122 121
123 # By default, search one month back
124 tag = None
125 if len(sys.argv) > 1:
126 try:
127 days = int(sys.argv[1])
128 except:
129 tag = sys.argv[1]
122 parser = ArgumentParser()
123 parser.add_argument('--since-tag', type=str,
124 help="The git tag to use for the starting point (typically the last major release)."
125 )
126 parser.add_argument('--milestone', type=str,
127 help="The GitHub milestone to use for filtering issues [optional]."
128 )
129 parser.add_argument('--days', type=int,
130 help="The number of days of data to summarize (use this or --since-tag)."
131 )
132
133 opts = parser.parse_args()
134 tag = opts.since_tag
135
136 # set `since` from days or git tag
137 if opts.days:
138 since = datetime.utcnow() - timedelta(days=opts.days)
130 139 else:
140 if not tag:
131 141 tag = check_output(['git', 'describe', '--abbrev=0']).strip()
132
133 if tag:
134 142 cmd = ['git', 'log', '-1', '--format=%ai', tag]
135 143 tagday, tz = check_output(cmd).strip().rsplit(' ', 1)
136 144 since = datetime.strptime(tagday, "%Y-%m-%d %H:%M:%S")
137 145 h = int(tz[1:3])
138 146 m = int(tz[3:])
139 147 td = timedelta(hours=h, minutes=m)
140 148 if tz[0] == '-':
141 149 since += td
142 150 else:
143 151 since -= td
144 else:
145 since = datetime.utcnow() - timedelta(days=days)
146 152
147 153 since = round_hour(since)
148 154
149 print("fetching GitHub stats since %s (tag: %s)" % (since, tag), file=sys.stderr)
150 # turn off to play interactively without redownloading, use %run -i
151 if 1:
155 milestone = opts.milestone
156
157 print("fetching GitHub stats since %s (tag: %s, milestone: %s)" % (since, tag, milestone), file=sys.stderr)
158 if milestone:
159 milestone_id = get_milestone_id("ipython/ipython", milestone,
160 auth=True)
161 issues = get_issues_list("ipython/ipython",
162 milestone=milestone_id,
163 state='closed',
164 auth=True,
165 )
166 else:
152 167 issues = issues_closed_since(since, pulls=False)
153 168 pulls = issues_closed_since(since, pulls=True)
154 169
155 170 # For regular reports, it's nice to show them in reverse chronological order
156 171 issues = sorted_by_field(issues, reverse=True)
157 172 pulls = sorted_by_field(pulls, reverse=True)
158 173
159 174 n_issues, n_pulls = map(len, (issues, pulls))
160 175 n_total = n_issues + n_pulls
161 176
162 177 # Print summary report we can directly include into release notes.
163 178
164 179 print()
165 180 since_day = since.strftime("%Y/%m/%d")
166 181 today = datetime.today().strftime("%Y/%m/%d")
167 182 print("GitHub stats for %s - %s (tag: %s)" % (since_day, today, tag))
168 183 print()
169 184 print("These lists are automatically generated, and may be incomplete or contain duplicates.")
170 185 print()
171 186 if tag:
172 187 # print git info, in addition to GitHub info:
173 188 since_tag = tag+'..'
174 189 cmd = ['git', 'log', '--oneline', since_tag]
175 190 ncommits = len(check_output(cmd).splitlines())
176 191
177 192 author_cmd = ['git', 'log', '--use-mailmap', "--format=* %aN", since_tag]
178 193 all_authors = check_output(author_cmd).decode('utf-8', 'replace').splitlines()
179 194 unique_authors = sorted(set(all_authors), key=lambda s: s.lower())
180 195 print("The following %i authors contributed %i commits." % (len(unique_authors), ncommits))
181 196 print()
182 197 print('\n'.join(unique_authors))
183 198 print()
184 199
185 200 print()
186 201 print("We closed a total of %d issues, %d pull requests and %d regular issues;\n"
187 202 "this is the full list (generated with the script \n"
188 203 ":file:`tools/github_stats.py`):" % (n_total, n_pulls, n_issues))
189 204 print()
190 205 print('Pull Requests (%d):\n' % n_pulls)
191 206 report(pulls, show_urls)
192 207 print()
193 208 print('Issues (%d):\n' % n_issues)
194 209 report(issues, show_urls)
General Comments 0
You need to be logged in to leave comments. Login now