##// END OF EJS Templates
remove quotes from github_stats
MinRK -
Show More
@@ -1,200 +1,200 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 """Simple tools to query github.com and gather stats about issues.
2 """Simple tools to query github.com and gather stats about issues.
3 """
3 """
4 #-----------------------------------------------------------------------------
4 #-----------------------------------------------------------------------------
5 # Imports
5 # Imports
6 #-----------------------------------------------------------------------------
6 #-----------------------------------------------------------------------------
7
7
8 from __future__ import print_function
8 from __future__ import print_function
9
9
10 import json
10 import json
11 import re
11 import re
12 import sys
12 import sys
13
13
14 from datetime import datetime, timedelta
14 from datetime import datetime, timedelta
15 from subprocess import check_output
15 from subprocess import check_output
16 from gh_api import get_paged_request, make_auth_header, get_pull_request
16 from gh_api import get_paged_request, make_auth_header, get_pull_request
17
17
18 #-----------------------------------------------------------------------------
18 #-----------------------------------------------------------------------------
19 # Globals
19 # Globals
20 #-----------------------------------------------------------------------------
20 #-----------------------------------------------------------------------------
21
21
22 ISO8601 = "%Y-%m-%dT%H:%M:%SZ"
22 ISO8601 = "%Y-%m-%dT%H:%M:%SZ"
23 PER_PAGE = 100
23 PER_PAGE = 100
24
24
25 #-----------------------------------------------------------------------------
25 #-----------------------------------------------------------------------------
26 # Functions
26 # Functions
27 #-----------------------------------------------------------------------------
27 #-----------------------------------------------------------------------------
28
28
29 def get_issues(project="ipython/ipython", state="closed", pulls=False):
29 def get_issues(project="ipython/ipython", state="closed", pulls=False):
30 """Get a list of the issues from the Github API."""
30 """Get a list of the issues from the Github API."""
31 which = 'pulls' if pulls else 'issues'
31 which = 'pulls' if pulls else 'issues'
32 url = "https://api.github.com/repos/%s/%s?state=%s&per_page=%i" % (project, which, state, PER_PAGE)
32 url = "https://api.github.com/repos/%s/%s?state=%s&per_page=%i" % (project, which, state, PER_PAGE)
33 return get_paged_request(url, headers=make_auth_header())
33 return get_paged_request(url, headers=make_auth_header())
34
34
35 def round_hour(dt):
35 def round_hour(dt):
36 return dt.replace(minute=0,second=0,microsecond=0)
36 return dt.replace(minute=0,second=0,microsecond=0)
37
37
38 def _parse_datetime(s):
38 def _parse_datetime(s):
39 """Parse dates in the format returned by the Github API."""
39 """Parse dates in the format returned by the Github API."""
40 if s:
40 if s:
41 return datetime.strptime(s, ISO8601)
41 return datetime.strptime(s, ISO8601)
42 else:
42 else:
43 return datetime.fromtimestamp(0)
43 return datetime.fromtimestamp(0)
44
44
45
45
46 def issues2dict(issues):
46 def issues2dict(issues):
47 """Convert a list of issues to a dict, keyed by issue number."""
47 """Convert a list of issues to a dict, keyed by issue number."""
48 idict = {}
48 idict = {}
49 for i in issues:
49 for i in issues:
50 idict[i['number']] = i
50 idict[i['number']] = i
51 return idict
51 return idict
52
52
53
53
54 def is_pull_request(issue):
54 def is_pull_request(issue):
55 """Return True if the given issue is a pull request."""
55 """Return True if the given issue is a pull request."""
56 return bool(issue.get('pull_request', {}).get('html_url', None))
56 return bool(issue.get('pull_request', {}).get('html_url', None))
57
57
58
58
59 def split_pulls(all_issues, project="ipython/ipython"):
59 def split_pulls(all_issues, project="ipython/ipython"):
60 """split a list of closed issues into non-PR Issues and Pull Requests"""
60 """split a list of closed issues into non-PR Issues and Pull Requests"""
61 pulls = []
61 pulls = []
62 issues = []
62 issues = []
63 for i in all_issues:
63 for i in all_issues:
64 if is_pull_request(i):
64 if is_pull_request(i):
65 pull = get_pull_request(project, i['number'], auth=True)
65 pull = get_pull_request(project, i['number'], auth=True)
66 pulls.append(pull)
66 pulls.append(pull)
67 else:
67 else:
68 issues.append(i)
68 issues.append(i)
69 return issues, pulls
69 return issues, pulls
70
70
71
71
72
72
73 def issues_closed_since(period=timedelta(days=365), project="ipython/ipython", pulls=False):
73 def issues_closed_since(period=timedelta(days=365), project="ipython/ipython", pulls=False):
74 """Get all issues closed since a particular point in time. period
74 """Get all issues closed since a particular point in time. period
75 can either be a datetime object, or a timedelta object. In the
75 can either be a datetime object, or a timedelta object. In the
76 latter case, it is used as a time before the present.
76 latter case, it is used as a time before the present.
77 """
77 """
78
78
79 which = 'pulls' if pulls else 'issues'
79 which = 'pulls' if pulls else 'issues'
80
80
81 if isinstance(period, timedelta):
81 if isinstance(period, timedelta):
82 since = round_hour(datetime.utcnow() - period)
82 since = round_hour(datetime.utcnow() - period)
83 else:
83 else:
84 since = period
84 since = period
85 url = "https://api.github.com/repos/%s/%s?state=closed&sort=updated&since=%s&per_page=%i" % (project, which, since.strftime(ISO8601), PER_PAGE)
85 url = "https://api.github.com/repos/%s/%s?state=closed&sort=updated&since=%s&per_page=%i" % (project, which, since.strftime(ISO8601), PER_PAGE)
86 allclosed = get_paged_request(url, headers=make_auth_header())
86 allclosed = get_paged_request(url, headers=make_auth_header())
87
87
88 filtered = [ i for i in allclosed if _parse_datetime(i['closed_at']) > since ]
88 filtered = [ i for i in allclosed if _parse_datetime(i['closed_at']) > since ]
89 if pulls:
89 if pulls:
90 filtered = [ i for i in filtered if _parse_datetime(i['merged_at']) > since ]
90 filtered = [ i for i in filtered if _parse_datetime(i['merged_at']) > since ]
91 # filter out PRs not against master (backports)
91 # filter out PRs not against master (backports)
92 filtered = [ i for i in filtered if i['base']['ref'] == 'master' ]
92 filtered = [ i for i in filtered if i['base']['ref'] == 'master' ]
93 else:
93 else:
94 filtered = [ i for i in filtered if not is_pull_request(i) ]
94 filtered = [ i for i in filtered if not is_pull_request(i) ]
95
95
96 return filtered
96 return filtered
97
97
98
98
99 def sorted_by_field(issues, field='closed_at', reverse=False):
99 def sorted_by_field(issues, field='closed_at', reverse=False):
100 """Return a list of issues sorted by closing date date."""
100 """Return a list of issues sorted by closing date date."""
101 return sorted(issues, key = lambda i:i[field], reverse=reverse)
101 return sorted(issues, key = lambda i:i[field], reverse=reverse)
102
102
103
103
104 def report(issues, show_urls=False):
104 def report(issues, show_urls=False):
105 """Summary report about a list of issues, printing number and title.
105 """Summary report about a list of issues, printing number and title.
106 """
106 """
107 # titles may have unicode in them, so we must encode everything below
107 # titles may have unicode in them, so we must encode everything below
108 if show_urls:
108 if show_urls:
109 for i in issues:
109 for i in issues:
110 role = 'ghpull' if 'merged_at' in i else 'ghissue'
110 role = 'ghpull' if 'merged_at' in i else 'ghissue'
111 print('* :%s:`%d`: %s' % (role, i['number'],
111 print('* :%s:`%d`: %s' % (role, i['number'],
112 i['title'].encode('utf-8')))
112 i['title'].encode('utf-8')))
113 else:
113 else:
114 for i in issues:
114 for i in issues:
115 print('* %d: %s' % (i['number'], i['title'].encode('utf-8')))
115 print('* %d: %s' % (i['number'], i['title'].encode('utf-8')))
116
116
117 #-----------------------------------------------------------------------------
117 #-----------------------------------------------------------------------------
118 # Main script
118 # Main script
119 #-----------------------------------------------------------------------------
119 #-----------------------------------------------------------------------------
120
120
121 if __name__ == "__main__":
121 if __name__ == "__main__":
122 # deal with unicode
122 # deal with unicode
123 import codecs
123 import codecs
124 sys.stdout = codecs.getwriter('utf8')(sys.stdout)
124 sys.stdout = codecs.getwriter('utf8')(sys.stdout)
125
125
126 # Whether to add reST urls for all issues in printout.
126 # Whether to add reST urls for all issues in printout.
127 show_urls = True
127 show_urls = True
128
128
129 # By default, search one month back
129 # By default, search one month back
130 tag = None
130 tag = None
131 if len(sys.argv) > 1:
131 if len(sys.argv) > 1:
132 try:
132 try:
133 days = int(sys.argv[1])
133 days = int(sys.argv[1])
134 except:
134 except:
135 tag = sys.argv[1]
135 tag = sys.argv[1]
136 else:
136 else:
137 tag = check_output(['git', 'describe', '--abbrev=0']).strip()
137 tag = check_output(['git', 'describe', '--abbrev=0']).strip()
138
138
139 if tag:
139 if tag:
140 cmd = ['git', 'log', '-1', '--format=%ai', tag]
140 cmd = ['git', 'log', '-1', '--format=%ai', tag]
141 tagday, tz = check_output(cmd).strip().rsplit(' ', 1)
141 tagday, tz = check_output(cmd).strip().rsplit(' ', 1)
142 since = datetime.strptime(tagday, "%Y-%m-%d %H:%M:%S")
142 since = datetime.strptime(tagday, "%Y-%m-%d %H:%M:%S")
143 h = int(tz[1:3])
143 h = int(tz[1:3])
144 m = int(tz[3:])
144 m = int(tz[3:])
145 td = timedelta(hours=h, minutes=m)
145 td = timedelta(hours=h, minutes=m)
146 if tz[0] == '-':
146 if tz[0] == '-':
147 since += td
147 since += td
148 else:
148 else:
149 since -= td
149 since -= td
150 else:
150 else:
151 since = datetime.utcnow() - timedelta(days=days)
151 since = datetime.utcnow() - timedelta(days=days)
152
152
153 since = round_hour(since)
153 since = round_hour(since)
154
154
155 print("fetching GitHub stats since %s (tag: %s)" % (since, tag), file=sys.stderr)
155 print("fetching GitHub stats since %s (tag: %s)" % (since, tag), file=sys.stderr)
156 # turn off to play interactively without redownloading, use %run -i
156 # turn off to play interactively without redownloading, use %run -i
157 if 1:
157 if 1:
158 issues = issues_closed_since(since, pulls=False)
158 issues = issues_closed_since(since, pulls=False)
159 pulls = issues_closed_since(since, pulls=True)
159 pulls = issues_closed_since(since, pulls=True)
160
160
161 # For regular reports, it's nice to show them in reverse chronological order
161 # For regular reports, it's nice to show them in reverse chronological order
162 issues = sorted_by_field(issues, reverse=True)
162 issues = sorted_by_field(issues, reverse=True)
163 pulls = sorted_by_field(pulls, reverse=True)
163 pulls = sorted_by_field(pulls, reverse=True)
164
164
165 n_issues, n_pulls = map(len, (issues, pulls))
165 n_issues, n_pulls = map(len, (issues, pulls))
166 n_total = n_issues + n_pulls
166 n_total = n_issues + n_pulls
167
167
168 # Print summary report we can directly include into release notes.
168 # Print summary report we can directly include into release notes.
169
169
170 print()
170 print()
171 since_day = since.strftime("%Y/%m/%d")
171 since_day = since.strftime("%Y/%m/%d")
172 today = datetime.today().strftime("%Y/%m/%d")
172 today = datetime.today().strftime("%Y/%m/%d")
173 print("GitHub stats for %s - %s (tag: %s)" % (since_day, today, tag))
173 print("GitHub stats for %s - %s (tag: %s)" % (since_day, today, tag))
174 print()
174 print()
175 print("These lists are automatically generated, and may be incomplete or contain duplicates.")
175 print("These lists are automatically generated, and may be incomplete or contain duplicates.")
176 print()
176 print()
177 if tag:
177 if tag:
178 # print git info, in addition to GitHub info:
178 # print git info, in addition to GitHub info:
179 since_tag = tag+'..'
179 since_tag = tag+'..'
180 cmd = ['git', 'log', '--oneline', since_tag]
180 cmd = ['git', 'log', '--oneline', since_tag]
181 ncommits = len(check_output(cmd).splitlines())
181 ncommits = len(check_output(cmd).splitlines())
182
182
183 author_cmd = ['git', 'log', '--use-mailmap', "--format='* %aN'", since_tag]
183 author_cmd = ['git', 'log', '--use-mailmap', "--format=* %aN", since_tag]
184 all_authors = check_output(author_cmd).decode('utf-8', 'replace').splitlines()
184 all_authors = check_output(author_cmd).decode('utf-8', 'replace').splitlines()
185 unique_authors = sorted(set(all_authors), key=lambda s: s.lower())
185 unique_authors = sorted(set(all_authors), key=lambda s: s.lower())
186 print("The following %i authors contributed %i commits." % (len(unique_authors), ncommits))
186 print("The following %i authors contributed %i commits." % (len(unique_authors), ncommits))
187 print()
187 print()
188 print('\n'.join(unique_authors))
188 print('\n'.join(unique_authors))
189 print()
189 print()
190
190
191 print()
191 print()
192 print("We closed a total of %d issues, %d pull requests and %d regular issues;\n"
192 print("We closed a total of %d issues, %d pull requests and %d regular issues;\n"
193 "this is the full list (generated with the script \n"
193 "this is the full list (generated with the script \n"
194 ":file:`tools/github_stats.py`):" % (n_total, n_pulls, n_issues))
194 ":file:`tools/github_stats.py`):" % (n_total, n_pulls, n_issues))
195 print()
195 print()
196 print('Pull Requests (%d):\n' % n_pulls)
196 print('Pull Requests (%d):\n' % n_pulls)
197 report(pulls, show_urls)
197 report(pulls, show_urls)
198 print()
198 print()
199 print('Issues (%d):\n' % n_issues)
199 print('Issues (%d):\n' % n_issues)
200 report(issues, show_urls)
200 report(issues, show_urls)
General Comments 0
You need to be logged in to leave comments. Login now