##// END OF EJS Templates
updates to github stats...
MinRK -
Show More
@@ -1,180 +1,200 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 """Simple tools to query github.com and gather stats about issues.
2 """Simple tools to query github.com and gather stats about issues.
3 """
3 """
4 #-----------------------------------------------------------------------------
4 #-----------------------------------------------------------------------------
5 # Imports
5 # Imports
6 #-----------------------------------------------------------------------------
6 #-----------------------------------------------------------------------------
7
7
8 from __future__ import print_function
8 from __future__ import print_function
9
9
10 import json
10 import json
11 import re
11 import re
12 import sys
12 import sys
13
13
14 from datetime import datetime, timedelta
14 from datetime import datetime, timedelta
15 from subprocess import check_output
15 from subprocess import check_output
16 from gh_api import get_paged_request, make_auth_header, get_pull_request
16 from gh_api import get_paged_request, make_auth_header, get_pull_request
17
17
18 #-----------------------------------------------------------------------------
18 #-----------------------------------------------------------------------------
19 # Globals
19 # Globals
20 #-----------------------------------------------------------------------------
20 #-----------------------------------------------------------------------------
21
21
22 ISO8601 = "%Y-%m-%dT%H:%M:%SZ"
22 ISO8601 = "%Y-%m-%dT%H:%M:%SZ"
23 PER_PAGE = 100
23 PER_PAGE = 100
24
24
25 #-----------------------------------------------------------------------------
25 #-----------------------------------------------------------------------------
26 # Functions
26 # Functions
27 #-----------------------------------------------------------------------------
27 #-----------------------------------------------------------------------------
28
28
29 def get_issues(project="ipython/ipython", state="closed", pulls=False):
29 def get_issues(project="ipython/ipython", state="closed", pulls=False):
30 """Get a list of the issues from the Github API."""
30 """Get a list of the issues from the Github API."""
31 which = 'pulls' if pulls else 'issues'
31 which = 'pulls' if pulls else 'issues'
32 url = "https://api.github.com/repos/%s/%s?state=%s&per_page=%i" % (project, which, state, PER_PAGE)
32 url = "https://api.github.com/repos/%s/%s?state=%s&per_page=%i" % (project, which, state, PER_PAGE)
33 return get_paged_request(url, headers=make_auth_header())
33 return get_paged_request(url, headers=make_auth_header())
34
34
35 def round_hour(dt):
36 return dt.replace(minute=0,second=0,microsecond=0)
35
37
36 def _parse_datetime(s):
38 def _parse_datetime(s):
37 """Parse dates in the format returned by the Github API."""
39 """Parse dates in the format returned by the Github API."""
38 if s:
40 if s:
39 return datetime.strptime(s, ISO8601)
41 return datetime.strptime(s, ISO8601)
40 else:
42 else:
41 return datetime.fromtimestamp(0)
43 return datetime.fromtimestamp(0)
42
44
43
45
44 def issues2dict(issues):
46 def issues2dict(issues):
45 """Convert a list of issues to a dict, keyed by issue number."""
47 """Convert a list of issues to a dict, keyed by issue number."""
46 idict = {}
48 idict = {}
47 for i in issues:
49 for i in issues:
48 idict[i['number']] = i
50 idict[i['number']] = i
49 return idict
51 return idict
50
52
51
53
52 def is_pull_request(issue):
54 def is_pull_request(issue):
53 """Return True if the given issue is a pull request."""
55 """Return True if the given issue is a pull request."""
54 return bool(issue.get('pull_request', {}).get('html_url', None))
56 return bool(issue.get('pull_request', {}).get('html_url', None))
55
57
56
58
57 def split_pulls(all_issues, project="ipython/ipython"):
59 def split_pulls(all_issues, project="ipython/ipython"):
58 """split a list of closed issues into non-PR Issues and Pull Requests"""
60 """split a list of closed issues into non-PR Issues and Pull Requests"""
59 pulls = []
61 pulls = []
60 issues = []
62 issues = []
61 for i in all_issues:
63 for i in all_issues:
62 if is_pull_request(i):
64 if is_pull_request(i):
63 pull = get_pull_request(project, i['number'], auth=True)
65 pull = get_pull_request(project, i['number'], auth=True)
64 pulls.append(pull)
66 pulls.append(pull)
65 else:
67 else:
66 issues.append(i)
68 issues.append(i)
67 return issues, pulls
69 return issues, pulls
68
70
69
71
70
72
71 def issues_closed_since(period=timedelta(days=365), project="ipython/ipython"):
73 def issues_closed_since(period=timedelta(days=365), project="ipython/ipython", pulls=False):
72 """Get all issues closed since a particular point in time. period
74 """Get all issues closed since a particular point in time. period
73 can either be a datetime object, or a timedelta object. In the
75 can either be a datetime object, or a timedelta object. In the
74 latter case, it is used as a time before the present.
76 latter case, it is used as a time before the present.
75 """
77 """
76
78
77 which = 'issues'
79 which = 'pulls' if pulls else 'issues'
78
80
79 if isinstance(period, timedelta):
81 if isinstance(period, timedelta):
80 since = datetime.now() - period
82 since = round_hour(datetime.utcnow() - period)
81 else:
83 else:
82 since = period
84 since = period
83 url = "https://api.github.com/repos/%s/%s?state=closed&sort=updated&since=%s&per_page=%i" % (project, which, since.strftime(ISO8601), PER_PAGE)
85 url = "https://api.github.com/repos/%s/%s?state=closed&sort=updated&since=%s&per_page=%i" % (project, which, since.strftime(ISO8601), PER_PAGE)
84 allclosed = get_paged_request(url, headers=make_auth_header())
86 allclosed = get_paged_request(url, headers=make_auth_header())
85
87
86 issues, pulls = split_pulls(allclosed, project=project)
88 filtered = [ i for i in allclosed if _parse_datetime(i['closed_at']) > since ]
87 issues = [i for i in issues if _parse_datetime(i['closed_at']) > since]
89 if pulls:
88 pulls = [p for p in pulls if p['merged_at'] and _parse_datetime(p['merged_at']) > since]
90 filtered = [ i for i in filtered if _parse_datetime(i['merged_at']) > since ]
91 # filter out PRs not against master (backports)
92 filtered = [ i for i in filtered if i['base']['ref'] == 'master' ]
93 else:
94 filtered = [ i for i in filtered if not is_pull_request(i) ]
89
95
90 return issues, pulls
96 return filtered
91
97
92
98
93 def sorted_by_field(issues, field='closed_at', reverse=False):
99 def sorted_by_field(issues, field='closed_at', reverse=False):
94 """Return a list of issues sorted by closing date date."""
100 """Return a list of issues sorted by closing date date."""
95 return sorted(issues, key = lambda i:i[field], reverse=reverse)
101 return sorted(issues, key = lambda i:i[field], reverse=reverse)
96
102
97
103
98 def report(issues, show_urls=False):
104 def report(issues, show_urls=False):
99 """Summary report about a list of issues, printing number and title.
105 """Summary report about a list of issues, printing number and title.
100 """
106 """
101 # titles may have unicode in them, so we must encode everything below
107 # titles may have unicode in them, so we must encode everything below
102 if show_urls:
108 if show_urls:
103 for i in issues:
109 for i in issues:
104 role = 'ghpull' if 'merged_at' in i else 'ghissue'
110 role = 'ghpull' if 'merged_at' in i else 'ghissue'
105 print('* :%s:`%d`: %s' % (role, i['number'],
111 print('* :%s:`%d`: %s' % (role, i['number'],
106 i['title'].encode('utf-8')))
112 i['title'].encode('utf-8')))
107 else:
113 else:
108 for i in issues:
114 for i in issues:
109 print('* %d: %s' % (i['number'], i['title'].encode('utf-8')))
115 print('* %d: %s' % (i['number'], i['title'].encode('utf-8')))
110
116
111 #-----------------------------------------------------------------------------
117 #-----------------------------------------------------------------------------
112 # Main script
118 # Main script
113 #-----------------------------------------------------------------------------
119 #-----------------------------------------------------------------------------
114
120
115 if __name__ == "__main__":
121 if __name__ == "__main__":
122 # deal with unicode
123 import codecs
124 sys.stdout = codecs.getwriter('utf8')(sys.stdout)
125
116 # Whether to add reST urls for all issues in printout.
126 # Whether to add reST urls for all issues in printout.
117 show_urls = True
127 show_urls = True
118
128
119 # By default, search one month back
129 # By default, search one month back
120 tag = None
130 tag = None
121 if len(sys.argv) > 1:
131 if len(sys.argv) > 1:
122 try:
132 try:
123 days = int(sys.argv[1])
133 days = int(sys.argv[1])
124 except:
134 except:
125 tag = sys.argv[1]
135 tag = sys.argv[1]
126 else:
136 else:
127 tag = check_output(['git', 'describe', '--abbrev=0']).strip()
137 tag = check_output(['git', 'describe', '--abbrev=0']).strip()
128
138
129 if tag:
139 if tag:
130 cmd = ['git', 'log', '-1', '--format=%ai', tag]
140 cmd = ['git', 'log', '-1', '--format=%ai', tag]
131 tagday, tz = check_output(cmd).strip().rsplit(' ', 1)
141 tagday, tz = check_output(cmd).strip().rsplit(' ', 1)
132 since = datetime.strptime(tagday, "%Y-%m-%d %H:%M:%S")# - timedelta(days=30 * 6)
142 since = datetime.strptime(tagday, "%Y-%m-%d %H:%M:%S")
143 h = int(tz[1:3])
144 m = int(tz[3:])
145 td = timedelta(hours=h, minutes=m)
146 if tz[0] == '-':
147 since += td
133 else:
148 else:
134 since = datetime.now() - timedelta(days=days)
149 since -= td
150 else:
151 since = datetime.utcnow() - timedelta(days=days)
152
153 since = round_hour(since)
135
154
136 print("fetching GitHub stats since %s (tag: %s)" % (since, tag), file=sys.stderr)
155 print("fetching GitHub stats since %s (tag: %s)" % (since, tag), file=sys.stderr)
137 # turn off to play interactively without redownloading, use %run -i
156 # turn off to play interactively without redownloading, use %run -i
138 if 1:
157 if 1:
139 issues, pulls = issues_closed_since(since)
158 issues = issues_closed_since(since, pulls=False)
159 pulls = issues_closed_since(since, pulls=True)
140
160
141 # For regular reports, it's nice to show them in reverse chronological order
161 # For regular reports, it's nice to show them in reverse chronological order
142 issues = sorted_by_field(issues, reverse=True)
162 issues = sorted_by_field(issues, reverse=True)
143 pulls = sorted_by_field(pulls, reverse=True)
163 pulls = sorted_by_field(pulls, reverse=True)
144
164
145 n_issues, n_pulls = map(len, (issues, pulls))
165 n_issues, n_pulls = map(len, (issues, pulls))
146 n_total = n_issues + n_pulls
166 n_total = n_issues + n_pulls
147
167
148 # Print summary report we can directly include into release notes.
168 # Print summary report we can directly include into release notes.
169
149 print()
170 print()
150 since_day = since.strftime("%Y/%m/%d")
171 since_day = since.strftime("%Y/%m/%d")
151 today = datetime.today().strftime("%Y/%m/%d")
172 today = datetime.today().strftime("%Y/%m/%d")
152 print("GitHub stats for %s - %s (tag: %s)" % (since_day, today, tag))
173 print("GitHub stats for %s - %s (tag: %s)" % (since_day, today, tag))
153 print()
174 print()
154 print("These lists are automatically generated, and may be incomplete or contain duplicates.")
175 print("These lists are automatically generated, and may be incomplete or contain duplicates.")
155 print()
176 print()
156 if tag:
177 if tag:
157 # print git info, in addition to GitHub info:
178 # print git info, in addition to GitHub info:
158 since_tag = tag+'..'
179 since_tag = tag+'..'
159 cmd = ['git', 'log', '--oneline', since_tag]
180 cmd = ['git', 'log', '--oneline', since_tag]
160 ncommits = len(check_output(cmd).splitlines())
181 ncommits = len(check_output(cmd).splitlines())
161
182
162 author_cmd = ['git', 'log', '--format=* %aN', since_tag]
183 author_cmd = ['git', 'log', '--use-mailmap', "--format='* %aN'", since_tag]
163 all_authors = check_output(author_cmd).splitlines()
184 all_authors = check_output(author_cmd).decode('utf-8', 'replace').splitlines()
164 unique_authors = sorted(set(all_authors))
185 unique_authors = sorted(set(all_authors), key=lambda s: s.lower())
165
166 print("The following %i authors contributed %i commits." % (len(unique_authors), ncommits))
186 print("The following %i authors contributed %i commits." % (len(unique_authors), ncommits))
167 print()
187 print()
168 print('\n'.join(unique_authors))
188 print('\n'.join(unique_authors))
169 print()
189 print()
170
190
171 print()
191 print()
172 print("We closed a total of %d issues, %d pull requests and %d regular issues;\n"
192 print("We closed a total of %d issues, %d pull requests and %d regular issues;\n"
173 "this is the full list (generated with the script \n"
193 "this is the full list (generated with the script \n"
174 ":file:`tools/github_stats.py`):" % (n_total, n_pulls, n_issues))
194 ":file:`tools/github_stats.py`):" % (n_total, n_pulls, n_issues))
175 print()
195 print()
176 print('Pull Requests (%d):\n' % n_pulls)
196 print('Pull Requests (%d):\n' % n_pulls)
177 report(pulls, show_urls)
197 report(pulls, show_urls)
178 print()
198 print()
179 print('Issues (%d):\n' % n_issues)
199 print('Issues (%d):\n' % n_issues)
180 report(issues, show_urls)
200 report(issues, show_urls)
General Comments 0
You need to be logged in to leave comments. Login now