##// END OF EJS Templates
a bit of cache and logging cleanup in gh_api
MinRK -
Show More
@@ -1,276 +1,280 b''
1 """Functions for Github API requests."""
1 """Functions for Github API requests."""
2 from __future__ import print_function
2 from __future__ import print_function
3
3
4 try:
4 try:
5 input = raw_input
5 input = raw_input
6 except NameError:
6 except NameError:
7 pass
7 pass
8
8
9 import os
9 import os
10 import re
10 import re
11 import sys
11 import sys
12
12
13 import requests
13 import requests
14 import getpass
14 import getpass
15 import json
15 import json
16
16
17 try:
17 try:
18 import requests_cache
18 import requests_cache
19 except ImportError:
19 except ImportError:
20 print("no cache")
20 print("no cache", file=sys.stderr)
21 else:
21 else:
22 requests_cache.install_cache("gh_api")
22 requests_cache.install_cache("gh_api", expire_after=3600)
23
23
24 # Keyring stores passwords by a 'username', but we're not storing a username and
24 # Keyring stores passwords by a 'username', but we're not storing a username and
25 # password
25 # password
26 fake_username = 'ipython_tools'
26 fake_username = 'ipython_tools'
27
27
28 class Obj(dict):
28 class Obj(dict):
29 """Dictionary with attribute access to names."""
29 """Dictionary with attribute access to names."""
30 def __getattr__(self, name):
30 def __getattr__(self, name):
31 try:
31 try:
32 return self[name]
32 return self[name]
33 except KeyError:
33 except KeyError:
34 raise AttributeError(name)
34 raise AttributeError(name)
35
35
36 def __setattr__(self, name, val):
36 def __setattr__(self, name, val):
37 self[name] = val
37 self[name] = val
38
38
39 token = None
39 token = None
40 def get_auth_token():
40 def get_auth_token():
41 global token
41 global token
42
42
43 if token is not None:
43 if token is not None:
44 return token
44 return token
45
45
46 import keyring
46 import keyring
47 token = keyring.get_password('github', fake_username)
47 token = keyring.get_password('github', fake_username)
48 if token is not None:
48 if token is not None:
49 return token
49 return token
50
50
51 print("Please enter your github username and password. These are not "
51 print("Please enter your github username and password. These are not "
52 "stored, only used to get an oAuth token. You can revoke this at "
52 "stored, only used to get an oAuth token. You can revoke this at "
53 "any time on Github.")
53 "any time on Github.")
54 user = input("Username: ")
54 user = input("Username: ")
55 pw = getpass.getpass("Password: ")
55 pw = getpass.getpass("Password: ")
56
56
57 auth_request = {
57 auth_request = {
58 "scopes": [
58 "scopes": [
59 "public_repo",
59 "public_repo",
60 "gist"
60 "gist"
61 ],
61 ],
62 "note": "IPython tools",
62 "note": "IPython tools",
63 "note_url": "https://github.com/ipython/ipython/tree/master/tools",
63 "note_url": "https://github.com/ipython/ipython/tree/master/tools",
64 }
64 }
65 response = requests.post('https://api.github.com/authorizations',
65 response = requests.post('https://api.github.com/authorizations',
66 auth=(user, pw), data=json.dumps(auth_request))
66 auth=(user, pw), data=json.dumps(auth_request))
67 response.raise_for_status()
67 response.raise_for_status()
68 token = json.loads(response.text)['token']
68 token = json.loads(response.text)['token']
69 keyring.set_password('github', fake_username, token)
69 keyring.set_password('github', fake_username, token)
70 return token
70 return token
71
71
72 def make_auth_header():
72 def make_auth_header():
73 return {'Authorization': 'token ' + get_auth_token()}
73 return {'Authorization': 'token ' + get_auth_token()}
74
74
75 def post_issue_comment(project, num, body):
75 def post_issue_comment(project, num, body):
76 url = 'https://api.github.com/repos/{project}/issues/{num}/comments'.format(project=project, num=num)
76 url = 'https://api.github.com/repos/{project}/issues/{num}/comments'.format(project=project, num=num)
77 payload = json.dumps({'body': body})
77 payload = json.dumps({'body': body})
78 requests.post(url, data=payload, headers=make_auth_header())
78 requests.post(url, data=payload, headers=make_auth_header())
79
79
80 def post_gist(content, description='', filename='file', auth=False):
80 def post_gist(content, description='', filename='file', auth=False):
81 """Post some text to a Gist, and return the URL."""
81 """Post some text to a Gist, and return the URL."""
82 post_data = json.dumps({
82 post_data = json.dumps({
83 "description": description,
83 "description": description,
84 "public": True,
84 "public": True,
85 "files": {
85 "files": {
86 filename: {
86 filename: {
87 "content": content
87 "content": content
88 }
88 }
89 }
89 }
90 }).encode('utf-8')
90 }).encode('utf-8')
91
91
92 headers = make_auth_header() if auth else {}
92 headers = make_auth_header() if auth else {}
93 response = requests.post("https://api.github.com/gists", data=post_data, headers=headers)
93 response = requests.post("https://api.github.com/gists", data=post_data, headers=headers)
94 response.raise_for_status()
94 response.raise_for_status()
95 response_data = json.loads(response.text)
95 response_data = json.loads(response.text)
96 return response_data['html_url']
96 return response_data['html_url']
97
97
98 def get_pull_request(project, num, auth=False):
98 def get_pull_request(project, num, auth=False):
99 """get pull request info by number
99 """get pull request info by number
100 """
100 """
101 url = "https://api.github.com/repos/{project}/pulls/{num}".format(project=project, num=num)
101 url = "https://api.github.com/repos/{project}/pulls/{num}".format(project=project, num=num)
102 if auth:
102 if auth:
103 header = make_auth_header()
103 header = make_auth_header()
104 else:
104 else:
105 header = None
105 header = None
106 response = requests.get(url, headers=header)
106 response = requests.get(url, headers=header)
107 response.raise_for_status()
107 response.raise_for_status()
108 return json.loads(response.text, object_hook=Obj)
108 return json.loads(response.text, object_hook=Obj)
109
109
110 def get_pull_request_files(project, num, auth=False):
110 def get_pull_request_files(project, num, auth=False):
111 """get list of files in a pull request"""
111 """get list of files in a pull request"""
112 url = "https://api.github.com/repos/{project}/pulls/{num}/files".format(project=project, num=num)
112 url = "https://api.github.com/repos/{project}/pulls/{num}/files".format(project=project, num=num)
113 if auth:
113 if auth:
114 header = make_auth_header()
114 header = make_auth_header()
115 else:
115 else:
116 header = None
116 header = None
117 return get_paged_request(url, headers=header)
117 return get_paged_request(url, headers=header)
118
118
119 element_pat = re.compile(r'<(.+?)>')
119 element_pat = re.compile(r'<(.+?)>')
120 rel_pat = re.compile(r'rel=[\'"](\w+)[\'"]')
120 rel_pat = re.compile(r'rel=[\'"](\w+)[\'"]')
121
121
122 def get_paged_request(url, headers=None, **params):
122 def get_paged_request(url, headers=None, **params):
123 """get a full list, handling APIv3's paging"""
123 """get a full list, handling APIv3's paging"""
124 results = []
124 results = []
125 params.setdefault("per_page", 100)
125 params.setdefault("per_page", 100)
126 while True:
126 while True:
127 if '?' in url:
128 params = None
129 print("fetching %s" % url, file=sys.stderr)
130 else:
127 print("fetching %s with %s" % (url, params), file=sys.stderr)
131 print("fetching %s with %s" % (url, params), file=sys.stderr)
128 response = requests.get(url, headers=headers, params=params)
132 response = requests.get(url, headers=headers, params=params)
129 response.raise_for_status()
133 response.raise_for_status()
130 results.extend(response.json())
134 results.extend(response.json())
131 if 'next' in response.links:
135 if 'next' in response.links:
132 url = response.links['next']['url']
136 url = response.links['next']['url']
133 else:
137 else:
134 break
138 break
135 return results
139 return results
136
140
137 def get_pulls_list(project, auth=False, **params):
141 def get_pulls_list(project, auth=False, **params):
138 """get pull request list"""
142 """get pull request list"""
139 params.setdefault("state", "closed")
143 params.setdefault("state", "closed")
140 url = "https://api.github.com/repos/{project}/pulls".format(project=project)
144 url = "https://api.github.com/repos/{project}/pulls".format(project=project)
141 if auth:
145 if auth:
142 headers = make_auth_header()
146 headers = make_auth_header()
143 else:
147 else:
144 headers = None
148 headers = None
145 pages = get_paged_request(url, headers=headers, **params)
149 pages = get_paged_request(url, headers=headers, **params)
146 return pages
150 return pages
147
151
148 def get_issues_list(project, auth=False, **params):
152 def get_issues_list(project, auth=False, **params):
149 """get issues list"""
153 """get issues list"""
150 params.setdefault("state", "closed")
154 params.setdefault("state", "closed")
151 url = "https://api.github.com/repos/{project}/issues".format(project=project)
155 url = "https://api.github.com/repos/{project}/issues".format(project=project)
152 if auth:
156 if auth:
153 headers = make_auth_header()
157 headers = make_auth_header()
154 else:
158 else:
155 headers = None
159 headers = None
156 pages = get_paged_request(url, headers=headers, **params)
160 pages = get_paged_request(url, headers=headers, **params)
157 return pages
161 return pages
158
162
159 def get_milestones(project, auth=False, **params):
163 def get_milestones(project, auth=False, **params):
160 url = "https://api.github.com/repos/{project}/milestones".format(project=project)
164 url = "https://api.github.com/repos/{project}/milestones".format(project=project)
161 if auth:
165 if auth:
162 headers = make_auth_header()
166 headers = make_auth_header()
163 else:
167 else:
164 headers = None
168 headers = None
165 milestones = get_paged_request(url, headers=headers, **params)
169 milestones = get_paged_request(url, headers=headers, **params)
166 return milestones
170 return milestones
167
171
168 def get_milestone_id(project, milestone, auth=False, **params):
172 def get_milestone_id(project, milestone, auth=False, **params):
169 milestones = get_milestones(project, auth=auth, **params)
173 milestones = get_milestones(project, auth=auth, **params)
170 for mstone in milestones:
174 for mstone in milestones:
171 if mstone['title'] == milestone:
175 if mstone['title'] == milestone:
172 return mstone['number']
176 return mstone['number']
173 else:
177 else:
174 raise ValueError("milestone %s not found" % milestone)
178 raise ValueError("milestone %s not found" % milestone)
175
179
176 def is_pull_request(issue):
180 def is_pull_request(issue):
177 """Return True if the given issue is a pull request."""
181 """Return True if the given issue is a pull request."""
178 return bool(issue.get('pull_request', {}).get('html_url', None))
182 return bool(issue.get('pull_request', {}).get('html_url', None))
179
183
180 # encode_multipart_formdata is from urllib3.filepost
184 # encode_multipart_formdata is from urllib3.filepost
181 # The only change is to iter_fields, to enforce S3's required key ordering
185 # The only change is to iter_fields, to enforce S3's required key ordering
182
186
183 def iter_fields(fields):
187 def iter_fields(fields):
184 fields = fields.copy()
188 fields = fields.copy()
185 for key in ('key', 'acl', 'Filename', 'success_action_status', 'AWSAccessKeyId',
189 for key in ('key', 'acl', 'Filename', 'success_action_status', 'AWSAccessKeyId',
186 'Policy', 'Signature', 'Content-Type', 'file'):
190 'Policy', 'Signature', 'Content-Type', 'file'):
187 yield (key, fields.pop(key))
191 yield (key, fields.pop(key))
188 for (k,v) in fields.items():
192 for (k,v) in fields.items():
189 yield k,v
193 yield k,v
190
194
191 def encode_multipart_formdata(fields, boundary=None):
195 def encode_multipart_formdata(fields, boundary=None):
192 """
196 """
193 Encode a dictionary of ``fields`` using the multipart/form-data mime format.
197 Encode a dictionary of ``fields`` using the multipart/form-data mime format.
194
198
195 :param fields:
199 :param fields:
196 Dictionary of fields or list of (key, value) field tuples. The key is
200 Dictionary of fields or list of (key, value) field tuples. The key is
197 treated as the field name, and the value as the body of the form-data
201 treated as the field name, and the value as the body of the form-data
198 bytes. If the value is a tuple of two elements, then the first element
202 bytes. If the value is a tuple of two elements, then the first element
199 is treated as the filename of the form-data section.
203 is treated as the filename of the form-data section.
200
204
201 Field names and filenames must be unicode.
205 Field names and filenames must be unicode.
202
206
203 :param boundary:
207 :param boundary:
204 If not specified, then a random boundary will be generated using
208 If not specified, then a random boundary will be generated using
205 :func:`mimetools.choose_boundary`.
209 :func:`mimetools.choose_boundary`.
206 """
210 """
207 # copy requests imports in here:
211 # copy requests imports in here:
208 from io import BytesIO
212 from io import BytesIO
209 from requests.packages.urllib3.filepost import (
213 from requests.packages.urllib3.filepost import (
210 choose_boundary, six, writer, b, get_content_type
214 choose_boundary, six, writer, b, get_content_type
211 )
215 )
212 body = BytesIO()
216 body = BytesIO()
213 if boundary is None:
217 if boundary is None:
214 boundary = choose_boundary()
218 boundary = choose_boundary()
215
219
216 for fieldname, value in iter_fields(fields):
220 for fieldname, value in iter_fields(fields):
217 body.write(b('--%s\r\n' % (boundary)))
221 body.write(b('--%s\r\n' % (boundary)))
218
222
219 if isinstance(value, tuple):
223 if isinstance(value, tuple):
220 filename, data = value
224 filename, data = value
221 writer(body).write('Content-Disposition: form-data; name="%s"; '
225 writer(body).write('Content-Disposition: form-data; name="%s"; '
222 'filename="%s"\r\n' % (fieldname, filename))
226 'filename="%s"\r\n' % (fieldname, filename))
223 body.write(b('Content-Type: %s\r\n\r\n' %
227 body.write(b('Content-Type: %s\r\n\r\n' %
224 (get_content_type(filename))))
228 (get_content_type(filename))))
225 else:
229 else:
226 data = value
230 data = value
227 writer(body).write('Content-Disposition: form-data; name="%s"\r\n'
231 writer(body).write('Content-Disposition: form-data; name="%s"\r\n'
228 % (fieldname))
232 % (fieldname))
229 body.write(b'Content-Type: text/plain\r\n\r\n')
233 body.write(b'Content-Type: text/plain\r\n\r\n')
230
234
231 if isinstance(data, int):
235 if isinstance(data, int):
232 data = str(data) # Backwards compatibility
236 data = str(data) # Backwards compatibility
233 if isinstance(data, six.text_type):
237 if isinstance(data, six.text_type):
234 writer(body).write(data)
238 writer(body).write(data)
235 else:
239 else:
236 body.write(data)
240 body.write(data)
237
241
238 body.write(b'\r\n')
242 body.write(b'\r\n')
239
243
240 body.write(b('--%s--\r\n' % (boundary)))
244 body.write(b('--%s--\r\n' % (boundary)))
241
245
242 content_type = b('multipart/form-data; boundary=%s' % boundary)
246 content_type = b('multipart/form-data; boundary=%s' % boundary)
243
247
244 return body.getvalue(), content_type
248 return body.getvalue(), content_type
245
249
246
250
247 def post_download(project, filename, name=None, description=""):
251 def post_download(project, filename, name=None, description=""):
248 """Upload a file to the GitHub downloads area"""
252 """Upload a file to the GitHub downloads area"""
249 if name is None:
253 if name is None:
250 name = os.path.basename(filename)
254 name = os.path.basename(filename)
251 with open(filename, 'rb') as f:
255 with open(filename, 'rb') as f:
252 filedata = f.read()
256 filedata = f.read()
253
257
254 url = "https://api.github.com/repos/{project}/downloads".format(project=project)
258 url = "https://api.github.com/repos/{project}/downloads".format(project=project)
255
259
256 payload = json.dumps(dict(name=name, size=len(filedata),
260 payload = json.dumps(dict(name=name, size=len(filedata),
257 description=description))
261 description=description))
258 response = requests.post(url, data=payload, headers=make_auth_header())
262 response = requests.post(url, data=payload, headers=make_auth_header())
259 response.raise_for_status()
263 response.raise_for_status()
260 reply = json.loads(response.content)
264 reply = json.loads(response.content)
261 s3_url = reply['s3_url']
265 s3_url = reply['s3_url']
262
266
263 fields = dict(
267 fields = dict(
264 key=reply['path'],
268 key=reply['path'],
265 acl=reply['acl'],
269 acl=reply['acl'],
266 success_action_status=201,
270 success_action_status=201,
267 Filename=reply['name'],
271 Filename=reply['name'],
268 AWSAccessKeyId=reply['accesskeyid'],
272 AWSAccessKeyId=reply['accesskeyid'],
269 Policy=reply['policy'],
273 Policy=reply['policy'],
270 Signature=reply['signature'],
274 Signature=reply['signature'],
271 file=(reply['name'], filedata),
275 file=(reply['name'], filedata),
272 )
276 )
273 fields['Content-Type'] = reply['mime_type']
277 fields['Content-Type'] = reply['mime_type']
274 data, content_type = encode_multipart_formdata(fields)
278 data, content_type = encode_multipart_formdata(fields)
275 s3r = requests.post(s3_url, data=data, headers={'Content-Type': content_type})
279 s3r = requests.post(s3_url, data=data, headers={'Content-Type': content_type})
276 return s3r
280 return s3r
General Comments 0
You need to be logged in to leave comments. Login now