##// END OF EJS Templates
Changes from linting github tools
Thomas Kluyver -
Show More
@@ -1,201 +1,201 b''
1 1 """Functions for Github authorisation."""
2 2 from __future__ import print_function
3 3
4 4 try:
5 5 input = raw_input
6 6 except NameError:
7 7 pass
8 8
9 9 import os
10 10
11 11 import requests
12 12 import getpass
13 13 import json
14 14
15 15 # Keyring stores passwords by a 'username', but we're not storing a username and
16 16 # password
17 17 fake_username = 'ipython_tools'
18 18
19 19 class Obj(dict):
20 20 """Dictionary with attribute access to names."""
21 21 def __getattr__(self, name):
22 22 try:
23 23 return self[name]
24 24 except KeyError:
25 25 raise AttributeError(name)
26 26
27 27 def __setattr__(self, name, val):
28 28 self[name] = val
29 29
30 30 token = None
31 31 def get_auth_token():
32 32 global token
33 33
34 34 if token is not None:
35 35 return token
36 36
37 37 import keyring
38 38 token = keyring.get_password('github', fake_username)
39 39 if token is not None:
40 40 return token
41 41
42 42 print("Please enter your github username and password. These are not "
43 43 "stored, only used to get an oAuth token. You can revoke this at "
44 44 "any time on Github.")
45 45 user = input("Username: ")
46 46 pw = getpass.getpass("Password: ")
47 47
48 48 auth_request = {
49 49 "scopes": [
50 50 "public_repo",
51 51 "gist"
52 52 ],
53 53 "note": "IPython tools",
54 54 "note_url": "https://github.com/ipython/ipython/tree/master/tools",
55 55 }
56 56 response = requests.post('https://api.github.com/authorizations',
57 57 auth=(user, pw), data=json.dumps(auth_request))
58 58 response.raise_for_status()
59 59 token = json.loads(response.text)['token']
60 60 keyring.set_password('github', fake_username, token)
61 61 return token
62 62
63 63 def make_auth_header():
64 64 return {'Authorization': 'token ' + get_auth_token()}
65 65
66 66 def post_issue_comment(project, num, body):
67 67 url = 'https://api.github.com/repos/{project}/issues/{num}/comments'.format(project=project, num=num)
68 68 payload = json.dumps({'body': body})
69 r = requests.post(url, data=payload, headers=make_auth_header())
69 requests.post(url, data=payload, headers=make_auth_header())
70 70
71 71 def post_gist(content, description='', filename='file', auth=False):
72 72 """Post some text to a Gist, and return the URL."""
73 73 post_data = json.dumps({
74 74 "description": description,
75 75 "public": True,
76 76 "files": {
77 77 filename: {
78 78 "content": content
79 79 }
80 80 }
81 81 }).encode('utf-8')
82 82
83 83 headers = make_auth_header() if auth else {}
84 84 response = requests.post("https://api.github.com/gists", data=post_data, headers=headers)
85 85 response.raise_for_status()
86 86 response_data = json.loads(response.text)
87 87 return response_data['html_url']
88 88
89 89 def get_pull_request(project, num):
90 90 """get pull request info by number
91 91 """
92 92 url = "https://api.github.com/repos/{project}/pulls/{num}".format(project=project, num=num)
93 93 response = requests.get(url)
94 94 response.raise_for_status()
95 95 return json.loads(response.text, object_hook=Obj)
96 96
97 97 def get_pulls_list(project):
98 98 """get pull request list
99 99 """
100 100 url = "https://api.github.com/repos/{project}/pulls".format(project=project)
101 101 response = requests.get(url)
102 102 response.raise_for_status()
103 103 return json.loads(response.text)
104 104
105 105 # encode_multipart_formdata is from urllib3.filepost
106 106 # The only change is to iter_fields, to enforce S3's required key ordering
107 107
108 108 def iter_fields(fields):
109 109 fields = fields.copy()
110 110 for key in ('key', 'acl', 'Filename', 'success_action_status', 'AWSAccessKeyId',
111 111 'Policy', 'Signature', 'Content-Type', 'file'):
112 112 yield (key, fields.pop(key))
113 113 for (k,v) in fields.items():
114 114 yield k,v
115 115
116 116 def encode_multipart_formdata(fields, boundary=None):
117 117 """
118 118 Encode a dictionary of ``fields`` using the multipart/form-data mime format.
119 119
120 120 :param fields:
121 121 Dictionary of fields or list of (key, value) field tuples. The key is
122 122 treated as the field name, and the value as the body of the form-data
123 123 bytes. If the value is a tuple of two elements, then the first element
124 124 is treated as the filename of the form-data section.
125 125
126 126 Field names and filenames must be unicode.
127 127
128 128 :param boundary:
129 129 If not specified, then a random boundary will be generated using
130 130 :func:`mimetools.choose_boundary`.
131 131 """
132 132 # copy requests imports in here:
133 133 from io import BytesIO
134 134 from requests.packages.urllib3.filepost import (
135 135 choose_boundary, six, writer, b, get_content_type
136 136 )
137 137 body = BytesIO()
138 138 if boundary is None:
139 139 boundary = choose_boundary()
140 140
141 141 for fieldname, value in iter_fields(fields):
142 142 body.write(b('--%s\r\n' % (boundary)))
143 143
144 144 if isinstance(value, tuple):
145 145 filename, data = value
146 146 writer(body).write('Content-Disposition: form-data; name="%s"; '
147 147 'filename="%s"\r\n' % (fieldname, filename))
148 148 body.write(b('Content-Type: %s\r\n\r\n' %
149 149 (get_content_type(filename))))
150 150 else:
151 151 data = value
152 152 writer(body).write('Content-Disposition: form-data; name="%s"\r\n'
153 153 % (fieldname))
154 154 body.write(b'Content-Type: text/plain\r\n\r\n')
155 155
156 156 if isinstance(data, int):
157 157 data = str(data) # Backwards compatibility
158 158 if isinstance(data, six.text_type):
159 159 writer(body).write(data)
160 160 else:
161 161 body.write(data)
162 162
163 163 body.write(b'\r\n')
164 164
165 165 body.write(b('--%s--\r\n' % (boundary)))
166 166
167 167 content_type = b('multipart/form-data; boundary=%s' % boundary)
168 168
169 169 return body.getvalue(), content_type
170 170
171 171
172 172 def post_download(project, filename, name=None, description=""):
173 173 """Upload a file to the GitHub downloads area"""
174 174 if name is None:
175 175 name = os.path.basename(filename)
176 176 with open(filename, 'rb') as f:
177 177 filedata = f.read()
178 178
179 179 url = "https://api.github.com/repos/{project}/downloads".format(project=project)
180 180
181 181 payload = json.dumps(dict(name=name, size=len(filedata),
182 182 description=description))
183 183 response = requests.post(url, data=payload, headers=make_auth_header())
184 184 response.raise_for_status()
185 185 reply = json.loads(response.content)
186 186 s3_url = reply['s3_url']
187 187
188 188 fields = dict(
189 189 key=reply['path'],
190 190 acl=reply['acl'],
191 191 success_action_status=201,
192 192 Filename=reply['name'],
193 193 AWSAccessKeyId=reply['accesskeyid'],
194 194 Policy=reply['policy'],
195 195 Signature=reply['signature'],
196 196 file=(reply['name'], filedata),
197 197 )
198 198 fields['Content-Type'] = reply['mime_type']
199 199 data, content_type = encode_multipart_formdata(fields)
200 200 s3r = requests.post(s3_url, data=data, headers={'Content-Type': content_type})
201 201 return s3r
@@ -1,291 +1,289 b''
1 1 #!/usr/bin/env python
2 2 """
3 3 This is a script for testing pull requests for IPython. It merges the pull
4 4 request with current master, installs and tests on all available versions of
5 5 Python, and posts the results to Gist if any tests fail.
6 6
7 7 Usage:
8 8 python test_pr.py 1657
9 9 """
10 10 from __future__ import print_function
11 11
12 12 import errno
13 13 from glob import glob
14 14 import io
15 import json
16 15 import os
17 16 import pickle
18 17 import re
19 import requests
20 18 import shutil
21 19 import time
22 20 from subprocess import call, check_call, check_output, PIPE, STDOUT, CalledProcessError
23 21 import sys
24 22
25 23 import gh_api
26 24 from gh_api import Obj
27 25
28 26 basedir = os.path.join(os.path.expanduser("~"), ".ipy_pr_tests")
29 27 repodir = os.path.join(basedir, "ipython")
30 28 ipy_repository = 'git://github.com/ipython/ipython.git'
31 29 ipy_http_repository = 'http://github.com/ipython/ipython.git'
32 30 gh_project="ipython/ipython"
33 31
34 32 supported_pythons = ['python2.6', 'python2.7', 'python3.2']
35 33
36 34 missing_libs_re = re.compile(r"Tools and libraries NOT available at test time:\n"
37 35 r"\s*(.*?)\n")
38 36 def get_missing_libraries(log):
39 37 m = missing_libs_re.search(log)
40 38 if m:
41 39 return m.group(1)
42 40
43 41 class TestRun(object):
44 42 def __init__(self, pr_num, extra_args):
45 43 self.unavailable_pythons = []
46 44 self.venvs = []
47 45 self.pr_num = pr_num
48 46 self.extra_args = extra_args
49 47
50 48 self.pr = gh_api.get_pull_request(gh_project, pr_num)
51 49
52 50 self.setup()
53 51
54 52 self.results = []
55 53
56 54 def available_python_versions(self):
57 55 """Get the executable names of available versions of Python on the system.
58 56 """
59 57 for py in supported_pythons:
60 58 try:
61 59 check_call([py, '-c', 'import nose'], stdout=PIPE)
62 60 yield py
63 61 except (OSError, CalledProcessError):
64 62 self.unavailable_pythons.append(py)
65 63
66 64 def setup(self):
67 65 """Prepare the repository and virtualenvs."""
68 66 try:
69 67 os.mkdir(basedir)
70 68 except OSError as e:
71 69 if e.errno != errno.EEXIST:
72 70 raise
73 71 os.chdir(basedir)
74 72
75 73 # Delete virtualenvs and recreate
76 74 for venv in glob('venv-*'):
77 75 shutil.rmtree(venv)
78 76 for py in self.available_python_versions():
79 77 check_call(['virtualenv', '-p', py, '--system-site-packages', 'venv-%s' % py])
80 78 self.venvs.append((py, 'venv-%s' % py))
81 79
82 80 # Check out and update the repository
83 81 if not os.path.exists('ipython'):
84 82 try :
85 83 check_call(['git', 'clone', ipy_repository])
86 84 except CalledProcessError :
87 85 check_call(['git', 'clone', ipy_http_repository])
88 86 os.chdir(repodir)
89 87 check_call(['git', 'checkout', 'master'])
90 88 try :
91 89 check_call(['git', 'pull', 'origin', 'master'])
92 90 except CalledProcessError :
93 91 check_call(['git', 'pull', ipy_http_repository, 'master'])
94 92 self.master_sha = check_output(['git', 'log', '-1', '--format=%h']).decode('ascii').strip()
95 93 os.chdir(basedir)
96 94
97 95 def get_branch(self):
98 96 repo = self.pr['head']['repo']['clone_url']
99 97 branch = self.pr['head']['ref']
100 98 owner = self.pr['head']['repo']['owner']['login']
101 99 mergeable = self.pr['mergeable']
102 100
103 101 os.chdir(repodir)
104 102 if mergeable:
105 103 merged_branch = "%s-%s" % (owner, branch)
106 104 # Delete the branch first
107 105 call(['git', 'branch', '-D', merged_branch])
108 106 check_call(['git', 'checkout', '-b', merged_branch])
109 107 check_call(['git', 'pull', '--no-ff', '--no-commit', repo, branch])
110 108 check_call(['git', 'commit', '-m', "merge %s/%s" % (repo, branch)])
111 109 else:
112 110 # Fetch the branch without merging it.
113 111 check_call(['git', 'fetch', repo, branch])
114 112 check_call(['git', 'checkout', 'FETCH_HEAD'])
115 113 os.chdir(basedir)
116 114
117 115 def markdown_format(self):
118 116 def format_result(result):
119 117 s = "* %s: " % result.py
120 118 if result.passed:
121 119 s += "OK"
122 120 else:
123 121 s += "Failed, log at %s" % result.log_url
124 122 if result.missing_libraries:
125 123 s += " (libraries not available: " + result.missing_libraries + ")"
126 124 return s
127 125
128 126 if self.pr['mergeable']:
129 127 com = self.pr['head']['sha'][:7] + " merged into master (%s)" % self.master_sha
130 128 else:
131 129 com = self.pr['head']['sha'][:7] + " (can't merge cleanly)"
132 130 lines = ["**Test results for commit %s**" % com,
133 131 "Platform: " + sys.platform,
134 132 ""] + \
135 133 [format_result(r) for r in self.results] + \
136 134 [""]
137 135 if self.extra_args:
138 136 lines.append("Extra args: %r" % self.extra_args),
139 137 lines.append("Not available for testing: " + ", ".join(self.unavailable_pythons))
140 138 return "\n".join(lines)
141 139
142 140 def post_results_comment(self):
143 141 body = self.markdown_format()
144 142 gh_api.post_issue_comment(gh_project, self.pr_num, body)
145 143
146 144 def print_results(self):
147 145 pr = self.pr
148 146
149 147 print("\n")
150 148 msg = "**Test results for commit %s" % pr['head']['sha'][:7]
151 149 if pr['mergeable']:
152 150 msg += " merged into master (%s)**" % self.master_sha
153 151 else:
154 152 msg += " (can't merge cleanly)**"
155 153 print(msg)
156 154 print("Platform:", sys.platform)
157 155 for result in self.results:
158 156 if result.passed:
159 157 print(result.py, ":", "OK")
160 158 else:
161 159 print(result.py, ":", "Failed")
162 160 print(" Test log:", result.get('log_url') or result.log_file)
163 161 if result.missing_libraries:
164 162 print(" Libraries not available:", result.missing_libraries)
165 163
166 164 if self.extra_args:
167 165 print("Extra args:", self.extra_args)
168 166 print("Not available for testing:", ", ".join(self.unavailable_pythons))
169 167
170 168 def dump_results(self):
171 169 with open(os.path.join(basedir, 'lastresults.pkl'), 'wb') as f:
172 170 pickle.dump(self, f)
173 171
174 172 @staticmethod
175 173 def load_results():
176 174 with open(os.path.join(basedir, 'lastresults.pkl'), 'rb') as f:
177 175 return pickle.load(f)
178 176
179 177 def save_logs(self):
180 178 for result in self.results:
181 179 if not result.passed:
182 180 result_locn = os.path.abspath(os.path.join('venv-%s' % result.py,
183 181 self.pr['head']['sha'][:7]+".log"))
184 182 with io.open(result_locn, 'w', encoding='utf-8') as f:
185 183 f.write(result.log)
186 184
187 185 result.log_file = result_locn
188 186
189 187 def post_logs(self):
190 188 for result in self.results:
191 189 if not result.passed:
192 190 result.log_url = gh_api.post_gist(result.log,
193 191 description='IPython test log',
194 192 filename="results.log", auth=True)
195 193
196 194 def run(self):
197 195 for py, venv in self.venvs:
198 196 tic = time.time()
199 197 passed, log = run_tests(venv, self.extra_args)
200 198 elapsed = int(time.time() - tic)
201 199 print("Ran tests with %s in %is" % (py, elapsed))
202 200 missing_libraries = get_missing_libraries(log)
203 201
204 202 self.results.append(Obj(py=py,
205 203 passed=passed,
206 204 log=log,
207 205 missing_libraries=missing_libraries
208 206 )
209 207 )
210 208
211 209
212 210 def run_tests(venv, extra_args):
213 211 py = os.path.join(basedir, venv, 'bin', 'python')
214 212 print(py)
215 213 os.chdir(repodir)
216 214 # cleanup build-dir
217 215 if os.path.exists('build'):
218 216 shutil.rmtree('build')
219 217 tic = time.time()
220 218 print ("\nInstalling IPython with %s" % py)
221 219 logfile = os.path.join(basedir, venv, 'install.log')
222 220 print ("Install log at %s" % logfile)
223 221 with open(logfile, 'wb') as f:
224 222 check_call([py, 'setup.py', 'install'], stdout=f)
225 223 toc = time.time()
226 224 print ("Installed IPython in %.1fs" % (toc-tic))
227 225 os.chdir(basedir)
228 226
229 227 # Environment variables:
230 228 orig_path = os.environ["PATH"]
231 229 os.environ["PATH"] = os.path.join(basedir, venv, 'bin') + ':' + os.environ["PATH"]
232 230 os.environ.pop("PYTHONPATH", None)
233 231
234 232 # check that the right IPython is imported
235 233 ipython_file = check_output([py, '-c', 'import IPython; print (IPython.__file__)'])
236 234 ipython_file = ipython_file.strip().decode('utf-8')
237 235 if not ipython_file.startswith(os.path.join(basedir, venv)):
238 236 msg = "IPython does not appear to be in the venv: %s" % ipython_file
239 237 msg += "\nDo you use setupegg.py develop?"
240 238 print(msg, file=sys.stderr)
241 239 return False, msg
242 240
243 241 iptest = os.path.join(basedir, venv, 'bin', 'iptest')
244 242 if not os.path.exists(iptest):
245 243 iptest = os.path.join(basedir, venv, 'bin', 'iptest3')
246 244
247 245 print("\nRunning tests, this typically takes a few minutes...")
248 246 try:
249 247 return True, check_output([iptest] + extra_args, stderr=STDOUT).decode('utf-8')
250 248 except CalledProcessError as e:
251 249 return False, e.output.decode('utf-8')
252 250 finally:
253 251 # Restore $PATH
254 252 os.environ["PATH"] = orig_path
255 253
256 254
257 255 def test_pr(num, post_results=True, extra_args=None):
258 256 # Get Github authorisation first, so that the user is prompted straight away
259 257 # if their login is needed.
260 258 if post_results:
261 259 gh_api.get_auth_token()
262 260
263 261 testrun = TestRun(num, extra_args or [])
264 262
265 263 testrun.get_branch()
266 264
267 265 testrun.run()
268 266
269 267 testrun.dump_results()
270 268
271 269 testrun.save_logs()
272 270 testrun.print_results()
273 271
274 272 if post_results:
275 273 testrun.post_logs()
276 274 testrun.post_results_comment()
277 275 print("(Posted to Github)")
278 276 else:
279 277 post_script = os.path.join(os.path.dirname(sys.argv[0]), "post_pr_test.py")
280 278 print("To post the results to Github, run", post_script)
281 279
282 280
283 281 if __name__ == '__main__':
284 282 import argparse
285 283 parser = argparse.ArgumentParser(description="Test an IPython pull request")
286 284 parser.add_argument('-p', '--publish', action='store_true',
287 285 help="Publish the results to Github")
288 286 parser.add_argument('number', type=int, help="The pull request number")
289 287
290 288 args, extra_args = parser.parse_known_args()
291 289 test_pr(args.number, post_results=args.publish, extra_args=extra_args)
General Comments 0
You need to be logged in to leave comments. Login now