##// END OF EJS Templates
time test run
MinRK -
Show More
@@ -1,239 +1,243 b''
1 """
1 """
2 This is a script for testing pull requests for IPython. It merges the pull
2 This is a script for testing pull requests for IPython. It merges the pull
3 request with current master, installs and tests on all available versions of
3 request with current master, installs and tests on all available versions of
4 Python, and posts the results to Gist if any tests fail.
4 Python, and posts the results to Gist if any tests fail.
5
5
6 Usage:
6 Usage:
7 python test_pr.py 1657
7 python test_pr.py 1657
8 """
8 """
9 from __future__ import print_function
9 from __future__ import print_function
10
10
11 import errno
11 import errno
12 from glob import glob
12 from glob import glob
13 import io
13 import io
14 import json
14 import json
15 import os
15 import os
16 import pickle
16 import pickle
17 import re
17 import re
18 import requests
18 import requests
19 import shutil
19 import shutil
20 import time
20 from subprocess import call, check_call, check_output, PIPE, STDOUT, CalledProcessError
21 from subprocess import call, check_call, check_output, PIPE, STDOUT, CalledProcessError
21 import sys
22 import sys
22
23
23 import gh_api
24 import gh_api
24
25
25 basedir = os.path.join(os.path.expanduser("~"), ".ipy_pr_tests")
26 basedir = os.path.join(os.path.expanduser("~"), ".ipy_pr_tests")
26 repodir = os.path.join(basedir, "ipython")
27 repodir = os.path.join(basedir, "ipython")
27 ipy_repository = 'git://github.com/ipython/ipython.git'
28 ipy_repository = 'git://github.com/ipython/ipython.git'
28 gh_project="ipython/ipython"
29 gh_project="ipython/ipython"
29
30
30 supported_pythons = ['python2.6', 'python2.7', 'python3.1', 'python3.2']
31 supported_pythons = ['python2.6', 'python2.7', 'python3.1', 'python3.2']
31 unavailable_pythons = []
32 unavailable_pythons = []
32
33
33 def available_python_versions():
34 def available_python_versions():
34 """Get the executable names of available versions of Python on the system.
35 """Get the executable names of available versions of Python on the system.
35 """
36 """
36 del unavailable_pythons[:]
37 del unavailable_pythons[:]
37 for py in supported_pythons:
38 for py in supported_pythons:
38 try:
39 try:
39 check_call([py, '-c', 'import nose'], stdout=PIPE)
40 check_call([py, '-c', 'import nose'], stdout=PIPE)
40 yield py
41 yield py
41 except (OSError, CalledProcessError):
42 except (OSError, CalledProcessError):
42 unavailable_pythons.append(py)
43 unavailable_pythons.append(py)
43
44
44 venvs = []
45 venvs = []
45
46
46 def setup():
47 def setup():
47 """Prepare the repository and virtualenvs."""
48 """Prepare the repository and virtualenvs."""
48 global venvs
49 global venvs
49
50
50 try:
51 try:
51 os.mkdir(basedir)
52 os.mkdir(basedir)
52 except OSError as e:
53 except OSError as e:
53 if e.errno != errno.EEXIST:
54 if e.errno != errno.EEXIST:
54 raise
55 raise
55 os.chdir(basedir)
56 os.chdir(basedir)
56
57
57 # Delete virtualenvs and recreate
58 # Delete virtualenvs and recreate
58 for venv in glob('venv-*'):
59 for venv in glob('venv-*'):
59 shutil.rmtree(venv)
60 shutil.rmtree(venv)
60 for py in available_python_versions():
61 for py in available_python_versions():
61 check_call(['virtualenv', '-p', py, '--system-site-packages', 'venv-%s' % py])
62 check_call(['virtualenv', '-p', py, '--system-site-packages', 'venv-%s' % py])
62 venvs.append((py, 'venv-%s' % py))
63 venvs.append((py, 'venv-%s' % py))
63
64
64 # Check out and update the repository
65 # Check out and update the repository
65 if not os.path.exists('ipython'):
66 if not os.path.exists('ipython'):
66 check_call(['git', 'clone', ipy_repository])
67 check_call(['git', 'clone', ipy_repository])
67 os.chdir(repodir)
68 os.chdir(repodir)
68 check_call(['git', 'checkout', 'master'])
69 check_call(['git', 'checkout', 'master'])
69 check_call(['git', 'pull', ipy_repository, 'master'])
70 check_call(['git', 'pull', ipy_repository, 'master'])
70 os.chdir(basedir)
71 os.chdir(basedir)
71
72
72 missing_libs_re = re.compile(r"Tools and libraries NOT available at test time:\n"
73 missing_libs_re = re.compile(r"Tools and libraries NOT available at test time:\n"
73 r"\s*(.*?)\n")
74 r"\s*(.*?)\n")
74 def get_missing_libraries(log):
75 def get_missing_libraries(log):
75 m = missing_libs_re.search(log)
76 m = missing_libs_re.search(log)
76 if m:
77 if m:
77 return m.group(1)
78 return m.group(1)
78
79
79 def get_branch(repo, branch, owner, mergeable):
80 def get_branch(repo, branch, owner, mergeable):
80 os.chdir(repodir)
81 os.chdir(repodir)
81 if mergeable:
82 if mergeable:
82 merged_branch = "%s-%s" % (owner, branch)
83 merged_branch = "%s-%s" % (owner, branch)
83 # Delete the branch first
84 # Delete the branch first
84 call(['git', 'branch', '-D', merged_branch])
85 call(['git', 'branch', '-D', merged_branch])
85 check_call(['git', 'checkout', '-b', merged_branch])
86 check_call(['git', 'checkout', '-b', merged_branch])
86 check_call(['git', 'pull', '--no-commit', repo, branch])
87 check_call(['git', 'pull', '--no-commit', repo, branch])
87 check_call(['git', 'commit', '-m', "merge %s/%s" % (repo, branch)])
88 check_call(['git', 'commit', '-m', "merge %s/%s" % (repo, branch)])
88 else:
89 else:
89 # Fetch the branch without merging it.
90 # Fetch the branch without merging it.
90 check_call(['git', 'fetch', repo, branch])
91 check_call(['git', 'fetch', repo, branch])
91 check_call(['git', 'checkout', 'FETCH_HEAD'])
92 check_call(['git', 'checkout', 'FETCH_HEAD'])
92 os.chdir(basedir)
93 os.chdir(basedir)
93
94
94 def run_tests(venv):
95 def run_tests(venv):
95 py = os.path.join(basedir, venv, 'bin', 'python')
96 py = os.path.join(basedir, venv, 'bin', 'python')
96 print(py)
97 print(py)
97 os.chdir(repodir)
98 os.chdir(repodir)
98 # cleanup build-dir
99 # cleanup build-dir
99 if os.path.exists('build'):
100 if os.path.exists('build'):
100 shutil.rmtree('build')
101 shutil.rmtree('build')
101 check_call([py, 'setup.py', 'install'])
102 check_call([py, 'setup.py', 'install'])
102 os.chdir(basedir)
103 os.chdir(basedir)
103
104
104 iptest = os.path.join(basedir, venv, 'bin', 'iptest')
105 iptest = os.path.join(basedir, venv, 'bin', 'iptest')
105 if not os.path.exists(iptest):
106 if not os.path.exists(iptest):
106 iptest = os.path.join(basedir, venv, 'bin', 'iptest3')
107 iptest = os.path.join(basedir, venv, 'bin', 'iptest3')
107
108
108 print("\nRunning tests, this typically takes a few minutes...")
109 print("\nRunning tests, this typically takes a few minutes...")
109 try:
110 try:
110 return True, check_output([iptest], stderr=STDOUT).decode('utf-8')
111 return True, check_output([iptest], stderr=STDOUT).decode('utf-8')
111 except CalledProcessError as e:
112 except CalledProcessError as e:
112 return False, e.output.decode('utf-8')
113 return False, e.output.decode('utf-8')
113
114
114 def markdown_format(pr, results_urls):
115 def markdown_format(pr, results_urls):
115 def format_result(py, passed, gist_url, missing_libraries):
116 def format_result(py, passed, gist_url, missing_libraries):
116 s = "* %s: " % py
117 s = "* %s: " % py
117 if passed:
118 if passed:
118 s += "OK"
119 s += "OK"
119 else:
120 else:
120 s += "Failed, log at %s" % gist_url
121 s += "Failed, log at %s" % gist_url
121 if missing_libraries:
122 if missing_libraries:
122 s += " (libraries not available: " + missing_libraries + ")"
123 s += " (libraries not available: " + missing_libraries + ")"
123 return s
124 return s
124
125
125 if pr['mergeable']:
126 if pr['mergeable']:
126 com = pr['head']['sha'][:7] + " merged into master"
127 com = pr['head']['sha'][:7] + " merged into master"
127 else:
128 else:
128 com = pr['head']['sha'][:7] + " (can't merge cleanly)"
129 com = pr['head']['sha'][:7] + " (can't merge cleanly)"
129 lines = ["**Test results for commit %s**" % com,
130 lines = ["**Test results for commit %s**" % com,
130 "Platform: " + sys.platform,
131 "Platform: " + sys.platform,
131 ""] + \
132 ""] + \
132 [format_result(*r) for r in results_urls] + \
133 [format_result(*r) for r in results_urls] + \
133 ["",
134 ["",
134 "Not available for testing: " + ", ".join(unavailable_pythons)]
135 "Not available for testing: " + ", ".join(unavailable_pythons)]
135 return "\n".join(lines)
136 return "\n".join(lines)
136
137
137 def post_results_comment(pr, results, num):
138 def post_results_comment(pr, results, num):
138 body = markdown_format(pr, results)
139 body = markdown_format(pr, results)
139 gh_api.post_issue_comment(gh_project, num, body)
140 gh_api.post_issue_comment(gh_project, num, body)
140
141
141 def print_results(pr, results_urls):
142 def print_results(pr, results_urls):
142 print("\n")
143 print("\n")
143 if pr['mergeable']:
144 if pr['mergeable']:
144 print("**Test results for commit %s merged into master**" % pr['head']['sha'][:7])
145 print("**Test results for commit %s merged into master**" % pr['head']['sha'][:7])
145 else:
146 else:
146 print("**Test results for commit %s (can't merge cleanly)**" % pr['head']['sha'][:7])
147 print("**Test results for commit %s (can't merge cleanly)**" % pr['head']['sha'][:7])
147 print("Platform:", sys.platform)
148 print("Platform:", sys.platform)
148 for py, passed, gist_url, missing_libraries in results_urls:
149 for py, passed, gist_url, missing_libraries in results_urls:
149 if passed:
150 if passed:
150 print(py, ":", "OK")
151 print(py, ":", "OK")
151 else:
152 else:
152 print(py, ":", "Failed")
153 print(py, ":", "Failed")
153 print(" Test log:", gist_url)
154 print(" Test log:", gist_url)
154 if missing_libraries:
155 if missing_libraries:
155 print(" Libraries not available:", missing_libraries)
156 print(" Libraries not available:", missing_libraries)
156 print("Not available for testing:", ", ".join(unavailable_pythons))
157 print("Not available for testing:", ", ".join(unavailable_pythons))
157
158
158 def dump_results(num, results, pr):
159 def dump_results(num, results, pr):
159 with open(os.path.join(basedir, 'lastresults.pkl'), 'wb') as f:
160 with open(os.path.join(basedir, 'lastresults.pkl'), 'wb') as f:
160 pickle.dump((num, results, pr), f)
161 pickle.dump((num, results, pr), f)
161
162
162 def load_results():
163 def load_results():
163 with open(os.path.join(basedir, 'lastresults.pkl'), 'rb') as f:
164 with open(os.path.join(basedir, 'lastresults.pkl'), 'rb') as f:
164 return pickle.load(f)
165 return pickle.load(f)
165
166
166 def save_logs(results, pr):
167 def save_logs(results, pr):
167 results_paths = []
168 results_paths = []
168 for py, passed, log, missing_libraries in results:
169 for py, passed, log, missing_libraries in results:
169 if passed:
170 if passed:
170 results_paths.append((py, passed, None, missing_libraries))
171 results_paths.append((py, passed, None, missing_libraries))
171 else:
172 else:
172
173
173 result_locn = os.path.abspath(os.path.join('venv-%s' % py,
174 result_locn = os.path.abspath(os.path.join('venv-%s' % py,
174 pr['head']['sha'][:7]+".log"))
175 pr['head']['sha'][:7]+".log"))
175 with io.open(result_locn, 'w', encoding='utf-8') as f:
176 with io.open(result_locn, 'w', encoding='utf-8') as f:
176 f.write(log)
177 f.write(log)
177
178
178 results_paths.append((py, False, result_locn, missing_libraries))
179 results_paths.append((py, False, result_locn, missing_libraries))
179
180
180 return results_paths
181 return results_paths
181
182
182 def post_logs(results):
183 def post_logs(results):
183 results_urls = []
184 results_urls = []
184 for py, passed, log, missing_libraries in results:
185 for py, passed, log, missing_libraries in results:
185 if passed:
186 if passed:
186 results_urls.append((py, passed, None, missing_libraries))
187 results_urls.append((py, passed, None, missing_libraries))
187 else:
188 else:
188 result_locn = gh_api.post_gist(log, description='IPython test log',
189 result_locn = gh_api.post_gist(log, description='IPython test log',
189 filename="results.log", auth=True)
190 filename="results.log", auth=True)
190 results_urls.append((py, False, result_locn, missing_libraries))
191 results_urls.append((py, False, result_locn, missing_libraries))
191
192
192 return results_urls
193 return results_urls
193
194
194 def test_pr(num, post_results=True):
195 def test_pr(num, post_results=True):
195 # Get Github authorisation first, so that the user is prompted straight away
196 # Get Github authorisation first, so that the user is prompted straight away
196 # if their login is needed.
197 # if their login is needed.
197 if post_results:
198 if post_results:
198 gh_api.get_auth_token()
199 gh_api.get_auth_token()
199
200
200 setup()
201 setup()
201 pr = gh_api.get_pull_request(gh_project, num)
202 pr = gh_api.get_pull_request(gh_project, num)
202 get_branch(repo=pr['head']['repo']['clone_url'],
203 get_branch(repo=pr['head']['repo']['clone_url'],
203 branch=pr['head']['ref'],
204 branch=pr['head']['ref'],
204 owner=pr['head']['repo']['owner']['login'],
205 owner=pr['head']['repo']['owner']['login'],
205 mergeable=pr['mergeable'],
206 mergeable=pr['mergeable'],
206 )
207 )
207
208
208 results = []
209 results = []
209 for py, venv in venvs:
210 for py, venv in venvs:
211 tic = time.time()
210 passed, log = run_tests(venv)
212 passed, log = run_tests(venv)
213 elapsed = int(time.time() - tic)
214 print("Ran tests with %s in %is" % (py, elapsed))
211 missing_libraries = get_missing_libraries(log)
215 missing_libraries = get_missing_libraries(log)
212 if passed:
216 if passed:
213 results.append((py, True, None, missing_libraries))
217 results.append((py, True, None, missing_libraries))
214 else:
218 else:
215 results.append((py, False, log, missing_libraries))
219 results.append((py, False, log, missing_libraries))
216
220
217 dump_results(num, results, pr)
221 dump_results(num, results, pr)
218
222
219 results_paths = save_logs(results, pr)
223 results_paths = save_logs(results, pr)
220 print_results(pr, results_paths)
224 print_results(pr, results_paths)
221
225
222 if post_results:
226 if post_results:
223 results_urls = post_logs(results)
227 results_urls = post_logs(results)
224 post_results_comment(pr, results_urls, num)
228 post_results_comment(pr, results_urls, num)
225 print("(Posted to Github)")
229 print("(Posted to Github)")
226 else:
230 else:
227 post_script = os.path.join(os.path.dirname(sys.argv[0]), "post_pr_test.py")
231 post_script = os.path.join(os.path.dirname(sys.argv[0]), "post_pr_test.py")
228 print("To post the results to Github, run", post_script)
232 print("To post the results to Github, run", post_script)
229
233
230
234
231 if __name__ == '__main__':
235 if __name__ == '__main__':
232 import argparse
236 import argparse
233 parser = argparse.ArgumentParser(description="Test an IPython pull request")
237 parser = argparse.ArgumentParser(description="Test an IPython pull request")
234 parser.add_argument('-p', '--publish', action='store_true',
238 parser.add_argument('-p', '--publish', action='store_true',
235 help="Publish the results to Github")
239 help="Publish the results to Github")
236 parser.add_argument('number', type=int, help="The pull request number")
240 parser.add_argument('number', type=int, help="The pull request number")
237
241
238 args = parser.parse_args()
242 args = parser.parse_args()
239 test_pr(args.number, post_results=args.publish)
243 test_pr(args.number, post_results=args.publish)
General Comments 0
You need to be logged in to leave comments. Login now