##// END OF EJS Templates
Pickle PR test logs so they can be posted without rerunning tests.
Thomas Kluyver -
Show More
@@ -0,0 +1,10 b''
1 from test_pr import load_results, post_logs, post_results_comment, print_results
2
3 num, results, pr = load_results()
4 results_urls = post_logs(results)
5 print_results(pr, results_urls)
6 post_results_comment(pr, results_urls, num)
7
8 print()
9 print("Posted test results to pull request")
10 print(" " + pr['html_url'])
@@ -1,198 +1,235 b''
1 """
1 """
2 This is a script for testing pull requests for IPython. It merges the pull
2 This is a script for testing pull requests for IPython. It merges the pull
3 request with current master, installs and tests on all available versions of
3 request with current master, installs and tests on all available versions of
4 Python, and posts the results to Gist if any tests fail.
4 Python, and posts the results to Gist if any tests fail.
5
5
6 Usage:
6 Usage:
7 python test_pr.py 1657
7 python test_pr.py 1657
8 """
8 """
9 from __future__ import print_function
9 from __future__ import print_function
10
10
11 import errno
11 import errno
12 from glob import glob
12 from glob import glob
13 import io
13 import io
14 import json
14 import json
15 import os
15 import os
16 import pickle
16 import re
17 import re
17 import requests
18 import requests
18 import shutil
19 import shutil
19 from subprocess import call, check_call, check_output, PIPE, STDOUT, CalledProcessError
20 from subprocess import call, check_call, check_output, PIPE, STDOUT, CalledProcessError
20 import sys
21 import sys
21
22
22 import gh_api
23 import gh_api
23
24
24 basedir = os.path.join(os.path.expanduser("~"), ".ipy_pr_tests")
25 basedir = os.path.join(os.path.expanduser("~"), ".ipy_pr_tests")
25 repodir = os.path.join(basedir, "ipython")
26 repodir = os.path.join(basedir, "ipython")
26 ipy_repository = 'git://github.com/ipython/ipython.git'
27 ipy_repository = 'git://github.com/ipython/ipython.git'
27 gh_project="ipython/ipython"
28 gh_project="ipython/ipython"
28
29
29 supported_pythons = ['python2.6', 'python2.7', 'python3.1', 'python3.2']
30 supported_pythons = ['python2.6', 'python2.7', 'python3.1', 'python3.2']
30 unavailable_pythons = []
31 unavailable_pythons = []
31
32
32 def available_python_versions():
33 def available_python_versions():
33 """Get the executable names of available versions of Python on the system.
34 """Get the executable names of available versions of Python on the system.
34 """
35 """
35 del unavailable_pythons[:]
36 del unavailable_pythons[:]
36 for py in supported_pythons:
37 for py in supported_pythons:
37 try:
38 try:
38 check_call([py, '-c', 'import nose'], stdout=PIPE)
39 check_call([py, '-c', 'import nose'], stdout=PIPE)
39 yield py
40 yield py
40 except (OSError, CalledProcessError):
41 except (OSError, CalledProcessError):
41 unavailable_pythons.append(py)
42 unavailable_pythons.append(py)
42
43
43 venvs = []
44 venvs = []
44
45
45 def setup():
46 def setup():
46 """Prepare the repository and virtualenvs."""
47 """Prepare the repository and virtualenvs."""
47 global venvs
48 global venvs
48
49
49 try:
50 try:
50 os.mkdir(basedir)
51 os.mkdir(basedir)
51 except OSError as e:
52 except OSError as e:
52 if e.errno != errno.EEXIST:
53 if e.errno != errno.EEXIST:
53 raise
54 raise
54 os.chdir(basedir)
55 os.chdir(basedir)
55
56
56 # Delete virtualenvs and recreate
57 # Delete virtualenvs and recreate
57 for venv in glob('venv-*'):
58 for venv in glob('venv-*'):
58 shutil.rmtree(venv)
59 shutil.rmtree(venv)
59 for py in available_python_versions():
60 for py in available_python_versions():
60 check_call(['virtualenv', '-p', py, '--system-site-packages', 'venv-%s' % py])
61 check_call(['virtualenv', '-p', py, '--system-site-packages', 'venv-%s' % py])
61 venvs.append((py, 'venv-%s' % py))
62 venvs.append((py, 'venv-%s' % py))
62
63
63 # Check out and update the repository
64 # Check out and update the repository
64 if not os.path.exists('ipython'):
65 if not os.path.exists('ipython'):
65 check_call(['git', 'clone', ipy_repository])
66 check_call(['git', 'clone', ipy_repository])
66 os.chdir(repodir)
67 os.chdir(repodir)
67 check_call(['git', 'checkout', 'master'])
68 check_call(['git', 'checkout', 'master'])
68 check_call(['git', 'pull', ipy_repository, 'master'])
69 check_call(['git', 'pull', ipy_repository, 'master'])
69 os.chdir(basedir)
70 os.chdir(basedir)
70
71
71 missing_libs_re = re.compile(r"Tools and libraries NOT available at test time:\n"
72 missing_libs_re = re.compile(r"Tools and libraries NOT available at test time:\n"
72 r"\s*(.*?)\n")
73 r"\s*(.*?)\n")
73 def get_missing_libraries(log):
74 def get_missing_libraries(log):
74 m = missing_libs_re.search(log)
75 m = missing_libs_re.search(log)
75 if m:
76 if m:
76 return m.group(1)
77 return m.group(1)
77
78
78 def get_branch(repo, branch, owner, mergeable):
79 def get_branch(repo, branch, owner, mergeable):
79 os.chdir(repodir)
80 os.chdir(repodir)
80 if mergeable:
81 if mergeable:
81 merged_branch = "%s-%s" % (owner, branch)
82 merged_branch = "%s-%s" % (owner, branch)
82 # Delete the branch first
83 # Delete the branch first
83 call(['git', 'branch', '-D', merged_branch])
84 call(['git', 'branch', '-D', merged_branch])
84 check_call(['git', 'checkout', '-b', merged_branch])
85 check_call(['git', 'checkout', '-b', merged_branch])
85 check_call(['git', 'pull', repo, branch])
86 check_call(['git', 'pull', repo, branch])
86 else:
87 else:
87 # Fetch the branch without merging it.
88 # Fetch the branch without merging it.
88 check_call(['git', 'fetch', repo, branch])
89 check_call(['git', 'fetch', repo, branch])
89 check_call(['git', 'checkout', 'FETCH_HEAD'])
90 check_call(['git', 'checkout', 'FETCH_HEAD'])
90 os.chdir(basedir)
91 os.chdir(basedir)
91
92
92 def run_tests(venv):
93 def run_tests(venv):
93 py = os.path.join(basedir, venv, 'bin', 'python')
94 py = os.path.join(basedir, venv, 'bin', 'python')
94 print(py)
95 print(py)
95 os.chdir(repodir)
96 os.chdir(repodir)
96 check_call([py, 'setup.py', 'install'])
97 check_call([py, 'setup.py', 'install'])
97 os.chdir(basedir)
98 os.chdir(basedir)
98
99
99 iptest = os.path.join(basedir, venv, 'bin', 'iptest')
100 iptest = os.path.join(basedir, venv, 'bin', 'iptest')
100 if not os.path.exists(iptest):
101 if not os.path.exists(iptest):
101 iptest = os.path.join(basedir, venv, 'bin', 'iptest3')
102 iptest = os.path.join(basedir, venv, 'bin', 'iptest3')
102
103
103 print("\nRunning tests, this typically takes a few minutes...")
104 print("\nRunning tests, this typically takes a few minutes...")
104 try:
105 try:
105 return True, check_output([iptest], stderr=STDOUT).decode('utf-8')
106 return True, check_output([iptest], stderr=STDOUT).decode('utf-8')
106 except CalledProcessError as e:
107 except CalledProcessError as e:
107 return False, e.output.decode('utf-8')
108 return False, e.output.decode('utf-8')
108
109
109 def markdown_format(pr, results):
110 def markdown_format(pr, results_urls):
110 def format_result(py, passed, gist_url, missing_libraries):
111 def format_result(py, passed, gist_url, missing_libraries):
111 s = "* %s: " % py
112 s = "* %s: " % py
112 if passed:
113 if passed:
113 s += "OK"
114 s += "OK"
114 else:
115 else:
115 s += "Failed, log at %s" % gist_url
116 s += "Failed, log at %s" % gist_url
116 if missing_libraries:
117 if missing_libraries:
117 s += " (libraries not available: " + missing_libraries + ")"
118 s += " (libraries not available: " + missing_libraries + ")"
118 return s
119 return s
119
120
120 if pr['mergeable']:
121 if pr['mergeable']:
121 com = pr['head']['sha'][:7] + " merged into master"
122 com = pr['head']['sha'][:7] + " merged into master"
122 else:
123 else:
123 com = pr['head']['sha'][:7] + " (can't merge cleanly)"
124 com = pr['head']['sha'][:7] + " (can't merge cleanly)"
124 lines = ["**Test results for commit %s**" % com,
125 lines = ["**Test results for commit %s**" % com,
125 "Platform: " + sys.platform,
126 "Platform: " + sys.platform,
126 ""] + \
127 ""] + \
127 [format_result(*r) for r in results] + \
128 [format_result(*r) for r in results_urls] + \
128 ["",
129 ["",
129 "Not available for testing: " + ", ".join(unavailable_pythons)]
130 "Not available for testing: " + ", ".join(unavailable_pythons)]
130 return "\n".join(lines)
131 return "\n".join(lines)
131
132
132 def post_results_comment(pr, results, num):
133 def post_results_comment(pr, results, num):
133 body = markdown_format(pr, results)
134 body = markdown_format(pr, results)
134 gh_api.post_issue_comment(gh_project, num, body)
135 gh_api.post_issue_comment(gh_project, num, body)
135
136
136 def print_results(pr, results):
137 def print_results(pr, results_urls):
137 print("\n")
138 print("\n")
138 if pr['mergeable']:
139 if pr['mergeable']:
139 print("**Test results for commit %s merged into master**" % pr['head']['sha'][:7])
140 print("**Test results for commit %s merged into master**" % pr['head']['sha'][:7])
140 else:
141 else:
141 print("**Test results for commit %s (can't merge cleanly)**" % pr['head']['sha'][:7])
142 print("**Test results for commit %s (can't merge cleanly)**" % pr['head']['sha'][:7])
142 print("Platform:", sys.platform)
143 print("Platform:", sys.platform)
143 for py, passed, gist_url, missing_libraries in results:
144 for py, passed, gist_url, missing_libraries in results_urls:
144 if passed:
145 if passed:
145 print(py, ":", "OK")
146 print(py, ":", "OK")
146 else:
147 else:
147 print(py, ":", "Failed")
148 print(py, ":", "Failed")
148 print(" Test log:", gist_url)
149 print(" Test log:", gist_url)
149 if missing_libraries:
150 if missing_libraries:
150 print(" Libraries not available:", missing_libraries)
151 print(" Libraries not available:", missing_libraries)
151 print("Not available for testing:", ", ".join(unavailable_pythons))
152 print("Not available for testing:", ", ".join(unavailable_pythons))
152
153
154 def dump_results(num, results, pr):
155 with open(os.path.join(basedir, 'lastresults.pkl'), 'wb') as f:
156 pickle.dump((num, results, pr), f)
157
158 def load_results():
159 with open(os.path.join(basedir, 'lastresults.pkl'), 'rb') as f:
160 return pickle.load(f)
161
162 def save_logs(results, pr):
163 results_paths = []
164 for py, passed, log, missing_libraries in results:
165 if passed:
166 results_paths.append((py, passed, None, missing_libraries))
167 else:
168
169 result_locn = os.path.abspath(os.path.join('venv-%s' % py,
170 pr['head']['sha'][:7]+".log"))
171 with io.open(result_locn, 'w', encoding='utf-8') as f:
172 f.write(log)
173
174 results_paths.append((py, False, result_locn, missing_libraries))
175
176 return results_paths
177
178 def post_logs(results):
179 results_urls = []
180 for py, passed, log, missing_libraries in results:
181 if passed:
182 results_urls.append((py, passed, None, missing_libraries))
183 else:
184 result_locn = gh_api.post_gist(log, description='IPython test log',
185 filename="results.log", auth=True)
186 results_urls.append((py, False, result_locn, missing_libraries))
187
188 return results_urls
189
153 def test_pr(num, post_results=True):
190 def test_pr(num, post_results=True):
154 # Get Github authorisation first, so that the user is prompted straight away
191 # Get Github authorisation first, so that the user is prompted straight away
155 # if their login is needed.
192 # if their login is needed.
156 if post_results:
193 if post_results:
157 gh_api.get_auth_token()
194 gh_api.get_auth_token()
158
195
159 setup()
196 setup()
160 pr = gh_api.get_pull_request(gh_project, num)
197 pr = gh_api.get_pull_request(gh_project, num)
161 get_branch(repo=pr['head']['repo']['clone_url'],
198 get_branch(repo=pr['head']['repo']['clone_url'],
162 branch=pr['head']['ref'],
199 branch=pr['head']['ref'],
163 owner=pr['head']['repo']['owner']['login'],
200 owner=pr['head']['repo']['owner']['login'],
164 mergeable=pr['mergeable'],
201 mergeable=pr['mergeable'],
165 )
202 )
166
203
167 results = []
204 results = []
168 for py, venv in venvs:
205 for py, venv in venvs:
169 passed, log = run_tests(venv)
206 passed, log = run_tests(venv)
170 missing_libraries = get_missing_libraries(log)
207 missing_libraries = get_missing_libraries(log)
171 if passed:
208 if passed:
172 results.append((py, True, None, missing_libraries))
209 results.append((py, True, None, missing_libraries))
173 else:
210 else:
174 if post_results:
211 results.append((py, False, log, missing_libraries))
175 result_locn = gh_api.post_gist(log, description='IPython test log',
212
176 filename="results.log", auth=True)
213 dump_results(num, results, pr)
177 else:
214
178 result_locn = os.path.abspath(os.path.join(venv,
215 results_paths = save_logs(results, pr)
179 pr['head']['sha'][:7]+".log"))
216 print_results(pr, results_paths)
180 with io.open(result_locn, 'w', encoding='utf-8') as f:
181 f.write(log)
182 results.append((py, False, result_locn, missing_libraries))
183
217
184 print_results(pr, results)
185 if post_results:
218 if post_results:
186 post_results_comment(pr, results, num)
219 results_urls = post_logs(results)
220 post_results_comment(pr, results_urls, num)
187 print("(Posted to Github)")
221 print("(Posted to Github)")
222 else:
223 post_script = os.path.join(os.path.dirname(sys.argv[0]), "post_pr_test.py")
224 print("To post the results to Github, run", post_script)
188
225
189
226
190 if __name__ == '__main__':
227 if __name__ == '__main__':
191 import argparse
228 import argparse
192 parser = argparse.ArgumentParser(description="Test an IPython pull request")
229 parser = argparse.ArgumentParser(description="Test an IPython pull request")
193 parser.add_argument('-l', '--local', action='store_true',
230 parser.add_argument('-p', '--publish', action='store_true',
194 help="Don't publish the results to Github")
231 help="Publish the results to Github")
195 parser.add_argument('number', type=int, help="The pull request number")
232 parser.add_argument('number', type=int, help="The pull request number")
196
233
197 args = parser.parse_args()
234 args = parser.parse_args()
198 test_pr(args.number, post_results=(not args.local))
235 test_pr(args.number, post_results=args.publish)
General Comments 0
You need to be logged in to leave comments. Login now