##// END OF EJS Templates
Pickle PR test logs so they can be posted without rerunning tests.
Thomas Kluyver -
Show More
@@ -0,0 +1,10 b''
1 from test_pr import load_results, post_logs, post_results_comment, print_results
2
3 num, results, pr = load_results()
4 results_urls = post_logs(results)
5 print_results(pr, results_urls)
6 post_results_comment(pr, results_urls, num)
7
8 print()
9 print("Posted test results to pull request")
10 print(" " + pr['html_url'])
@@ -1,198 +1,235 b''
1 1 """
2 2 This is a script for testing pull requests for IPython. It merges the pull
3 3 request with current master, installs and tests on all available versions of
4 4 Python, and posts the results to Gist if any tests fail.
5 5
6 6 Usage:
7 7 python test_pr.py 1657
8 8 """
9 9 from __future__ import print_function
10 10
11 11 import errno
12 12 from glob import glob
13 13 import io
14 14 import json
15 15 import os
16 import pickle
16 17 import re
17 18 import requests
18 19 import shutil
19 20 from subprocess import call, check_call, check_output, PIPE, STDOUT, CalledProcessError
20 21 import sys
21 22
22 23 import gh_api
23 24
24 25 basedir = os.path.join(os.path.expanduser("~"), ".ipy_pr_tests")
25 26 repodir = os.path.join(basedir, "ipython")
26 27 ipy_repository = 'git://github.com/ipython/ipython.git'
27 28 gh_project="ipython/ipython"
28 29
29 30 supported_pythons = ['python2.6', 'python2.7', 'python3.1', 'python3.2']
30 31 unavailable_pythons = []
31 32
32 33 def available_python_versions():
33 34 """Get the executable names of available versions of Python on the system.
34 35 """
35 36 del unavailable_pythons[:]
36 37 for py in supported_pythons:
37 38 try:
38 39 check_call([py, '-c', 'import nose'], stdout=PIPE)
39 40 yield py
40 41 except (OSError, CalledProcessError):
41 42 unavailable_pythons.append(py)
42 43
43 44 venvs = []
44 45
45 46 def setup():
46 47 """Prepare the repository and virtualenvs."""
47 48 global venvs
48 49
49 50 try:
50 51 os.mkdir(basedir)
51 52 except OSError as e:
52 53 if e.errno != errno.EEXIST:
53 54 raise
54 55 os.chdir(basedir)
55 56
56 57 # Delete virtualenvs and recreate
57 58 for venv in glob('venv-*'):
58 59 shutil.rmtree(venv)
59 60 for py in available_python_versions():
60 61 check_call(['virtualenv', '-p', py, '--system-site-packages', 'venv-%s' % py])
61 62 venvs.append((py, 'venv-%s' % py))
62 63
63 64 # Check out and update the repository
64 65 if not os.path.exists('ipython'):
65 66 check_call(['git', 'clone', ipy_repository])
66 67 os.chdir(repodir)
67 68 check_call(['git', 'checkout', 'master'])
68 69 check_call(['git', 'pull', ipy_repository, 'master'])
69 70 os.chdir(basedir)
70 71
71 72 missing_libs_re = re.compile(r"Tools and libraries NOT available at test time:\n"
72 73 r"\s*(.*?)\n")
73 74 def get_missing_libraries(log):
74 75 m = missing_libs_re.search(log)
75 76 if m:
76 77 return m.group(1)
77 78
78 79 def get_branch(repo, branch, owner, mergeable):
79 80 os.chdir(repodir)
80 81 if mergeable:
81 82 merged_branch = "%s-%s" % (owner, branch)
82 83 # Delete the branch first
83 84 call(['git', 'branch', '-D', merged_branch])
84 85 check_call(['git', 'checkout', '-b', merged_branch])
85 86 check_call(['git', 'pull', repo, branch])
86 87 else:
87 88 # Fetch the branch without merging it.
88 89 check_call(['git', 'fetch', repo, branch])
89 90 check_call(['git', 'checkout', 'FETCH_HEAD'])
90 91 os.chdir(basedir)
91 92
92 93 def run_tests(venv):
93 94 py = os.path.join(basedir, venv, 'bin', 'python')
94 95 print(py)
95 96 os.chdir(repodir)
96 97 check_call([py, 'setup.py', 'install'])
97 98 os.chdir(basedir)
98 99
99 100 iptest = os.path.join(basedir, venv, 'bin', 'iptest')
100 101 if not os.path.exists(iptest):
101 102 iptest = os.path.join(basedir, venv, 'bin', 'iptest3')
102 103
103 104 print("\nRunning tests, this typically takes a few minutes...")
104 105 try:
105 106 return True, check_output([iptest], stderr=STDOUT).decode('utf-8')
106 107 except CalledProcessError as e:
107 108 return False, e.output.decode('utf-8')
108 109
109 def markdown_format(pr, results):
110 def markdown_format(pr, results_urls):
110 111 def format_result(py, passed, gist_url, missing_libraries):
111 112 s = "* %s: " % py
112 113 if passed:
113 114 s += "OK"
114 115 else:
115 116 s += "Failed, log at %s" % gist_url
116 117 if missing_libraries:
117 118 s += " (libraries not available: " + missing_libraries + ")"
118 119 return s
119 120
120 121 if pr['mergeable']:
121 122 com = pr['head']['sha'][:7] + " merged into master"
122 123 else:
123 124 com = pr['head']['sha'][:7] + " (can't merge cleanly)"
124 125 lines = ["**Test results for commit %s**" % com,
125 126 "Platform: " + sys.platform,
126 127 ""] + \
127 [format_result(*r) for r in results] + \
128 [format_result(*r) for r in results_urls] + \
128 129 ["",
129 130 "Not available for testing: " + ", ".join(unavailable_pythons)]
130 131 return "\n".join(lines)
131 132
132 133 def post_results_comment(pr, results, num):
133 134 body = markdown_format(pr, results)
134 135 gh_api.post_issue_comment(gh_project, num, body)
135 136
136 def print_results(pr, results):
137 def print_results(pr, results_urls):
137 138 print("\n")
138 139 if pr['mergeable']:
139 140 print("**Test results for commit %s merged into master**" % pr['head']['sha'][:7])
140 141 else:
141 142 print("**Test results for commit %s (can't merge cleanly)**" % pr['head']['sha'][:7])
142 143 print("Platform:", sys.platform)
143 for py, passed, gist_url, missing_libraries in results:
144 for py, passed, gist_url, missing_libraries in results_urls:
144 145 if passed:
145 146 print(py, ":", "OK")
146 147 else:
147 148 print(py, ":", "Failed")
148 149 print(" Test log:", gist_url)
149 150 if missing_libraries:
150 151 print(" Libraries not available:", missing_libraries)
151 152 print("Not available for testing:", ", ".join(unavailable_pythons))
152 153
154 def dump_results(num, results, pr):
155 with open(os.path.join(basedir, 'lastresults.pkl'), 'wb') as f:
156 pickle.dump((num, results, pr), f)
157
158 def load_results():
159 with open(os.path.join(basedir, 'lastresults.pkl'), 'rb') as f:
160 return pickle.load(f)
161
162 def save_logs(results, pr):
163 results_paths = []
164 for py, passed, log, missing_libraries in results:
165 if passed:
166 results_paths.append((py, passed, None, missing_libraries))
167 else:
168
169 result_locn = os.path.abspath(os.path.join('venv-%s' % py,
170 pr['head']['sha'][:7]+".log"))
171 with io.open(result_locn, 'w', encoding='utf-8') as f:
172 f.write(log)
173
174 results_paths.append((py, False, result_locn, missing_libraries))
175
176 return results_paths
177
178 def post_logs(results):
179 results_urls = []
180 for py, passed, log, missing_libraries in results:
181 if passed:
182 results_urls.append((py, passed, None, missing_libraries))
183 else:
184 result_locn = gh_api.post_gist(log, description='IPython test log',
185 filename="results.log", auth=True)
186 results_urls.append((py, False, result_locn, missing_libraries))
187
188 return results_urls
189
153 190 def test_pr(num, post_results=True):
154 191 # Get Github authorisation first, so that the user is prompted straight away
155 192 # if their login is needed.
156 193 if post_results:
157 194 gh_api.get_auth_token()
158 195
159 196 setup()
160 197 pr = gh_api.get_pull_request(gh_project, num)
161 198 get_branch(repo=pr['head']['repo']['clone_url'],
162 199 branch=pr['head']['ref'],
163 200 owner=pr['head']['repo']['owner']['login'],
164 201 mergeable=pr['mergeable'],
165 202 )
166 203
167 204 results = []
168 205 for py, venv in venvs:
169 206 passed, log = run_tests(venv)
170 207 missing_libraries = get_missing_libraries(log)
171 208 if passed:
172 209 results.append((py, True, None, missing_libraries))
173 210 else:
174 if post_results:
175 result_locn = gh_api.post_gist(log, description='IPython test log',
176 filename="results.log", auth=True)
177 else:
178 result_locn = os.path.abspath(os.path.join(venv,
179 pr['head']['sha'][:7]+".log"))
180 with io.open(result_locn, 'w', encoding='utf-8') as f:
181 f.write(log)
182 results.append((py, False, result_locn, missing_libraries))
211 results.append((py, False, log, missing_libraries))
212
213 dump_results(num, results, pr)
214
215 results_paths = save_logs(results, pr)
216 print_results(pr, results_paths)
183 217
184 print_results(pr, results)
185 218 if post_results:
186 post_results_comment(pr, results, num)
219 results_urls = post_logs(results)
220 post_results_comment(pr, results_urls, num)
187 221 print("(Posted to Github)")
222 else:
223 post_script = os.path.join(os.path.dirname(sys.argv[0]), "post_pr_test.py")
224 print("To post the results to Github, run", post_script)
188 225
189 226
190 227 if __name__ == '__main__':
191 228 import argparse
192 229 parser = argparse.ArgumentParser(description="Test an IPython pull request")
193 parser.add_argument('-l', '--local', action='store_true',
194 help="Don't publish the results to Github")
230 parser.add_argument('-p', '--publish', action='store_true',
231 help="Publish the results to Github")
195 232 parser.add_argument('number', type=int, help="The pull request number")
196 233
197 234 args = parser.parse_args()
198 test_pr(args.number, post_results=(not args.local))
235 test_pr(args.number, post_results=args.publish)
General Comments 0
You need to be logged in to leave comments. Login now