##// END OF EJS Templates
Give test_pr proper env call to python interpreter....
Fernando Perez -
Show More
@@ -1,243 +1,244 b''
1 #!/usr/bin/env python
1 """
2 """
2 This is a script for testing pull requests for IPython. It merges the pull
3 This is a script for testing pull requests for IPython. It merges the pull
3 request with current master, installs and tests on all available versions of
4 request with current master, installs and tests on all available versions of
4 Python, and posts the results to Gist if any tests fail.
5 Python, and posts the results to Gist if any tests fail.
5
6
6 Usage:
7 Usage:
7 python test_pr.py 1657
8 python test_pr.py 1657
8 """
9 """
9 from __future__ import print_function
10 from __future__ import print_function
10
11
11 import errno
12 import errno
12 from glob import glob
13 from glob import glob
13 import io
14 import io
14 import json
15 import json
15 import os
16 import os
16 import pickle
17 import pickle
17 import re
18 import re
18 import requests
19 import requests
19 import shutil
20 import shutil
20 import time
21 import time
21 from subprocess import call, check_call, check_output, PIPE, STDOUT, CalledProcessError
22 from subprocess import call, check_call, check_output, PIPE, STDOUT, CalledProcessError
22 import sys
23 import sys
23
24
24 import gh_api
25 import gh_api
25
26
26 basedir = os.path.join(os.path.expanduser("~"), ".ipy_pr_tests")
27 basedir = os.path.join(os.path.expanduser("~"), ".ipy_pr_tests")
27 repodir = os.path.join(basedir, "ipython")
28 repodir = os.path.join(basedir, "ipython")
28 ipy_repository = 'git://github.com/ipython/ipython.git'
29 ipy_repository = 'git://github.com/ipython/ipython.git'
29 gh_project="ipython/ipython"
30 gh_project="ipython/ipython"
30
31
31 supported_pythons = ['python2.6', 'python2.7', 'python3.1', 'python3.2']
32 supported_pythons = ['python2.6', 'python2.7', 'python3.1', 'python3.2']
32 unavailable_pythons = []
33 unavailable_pythons = []
33
34
34 def available_python_versions():
35 def available_python_versions():
35 """Get the executable names of available versions of Python on the system.
36 """Get the executable names of available versions of Python on the system.
36 """
37 """
37 del unavailable_pythons[:]
38 del unavailable_pythons[:]
38 for py in supported_pythons:
39 for py in supported_pythons:
39 try:
40 try:
40 check_call([py, '-c', 'import nose'], stdout=PIPE)
41 check_call([py, '-c', 'import nose'], stdout=PIPE)
41 yield py
42 yield py
42 except (OSError, CalledProcessError):
43 except (OSError, CalledProcessError):
43 unavailable_pythons.append(py)
44 unavailable_pythons.append(py)
44
45
45 venvs = []
46 venvs = []
46
47
47 def setup():
48 def setup():
48 """Prepare the repository and virtualenvs."""
49 """Prepare the repository and virtualenvs."""
49 global venvs
50 global venvs
50
51
51 try:
52 try:
52 os.mkdir(basedir)
53 os.mkdir(basedir)
53 except OSError as e:
54 except OSError as e:
54 if e.errno != errno.EEXIST:
55 if e.errno != errno.EEXIST:
55 raise
56 raise
56 os.chdir(basedir)
57 os.chdir(basedir)
57
58
58 # Delete virtualenvs and recreate
59 # Delete virtualenvs and recreate
59 for venv in glob('venv-*'):
60 for venv in glob('venv-*'):
60 shutil.rmtree(venv)
61 shutil.rmtree(venv)
61 for py in available_python_versions():
62 for py in available_python_versions():
62 check_call(['virtualenv', '-p', py, '--system-site-packages', 'venv-%s' % py])
63 check_call(['virtualenv', '-p', py, '--system-site-packages', 'venv-%s' % py])
63 venvs.append((py, 'venv-%s' % py))
64 venvs.append((py, 'venv-%s' % py))
64
65
65 # Check out and update the repository
66 # Check out and update the repository
66 if not os.path.exists('ipython'):
67 if not os.path.exists('ipython'):
67 check_call(['git', 'clone', ipy_repository])
68 check_call(['git', 'clone', ipy_repository])
68 os.chdir(repodir)
69 os.chdir(repodir)
69 check_call(['git', 'checkout', 'master'])
70 check_call(['git', 'checkout', 'master'])
70 check_call(['git', 'pull', ipy_repository, 'master'])
71 check_call(['git', 'pull', ipy_repository, 'master'])
71 os.chdir(basedir)
72 os.chdir(basedir)
72
73
73 missing_libs_re = re.compile(r"Tools and libraries NOT available at test time:\n"
74 missing_libs_re = re.compile(r"Tools and libraries NOT available at test time:\n"
74 r"\s*(.*?)\n")
75 r"\s*(.*?)\n")
75 def get_missing_libraries(log):
76 def get_missing_libraries(log):
76 m = missing_libs_re.search(log)
77 m = missing_libs_re.search(log)
77 if m:
78 if m:
78 return m.group(1)
79 return m.group(1)
79
80
80 def get_branch(repo, branch, owner, mergeable):
81 def get_branch(repo, branch, owner, mergeable):
81 os.chdir(repodir)
82 os.chdir(repodir)
82 if mergeable:
83 if mergeable:
83 merged_branch = "%s-%s" % (owner, branch)
84 merged_branch = "%s-%s" % (owner, branch)
84 # Delete the branch first
85 # Delete the branch first
85 call(['git', 'branch', '-D', merged_branch])
86 call(['git', 'branch', '-D', merged_branch])
86 check_call(['git', 'checkout', '-b', merged_branch])
87 check_call(['git', 'checkout', '-b', merged_branch])
87 check_call(['git', 'pull', '--no-commit', repo, branch])
88 check_call(['git', 'pull', '--no-commit', repo, branch])
88 check_call(['git', 'commit', '-m', "merge %s/%s" % (repo, branch)])
89 check_call(['git', 'commit', '-m', "merge %s/%s" % (repo, branch)])
89 else:
90 else:
90 # Fetch the branch without merging it.
91 # Fetch the branch without merging it.
91 check_call(['git', 'fetch', repo, branch])
92 check_call(['git', 'fetch', repo, branch])
92 check_call(['git', 'checkout', 'FETCH_HEAD'])
93 check_call(['git', 'checkout', 'FETCH_HEAD'])
93 os.chdir(basedir)
94 os.chdir(basedir)
94
95
95 def run_tests(venv):
96 def run_tests(venv):
96 py = os.path.join(basedir, venv, 'bin', 'python')
97 py = os.path.join(basedir, venv, 'bin', 'python')
97 print(py)
98 print(py)
98 os.chdir(repodir)
99 os.chdir(repodir)
99 # cleanup build-dir
100 # cleanup build-dir
100 if os.path.exists('build'):
101 if os.path.exists('build'):
101 shutil.rmtree('build')
102 shutil.rmtree('build')
102 check_call([py, 'setup.py', 'install'])
103 check_call([py, 'setup.py', 'install'])
103 os.chdir(basedir)
104 os.chdir(basedir)
104
105
105 iptest = os.path.join(basedir, venv, 'bin', 'iptest')
106 iptest = os.path.join(basedir, venv, 'bin', 'iptest')
106 if not os.path.exists(iptest):
107 if not os.path.exists(iptest):
107 iptest = os.path.join(basedir, venv, 'bin', 'iptest3')
108 iptest = os.path.join(basedir, venv, 'bin', 'iptest3')
108
109
109 print("\nRunning tests, this typically takes a few minutes...")
110 print("\nRunning tests, this typically takes a few minutes...")
110 try:
111 try:
111 return True, check_output([iptest], stderr=STDOUT).decode('utf-8')
112 return True, check_output([iptest], stderr=STDOUT).decode('utf-8')
112 except CalledProcessError as e:
113 except CalledProcessError as e:
113 return False, e.output.decode('utf-8')
114 return False, e.output.decode('utf-8')
114
115
115 def markdown_format(pr, results_urls):
116 def markdown_format(pr, results_urls):
116 def format_result(py, passed, gist_url, missing_libraries):
117 def format_result(py, passed, gist_url, missing_libraries):
117 s = "* %s: " % py
118 s = "* %s: " % py
118 if passed:
119 if passed:
119 s += "OK"
120 s += "OK"
120 else:
121 else:
121 s += "Failed, log at %s" % gist_url
122 s += "Failed, log at %s" % gist_url
122 if missing_libraries:
123 if missing_libraries:
123 s += " (libraries not available: " + missing_libraries + ")"
124 s += " (libraries not available: " + missing_libraries + ")"
124 return s
125 return s
125
126
126 if pr['mergeable']:
127 if pr['mergeable']:
127 com = pr['head']['sha'][:7] + " merged into master"
128 com = pr['head']['sha'][:7] + " merged into master"
128 else:
129 else:
129 com = pr['head']['sha'][:7] + " (can't merge cleanly)"
130 com = pr['head']['sha'][:7] + " (can't merge cleanly)"
130 lines = ["**Test results for commit %s**" % com,
131 lines = ["**Test results for commit %s**" % com,
131 "Platform: " + sys.platform,
132 "Platform: " + sys.platform,
132 ""] + \
133 ""] + \
133 [format_result(*r) for r in results_urls] + \
134 [format_result(*r) for r in results_urls] + \
134 ["",
135 ["",
135 "Not available for testing: " + ", ".join(unavailable_pythons)]
136 "Not available for testing: " + ", ".join(unavailable_pythons)]
136 return "\n".join(lines)
137 return "\n".join(lines)
137
138
138 def post_results_comment(pr, results, num):
139 def post_results_comment(pr, results, num):
139 body = markdown_format(pr, results)
140 body = markdown_format(pr, results)
140 gh_api.post_issue_comment(gh_project, num, body)
141 gh_api.post_issue_comment(gh_project, num, body)
141
142
142 def print_results(pr, results_urls):
143 def print_results(pr, results_urls):
143 print("\n")
144 print("\n")
144 if pr['mergeable']:
145 if pr['mergeable']:
145 print("**Test results for commit %s merged into master**" % pr['head']['sha'][:7])
146 print("**Test results for commit %s merged into master**" % pr['head']['sha'][:7])
146 else:
147 else:
147 print("**Test results for commit %s (can't merge cleanly)**" % pr['head']['sha'][:7])
148 print("**Test results for commit %s (can't merge cleanly)**" % pr['head']['sha'][:7])
148 print("Platform:", sys.platform)
149 print("Platform:", sys.platform)
149 for py, passed, gist_url, missing_libraries in results_urls:
150 for py, passed, gist_url, missing_libraries in results_urls:
150 if passed:
151 if passed:
151 print(py, ":", "OK")
152 print(py, ":", "OK")
152 else:
153 else:
153 print(py, ":", "Failed")
154 print(py, ":", "Failed")
154 print(" Test log:", gist_url)
155 print(" Test log:", gist_url)
155 if missing_libraries:
156 if missing_libraries:
156 print(" Libraries not available:", missing_libraries)
157 print(" Libraries not available:", missing_libraries)
157 print("Not available for testing:", ", ".join(unavailable_pythons))
158 print("Not available for testing:", ", ".join(unavailable_pythons))
158
159
159 def dump_results(num, results, pr):
160 def dump_results(num, results, pr):
160 with open(os.path.join(basedir, 'lastresults.pkl'), 'wb') as f:
161 with open(os.path.join(basedir, 'lastresults.pkl'), 'wb') as f:
161 pickle.dump((num, results, pr), f)
162 pickle.dump((num, results, pr), f)
162
163
163 def load_results():
164 def load_results():
164 with open(os.path.join(basedir, 'lastresults.pkl'), 'rb') as f:
165 with open(os.path.join(basedir, 'lastresults.pkl'), 'rb') as f:
165 return pickle.load(f)
166 return pickle.load(f)
166
167
167 def save_logs(results, pr):
168 def save_logs(results, pr):
168 results_paths = []
169 results_paths = []
169 for py, passed, log, missing_libraries in results:
170 for py, passed, log, missing_libraries in results:
170 if passed:
171 if passed:
171 results_paths.append((py, passed, None, missing_libraries))
172 results_paths.append((py, passed, None, missing_libraries))
172 else:
173 else:
173
174
174 result_locn = os.path.abspath(os.path.join('venv-%s' % py,
175 result_locn = os.path.abspath(os.path.join('venv-%s' % py,
175 pr['head']['sha'][:7]+".log"))
176 pr['head']['sha'][:7]+".log"))
176 with io.open(result_locn, 'w', encoding='utf-8') as f:
177 with io.open(result_locn, 'w', encoding='utf-8') as f:
177 f.write(log)
178 f.write(log)
178
179
179 results_paths.append((py, False, result_locn, missing_libraries))
180 results_paths.append((py, False, result_locn, missing_libraries))
180
181
181 return results_paths
182 return results_paths
182
183
183 def post_logs(results):
184 def post_logs(results):
184 results_urls = []
185 results_urls = []
185 for py, passed, log, missing_libraries in results:
186 for py, passed, log, missing_libraries in results:
186 if passed:
187 if passed:
187 results_urls.append((py, passed, None, missing_libraries))
188 results_urls.append((py, passed, None, missing_libraries))
188 else:
189 else:
189 result_locn = gh_api.post_gist(log, description='IPython test log',
190 result_locn = gh_api.post_gist(log, description='IPython test log',
190 filename="results.log", auth=True)
191 filename="results.log", auth=True)
191 results_urls.append((py, False, result_locn, missing_libraries))
192 results_urls.append((py, False, result_locn, missing_libraries))
192
193
193 return results_urls
194 return results_urls
194
195
195 def test_pr(num, post_results=True):
196 def test_pr(num, post_results=True):
196 # Get Github authorisation first, so that the user is prompted straight away
197 # Get Github authorisation first, so that the user is prompted straight away
197 # if their login is needed.
198 # if their login is needed.
198 if post_results:
199 if post_results:
199 gh_api.get_auth_token()
200 gh_api.get_auth_token()
200
201
201 setup()
202 setup()
202 pr = gh_api.get_pull_request(gh_project, num)
203 pr = gh_api.get_pull_request(gh_project, num)
203 get_branch(repo=pr['head']['repo']['clone_url'],
204 get_branch(repo=pr['head']['repo']['clone_url'],
204 branch=pr['head']['ref'],
205 branch=pr['head']['ref'],
205 owner=pr['head']['repo']['owner']['login'],
206 owner=pr['head']['repo']['owner']['login'],
206 mergeable=pr['mergeable'],
207 mergeable=pr['mergeable'],
207 )
208 )
208
209
209 results = []
210 results = []
210 for py, venv in venvs:
211 for py, venv in venvs:
211 tic = time.time()
212 tic = time.time()
212 passed, log = run_tests(venv)
213 passed, log = run_tests(venv)
213 elapsed = int(time.time() - tic)
214 elapsed = int(time.time() - tic)
214 print("Ran tests with %s in %is" % (py, elapsed))
215 print("Ran tests with %s in %is" % (py, elapsed))
215 missing_libraries = get_missing_libraries(log)
216 missing_libraries = get_missing_libraries(log)
216 if passed:
217 if passed:
217 results.append((py, True, None, missing_libraries))
218 results.append((py, True, None, missing_libraries))
218 else:
219 else:
219 results.append((py, False, log, missing_libraries))
220 results.append((py, False, log, missing_libraries))
220
221
221 dump_results(num, results, pr)
222 dump_results(num, results, pr)
222
223
223 results_paths = save_logs(results, pr)
224 results_paths = save_logs(results, pr)
224 print_results(pr, results_paths)
225 print_results(pr, results_paths)
225
226
226 if post_results:
227 if post_results:
227 results_urls = post_logs(results)
228 results_urls = post_logs(results)
228 post_results_comment(pr, results_urls, num)
229 post_results_comment(pr, results_urls, num)
229 print("(Posted to Github)")
230 print("(Posted to Github)")
230 else:
231 else:
231 post_script = os.path.join(os.path.dirname(sys.argv[0]), "post_pr_test.py")
232 post_script = os.path.join(os.path.dirname(sys.argv[0]), "post_pr_test.py")
232 print("To post the results to Github, run", post_script)
233 print("To post the results to Github, run", post_script)
233
234
234
235
235 if __name__ == '__main__':
236 if __name__ == '__main__':
236 import argparse
237 import argparse
237 parser = argparse.ArgumentParser(description="Test an IPython pull request")
238 parser = argparse.ArgumentParser(description="Test an IPython pull request")
238 parser.add_argument('-p', '--publish', action='store_true',
239 parser.add_argument('-p', '--publish', action='store_true',
239 help="Publish the results to Github")
240 help="Publish the results to Github")
240 parser.add_argument('number', type=int, help="The pull request number")
241 parser.add_argument('number', type=int, help="The pull request number")
241
242
242 args = parser.parse_args()
243 args = parser.parse_args()
243 test_pr(args.number, post_results=args.publish)
244 test_pr(args.number, post_results=args.publish)
General Comments 0
You need to be logged in to leave comments. Login now