##// END OF EJS Templates
run-tests.py: fix handling of symlink to the right python...
Mads Kiilerich -
r18244:5a3c71b0 default
parent child Browse files
Show More
@@ -1,1353 +1,1355 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 #
2 #
3 # run-tests.py - Run a set of tests on Mercurial
3 # run-tests.py - Run a set of tests on Mercurial
4 #
4 #
5 # Copyright 2006 Matt Mackall <mpm@selenic.com>
5 # Copyright 2006 Matt Mackall <mpm@selenic.com>
6 #
6 #
7 # This software may be used and distributed according to the terms of the
7 # This software may be used and distributed according to the terms of the
8 # GNU General Public License version 2 or any later version.
8 # GNU General Public License version 2 or any later version.
9
9
10 # Modifying this script is tricky because it has many modes:
10 # Modifying this script is tricky because it has many modes:
11 # - serial (default) vs parallel (-jN, N > 1)
11 # - serial (default) vs parallel (-jN, N > 1)
12 # - no coverage (default) vs coverage (-c, -C, -s)
12 # - no coverage (default) vs coverage (-c, -C, -s)
13 # - temp install (default) vs specific hg script (--with-hg, --local)
13 # - temp install (default) vs specific hg script (--with-hg, --local)
14 # - tests are a mix of shell scripts and Python scripts
14 # - tests are a mix of shell scripts and Python scripts
15 #
15 #
16 # If you change this script, it is recommended that you ensure you
16 # If you change this script, it is recommended that you ensure you
17 # haven't broken it by running it in various modes with a representative
17 # haven't broken it by running it in various modes with a representative
18 # sample of test scripts. For example:
18 # sample of test scripts. For example:
19 #
19 #
20 # 1) serial, no coverage, temp install:
20 # 1) serial, no coverage, temp install:
21 # ./run-tests.py test-s*
21 # ./run-tests.py test-s*
22 # 2) serial, no coverage, local hg:
22 # 2) serial, no coverage, local hg:
23 # ./run-tests.py --local test-s*
23 # ./run-tests.py --local test-s*
24 # 3) serial, coverage, temp install:
24 # 3) serial, coverage, temp install:
25 # ./run-tests.py -c test-s*
25 # ./run-tests.py -c test-s*
26 # 4) serial, coverage, local hg:
26 # 4) serial, coverage, local hg:
27 # ./run-tests.py -c --local test-s* # unsupported
27 # ./run-tests.py -c --local test-s* # unsupported
28 # 5) parallel, no coverage, temp install:
28 # 5) parallel, no coverage, temp install:
29 # ./run-tests.py -j2 test-s*
29 # ./run-tests.py -j2 test-s*
30 # 6) parallel, no coverage, local hg:
30 # 6) parallel, no coverage, local hg:
31 # ./run-tests.py -j2 --local test-s*
31 # ./run-tests.py -j2 --local test-s*
32 # 7) parallel, coverage, temp install:
32 # 7) parallel, coverage, temp install:
33 # ./run-tests.py -j2 -c test-s* # currently broken
33 # ./run-tests.py -j2 -c test-s* # currently broken
34 # 8) parallel, coverage, local install:
34 # 8) parallel, coverage, local install:
35 # ./run-tests.py -j2 -c --local test-s* # unsupported (and broken)
35 # ./run-tests.py -j2 -c --local test-s* # unsupported (and broken)
36 # 9) parallel, custom tmp dir:
36 # 9) parallel, custom tmp dir:
37 # ./run-tests.py -j2 --tmpdir /tmp/myhgtests
37 # ./run-tests.py -j2 --tmpdir /tmp/myhgtests
38 #
38 #
39 # (You could use any subset of the tests: test-s* happens to match
39 # (You could use any subset of the tests: test-s* happens to match
40 # enough that it's worth doing parallel runs, few enough that it
40 # enough that it's worth doing parallel runs, few enough that it
41 # completes fairly quickly, includes both shell and Python scripts, and
41 # completes fairly quickly, includes both shell and Python scripts, and
42 # includes some scripts that run daemon processes.)
42 # includes some scripts that run daemon processes.)
43
43
44 from distutils import version
44 from distutils import version
45 import difflib
45 import difflib
46 import errno
46 import errno
47 import optparse
47 import optparse
48 import os
48 import os
49 import shutil
49 import shutil
50 import subprocess
50 import subprocess
51 import signal
51 import signal
52 import sys
52 import sys
53 import tempfile
53 import tempfile
54 import time
54 import time
55 import re
55 import re
56 import threading
56 import threading
57 import killdaemons as killmod
57 import killdaemons as killmod
58 import cPickle as pickle
58 import cPickle as pickle
59 import Queue as queue
59 import Queue as queue
60
60
61 processlock = threading.Lock()
61 processlock = threading.Lock()
62
62
63 closefds = os.name == 'posix'
63 closefds = os.name == 'posix'
64 def Popen4(cmd, wd, timeout):
64 def Popen4(cmd, wd, timeout):
65 processlock.acquire()
65 processlock.acquire()
66 p = subprocess.Popen(cmd, shell=True, bufsize=-1, cwd=wd,
66 p = subprocess.Popen(cmd, shell=True, bufsize=-1, cwd=wd,
67 close_fds=closefds,
67 close_fds=closefds,
68 stdin=subprocess.PIPE, stdout=subprocess.PIPE,
68 stdin=subprocess.PIPE, stdout=subprocess.PIPE,
69 stderr=subprocess.STDOUT)
69 stderr=subprocess.STDOUT)
70 processlock.release()
70 processlock.release()
71
71
72 p.fromchild = p.stdout
72 p.fromchild = p.stdout
73 p.tochild = p.stdin
73 p.tochild = p.stdin
74 p.childerr = p.stderr
74 p.childerr = p.stderr
75
75
76 p.timeout = False
76 p.timeout = False
77 if timeout:
77 if timeout:
78 def t():
78 def t():
79 start = time.time()
79 start = time.time()
80 while time.time() - start < timeout and p.returncode is None:
80 while time.time() - start < timeout and p.returncode is None:
81 time.sleep(.1)
81 time.sleep(.1)
82 p.timeout = True
82 p.timeout = True
83 if p.returncode is None:
83 if p.returncode is None:
84 terminate(p)
84 terminate(p)
85 threading.Thread(target=t).start()
85 threading.Thread(target=t).start()
86
86
87 return p
87 return p
88
88
89 # reserved exit code to skip test (used by hghave)
89 # reserved exit code to skip test (used by hghave)
90 SKIPPED_STATUS = 80
90 SKIPPED_STATUS = 80
91 SKIPPED_PREFIX = 'skipped: '
91 SKIPPED_PREFIX = 'skipped: '
92 FAILED_PREFIX = 'hghave check failed: '
92 FAILED_PREFIX = 'hghave check failed: '
93 PYTHON = sys.executable.replace('\\', '/')
93 PYTHON = sys.executable.replace('\\', '/')
94 IMPL_PATH = 'PYTHONPATH'
94 IMPL_PATH = 'PYTHONPATH'
95 if 'java' in sys.platform:
95 if 'java' in sys.platform:
96 IMPL_PATH = 'JYTHONPATH'
96 IMPL_PATH = 'JYTHONPATH'
97
97
98 requiredtools = [os.path.basename(sys.executable), "diff", "grep", "unzip",
98 requiredtools = [os.path.basename(sys.executable), "diff", "grep", "unzip",
99 "gunzip", "bunzip2", "sed"]
99 "gunzip", "bunzip2", "sed"]
100
100
101 defaults = {
101 defaults = {
102 'jobs': ('HGTEST_JOBS', 1),
102 'jobs': ('HGTEST_JOBS', 1),
103 'timeout': ('HGTEST_TIMEOUT', 180),
103 'timeout': ('HGTEST_TIMEOUT', 180),
104 'port': ('HGTEST_PORT', 20059),
104 'port': ('HGTEST_PORT', 20059),
105 'shell': ('HGTEST_SHELL', 'sh'),
105 'shell': ('HGTEST_SHELL', 'sh'),
106 }
106 }
107
107
108 def parselistfiles(files, listtype, warn=True):
108 def parselistfiles(files, listtype, warn=True):
109 entries = dict()
109 entries = dict()
110 for filename in files:
110 for filename in files:
111 try:
111 try:
112 path = os.path.expanduser(os.path.expandvars(filename))
112 path = os.path.expanduser(os.path.expandvars(filename))
113 f = open(path, "r")
113 f = open(path, "r")
114 except IOError, err:
114 except IOError, err:
115 if err.errno != errno.ENOENT:
115 if err.errno != errno.ENOENT:
116 raise
116 raise
117 if warn:
117 if warn:
118 print "warning: no such %s file: %s" % (listtype, filename)
118 print "warning: no such %s file: %s" % (listtype, filename)
119 continue
119 continue
120
120
121 for line in f.readlines():
121 for line in f.readlines():
122 line = line.split('#', 1)[0].strip()
122 line = line.split('#', 1)[0].strip()
123 if line:
123 if line:
124 entries[line] = filename
124 entries[line] = filename
125
125
126 f.close()
126 f.close()
127 return entries
127 return entries
128
128
129 def parseargs():
129 def parseargs():
130 parser = optparse.OptionParser("%prog [options] [tests]")
130 parser = optparse.OptionParser("%prog [options] [tests]")
131
131
132 # keep these sorted
132 # keep these sorted
133 parser.add_option("--blacklist", action="append",
133 parser.add_option("--blacklist", action="append",
134 help="skip tests listed in the specified blacklist file")
134 help="skip tests listed in the specified blacklist file")
135 parser.add_option("--whitelist", action="append",
135 parser.add_option("--whitelist", action="append",
136 help="always run tests listed in the specified whitelist file")
136 help="always run tests listed in the specified whitelist file")
137 parser.add_option("-C", "--annotate", action="store_true",
137 parser.add_option("-C", "--annotate", action="store_true",
138 help="output files annotated with coverage")
138 help="output files annotated with coverage")
139 parser.add_option("--child", type="int",
139 parser.add_option("--child", type="int",
140 help="run as child process, summary to given fd")
140 help="run as child process, summary to given fd")
141 parser.add_option("-c", "--cover", action="store_true",
141 parser.add_option("-c", "--cover", action="store_true",
142 help="print a test coverage report")
142 help="print a test coverage report")
143 parser.add_option("-d", "--debug", action="store_true",
143 parser.add_option("-d", "--debug", action="store_true",
144 help="debug mode: write output of test scripts to console"
144 help="debug mode: write output of test scripts to console"
145 " rather than capturing and diff'ing it (disables timeout)")
145 " rather than capturing and diff'ing it (disables timeout)")
146 parser.add_option("-f", "--first", action="store_true",
146 parser.add_option("-f", "--first", action="store_true",
147 help="exit on the first test failure")
147 help="exit on the first test failure")
148 parser.add_option("-H", "--htmlcov", action="store_true",
148 parser.add_option("-H", "--htmlcov", action="store_true",
149 help="create an HTML report of the coverage of the files")
149 help="create an HTML report of the coverage of the files")
150 parser.add_option("--inotify", action="store_true",
150 parser.add_option("--inotify", action="store_true",
151 help="enable inotify extension when running tests")
151 help="enable inotify extension when running tests")
152 parser.add_option("-i", "--interactive", action="store_true",
152 parser.add_option("-i", "--interactive", action="store_true",
153 help="prompt to accept changed output")
153 help="prompt to accept changed output")
154 parser.add_option("-j", "--jobs", type="int",
154 parser.add_option("-j", "--jobs", type="int",
155 help="number of jobs to run in parallel"
155 help="number of jobs to run in parallel"
156 " (default: $%s or %d)" % defaults['jobs'])
156 " (default: $%s or %d)" % defaults['jobs'])
157 parser.add_option("--keep-tmpdir", action="store_true",
157 parser.add_option("--keep-tmpdir", action="store_true",
158 help="keep temporary directory after running tests")
158 help="keep temporary directory after running tests")
159 parser.add_option("-k", "--keywords",
159 parser.add_option("-k", "--keywords",
160 help="run tests matching keywords")
160 help="run tests matching keywords")
161 parser.add_option("-l", "--local", action="store_true",
161 parser.add_option("-l", "--local", action="store_true",
162 help="shortcut for --with-hg=<testdir>/../hg")
162 help="shortcut for --with-hg=<testdir>/../hg")
163 parser.add_option("-n", "--nodiff", action="store_true",
163 parser.add_option("-n", "--nodiff", action="store_true",
164 help="skip showing test changes")
164 help="skip showing test changes")
165 parser.add_option("-p", "--port", type="int",
165 parser.add_option("-p", "--port", type="int",
166 help="port on which servers should listen"
166 help="port on which servers should listen"
167 " (default: $%s or %d)" % defaults['port'])
167 " (default: $%s or %d)" % defaults['port'])
168 parser.add_option("--compiler", type="string",
168 parser.add_option("--compiler", type="string",
169 help="compiler to build with")
169 help="compiler to build with")
170 parser.add_option("--pure", action="store_true",
170 parser.add_option("--pure", action="store_true",
171 help="use pure Python code instead of C extensions")
171 help="use pure Python code instead of C extensions")
172 parser.add_option("-R", "--restart", action="store_true",
172 parser.add_option("-R", "--restart", action="store_true",
173 help="restart at last error")
173 help="restart at last error")
174 parser.add_option("-r", "--retest", action="store_true",
174 parser.add_option("-r", "--retest", action="store_true",
175 help="retest failed tests")
175 help="retest failed tests")
176 parser.add_option("-S", "--noskips", action="store_true",
176 parser.add_option("-S", "--noskips", action="store_true",
177 help="don't report skip tests verbosely")
177 help="don't report skip tests verbosely")
178 parser.add_option("--shell", type="string",
178 parser.add_option("--shell", type="string",
179 help="shell to use (default: $%s or %s)" % defaults['shell'])
179 help="shell to use (default: $%s or %s)" % defaults['shell'])
180 parser.add_option("-t", "--timeout", type="int",
180 parser.add_option("-t", "--timeout", type="int",
181 help="kill errant tests after TIMEOUT seconds"
181 help="kill errant tests after TIMEOUT seconds"
182 " (default: $%s or %d)" % defaults['timeout'])
182 " (default: $%s or %d)" % defaults['timeout'])
183 parser.add_option("--time", action="store_true",
183 parser.add_option("--time", action="store_true",
184 help="time how long each test takes")
184 help="time how long each test takes")
185 parser.add_option("--tmpdir", type="string",
185 parser.add_option("--tmpdir", type="string",
186 help="run tests in the given temporary directory"
186 help="run tests in the given temporary directory"
187 " (implies --keep-tmpdir)")
187 " (implies --keep-tmpdir)")
188 parser.add_option("-v", "--verbose", action="store_true",
188 parser.add_option("-v", "--verbose", action="store_true",
189 help="output verbose messages")
189 help="output verbose messages")
190 parser.add_option("--view", type="string",
190 parser.add_option("--view", type="string",
191 help="external diff viewer")
191 help="external diff viewer")
192 parser.add_option("--with-hg", type="string",
192 parser.add_option("--with-hg", type="string",
193 metavar="HG",
193 metavar="HG",
194 help="test using specified hg script rather than a "
194 help="test using specified hg script rather than a "
195 "temporary installation")
195 "temporary installation")
196 parser.add_option("-3", "--py3k-warnings", action="store_true",
196 parser.add_option("-3", "--py3k-warnings", action="store_true",
197 help="enable Py3k warnings on Python 2.6+")
197 help="enable Py3k warnings on Python 2.6+")
198 parser.add_option('--extra-config-opt', action="append",
198 parser.add_option('--extra-config-opt', action="append",
199 help='set the given config opt in the test hgrc')
199 help='set the given config opt in the test hgrc')
200
200
201 for option, (envvar, default) in defaults.items():
201 for option, (envvar, default) in defaults.items():
202 defaults[option] = type(default)(os.environ.get(envvar, default))
202 defaults[option] = type(default)(os.environ.get(envvar, default))
203 parser.set_defaults(**defaults)
203 parser.set_defaults(**defaults)
204 (options, args) = parser.parse_args()
204 (options, args) = parser.parse_args()
205
205
206 # jython is always pure
206 # jython is always pure
207 if 'java' in sys.platform or '__pypy__' in sys.modules:
207 if 'java' in sys.platform or '__pypy__' in sys.modules:
208 options.pure = True
208 options.pure = True
209
209
210 if options.with_hg:
210 if options.with_hg:
211 options.with_hg = os.path.expanduser(options.with_hg)
211 options.with_hg = os.path.expanduser(options.with_hg)
212 if not (os.path.isfile(options.with_hg) and
212 if not (os.path.isfile(options.with_hg) and
213 os.access(options.with_hg, os.X_OK)):
213 os.access(options.with_hg, os.X_OK)):
214 parser.error('--with-hg must specify an executable hg script')
214 parser.error('--with-hg must specify an executable hg script')
215 if not os.path.basename(options.with_hg) == 'hg':
215 if not os.path.basename(options.with_hg) == 'hg':
216 sys.stderr.write('warning: --with-hg should specify an hg script\n')
216 sys.stderr.write('warning: --with-hg should specify an hg script\n')
217 if options.local:
217 if options.local:
218 testdir = os.path.dirname(os.path.realpath(sys.argv[0]))
218 testdir = os.path.dirname(os.path.realpath(sys.argv[0]))
219 hgbin = os.path.join(os.path.dirname(testdir), 'hg')
219 hgbin = os.path.join(os.path.dirname(testdir), 'hg')
220 if os.name != 'nt' and not os.access(hgbin, os.X_OK):
220 if os.name != 'nt' and not os.access(hgbin, os.X_OK):
221 parser.error('--local specified, but %r not found or not executable'
221 parser.error('--local specified, but %r not found or not executable'
222 % hgbin)
222 % hgbin)
223 options.with_hg = hgbin
223 options.with_hg = hgbin
224
224
225 options.anycoverage = options.cover or options.annotate or options.htmlcov
225 options.anycoverage = options.cover or options.annotate or options.htmlcov
226 if options.anycoverage:
226 if options.anycoverage:
227 try:
227 try:
228 import coverage
228 import coverage
229 covver = version.StrictVersion(coverage.__version__).version
229 covver = version.StrictVersion(coverage.__version__).version
230 if covver < (3, 3):
230 if covver < (3, 3):
231 parser.error('coverage options require coverage 3.3 or later')
231 parser.error('coverage options require coverage 3.3 or later')
232 except ImportError:
232 except ImportError:
233 parser.error('coverage options now require the coverage package')
233 parser.error('coverage options now require the coverage package')
234
234
235 if options.anycoverage and options.local:
235 if options.anycoverage and options.local:
236 # this needs some path mangling somewhere, I guess
236 # this needs some path mangling somewhere, I guess
237 parser.error("sorry, coverage options do not work when --local "
237 parser.error("sorry, coverage options do not work when --local "
238 "is specified")
238 "is specified")
239
239
240 global vlog
240 global vlog
241 if options.verbose:
241 if options.verbose:
242 if options.jobs > 1 or options.child is not None:
242 if options.jobs > 1 or options.child is not None:
243 pid = "[%d]" % os.getpid()
243 pid = "[%d]" % os.getpid()
244 else:
244 else:
245 pid = None
245 pid = None
246 def vlog(*msg):
246 def vlog(*msg):
247 iolock.acquire()
247 iolock.acquire()
248 if pid:
248 if pid:
249 print pid,
249 print pid,
250 for m in msg:
250 for m in msg:
251 print m,
251 print m,
252 print
252 print
253 sys.stdout.flush()
253 sys.stdout.flush()
254 iolock.release()
254 iolock.release()
255 else:
255 else:
256 vlog = lambda *msg: None
256 vlog = lambda *msg: None
257
257
258 if options.tmpdir:
258 if options.tmpdir:
259 options.tmpdir = os.path.expanduser(options.tmpdir)
259 options.tmpdir = os.path.expanduser(options.tmpdir)
260
260
261 if options.jobs < 1:
261 if options.jobs < 1:
262 parser.error('--jobs must be positive')
262 parser.error('--jobs must be positive')
263 if options.interactive and options.jobs > 1:
263 if options.interactive and options.jobs > 1:
264 print '(--interactive overrides --jobs)'
264 print '(--interactive overrides --jobs)'
265 options.jobs = 1
265 options.jobs = 1
266 if options.interactive and options.debug:
266 if options.interactive and options.debug:
267 parser.error("-i/--interactive and -d/--debug are incompatible")
267 parser.error("-i/--interactive and -d/--debug are incompatible")
268 if options.debug:
268 if options.debug:
269 if options.timeout != defaults['timeout']:
269 if options.timeout != defaults['timeout']:
270 sys.stderr.write(
270 sys.stderr.write(
271 'warning: --timeout option ignored with --debug\n')
271 'warning: --timeout option ignored with --debug\n')
272 options.timeout = 0
272 options.timeout = 0
273 if options.time:
273 if options.time:
274 sys.stderr.write(
274 sys.stderr.write(
275 'warning: --time option ignored with --debug\n')
275 'warning: --time option ignored with --debug\n')
276 options.time = False
276 options.time = False
277 if options.py3k_warnings:
277 if options.py3k_warnings:
278 if sys.version_info[:2] < (2, 6) or sys.version_info[:2] >= (3, 0):
278 if sys.version_info[:2] < (2, 6) or sys.version_info[:2] >= (3, 0):
279 parser.error('--py3k-warnings can only be used on Python 2.6+')
279 parser.error('--py3k-warnings can only be used on Python 2.6+')
280 if options.blacklist:
280 if options.blacklist:
281 options.blacklist = parselistfiles(options.blacklist, 'blacklist')
281 options.blacklist = parselistfiles(options.blacklist, 'blacklist')
282 if options.whitelist:
282 if options.whitelist:
283 options.whitelisted = parselistfiles(options.whitelist, 'whitelist',
283 options.whitelisted = parselistfiles(options.whitelist, 'whitelist',
284 warn=options.child is None)
284 warn=options.child is None)
285 else:
285 else:
286 options.whitelisted = {}
286 options.whitelisted = {}
287
287
288 return (options, args)
288 return (options, args)
289
289
290 def rename(src, dst):
290 def rename(src, dst):
291 """Like os.rename(), trade atomicity and opened files friendliness
291 """Like os.rename(), trade atomicity and opened files friendliness
292 for existing destination support.
292 for existing destination support.
293 """
293 """
294 shutil.copy(src, dst)
294 shutil.copy(src, dst)
295 os.remove(src)
295 os.remove(src)
296
296
297 def parsehghaveoutput(lines):
297 def parsehghaveoutput(lines):
298 '''Parse hghave log lines.
298 '''Parse hghave log lines.
299 Return tuple of lists (missing, failed):
299 Return tuple of lists (missing, failed):
300 * the missing/unknown features
300 * the missing/unknown features
301 * the features for which existence check failed'''
301 * the features for which existence check failed'''
302 missing = []
302 missing = []
303 failed = []
303 failed = []
304 for line in lines:
304 for line in lines:
305 if line.startswith(SKIPPED_PREFIX):
305 if line.startswith(SKIPPED_PREFIX):
306 line = line.splitlines()[0]
306 line = line.splitlines()[0]
307 missing.append(line[len(SKIPPED_PREFIX):])
307 missing.append(line[len(SKIPPED_PREFIX):])
308 elif line.startswith(FAILED_PREFIX):
308 elif line.startswith(FAILED_PREFIX):
309 line = line.splitlines()[0]
309 line = line.splitlines()[0]
310 failed.append(line[len(FAILED_PREFIX):])
310 failed.append(line[len(FAILED_PREFIX):])
311
311
312 return missing, failed
312 return missing, failed
313
313
314 def showdiff(expected, output, ref, err):
314 def showdiff(expected, output, ref, err):
315 print
315 print
316 for line in difflib.unified_diff(expected, output, ref, err):
316 for line in difflib.unified_diff(expected, output, ref, err):
317 sys.stdout.write(line)
317 sys.stdout.write(line)
318
318
319 def findprogram(program):
319 def findprogram(program):
320 """Search PATH for a executable program"""
320 """Search PATH for a executable program"""
321 for p in os.environ.get('PATH', os.defpath).split(os.pathsep):
321 for p in os.environ.get('PATH', os.defpath).split(os.pathsep):
322 name = os.path.join(p, program)
322 name = os.path.join(p, program)
323 if os.name == 'nt' or os.access(name, os.X_OK):
323 if os.name == 'nt' or os.access(name, os.X_OK):
324 return name
324 return name
325 return None
325 return None
326
326
327 def checktools():
327 def checktools():
328 # Before we go any further, check for pre-requisite tools
328 # Before we go any further, check for pre-requisite tools
329 # stuff from coreutils (cat, rm, etc) are not tested
329 # stuff from coreutils (cat, rm, etc) are not tested
330 for p in requiredtools:
330 for p in requiredtools:
331 if os.name == 'nt' and not p.endswith('.exe'):
331 if os.name == 'nt' and not p.endswith('.exe'):
332 p += '.exe'
332 p += '.exe'
333 found = findprogram(p)
333 found = findprogram(p)
334 if found:
334 if found:
335 vlog("# Found prerequisite", p, "at", found)
335 vlog("# Found prerequisite", p, "at", found)
336 else:
336 else:
337 print "WARNING: Did not find prerequisite tool: "+p
337 print "WARNING: Did not find prerequisite tool: "+p
338
338
339 def terminate(proc):
339 def terminate(proc):
340 """Terminate subprocess (with fallback for Python versions < 2.6)"""
340 """Terminate subprocess (with fallback for Python versions < 2.6)"""
341 vlog('# Terminating process %d' % proc.pid)
341 vlog('# Terminating process %d' % proc.pid)
342 try:
342 try:
343 getattr(proc, 'terminate', lambda : os.kill(proc.pid, signal.SIGTERM))()
343 getattr(proc, 'terminate', lambda : os.kill(proc.pid, signal.SIGTERM))()
344 except OSError:
344 except OSError:
345 pass
345 pass
346
346
347 def killdaemons():
347 def killdaemons():
348 return killmod.killdaemons(DAEMON_PIDS, tryhard=False, remove=True,
348 return killmod.killdaemons(DAEMON_PIDS, tryhard=False, remove=True,
349 logfn=vlog)
349 logfn=vlog)
350
350
351 def cleanup(options):
351 def cleanup(options):
352 if not options.keep_tmpdir:
352 if not options.keep_tmpdir:
353 vlog("# Cleaning up HGTMP", HGTMP)
353 vlog("# Cleaning up HGTMP", HGTMP)
354 shutil.rmtree(HGTMP, True)
354 shutil.rmtree(HGTMP, True)
355
355
356 def usecorrectpython():
356 def usecorrectpython():
357 # some tests run python interpreter. they must use same
357 # some tests run python interpreter. they must use same
358 # interpreter we use or bad things will happen.
358 # interpreter we use or bad things will happen.
359 exedir, exename = os.path.split(sys.executable)
359 pyexename = sys.platform == 'win32' and 'python.exe' or 'python'
360 if exename in ('python', 'python.exe'):
361 path = findprogram(exename)
362 if os.path.dirname(path) == exedir:
363 return
364 else:
365 exename = 'python'
366 if sys.platform == 'win32':
367 exename = 'python.exe'
368 if getattr(os, 'symlink', None):
360 if getattr(os, 'symlink', None):
369 vlog("# Making python executable in test path a symlink to '%s'" %
361 vlog("# Making python executable in test path a symlink to '%s'" %
370 sys.executable)
362 sys.executable)
371 mypython = os.path.join(BINDIR, exename)
363 mypython = os.path.join(BINDIR, pyexename)
372 try:
364 try:
373 os.symlink(sys.executable, mypython)
365 if os.readlink(mypython) == sys.executable:
366 return
367 os.unlink(mypython)
374 except OSError, err:
368 except OSError, err:
375 # child processes may race, which is harmless
369 if err.errno != errno.ENOENT:
376 if err.errno != errno.EEXIST:
377 raise
370 raise
371 if findprogram(pyexename) != sys.executable:
372 try:
373 os.symlink(sys.executable, mypython)
374 except OSError, err:
375 # child processes may race, which is harmless
376 if err.errno != errno.EEXIST:
377 raise
378 else:
378 else:
379 vlog("# Modifying search path to find %s in '%s'" % (exename, exedir))
379 exedir, exename = os.path.split(sys.executable)
380 vlog("# Modifying search path to find %s as %s in '%s'" %
381 (exename, pyexename, exedir))
380 path = os.environ['PATH'].split(os.pathsep)
382 path = os.environ['PATH'].split(os.pathsep)
381 while exedir in path:
383 while exedir in path:
382 path.remove(exedir)
384 path.remove(exedir)
383 os.environ['PATH'] = os.pathsep.join([exedir] + path)
385 os.environ['PATH'] = os.pathsep.join([exedir] + path)
384 if not findprogram(exename):
386 if not findprogram(pyexename):
385 print "WARNING: Cannot find %s in search path" % exename
387 print "WARNING: Cannot find %s in search path" % pyexename
386
388
387 def installhg(options):
389 def installhg(options):
388 vlog("# Performing temporary installation of HG")
390 vlog("# Performing temporary installation of HG")
389 installerrs = os.path.join("tests", "install.err")
391 installerrs = os.path.join("tests", "install.err")
390 compiler = ''
392 compiler = ''
391 if options.compiler:
393 if options.compiler:
392 compiler = '--compiler ' + options.compiler
394 compiler = '--compiler ' + options.compiler
393 pure = options.pure and "--pure" or ""
395 pure = options.pure and "--pure" or ""
394
396
395 # Run installer in hg root
397 # Run installer in hg root
396 script = os.path.realpath(sys.argv[0])
398 script = os.path.realpath(sys.argv[0])
397 hgroot = os.path.dirname(os.path.dirname(script))
399 hgroot = os.path.dirname(os.path.dirname(script))
398 os.chdir(hgroot)
400 os.chdir(hgroot)
399 nohome = '--home=""'
401 nohome = '--home=""'
400 if os.name == 'nt':
402 if os.name == 'nt':
401 # The --home="" trick works only on OS where os.sep == '/'
403 # The --home="" trick works only on OS where os.sep == '/'
402 # because of a distutils convert_path() fast-path. Avoid it at
404 # because of a distutils convert_path() fast-path. Avoid it at
403 # least on Windows for now, deal with .pydistutils.cfg bugs
405 # least on Windows for now, deal with .pydistutils.cfg bugs
404 # when they happen.
406 # when they happen.
405 nohome = ''
407 nohome = ''
406 cmd = ('%(exe)s setup.py %(pure)s clean --all'
408 cmd = ('%(exe)s setup.py %(pure)s clean --all'
407 ' build %(compiler)s --build-base="%(base)s"'
409 ' build %(compiler)s --build-base="%(base)s"'
408 ' install --force --prefix="%(prefix)s" --install-lib="%(libdir)s"'
410 ' install --force --prefix="%(prefix)s" --install-lib="%(libdir)s"'
409 ' --install-scripts="%(bindir)s" %(nohome)s >%(logfile)s 2>&1'
411 ' --install-scripts="%(bindir)s" %(nohome)s >%(logfile)s 2>&1'
410 % dict(exe=sys.executable, pure=pure, compiler=compiler,
412 % dict(exe=sys.executable, pure=pure, compiler=compiler,
411 base=os.path.join(HGTMP, "build"),
413 base=os.path.join(HGTMP, "build"),
412 prefix=INST, libdir=PYTHONDIR, bindir=BINDIR,
414 prefix=INST, libdir=PYTHONDIR, bindir=BINDIR,
413 nohome=nohome, logfile=installerrs))
415 nohome=nohome, logfile=installerrs))
414 vlog("# Running", cmd)
416 vlog("# Running", cmd)
415 if os.system(cmd) == 0:
417 if os.system(cmd) == 0:
416 if not options.verbose:
418 if not options.verbose:
417 os.remove(installerrs)
419 os.remove(installerrs)
418 else:
420 else:
419 f = open(installerrs)
421 f = open(installerrs)
420 for line in f:
422 for line in f:
421 print line,
423 print line,
422 f.close()
424 f.close()
423 sys.exit(1)
425 sys.exit(1)
424 os.chdir(TESTDIR)
426 os.chdir(TESTDIR)
425
427
426 usecorrectpython()
428 usecorrectpython()
427
429
428 vlog("# Installing dummy diffstat")
430 vlog("# Installing dummy diffstat")
429 f = open(os.path.join(BINDIR, 'diffstat'), 'w')
431 f = open(os.path.join(BINDIR, 'diffstat'), 'w')
430 f.write('#!' + sys.executable + '\n'
432 f.write('#!' + sys.executable + '\n'
431 'import sys\n'
433 'import sys\n'
432 'files = 0\n'
434 'files = 0\n'
433 'for line in sys.stdin:\n'
435 'for line in sys.stdin:\n'
434 ' if line.startswith("diff "):\n'
436 ' if line.startswith("diff "):\n'
435 ' files += 1\n'
437 ' files += 1\n'
436 'sys.stdout.write("files patched: %d\\n" % files)\n')
438 'sys.stdout.write("files patched: %d\\n" % files)\n')
437 f.close()
439 f.close()
438 os.chmod(os.path.join(BINDIR, 'diffstat'), 0700)
440 os.chmod(os.path.join(BINDIR, 'diffstat'), 0700)
439
441
440 if options.py3k_warnings and not options.anycoverage:
442 if options.py3k_warnings and not options.anycoverage:
441 vlog("# Updating hg command to enable Py3k Warnings switch")
443 vlog("# Updating hg command to enable Py3k Warnings switch")
442 f = open(os.path.join(BINDIR, 'hg'), 'r')
444 f = open(os.path.join(BINDIR, 'hg'), 'r')
443 lines = [line.rstrip() for line in f]
445 lines = [line.rstrip() for line in f]
444 lines[0] += ' -3'
446 lines[0] += ' -3'
445 f.close()
447 f.close()
446 f = open(os.path.join(BINDIR, 'hg'), 'w')
448 f = open(os.path.join(BINDIR, 'hg'), 'w')
447 for line in lines:
449 for line in lines:
448 f.write(line + '\n')
450 f.write(line + '\n')
449 f.close()
451 f.close()
450
452
451 hgbat = os.path.join(BINDIR, 'hg.bat')
453 hgbat = os.path.join(BINDIR, 'hg.bat')
452 if os.path.isfile(hgbat):
454 if os.path.isfile(hgbat):
453 # hg.bat expects to be put in bin/scripts while run-tests.py
455 # hg.bat expects to be put in bin/scripts while run-tests.py
454 # installation layout put it in bin/ directly. Fix it
456 # installation layout put it in bin/ directly. Fix it
455 f = open(hgbat, 'rb')
457 f = open(hgbat, 'rb')
456 data = f.read()
458 data = f.read()
457 f.close()
459 f.close()
458 if '"%~dp0..\python" "%~dp0hg" %*' in data:
460 if '"%~dp0..\python" "%~dp0hg" %*' in data:
459 data = data.replace('"%~dp0..\python" "%~dp0hg" %*',
461 data = data.replace('"%~dp0..\python" "%~dp0hg" %*',
460 '"%~dp0python" "%~dp0hg" %*')
462 '"%~dp0python" "%~dp0hg" %*')
461 f = open(hgbat, 'wb')
463 f = open(hgbat, 'wb')
462 f.write(data)
464 f.write(data)
463 f.close()
465 f.close()
464 else:
466 else:
465 print 'WARNING: cannot fix hg.bat reference to python.exe'
467 print 'WARNING: cannot fix hg.bat reference to python.exe'
466
468
467 if options.anycoverage:
469 if options.anycoverage:
468 custom = os.path.join(TESTDIR, 'sitecustomize.py')
470 custom = os.path.join(TESTDIR, 'sitecustomize.py')
469 target = os.path.join(PYTHONDIR, 'sitecustomize.py')
471 target = os.path.join(PYTHONDIR, 'sitecustomize.py')
470 vlog('# Installing coverage trigger to %s' % target)
472 vlog('# Installing coverage trigger to %s' % target)
471 shutil.copyfile(custom, target)
473 shutil.copyfile(custom, target)
472 rc = os.path.join(TESTDIR, '.coveragerc')
474 rc = os.path.join(TESTDIR, '.coveragerc')
473 vlog('# Installing coverage rc to %s' % rc)
475 vlog('# Installing coverage rc to %s' % rc)
474 os.environ['COVERAGE_PROCESS_START'] = rc
476 os.environ['COVERAGE_PROCESS_START'] = rc
475 fn = os.path.join(INST, '..', '.coverage')
477 fn = os.path.join(INST, '..', '.coverage')
476 os.environ['COVERAGE_FILE'] = fn
478 os.environ['COVERAGE_FILE'] = fn
477
479
478 def outputtimes(options):
480 def outputtimes(options):
479 vlog('# Producing time report')
481 vlog('# Producing time report')
480 times.sort(key=lambda t: (t[1], t[0]), reverse=True)
482 times.sort(key=lambda t: (t[1], t[0]), reverse=True)
481 cols = '%7.3f %s'
483 cols = '%7.3f %s'
482 print '\n%-7s %s' % ('Time', 'Test')
484 print '\n%-7s %s' % ('Time', 'Test')
483 for test, timetaken in times:
485 for test, timetaken in times:
484 print cols % (timetaken, test)
486 print cols % (timetaken, test)
485
487
486 def outputcoverage(options):
488 def outputcoverage(options):
487
489
488 vlog('# Producing coverage report')
490 vlog('# Producing coverage report')
489 os.chdir(PYTHONDIR)
491 os.chdir(PYTHONDIR)
490
492
491 def covrun(*args):
493 def covrun(*args):
492 cmd = 'coverage %s' % ' '.join(args)
494 cmd = 'coverage %s' % ' '.join(args)
493 vlog('# Running: %s' % cmd)
495 vlog('# Running: %s' % cmd)
494 os.system(cmd)
496 os.system(cmd)
495
497
496 if options.child:
498 if options.child:
497 return
499 return
498
500
499 covrun('-c')
501 covrun('-c')
500 omit = ','.join(os.path.join(x, '*') for x in [BINDIR, TESTDIR])
502 omit = ','.join(os.path.join(x, '*') for x in [BINDIR, TESTDIR])
501 covrun('-i', '-r', '"--omit=%s"' % omit) # report
503 covrun('-i', '-r', '"--omit=%s"' % omit) # report
502 if options.htmlcov:
504 if options.htmlcov:
503 htmldir = os.path.join(TESTDIR, 'htmlcov')
505 htmldir = os.path.join(TESTDIR, 'htmlcov')
504 covrun('-i', '-b', '"--directory=%s"' % htmldir, '"--omit=%s"' % omit)
506 covrun('-i', '-b', '"--directory=%s"' % htmldir, '"--omit=%s"' % omit)
505 if options.annotate:
507 if options.annotate:
506 adir = os.path.join(TESTDIR, 'annotated')
508 adir = os.path.join(TESTDIR, 'annotated')
507 if not os.path.isdir(adir):
509 if not os.path.isdir(adir):
508 os.mkdir(adir)
510 os.mkdir(adir)
509 covrun('-i', '-a', '"--directory=%s"' % adir, '"--omit=%s"' % omit)
511 covrun('-i', '-a', '"--directory=%s"' % adir, '"--omit=%s"' % omit)
510
512
511 def pytest(test, wd, options, replacements):
513 def pytest(test, wd, options, replacements):
512 py3kswitch = options.py3k_warnings and ' -3' or ''
514 py3kswitch = options.py3k_warnings and ' -3' or ''
513 cmd = '%s%s "%s"' % (PYTHON, py3kswitch, test)
515 cmd = '%s%s "%s"' % (PYTHON, py3kswitch, test)
514 vlog("# Running", cmd)
516 vlog("# Running", cmd)
515 if os.name == 'nt':
517 if os.name == 'nt':
516 replacements.append((r'\r\n', '\n'))
518 replacements.append((r'\r\n', '\n'))
517 return run(cmd, wd, options, replacements)
519 return run(cmd, wd, options, replacements)
518
520
519 needescape = re.compile(r'[\x00-\x08\x0b-\x1f\x7f-\xff]').search
521 needescape = re.compile(r'[\x00-\x08\x0b-\x1f\x7f-\xff]').search
520 escapesub = re.compile(r'[\x00-\x08\x0b-\x1f\\\x7f-\xff]').sub
522 escapesub = re.compile(r'[\x00-\x08\x0b-\x1f\\\x7f-\xff]').sub
521 escapemap = dict((chr(i), r'\x%02x' % i) for i in range(256))
523 escapemap = dict((chr(i), r'\x%02x' % i) for i in range(256))
522 escapemap.update({'\\': '\\\\', '\r': r'\r'})
524 escapemap.update({'\\': '\\\\', '\r': r'\r'})
523 def escapef(m):
525 def escapef(m):
524 return escapemap[m.group(0)]
526 return escapemap[m.group(0)]
525 def stringescape(s):
527 def stringescape(s):
526 return escapesub(escapef, s)
528 return escapesub(escapef, s)
527
529
528 def rematch(el, l):
530 def rematch(el, l):
529 try:
531 try:
530 # use \Z to ensure that the regex matches to the end of the string
532 # use \Z to ensure that the regex matches to the end of the string
531 if os.name == 'nt':
533 if os.name == 'nt':
532 return re.match(el + r'\r?\n\Z', l)
534 return re.match(el + r'\r?\n\Z', l)
533 return re.match(el + r'\n\Z', l)
535 return re.match(el + r'\n\Z', l)
534 except re.error:
536 except re.error:
535 # el is an invalid regex
537 # el is an invalid regex
536 return False
538 return False
537
539
538 def globmatch(el, l):
540 def globmatch(el, l):
539 # The only supported special characters are * and ? plus / which also
541 # The only supported special characters are * and ? plus / which also
540 # matches \ on windows. Escaping of these caracters is supported.
542 # matches \ on windows. Escaping of these caracters is supported.
541 i, n = 0, len(el)
543 i, n = 0, len(el)
542 res = ''
544 res = ''
543 while i < n:
545 while i < n:
544 c = el[i]
546 c = el[i]
545 i += 1
547 i += 1
546 if c == '\\' and el[i] in '*?\\/':
548 if c == '\\' and el[i] in '*?\\/':
547 res += el[i - 1:i + 1]
549 res += el[i - 1:i + 1]
548 i += 1
550 i += 1
549 elif c == '*':
551 elif c == '*':
550 res += '.*'
552 res += '.*'
551 elif c == '?':
553 elif c == '?':
552 res += '.'
554 res += '.'
553 elif c == '/' and os.name == 'nt':
555 elif c == '/' and os.name == 'nt':
554 res += '[/\\\\]'
556 res += '[/\\\\]'
555 else:
557 else:
556 res += re.escape(c)
558 res += re.escape(c)
557 return rematch(res, l)
559 return rematch(res, l)
558
560
559 def linematch(el, l):
561 def linematch(el, l):
560 if el == l: # perfect match (fast)
562 if el == l: # perfect match (fast)
561 return True
563 return True
562 if el:
564 if el:
563 if el.endswith(" (esc)\n"):
565 if el.endswith(" (esc)\n"):
564 el = el[:-7].decode('string-escape') + '\n'
566 el = el[:-7].decode('string-escape') + '\n'
565 if el == l or os.name == 'nt' and el[:-1] + '\r\n' == l:
567 if el == l or os.name == 'nt' and el[:-1] + '\r\n' == l:
566 return True
568 return True
567 if (el.endswith(" (re)\n") and rematch(el[:-6], l) or
569 if (el.endswith(" (re)\n") and rematch(el[:-6], l) or
568 el.endswith(" (glob)\n") and globmatch(el[:-8], l)):
570 el.endswith(" (glob)\n") and globmatch(el[:-8], l)):
569 return True
571 return True
570 return False
572 return False
571
573
572 def tsttest(test, wd, options, replacements):
574 def tsttest(test, wd, options, replacements):
573 # We generate a shell script which outputs unique markers to line
575 # We generate a shell script which outputs unique markers to line
574 # up script results with our source. These markers include input
576 # up script results with our source. These markers include input
575 # line number and the last return code
577 # line number and the last return code
576 salt = "SALT" + str(time.time())
578 salt = "SALT" + str(time.time())
577 def addsalt(line, inpython):
579 def addsalt(line, inpython):
578 if inpython:
580 if inpython:
579 script.append('%s %d 0\n' % (salt, line))
581 script.append('%s %d 0\n' % (salt, line))
580 else:
582 else:
581 script.append('echo %s %s $?\n' % (salt, line))
583 script.append('echo %s %s $?\n' % (salt, line))
582
584
583 # After we run the shell script, we re-unify the script output
585 # After we run the shell script, we re-unify the script output
584 # with non-active parts of the source, with synchronization by our
586 # with non-active parts of the source, with synchronization by our
585 # SALT line number markers. The after table contains the
587 # SALT line number markers. The after table contains the
586 # non-active components, ordered by line number
588 # non-active components, ordered by line number
587 after = {}
589 after = {}
588 pos = prepos = -1
590 pos = prepos = -1
589
591
590 # Expected shellscript output
592 # Expected shellscript output
591 expected = {}
593 expected = {}
592
594
593 # We keep track of whether or not we're in a Python block so we
595 # We keep track of whether or not we're in a Python block so we
594 # can generate the surrounding doctest magic
596 # can generate the surrounding doctest magic
595 inpython = False
597 inpython = False
596
598
597 # True or False when in a true or false conditional section
599 # True or False when in a true or false conditional section
598 skipping = None
600 skipping = None
599
601
600 def hghave(reqs):
602 def hghave(reqs):
601 # TODO: do something smarter when all other uses of hghave is gone
603 # TODO: do something smarter when all other uses of hghave is gone
602 tdir = TESTDIR.replace('\\', '/')
604 tdir = TESTDIR.replace('\\', '/')
603 proc = Popen4('%s -c "%s/hghave %s"' %
605 proc = Popen4('%s -c "%s/hghave %s"' %
604 (options.shell, tdir, ' '.join(reqs)), wd, 0)
606 (options.shell, tdir, ' '.join(reqs)), wd, 0)
605 stdout, stderr = proc.communicate()
607 stdout, stderr = proc.communicate()
606 ret = proc.wait()
608 ret = proc.wait()
607 if wifexited(ret):
609 if wifexited(ret):
608 ret = os.WEXITSTATUS(ret)
610 ret = os.WEXITSTATUS(ret)
609 if ret == 2:
611 if ret == 2:
610 print stdout
612 print stdout
611 sys.exit(1)
613 sys.exit(1)
612 return ret == 0
614 return ret == 0
613
615
614 f = open(test)
616 f = open(test)
615 t = f.readlines()
617 t = f.readlines()
616 f.close()
618 f.close()
617
619
618 script = []
620 script = []
619 if options.debug:
621 if options.debug:
620 script.append('set -x\n')
622 script.append('set -x\n')
621 if os.getenv('MSYSTEM'):
623 if os.getenv('MSYSTEM'):
622 script.append('alias pwd="pwd -W"\n')
624 script.append('alias pwd="pwd -W"\n')
623 for n, l in enumerate(t):
625 for n, l in enumerate(t):
624 if not l.endswith('\n'):
626 if not l.endswith('\n'):
625 l += '\n'
627 l += '\n'
626 if l.startswith('#if'):
628 if l.startswith('#if'):
627 if skipping is not None:
629 if skipping is not None:
628 after.setdefault(pos, []).append(' !!! nested #if\n')
630 after.setdefault(pos, []).append(' !!! nested #if\n')
629 skipping = not hghave(l.split()[1:])
631 skipping = not hghave(l.split()[1:])
630 after.setdefault(pos, []).append(l)
632 after.setdefault(pos, []).append(l)
631 elif l.startswith('#else'):
633 elif l.startswith('#else'):
632 if skipping is None:
634 if skipping is None:
633 after.setdefault(pos, []).append(' !!! missing #if\n')
635 after.setdefault(pos, []).append(' !!! missing #if\n')
634 skipping = not skipping
636 skipping = not skipping
635 after.setdefault(pos, []).append(l)
637 after.setdefault(pos, []).append(l)
636 elif l.startswith('#endif'):
638 elif l.startswith('#endif'):
637 if skipping is None:
639 if skipping is None:
638 after.setdefault(pos, []).append(' !!! missing #if\n')
640 after.setdefault(pos, []).append(' !!! missing #if\n')
639 skipping = None
641 skipping = None
640 after.setdefault(pos, []).append(l)
642 after.setdefault(pos, []).append(l)
641 elif skipping:
643 elif skipping:
642 after.setdefault(pos, []).append(l)
644 after.setdefault(pos, []).append(l)
643 elif l.startswith(' >>> '): # python inlines
645 elif l.startswith(' >>> '): # python inlines
644 after.setdefault(pos, []).append(l)
646 after.setdefault(pos, []).append(l)
645 prepos = pos
647 prepos = pos
646 pos = n
648 pos = n
647 if not inpython:
649 if not inpython:
648 # we've just entered a Python block, add the header
650 # we've just entered a Python block, add the header
649 inpython = True
651 inpython = True
650 addsalt(prepos, False) # make sure we report the exit code
652 addsalt(prepos, False) # make sure we report the exit code
651 script.append('%s -m heredoctest <<EOF\n' % PYTHON)
653 script.append('%s -m heredoctest <<EOF\n' % PYTHON)
652 addsalt(n, True)
654 addsalt(n, True)
653 script.append(l[2:])
655 script.append(l[2:])
654 elif l.startswith(' ... '): # python inlines
656 elif l.startswith(' ... '): # python inlines
655 after.setdefault(prepos, []).append(l)
657 after.setdefault(prepos, []).append(l)
656 script.append(l[2:])
658 script.append(l[2:])
657 elif l.startswith(' $ '): # commands
659 elif l.startswith(' $ '): # commands
658 if inpython:
660 if inpython:
659 script.append("EOF\n")
661 script.append("EOF\n")
660 inpython = False
662 inpython = False
661 after.setdefault(pos, []).append(l)
663 after.setdefault(pos, []).append(l)
662 prepos = pos
664 prepos = pos
663 pos = n
665 pos = n
664 addsalt(n, False)
666 addsalt(n, False)
665 cmd = l[4:].split()
667 cmd = l[4:].split()
666 if len(cmd) == 2 and cmd[0] == 'cd':
668 if len(cmd) == 2 and cmd[0] == 'cd':
667 l = ' $ cd %s || exit 1\n' % cmd[1]
669 l = ' $ cd %s || exit 1\n' % cmd[1]
668 script.append(l[4:])
670 script.append(l[4:])
669 elif l.startswith(' > '): # continuations
671 elif l.startswith(' > '): # continuations
670 after.setdefault(prepos, []).append(l)
672 after.setdefault(prepos, []).append(l)
671 script.append(l[4:])
673 script.append(l[4:])
672 elif l.startswith(' '): # results
674 elif l.startswith(' '): # results
673 # queue up a list of expected results
675 # queue up a list of expected results
674 expected.setdefault(pos, []).append(l[2:])
676 expected.setdefault(pos, []).append(l[2:])
675 else:
677 else:
676 if inpython:
678 if inpython:
677 script.append("EOF\n")
679 script.append("EOF\n")
678 inpython = False
680 inpython = False
679 # non-command/result - queue up for merged output
681 # non-command/result - queue up for merged output
680 after.setdefault(pos, []).append(l)
682 after.setdefault(pos, []).append(l)
681
683
682 if inpython:
684 if inpython:
683 script.append("EOF\n")
685 script.append("EOF\n")
684 if skipping is not None:
686 if skipping is not None:
685 after.setdefault(pos, []).append(' !!! missing #endif\n')
687 after.setdefault(pos, []).append(' !!! missing #endif\n')
686 addsalt(n + 1, False)
688 addsalt(n + 1, False)
687
689
688 # Write out the script and execute it
690 # Write out the script and execute it
689 fd, name = tempfile.mkstemp(suffix='hg-tst')
691 fd, name = tempfile.mkstemp(suffix='hg-tst')
690 try:
692 try:
691 for l in script:
693 for l in script:
692 os.write(fd, l)
694 os.write(fd, l)
693 os.close(fd)
695 os.close(fd)
694
696
695 cmd = '%s "%s"' % (options.shell, name)
697 cmd = '%s "%s"' % (options.shell, name)
696 vlog("# Running", cmd)
698 vlog("# Running", cmd)
697 exitcode, output = run(cmd, wd, options, replacements)
699 exitcode, output = run(cmd, wd, options, replacements)
698 # do not merge output if skipped, return hghave message instead
700 # do not merge output if skipped, return hghave message instead
699 # similarly, with --debug, output is None
701 # similarly, with --debug, output is None
700 if exitcode == SKIPPED_STATUS or output is None:
702 if exitcode == SKIPPED_STATUS or output is None:
701 return exitcode, output
703 return exitcode, output
702 finally:
704 finally:
703 os.remove(name)
705 os.remove(name)
704
706
705 # Merge the script output back into a unified test
707 # Merge the script output back into a unified test
706
708
707 pos = -1
709 pos = -1
708 postout = []
710 postout = []
709 ret = 0
711 ret = 0
710 for l in output:
712 for l in output:
711 lout, lcmd = l, None
713 lout, lcmd = l, None
712 if salt in l:
714 if salt in l:
713 lout, lcmd = l.split(salt, 1)
715 lout, lcmd = l.split(salt, 1)
714
716
715 if lout:
717 if lout:
716 if not lout.endswith('\n'):
718 if not lout.endswith('\n'):
717 lout += ' (no-eol)\n'
719 lout += ' (no-eol)\n'
718
720
719 # find the expected output at the current position
721 # find the expected output at the current position
720 el = None
722 el = None
721 if pos in expected and expected[pos]:
723 if pos in expected and expected[pos]:
722 el = expected[pos].pop(0)
724 el = expected[pos].pop(0)
723
725
724 if linematch(el, lout):
726 if linematch(el, lout):
725 postout.append(" " + el)
727 postout.append(" " + el)
726 else:
728 else:
727 if needescape(lout):
729 if needescape(lout):
728 lout = stringescape(lout.rstrip('\n')) + " (esc)\n"
730 lout = stringescape(lout.rstrip('\n')) + " (esc)\n"
729 postout.append(" " + lout) # let diff deal with it
731 postout.append(" " + lout) # let diff deal with it
730
732
731 if lcmd:
733 if lcmd:
732 # add on last return code
734 # add on last return code
733 ret = int(lcmd.split()[1])
735 ret = int(lcmd.split()[1])
734 if ret != 0:
736 if ret != 0:
735 postout.append(" [%s]\n" % ret)
737 postout.append(" [%s]\n" % ret)
736 if pos in after:
738 if pos in after:
737 # merge in non-active test bits
739 # merge in non-active test bits
738 postout += after.pop(pos)
740 postout += after.pop(pos)
739 pos = int(lcmd.split()[0])
741 pos = int(lcmd.split()[0])
740
742
741 if pos in after:
743 if pos in after:
742 postout += after.pop(pos)
744 postout += after.pop(pos)
743
745
744 return exitcode, postout
746 return exitcode, postout
745
747
746 wifexited = getattr(os, "WIFEXITED", lambda x: False)
748 wifexited = getattr(os, "WIFEXITED", lambda x: False)
747 def run(cmd, wd, options, replacements):
749 def run(cmd, wd, options, replacements):
748 """Run command in a sub-process, capturing the output (stdout and stderr).
750 """Run command in a sub-process, capturing the output (stdout and stderr).
749 Return a tuple (exitcode, output). output is None in debug mode."""
751 Return a tuple (exitcode, output). output is None in debug mode."""
750 # TODO: Use subprocess.Popen if we're running on Python 2.4
752 # TODO: Use subprocess.Popen if we're running on Python 2.4
751 if options.debug:
753 if options.debug:
752 proc = subprocess.Popen(cmd, shell=True, cwd=wd, stdin=subprocess.PIPE)
754 proc = subprocess.Popen(cmd, shell=True, cwd=wd, stdin=subprocess.PIPE)
753 proc.stdin.close()
755 proc.stdin.close()
754 ret = proc.wait()
756 ret = proc.wait()
755 return (ret, None)
757 return (ret, None)
756
758
757 proc = Popen4(cmd, wd, options.timeout)
759 proc = Popen4(cmd, wd, options.timeout)
758 def cleanup():
760 def cleanup():
759 terminate(proc)
761 terminate(proc)
760 ret = proc.wait()
762 ret = proc.wait()
761 if ret == 0:
763 if ret == 0:
762 ret = signal.SIGTERM << 8
764 ret = signal.SIGTERM << 8
763 killdaemons()
765 killdaemons()
764 return ret
766 return ret
765
767
766 output = ''
768 output = ''
767 proc.tochild.close()
769 proc.tochild.close()
768
770
769 try:
771 try:
770 output = proc.fromchild.read()
772 output = proc.fromchild.read()
771 except KeyboardInterrupt:
773 except KeyboardInterrupt:
772 vlog('# Handling keyboard interrupt')
774 vlog('# Handling keyboard interrupt')
773 cleanup()
775 cleanup()
774 raise
776 raise
775
777
776 ret = proc.wait()
778 ret = proc.wait()
777 if wifexited(ret):
779 if wifexited(ret):
778 ret = os.WEXITSTATUS(ret)
780 ret = os.WEXITSTATUS(ret)
779
781
780 if proc.timeout:
782 if proc.timeout:
781 ret = 'timeout'
783 ret = 'timeout'
782
784
783 if ret:
785 if ret:
784 killdaemons()
786 killdaemons()
785
787
786 for s, r in replacements:
788 for s, r in replacements:
787 output = re.sub(s, r, output)
789 output = re.sub(s, r, output)
788 return ret, output.splitlines(True)
790 return ret, output.splitlines(True)
789
791
790 def runone(options, test):
792 def runone(options, test):
791 '''tristate output:
793 '''tristate output:
792 None -> skipped
794 None -> skipped
793 True -> passed
795 True -> passed
794 False -> failed'''
796 False -> failed'''
795
797
796 global results, resultslock, iolock
798 global results, resultslock, iolock
797
799
798 testpath = os.path.join(TESTDIR, test)
800 testpath = os.path.join(TESTDIR, test)
799
801
800 def result(l, e):
802 def result(l, e):
801 resultslock.acquire()
803 resultslock.acquire()
802 results[l].append(e)
804 results[l].append(e)
803 resultslock.release()
805 resultslock.release()
804
806
805 def skip(msg):
807 def skip(msg):
806 if not options.verbose:
808 if not options.verbose:
807 result('s', (test, msg))
809 result('s', (test, msg))
808 else:
810 else:
809 iolock.acquire()
811 iolock.acquire()
810 print "\nSkipping %s: %s" % (testpath, msg)
812 print "\nSkipping %s: %s" % (testpath, msg)
811 iolock.release()
813 iolock.release()
812 return None
814 return None
813
815
814 def fail(msg, ret):
816 def fail(msg, ret):
815 if not options.nodiff:
817 if not options.nodiff:
816 iolock.acquire()
818 iolock.acquire()
817 print "\nERROR: %s %s" % (testpath, msg)
819 print "\nERROR: %s %s" % (testpath, msg)
818 iolock.release()
820 iolock.release()
819 if (not ret and options.interactive
821 if (not ret and options.interactive
820 and os.path.exists(testpath + ".err")):
822 and os.path.exists(testpath + ".err")):
821 iolock.acquire()
823 iolock.acquire()
822 print "Accept this change? [n] ",
824 print "Accept this change? [n] ",
823 answer = sys.stdin.readline().strip()
825 answer = sys.stdin.readline().strip()
824 iolock.release()
826 iolock.release()
825 if answer.lower() in "y yes".split():
827 if answer.lower() in "y yes".split():
826 if test.endswith(".t"):
828 if test.endswith(".t"):
827 rename(testpath + ".err", testpath)
829 rename(testpath + ".err", testpath)
828 else:
830 else:
829 rename(testpath + ".err", testpath + ".out")
831 rename(testpath + ".err", testpath + ".out")
830 result('p', test)
832 result('p', test)
831 return
833 return
832 result('f', (test, msg))
834 result('f', (test, msg))
833
835
834 def success():
836 def success():
835 result('p', test)
837 result('p', test)
836
838
837 def ignore(msg):
839 def ignore(msg):
838 result('i', (test, msg))
840 result('i', (test, msg))
839
841
840 if (os.path.basename(test).startswith("test-") and '~' not in test and
842 if (os.path.basename(test).startswith("test-") and '~' not in test and
841 ('.' not in test or test.endswith('.py') or
843 ('.' not in test or test.endswith('.py') or
842 test.endswith('.bat') or test.endswith('.t'))):
844 test.endswith('.bat') or test.endswith('.t'))):
843 if not os.path.exists(test):
845 if not os.path.exists(test):
844 skip("doesn't exist")
846 skip("doesn't exist")
845 return None
847 return None
846 else:
848 else:
847 vlog('# Test file', test, 'not supported, ignoring')
849 vlog('# Test file', test, 'not supported, ignoring')
848 return None # not a supported test, don't record
850 return None # not a supported test, don't record
849
851
850 if not (options.whitelisted and test in options.whitelisted):
852 if not (options.whitelisted and test in options.whitelisted):
851 if options.blacklist and test in options.blacklist:
853 if options.blacklist and test in options.blacklist:
852 skip("blacklisted")
854 skip("blacklisted")
853 return None
855 return None
854
856
855 if options.retest and not os.path.exists(test + ".err"):
857 if options.retest and not os.path.exists(test + ".err"):
856 ignore("not retesting")
858 ignore("not retesting")
857 return None
859 return None
858
860
859 if options.keywords:
861 if options.keywords:
860 fp = open(test)
862 fp = open(test)
861 t = fp.read().lower() + test.lower()
863 t = fp.read().lower() + test.lower()
862 fp.close()
864 fp.close()
863 for k in options.keywords.lower().split():
865 for k in options.keywords.lower().split():
864 if k in t:
866 if k in t:
865 break
867 break
866 else:
868 else:
867 ignore("doesn't match keyword")
869 ignore("doesn't match keyword")
868 return None
870 return None
869
871
870 vlog("# Test", test)
872 vlog("# Test", test)
871
873
872 # create a fresh hgrc
874 # create a fresh hgrc
873 hgrc = open(HGRCPATH, 'w+')
875 hgrc = open(HGRCPATH, 'w+')
874 hgrc.write('[ui]\n')
876 hgrc.write('[ui]\n')
875 hgrc.write('slash = True\n')
877 hgrc.write('slash = True\n')
876 hgrc.write('[defaults]\n')
878 hgrc.write('[defaults]\n')
877 hgrc.write('backout = -d "0 0"\n')
879 hgrc.write('backout = -d "0 0"\n')
878 hgrc.write('commit = -d "0 0"\n')
880 hgrc.write('commit = -d "0 0"\n')
879 hgrc.write('tag = -d "0 0"\n')
881 hgrc.write('tag = -d "0 0"\n')
880 if options.inotify:
882 if options.inotify:
881 hgrc.write('[extensions]\n')
883 hgrc.write('[extensions]\n')
882 hgrc.write('inotify=\n')
884 hgrc.write('inotify=\n')
883 hgrc.write('[inotify]\n')
885 hgrc.write('[inotify]\n')
884 hgrc.write('pidfile=%s\n' % DAEMON_PIDS)
886 hgrc.write('pidfile=%s\n' % DAEMON_PIDS)
885 hgrc.write('appendpid=True\n')
887 hgrc.write('appendpid=True\n')
886 if options.extra_config_opt:
888 if options.extra_config_opt:
887 for opt in options.extra_config_opt:
889 for opt in options.extra_config_opt:
888 section, key = opt.split('.', 1)
890 section, key = opt.split('.', 1)
889 assert '=' in key, ('extra config opt %s must '
891 assert '=' in key, ('extra config opt %s must '
890 'have an = for assignment' % opt)
892 'have an = for assignment' % opt)
891 hgrc.write('[%s]\n%s\n' % (section, key))
893 hgrc.write('[%s]\n%s\n' % (section, key))
892 hgrc.close()
894 hgrc.close()
893
895
894 ref = os.path.join(TESTDIR, test+".out")
896 ref = os.path.join(TESTDIR, test+".out")
895 err = os.path.join(TESTDIR, test+".err")
897 err = os.path.join(TESTDIR, test+".err")
896 if os.path.exists(err):
898 if os.path.exists(err):
897 os.remove(err) # Remove any previous output files
899 os.remove(err) # Remove any previous output files
898 try:
900 try:
899 tf = open(testpath)
901 tf = open(testpath)
900 firstline = tf.readline().rstrip()
902 firstline = tf.readline().rstrip()
901 tf.close()
903 tf.close()
902 except IOError:
904 except IOError:
903 firstline = ''
905 firstline = ''
904 lctest = test.lower()
906 lctest = test.lower()
905
907
906 if lctest.endswith('.py') or firstline == '#!/usr/bin/env python':
908 if lctest.endswith('.py') or firstline == '#!/usr/bin/env python':
907 runner = pytest
909 runner = pytest
908 elif lctest.endswith('.t'):
910 elif lctest.endswith('.t'):
909 runner = tsttest
911 runner = tsttest
910 ref = testpath
912 ref = testpath
911 else:
913 else:
912 return skip("unknown test type")
914 return skip("unknown test type")
913
915
914 # Make a tmp subdirectory to work in
916 # Make a tmp subdirectory to work in
915 testtmp = os.environ["TESTTMP"] = os.environ["HOME"] = \
917 testtmp = os.environ["TESTTMP"] = os.environ["HOME"] = \
916 os.path.join(HGTMP, os.path.basename(test))
918 os.path.join(HGTMP, os.path.basename(test))
917
919
918 replacements = [
920 replacements = [
919 (r':%s\b' % options.port, ':$HGPORT'),
921 (r':%s\b' % options.port, ':$HGPORT'),
920 (r':%s\b' % (options.port + 1), ':$HGPORT1'),
922 (r':%s\b' % (options.port + 1), ':$HGPORT1'),
921 (r':%s\b' % (options.port + 2), ':$HGPORT2'),
923 (r':%s\b' % (options.port + 2), ':$HGPORT2'),
922 ]
924 ]
923 if os.name == 'nt':
925 if os.name == 'nt':
924 replacements.append(
926 replacements.append(
925 (''.join(c.isalpha() and '[%s%s]' % (c.lower(), c.upper()) or
927 (''.join(c.isalpha() and '[%s%s]' % (c.lower(), c.upper()) or
926 c in '/\\' and r'[/\\]' or
928 c in '/\\' and r'[/\\]' or
927 c.isdigit() and c or
929 c.isdigit() and c or
928 '\\' + c
930 '\\' + c
929 for c in testtmp), '$TESTTMP'))
931 for c in testtmp), '$TESTTMP'))
930 else:
932 else:
931 replacements.append((re.escape(testtmp), '$TESTTMP'))
933 replacements.append((re.escape(testtmp), '$TESTTMP'))
932
934
933 os.mkdir(testtmp)
935 os.mkdir(testtmp)
934 if options.time:
936 if options.time:
935 starttime = time.time()
937 starttime = time.time()
936 ret, out = runner(testpath, testtmp, options, replacements)
938 ret, out = runner(testpath, testtmp, options, replacements)
937 if options.time:
939 if options.time:
938 endtime = time.time()
940 endtime = time.time()
939 times.append((test, endtime - starttime))
941 times.append((test, endtime - starttime))
940 vlog("# Ret was:", ret)
942 vlog("# Ret was:", ret)
941
943
942 killdaemons()
944 killdaemons()
943
945
944 mark = '.'
946 mark = '.'
945
947
946 skipped = (ret == SKIPPED_STATUS)
948 skipped = (ret == SKIPPED_STATUS)
947
949
948 # If we're not in --debug mode and reference output file exists,
950 # If we're not in --debug mode and reference output file exists,
949 # check test output against it.
951 # check test output against it.
950 if options.debug:
952 if options.debug:
951 refout = None # to match "out is None"
953 refout = None # to match "out is None"
952 elif os.path.exists(ref):
954 elif os.path.exists(ref):
953 f = open(ref, "r")
955 f = open(ref, "r")
954 refout = f.read().splitlines(True)
956 refout = f.read().splitlines(True)
955 f.close()
957 f.close()
956 else:
958 else:
957 refout = []
959 refout = []
958
960
959 if (ret != 0 or out != refout) and not skipped and not options.debug:
961 if (ret != 0 or out != refout) and not skipped and not options.debug:
960 # Save errors to a file for diagnosis
962 # Save errors to a file for diagnosis
961 f = open(err, "wb")
963 f = open(err, "wb")
962 for line in out:
964 for line in out:
963 f.write(line)
965 f.write(line)
964 f.close()
966 f.close()
965
967
966 def describe(ret):
968 def describe(ret):
967 if ret < 0:
969 if ret < 0:
968 return 'killed by signal %d' % -ret
970 return 'killed by signal %d' % -ret
969 return 'returned error code %d' % ret
971 return 'returned error code %d' % ret
970
972
971 if skipped:
973 if skipped:
972 mark = 's'
974 mark = 's'
973 if out is None: # debug mode: nothing to parse
975 if out is None: # debug mode: nothing to parse
974 missing = ['unknown']
976 missing = ['unknown']
975 failed = None
977 failed = None
976 else:
978 else:
977 missing, failed = parsehghaveoutput(out)
979 missing, failed = parsehghaveoutput(out)
978 if not missing:
980 if not missing:
979 missing = ['irrelevant']
981 missing = ['irrelevant']
980 if failed:
982 if failed:
981 fail("hghave failed checking for %s" % failed[-1], ret)
983 fail("hghave failed checking for %s" % failed[-1], ret)
982 skipped = False
984 skipped = False
983 else:
985 else:
984 skip(missing[-1])
986 skip(missing[-1])
985 elif ret == 'timeout':
987 elif ret == 'timeout':
986 mark = 't'
988 mark = 't'
987 fail("timed out", ret)
989 fail("timed out", ret)
988 elif out != refout:
990 elif out != refout:
989 mark = '!'
991 mark = '!'
990 if not options.nodiff:
992 if not options.nodiff:
991 iolock.acquire()
993 iolock.acquire()
992 if options.view:
994 if options.view:
993 os.system("%s %s %s" % (options.view, ref, err))
995 os.system("%s %s %s" % (options.view, ref, err))
994 else:
996 else:
995 showdiff(refout, out, ref, err)
997 showdiff(refout, out, ref, err)
996 iolock.release()
998 iolock.release()
997 if ret:
999 if ret:
998 fail("output changed and " + describe(ret), ret)
1000 fail("output changed and " + describe(ret), ret)
999 else:
1001 else:
1000 fail("output changed", ret)
1002 fail("output changed", ret)
1001 ret = 1
1003 ret = 1
1002 elif ret:
1004 elif ret:
1003 mark = '!'
1005 mark = '!'
1004 fail(describe(ret), ret)
1006 fail(describe(ret), ret)
1005 else:
1007 else:
1006 success()
1008 success()
1007
1009
1008 if not options.verbose:
1010 if not options.verbose:
1009 iolock.acquire()
1011 iolock.acquire()
1010 sys.stdout.write(mark)
1012 sys.stdout.write(mark)
1011 sys.stdout.flush()
1013 sys.stdout.flush()
1012 iolock.release()
1014 iolock.release()
1013
1015
1014 if not options.keep_tmpdir:
1016 if not options.keep_tmpdir:
1015 shutil.rmtree(testtmp, True)
1017 shutil.rmtree(testtmp, True)
1016 if skipped:
1018 if skipped:
1017 return None
1019 return None
1018 return ret == 0
1020 return ret == 0
1019
1021
1020 _hgpath = None
1022 _hgpath = None
1021
1023
1022 def _gethgpath():
1024 def _gethgpath():
1023 """Return the path to the mercurial package that is actually found by
1025 """Return the path to the mercurial package that is actually found by
1024 the current Python interpreter."""
1026 the current Python interpreter."""
1025 global _hgpath
1027 global _hgpath
1026 if _hgpath is not None:
1028 if _hgpath is not None:
1027 return _hgpath
1029 return _hgpath
1028
1030
1029 cmd = '%s -c "import mercurial; print mercurial.__path__[0]"'
1031 cmd = '%s -c "import mercurial; print mercurial.__path__[0]"'
1030 pipe = os.popen(cmd % PYTHON)
1032 pipe = os.popen(cmd % PYTHON)
1031 try:
1033 try:
1032 _hgpath = pipe.read().strip()
1034 _hgpath = pipe.read().strip()
1033 finally:
1035 finally:
1034 pipe.close()
1036 pipe.close()
1035 return _hgpath
1037 return _hgpath
1036
1038
1037 def _checkhglib(verb):
1039 def _checkhglib(verb):
1038 """Ensure that the 'mercurial' package imported by python is
1040 """Ensure that the 'mercurial' package imported by python is
1039 the one we expect it to be. If not, print a warning to stderr."""
1041 the one we expect it to be. If not, print a warning to stderr."""
1040 expecthg = os.path.join(PYTHONDIR, 'mercurial')
1042 expecthg = os.path.join(PYTHONDIR, 'mercurial')
1041 actualhg = _gethgpath()
1043 actualhg = _gethgpath()
1042 if os.path.abspath(actualhg) != os.path.abspath(expecthg):
1044 if os.path.abspath(actualhg) != os.path.abspath(expecthg):
1043 sys.stderr.write('warning: %s with unexpected mercurial lib: %s\n'
1045 sys.stderr.write('warning: %s with unexpected mercurial lib: %s\n'
1044 ' (expected %s)\n'
1046 ' (expected %s)\n'
1045 % (verb, actualhg, expecthg))
1047 % (verb, actualhg, expecthg))
1046
1048
1047 def runchildren(options, tests):
1049 def runchildren(options, tests):
1048 if INST:
1050 if INST:
1049 installhg(options)
1051 installhg(options)
1050 _checkhglib("Testing")
1052 _checkhglib("Testing")
1051 else:
1053 else:
1052 usecorrectpython()
1054 usecorrectpython()
1053
1055
1054 optcopy = dict(options.__dict__)
1056 optcopy = dict(options.__dict__)
1055 optcopy['jobs'] = 1
1057 optcopy['jobs'] = 1
1056
1058
1057 # Because whitelist has to override keyword matches, we have to
1059 # Because whitelist has to override keyword matches, we have to
1058 # actually load the whitelist in the children as well, so we allow
1060 # actually load the whitelist in the children as well, so we allow
1059 # the list of whitelist files to pass through and be parsed in the
1061 # the list of whitelist files to pass through and be parsed in the
1060 # children, but not the dict of whitelisted tests resulting from
1062 # children, but not the dict of whitelisted tests resulting from
1061 # the parse, used here to override blacklisted tests.
1063 # the parse, used here to override blacklisted tests.
1062 whitelist = optcopy['whitelisted'] or []
1064 whitelist = optcopy['whitelisted'] or []
1063 del optcopy['whitelisted']
1065 del optcopy['whitelisted']
1064
1066
1065 blacklist = optcopy['blacklist'] or []
1067 blacklist = optcopy['blacklist'] or []
1066 del optcopy['blacklist']
1068 del optcopy['blacklist']
1067 blacklisted = []
1069 blacklisted = []
1068
1070
1069 if optcopy['with_hg'] is None:
1071 if optcopy['with_hg'] is None:
1070 optcopy['with_hg'] = os.path.join(BINDIR, "hg")
1072 optcopy['with_hg'] = os.path.join(BINDIR, "hg")
1071 optcopy.pop('anycoverage', None)
1073 optcopy.pop('anycoverage', None)
1072
1074
1073 opts = []
1075 opts = []
1074 for opt, value in optcopy.iteritems():
1076 for opt, value in optcopy.iteritems():
1075 name = '--' + opt.replace('_', '-')
1077 name = '--' + opt.replace('_', '-')
1076 if value is True:
1078 if value is True:
1077 opts.append(name)
1079 opts.append(name)
1078 elif isinstance(value, list):
1080 elif isinstance(value, list):
1079 for v in value:
1081 for v in value:
1080 opts.append(name + '=' + str(v))
1082 opts.append(name + '=' + str(v))
1081 elif value is not None:
1083 elif value is not None:
1082 opts.append(name + '=' + str(value))
1084 opts.append(name + '=' + str(value))
1083
1085
1084 tests.reverse()
1086 tests.reverse()
1085 jobs = [[] for j in xrange(options.jobs)]
1087 jobs = [[] for j in xrange(options.jobs)]
1086 while tests:
1088 while tests:
1087 for job in jobs:
1089 for job in jobs:
1088 if not tests:
1090 if not tests:
1089 break
1091 break
1090 test = tests.pop()
1092 test = tests.pop()
1091 if test not in whitelist and test in blacklist:
1093 if test not in whitelist and test in blacklist:
1092 blacklisted.append(test)
1094 blacklisted.append(test)
1093 else:
1095 else:
1094 job.append(test)
1096 job.append(test)
1095
1097
1096 waitq = queue.Queue()
1098 waitq = queue.Queue()
1097
1099
1098 # windows lacks os.wait, so we must emulate it
1100 # windows lacks os.wait, so we must emulate it
1099 def waitfor(proc, rfd):
1101 def waitfor(proc, rfd):
1100 fp = os.fdopen(rfd, 'rb')
1102 fp = os.fdopen(rfd, 'rb')
1101 return lambda: waitq.put((proc.pid, proc.wait(), fp))
1103 return lambda: waitq.put((proc.pid, proc.wait(), fp))
1102
1104
1103 for j, job in enumerate(jobs):
1105 for j, job in enumerate(jobs):
1104 if not job:
1106 if not job:
1105 continue
1107 continue
1106 rfd, wfd = os.pipe()
1108 rfd, wfd = os.pipe()
1107 childopts = ['--child=%d' % wfd, '--port=%d' % (options.port + j * 3)]
1109 childopts = ['--child=%d' % wfd, '--port=%d' % (options.port + j * 3)]
1108 childtmp = os.path.join(HGTMP, 'child%d' % j)
1110 childtmp = os.path.join(HGTMP, 'child%d' % j)
1109 childopts += ['--tmpdir', childtmp]
1111 childopts += ['--tmpdir', childtmp]
1110 cmdline = [PYTHON, sys.argv[0]] + opts + childopts + job
1112 cmdline = [PYTHON, sys.argv[0]] + opts + childopts + job
1111 vlog(' '.join(cmdline))
1113 vlog(' '.join(cmdline))
1112 proc = subprocess.Popen(cmdline, executable=cmdline[0])
1114 proc = subprocess.Popen(cmdline, executable=cmdline[0])
1113 threading.Thread(target=waitfor(proc, rfd)).start()
1115 threading.Thread(target=waitfor(proc, rfd)).start()
1114 os.close(wfd)
1116 os.close(wfd)
1115 signal.signal(signal.SIGINT, signal.SIG_IGN)
1117 signal.signal(signal.SIGINT, signal.SIG_IGN)
1116 failures = 0
1118 failures = 0
1117 passed, skipped, failed = 0, 0, 0
1119 passed, skipped, failed = 0, 0, 0
1118 skips = []
1120 skips = []
1119 fails = []
1121 fails = []
1120 for job in jobs:
1122 for job in jobs:
1121 if not job:
1123 if not job:
1122 continue
1124 continue
1123 pid, status, fp = waitq.get()
1125 pid, status, fp = waitq.get()
1124 try:
1126 try:
1125 childresults = pickle.load(fp)
1127 childresults = pickle.load(fp)
1126 except pickle.UnpicklingError:
1128 except pickle.UnpicklingError:
1127 pass
1129 pass
1128 else:
1130 else:
1129 passed += len(childresults['p'])
1131 passed += len(childresults['p'])
1130 skipped += len(childresults['s'])
1132 skipped += len(childresults['s'])
1131 failed += len(childresults['f'])
1133 failed += len(childresults['f'])
1132 skips.extend(childresults['s'])
1134 skips.extend(childresults['s'])
1133 fails.extend(childresults['f'])
1135 fails.extend(childresults['f'])
1134 if options.time:
1136 if options.time:
1135 childtimes = pickle.load(fp)
1137 childtimes = pickle.load(fp)
1136 times.extend(childtimes)
1138 times.extend(childtimes)
1137
1139
1138 vlog('pid %d exited, status %d' % (pid, status))
1140 vlog('pid %d exited, status %d' % (pid, status))
1139 failures |= status
1141 failures |= status
1140 print
1142 print
1141 skipped += len(blacklisted)
1143 skipped += len(blacklisted)
1142 if not options.noskips:
1144 if not options.noskips:
1143 for s in skips:
1145 for s in skips:
1144 print "Skipped %s: %s" % (s[0], s[1])
1146 print "Skipped %s: %s" % (s[0], s[1])
1145 for s in blacklisted:
1147 for s in blacklisted:
1146 print "Skipped %s: blacklisted" % s
1148 print "Skipped %s: blacklisted" % s
1147 for s in fails:
1149 for s in fails:
1148 print "Failed %s: %s" % (s[0], s[1])
1150 print "Failed %s: %s" % (s[0], s[1])
1149
1151
1150 _checkhglib("Tested")
1152 _checkhglib("Tested")
1151 print "# Ran %d tests, %d skipped, %d failed." % (
1153 print "# Ran %d tests, %d skipped, %d failed." % (
1152 passed + failed, skipped, failed)
1154 passed + failed, skipped, failed)
1153
1155
1154 if options.time:
1156 if options.time:
1155 outputtimes(options)
1157 outputtimes(options)
1156 if options.anycoverage:
1158 if options.anycoverage:
1157 outputcoverage(options)
1159 outputcoverage(options)
1158 sys.exit(failures != 0)
1160 sys.exit(failures != 0)
1159
1161
1160 results = dict(p=[], f=[], s=[], i=[])
1162 results = dict(p=[], f=[], s=[], i=[])
1161 resultslock = threading.Lock()
1163 resultslock = threading.Lock()
1162 times = []
1164 times = []
1163 iolock = threading.Lock()
1165 iolock = threading.Lock()
1164
1166
1165 def runqueue(options, tests):
1167 def runqueue(options, tests):
1166 for test in tests:
1168 for test in tests:
1167 ret = runone(options, test)
1169 ret = runone(options, test)
1168 if options.first and ret is not None and not ret:
1170 if options.first and ret is not None and not ret:
1169 break
1171 break
1170
1172
1171 def runtests(options, tests):
1173 def runtests(options, tests):
1172 global DAEMON_PIDS, HGRCPATH
1174 global DAEMON_PIDS, HGRCPATH
1173 DAEMON_PIDS = os.environ["DAEMON_PIDS"] = os.path.join(HGTMP, 'daemon.pids')
1175 DAEMON_PIDS = os.environ["DAEMON_PIDS"] = os.path.join(HGTMP, 'daemon.pids')
1174 HGRCPATH = os.environ["HGRCPATH"] = os.path.join(HGTMP, '.hgrc')
1176 HGRCPATH = os.environ["HGRCPATH"] = os.path.join(HGTMP, '.hgrc')
1175
1177
1176 try:
1178 try:
1177 if INST:
1179 if INST:
1178 installhg(options)
1180 installhg(options)
1179 _checkhglib("Testing")
1181 _checkhglib("Testing")
1180 else:
1182 else:
1181 usecorrectpython()
1183 usecorrectpython()
1182
1184
1183 if options.restart:
1185 if options.restart:
1184 orig = list(tests)
1186 orig = list(tests)
1185 while tests:
1187 while tests:
1186 if os.path.exists(tests[0] + ".err"):
1188 if os.path.exists(tests[0] + ".err"):
1187 break
1189 break
1188 tests.pop(0)
1190 tests.pop(0)
1189 if not tests:
1191 if not tests:
1190 print "running all tests"
1192 print "running all tests"
1191 tests = orig
1193 tests = orig
1192
1194
1193 runqueue(options, tests)
1195 runqueue(options, tests)
1194
1196
1195 failed = len(results['f'])
1197 failed = len(results['f'])
1196 tested = len(results['p']) + failed
1198 tested = len(results['p']) + failed
1197 skipped = len(results['s'])
1199 skipped = len(results['s'])
1198 ignored = len(results['i'])
1200 ignored = len(results['i'])
1199
1201
1200 if options.child:
1202 if options.child:
1201 fp = os.fdopen(options.child, 'wb')
1203 fp = os.fdopen(options.child, 'wb')
1202 pickle.dump(results, fp, pickle.HIGHEST_PROTOCOL)
1204 pickle.dump(results, fp, pickle.HIGHEST_PROTOCOL)
1203 if options.time:
1205 if options.time:
1204 pickle.dump(times, fp, pickle.HIGHEST_PROTOCOL)
1206 pickle.dump(times, fp, pickle.HIGHEST_PROTOCOL)
1205 fp.close()
1207 fp.close()
1206 else:
1208 else:
1207 print
1209 print
1208 for s in results['s']:
1210 for s in results['s']:
1209 print "Skipped %s: %s" % s
1211 print "Skipped %s: %s" % s
1210 for s in results['f']:
1212 for s in results['f']:
1211 print "Failed %s: %s" % s
1213 print "Failed %s: %s" % s
1212 _checkhglib("Tested")
1214 _checkhglib("Tested")
1213 print "# Ran %d tests, %d skipped, %d failed." % (
1215 print "# Ran %d tests, %d skipped, %d failed." % (
1214 tested, skipped + ignored, failed)
1216 tested, skipped + ignored, failed)
1215 if options.time:
1217 if options.time:
1216 outputtimes(options)
1218 outputtimes(options)
1217
1219
1218 if options.anycoverage:
1220 if options.anycoverage:
1219 outputcoverage(options)
1221 outputcoverage(options)
1220 except KeyboardInterrupt:
1222 except KeyboardInterrupt:
1221 failed = True
1223 failed = True
1222 print "\ninterrupted!"
1224 print "\ninterrupted!"
1223
1225
1224 if failed:
1226 if failed:
1225 sys.exit(1)
1227 sys.exit(1)
1226
1228
1227 def main():
1229 def main():
1228 (options, args) = parseargs()
1230 (options, args) = parseargs()
1229 if not options.child:
1231 if not options.child:
1230 os.umask(022)
1232 os.umask(022)
1231
1233
1232 checktools()
1234 checktools()
1233
1235
1234 if len(args) == 0:
1236 if len(args) == 0:
1235 args = os.listdir(".")
1237 args = os.listdir(".")
1236 args.sort()
1238 args.sort()
1237
1239
1238 tests = args
1240 tests = args
1239
1241
1240 # Reset some environment variables to well-known values so that
1242 # Reset some environment variables to well-known values so that
1241 # the tests produce repeatable output.
1243 # the tests produce repeatable output.
1242 os.environ['LANG'] = os.environ['LC_ALL'] = os.environ['LANGUAGE'] = 'C'
1244 os.environ['LANG'] = os.environ['LC_ALL'] = os.environ['LANGUAGE'] = 'C'
1243 os.environ['TZ'] = 'GMT'
1245 os.environ['TZ'] = 'GMT'
1244 os.environ["EMAIL"] = "Foo Bar <foo.bar@example.com>"
1246 os.environ["EMAIL"] = "Foo Bar <foo.bar@example.com>"
1245 os.environ['CDPATH'] = ''
1247 os.environ['CDPATH'] = ''
1246 os.environ['COLUMNS'] = '80'
1248 os.environ['COLUMNS'] = '80'
1247 os.environ['GREP_OPTIONS'] = ''
1249 os.environ['GREP_OPTIONS'] = ''
1248 os.environ['http_proxy'] = ''
1250 os.environ['http_proxy'] = ''
1249 os.environ['no_proxy'] = ''
1251 os.environ['no_proxy'] = ''
1250 os.environ['NO_PROXY'] = ''
1252 os.environ['NO_PROXY'] = ''
1251 os.environ['TERM'] = 'xterm'
1253 os.environ['TERM'] = 'xterm'
1252
1254
1253 # unset env related to hooks
1255 # unset env related to hooks
1254 for k in os.environ.keys():
1256 for k in os.environ.keys():
1255 if k.startswith('HG_'):
1257 if k.startswith('HG_'):
1256 # can't remove on solaris
1258 # can't remove on solaris
1257 os.environ[k] = ''
1259 os.environ[k] = ''
1258 del os.environ[k]
1260 del os.environ[k]
1259 if 'HG' in os.environ:
1261 if 'HG' in os.environ:
1260 # can't remove on solaris
1262 # can't remove on solaris
1261 os.environ['HG'] = ''
1263 os.environ['HG'] = ''
1262 del os.environ['HG']
1264 del os.environ['HG']
1263
1265
1264 global TESTDIR, HGTMP, INST, BINDIR, PYTHONDIR, COVERAGE_FILE
1266 global TESTDIR, HGTMP, INST, BINDIR, PYTHONDIR, COVERAGE_FILE
1265 TESTDIR = os.environ["TESTDIR"] = os.getcwd()
1267 TESTDIR = os.environ["TESTDIR"] = os.getcwd()
1266 if options.tmpdir:
1268 if options.tmpdir:
1267 options.keep_tmpdir = True
1269 options.keep_tmpdir = True
1268 tmpdir = options.tmpdir
1270 tmpdir = options.tmpdir
1269 if os.path.exists(tmpdir):
1271 if os.path.exists(tmpdir):
1270 # Meaning of tmpdir has changed since 1.3: we used to create
1272 # Meaning of tmpdir has changed since 1.3: we used to create
1271 # HGTMP inside tmpdir; now HGTMP is tmpdir. So fail if
1273 # HGTMP inside tmpdir; now HGTMP is tmpdir. So fail if
1272 # tmpdir already exists.
1274 # tmpdir already exists.
1273 sys.exit("error: temp dir %r already exists" % tmpdir)
1275 sys.exit("error: temp dir %r already exists" % tmpdir)
1274
1276
1275 # Automatically removing tmpdir sounds convenient, but could
1277 # Automatically removing tmpdir sounds convenient, but could
1276 # really annoy anyone in the habit of using "--tmpdir=/tmp"
1278 # really annoy anyone in the habit of using "--tmpdir=/tmp"
1277 # or "--tmpdir=$HOME".
1279 # or "--tmpdir=$HOME".
1278 #vlog("# Removing temp dir", tmpdir)
1280 #vlog("# Removing temp dir", tmpdir)
1279 #shutil.rmtree(tmpdir)
1281 #shutil.rmtree(tmpdir)
1280 os.makedirs(tmpdir)
1282 os.makedirs(tmpdir)
1281 else:
1283 else:
1282 d = None
1284 d = None
1283 if os.name == 'nt':
1285 if os.name == 'nt':
1284 # without this, we get the default temp dir location, but
1286 # without this, we get the default temp dir location, but
1285 # in all lowercase, which causes troubles with paths (issue3490)
1287 # in all lowercase, which causes troubles with paths (issue3490)
1286 d = os.getenv('TMP')
1288 d = os.getenv('TMP')
1287 tmpdir = tempfile.mkdtemp('', 'hgtests.', d)
1289 tmpdir = tempfile.mkdtemp('', 'hgtests.', d)
1288 HGTMP = os.environ['HGTMP'] = os.path.realpath(tmpdir)
1290 HGTMP = os.environ['HGTMP'] = os.path.realpath(tmpdir)
1289 DAEMON_PIDS = None
1291 DAEMON_PIDS = None
1290 HGRCPATH = None
1292 HGRCPATH = None
1291
1293
1292 os.environ["HGEDITOR"] = sys.executable + ' -c "import sys; sys.exit(0)"'
1294 os.environ["HGEDITOR"] = sys.executable + ' -c "import sys; sys.exit(0)"'
1293 os.environ["HGMERGE"] = "internal:merge"
1295 os.environ["HGMERGE"] = "internal:merge"
1294 os.environ["HGUSER"] = "test"
1296 os.environ["HGUSER"] = "test"
1295 os.environ["HGENCODING"] = "ascii"
1297 os.environ["HGENCODING"] = "ascii"
1296 os.environ["HGENCODINGMODE"] = "strict"
1298 os.environ["HGENCODINGMODE"] = "strict"
1297 os.environ["HGPORT"] = str(options.port)
1299 os.environ["HGPORT"] = str(options.port)
1298 os.environ["HGPORT1"] = str(options.port + 1)
1300 os.environ["HGPORT1"] = str(options.port + 1)
1299 os.environ["HGPORT2"] = str(options.port + 2)
1301 os.environ["HGPORT2"] = str(options.port + 2)
1300
1302
1301 if options.with_hg:
1303 if options.with_hg:
1302 INST = None
1304 INST = None
1303 BINDIR = os.path.dirname(os.path.realpath(options.with_hg))
1305 BINDIR = os.path.dirname(os.path.realpath(options.with_hg))
1304
1306
1305 # This looks redundant with how Python initializes sys.path from
1307 # This looks redundant with how Python initializes sys.path from
1306 # the location of the script being executed. Needed because the
1308 # the location of the script being executed. Needed because the
1307 # "hg" specified by --with-hg is not the only Python script
1309 # "hg" specified by --with-hg is not the only Python script
1308 # executed in the test suite that needs to import 'mercurial'
1310 # executed in the test suite that needs to import 'mercurial'
1309 # ... which means it's not really redundant at all.
1311 # ... which means it's not really redundant at all.
1310 PYTHONDIR = BINDIR
1312 PYTHONDIR = BINDIR
1311 else:
1313 else:
1312 INST = os.path.join(HGTMP, "install")
1314 INST = os.path.join(HGTMP, "install")
1313 BINDIR = os.environ["BINDIR"] = os.path.join(INST, "bin")
1315 BINDIR = os.environ["BINDIR"] = os.path.join(INST, "bin")
1314 PYTHONDIR = os.path.join(INST, "lib", "python")
1316 PYTHONDIR = os.path.join(INST, "lib", "python")
1315
1317
1316 os.environ["BINDIR"] = BINDIR
1318 os.environ["BINDIR"] = BINDIR
1317 os.environ["PYTHON"] = PYTHON
1319 os.environ["PYTHON"] = PYTHON
1318
1320
1319 if not options.child:
1321 if not options.child:
1320 path = [BINDIR] + os.environ["PATH"].split(os.pathsep)
1322 path = [BINDIR] + os.environ["PATH"].split(os.pathsep)
1321 os.environ["PATH"] = os.pathsep.join(path)
1323 os.environ["PATH"] = os.pathsep.join(path)
1322
1324
1323 # Include TESTDIR in PYTHONPATH so that out-of-tree extensions
1325 # Include TESTDIR in PYTHONPATH so that out-of-tree extensions
1324 # can run .../tests/run-tests.py test-foo where test-foo
1326 # can run .../tests/run-tests.py test-foo where test-foo
1325 # adds an extension to HGRC
1327 # adds an extension to HGRC
1326 pypath = [PYTHONDIR, TESTDIR]
1328 pypath = [PYTHONDIR, TESTDIR]
1327 # We have to augment PYTHONPATH, rather than simply replacing
1329 # We have to augment PYTHONPATH, rather than simply replacing
1328 # it, in case external libraries are only available via current
1330 # it, in case external libraries are only available via current
1329 # PYTHONPATH. (In particular, the Subversion bindings on OS X
1331 # PYTHONPATH. (In particular, the Subversion bindings on OS X
1330 # are in /opt/subversion.)
1332 # are in /opt/subversion.)
1331 oldpypath = os.environ.get(IMPL_PATH)
1333 oldpypath = os.environ.get(IMPL_PATH)
1332 if oldpypath:
1334 if oldpypath:
1333 pypath.append(oldpypath)
1335 pypath.append(oldpypath)
1334 os.environ[IMPL_PATH] = os.pathsep.join(pypath)
1336 os.environ[IMPL_PATH] = os.pathsep.join(pypath)
1335
1337
1336 COVERAGE_FILE = os.path.join(TESTDIR, ".coverage")
1338 COVERAGE_FILE = os.path.join(TESTDIR, ".coverage")
1337
1339
1338 vlog("# Using TESTDIR", TESTDIR)
1340 vlog("# Using TESTDIR", TESTDIR)
1339 vlog("# Using HGTMP", HGTMP)
1341 vlog("# Using HGTMP", HGTMP)
1340 vlog("# Using PATH", os.environ["PATH"])
1342 vlog("# Using PATH", os.environ["PATH"])
1341 vlog("# Using", IMPL_PATH, os.environ[IMPL_PATH])
1343 vlog("# Using", IMPL_PATH, os.environ[IMPL_PATH])
1342
1344
1343 try:
1345 try:
1344 if len(tests) > 1 and options.jobs > 1:
1346 if len(tests) > 1 and options.jobs > 1:
1345 runchildren(options, tests)
1347 runchildren(options, tests)
1346 else:
1348 else:
1347 runtests(options, tests)
1349 runtests(options, tests)
1348 finally:
1350 finally:
1349 time.sleep(.1)
1351 time.sleep(.1)
1350 cleanup(options)
1352 cleanup(options)
1351
1353
1352 if __name__ == '__main__':
1354 if __name__ == '__main__':
1353 main()
1355 main()
General Comments 0
You need to be logged in to leave comments. Login now