##// END OF EJS Templates
upstream » mercurial-mirror
RSS

Clone from https://www.mercurial-scm.org/repo/hg-all

testrunner: add option to sort tests by previous run time...
Martin von Zweigbergk -
r36683:6276cbc7 default
parent child Browse files
Show More
Show file before | Show file after | Hide comments
@@ -1,3095 +1,3109 b''
1 1 #!/usr/bin/env python
2 2 #
3 3 # run-tests.py - Run a set of tests on Mercurial
4 4 #
5 5 # Copyright 2006 Matt Mackall <mpm@selenic.com>
6 6 #
7 7 # This software may be used and distributed according to the terms of the
8 8 # GNU General Public License version 2 or any later version.
9 9
10 10 # Modifying this script is tricky because it has many modes:
11 11 # - serial (default) vs parallel (-jN, N > 1)
12 12 # - no coverage (default) vs coverage (-c, -C, -s)
13 13 # - temp install (default) vs specific hg script (--with-hg, --local)
14 14 # - tests are a mix of shell scripts and Python scripts
15 15 #
16 16 # If you change this script, it is recommended that you ensure you
17 17 # haven't broken it by running it in various modes with a representative
18 18 # sample of test scripts. For example:
19 19 #
20 20 # 1) serial, no coverage, temp install:
21 21 # ./run-tests.py test-s*
22 22 # 2) serial, no coverage, local hg:
23 23 # ./run-tests.py --local test-s*
24 24 # 3) serial, coverage, temp install:
25 25 # ./run-tests.py -c test-s*
26 26 # 4) serial, coverage, local hg:
27 27 # ./run-tests.py -c --local test-s* # unsupported
28 28 # 5) parallel, no coverage, temp install:
29 29 # ./run-tests.py -j2 test-s*
30 30 # 6) parallel, no coverage, local hg:
31 31 # ./run-tests.py -j2 --local test-s*
32 32 # 7) parallel, coverage, temp install:
33 33 # ./run-tests.py -j2 -c test-s* # currently broken
34 34 # 8) parallel, coverage, local install:
35 35 # ./run-tests.py -j2 -c --local test-s* # unsupported (and broken)
36 36 # 9) parallel, custom tmp dir:
37 37 # ./run-tests.py -j2 --tmpdir /tmp/myhgtests
38 38 # 10) parallel, pure, tests that call run-tests:
39 39 # ./run-tests.py --pure `grep -l run-tests.py *.t`
40 40 #
41 41 # (You could use any subset of the tests: test-s* happens to match
42 42 # enough that it's worth doing parallel runs, few enough that it
43 43 # completes fairly quickly, includes both shell and Python scripts, and
44 44 # includes some scripts that run daemon processes.)
45 45
46 46 from __future__ import absolute_import, print_function
47 47
48 48 import argparse
49 49 import collections
50 50 import difflib
51 51 import distutils.version as version
52 52 import errno
53 53 import json
54 54 import os
55 55 import random
56 56 import re
57 57 import shutil
58 58 import signal
59 59 import socket
60 60 import subprocess
61 61 import sys
62 62 import sysconfig
63 63 import tempfile
64 64 import threading
65 65 import time
66 66 import unittest
67 67 import xml.dom.minidom as minidom
68 68
69 69 try:
70 70 import Queue as queue
71 71 except ImportError:
72 72 import queue
73 73
74 74 try:
75 75 import shlex
76 76 shellquote = shlex.quote
77 77 except (ImportError, AttributeError):
78 78 import pipes
79 79 shellquote = pipes.quote
80 80
81 81 if os.environ.get('RTUNICODEPEDANTRY', False):
82 82 try:
83 83 reload(sys)
84 84 sys.setdefaultencoding("undefined")
85 85 except NameError:
86 86 pass
87 87
88 88 origenviron = os.environ.copy()
89 89 osenvironb = getattr(os, 'environb', os.environ)
90 90 processlock = threading.Lock()
91 91
92 92 pygmentspresent = False
93 93 # ANSI color is unsupported prior to Windows 10
94 94 if os.name != 'nt':
95 95 try: # is pygments installed
96 96 import pygments
97 97 import pygments.lexers as lexers
98 98 import pygments.lexer as lexer
99 99 import pygments.formatters as formatters
100 100 import pygments.token as token
101 101 import pygments.style as style
102 102 pygmentspresent = True
103 103 difflexer = lexers.DiffLexer()
104 104 terminal256formatter = formatters.Terminal256Formatter()
105 105 except ImportError:
106 106 pass
107 107
108 108 if pygmentspresent:
109 109 class TestRunnerStyle(style.Style):
110 110 default_style = ""
111 111 skipped = token.string_to_tokentype("Token.Generic.Skipped")
112 112 failed = token.string_to_tokentype("Token.Generic.Failed")
113 113 skippedname = token.string_to_tokentype("Token.Generic.SName")
114 114 failedname = token.string_to_tokentype("Token.Generic.FName")
115 115 styles = {
116 116 skipped: '#e5e5e5',
117 117 skippedname: '#00ffff',
118 118 failed: '#7f0000',
119 119 failedname: '#ff0000',
120 120 }
121 121
122 122 class TestRunnerLexer(lexer.RegexLexer):
123 123 testpattern = r'[\w-]+\.(t|py)( \(case [\w-]+\))?'
124 124 tokens = {
125 125 'root': [
126 126 (r'^Skipped', token.Generic.Skipped, 'skipped'),
127 127 (r'^Failed ', token.Generic.Failed, 'failed'),
128 128 (r'^ERROR: ', token.Generic.Failed, 'failed'),
129 129 ],
130 130 'skipped': [
131 131 (testpattern, token.Generic.SName),
132 132 (r':.*', token.Generic.Skipped),
133 133 ],
134 134 'failed': [
135 135 (testpattern, token.Generic.FName),
136 136 (r'(:| ).*', token.Generic.Failed),
137 137 ]
138 138 }
139 139
140 140 runnerformatter = formatters.Terminal256Formatter(style=TestRunnerStyle)
141 141 runnerlexer = TestRunnerLexer()
142 142
143 143 if sys.version_info > (3, 5, 0):
144 144 PYTHON3 = True
145 145 xrange = range # we use xrange in one place, and we'd rather not use range
146 146 def _bytespath(p):
147 147 if p is None:
148 148 return p
149 149 return p.encode('utf-8')
150 150
151 151 def _strpath(p):
152 152 if p is None:
153 153 return p
154 154 return p.decode('utf-8')
155 155
156 156 elif sys.version_info >= (3, 0, 0):
157 157 print('%s is only supported on Python 3.5+ and 2.7, not %s' %
158 158 (sys.argv[0], '.'.join(str(v) for v in sys.version_info[:3])))
159 159 sys.exit(70) # EX_SOFTWARE from `man 3 sysexit`
160 160 else:
161 161 PYTHON3 = False
162 162
163 163 # In python 2.x, path operations are generally done using
164 164 # bytestrings by default, so we don't have to do any extra
165 165 # fiddling there. We define the wrapper functions anyway just to
166 166 # help keep code consistent between platforms.
167 167 def _bytespath(p):
168 168 return p
169 169
170 170 _strpath = _bytespath
171 171
172 172 # For Windows support
173 173 wifexited = getattr(os, "WIFEXITED", lambda x: False)
174 174
175 175 # Whether to use IPv6
176 176 def checksocketfamily(name, port=20058):
177 177 """return true if we can listen on localhost using family=name
178 178
179 179 name should be either 'AF_INET', or 'AF_INET6'.
180 180 port being used is okay - EADDRINUSE is considered as successful.
181 181 """
182 182 family = getattr(socket, name, None)
183 183 if family is None:
184 184 return False
185 185 try:
186 186 s = socket.socket(family, socket.SOCK_STREAM)
187 187 s.bind(('localhost', port))
188 188 s.close()
189 189 return True
190 190 except socket.error as exc:
191 191 if exc.errno == errno.EADDRINUSE:
192 192 return True
193 193 elif exc.errno in (errno.EADDRNOTAVAIL, errno.EPROTONOSUPPORT):
194 194 return False
195 195 else:
196 196 raise
197 197 else:
198 198 return False
199 199
200 200 # useipv6 will be set by parseargs
201 201 useipv6 = None
202 202
203 203 def checkportisavailable(port):
204 204 """return true if a port seems free to bind on localhost"""
205 205 if useipv6:
206 206 family = socket.AF_INET6
207 207 else:
208 208 family = socket.AF_INET
209 209 try:
210 210 s = socket.socket(family, socket.SOCK_STREAM)
211 211 s.bind(('localhost', port))
212 212 s.close()
213 213 return True
214 214 except socket.error as exc:
215 215 if exc.errno not in (errno.EADDRINUSE, errno.EADDRNOTAVAIL,
216 216 errno.EPROTONOSUPPORT):
217 217 raise
218 218 return False
219 219
220 220 closefds = os.name == 'posix'
221 221 def Popen4(cmd, wd, timeout, env=None):
222 222 processlock.acquire()
223 223 p = subprocess.Popen(cmd, shell=True, bufsize=-1, cwd=wd, env=env,
224 224 close_fds=closefds,
225 225 stdin=subprocess.PIPE, stdout=subprocess.PIPE,
226 226 stderr=subprocess.STDOUT)
227 227 processlock.release()
228 228
229 229 p.fromchild = p.stdout
230 230 p.tochild = p.stdin
231 231 p.childerr = p.stderr
232 232
233 233 p.timeout = False
234 234 if timeout:
235 235 def t():
236 236 start = time.time()
237 237 while time.time() - start < timeout and p.returncode is None:
238 238 time.sleep(.1)
239 239 p.timeout = True
240 240 if p.returncode is None:
241 241 terminate(p)
242 242 threading.Thread(target=t).start()
243 243
244 244 return p
245 245
246 246 PYTHON = _bytespath(sys.executable.replace('\\', '/'))
247 247 IMPL_PATH = b'PYTHONPATH'
248 248 if 'java' in sys.platform:
249 249 IMPL_PATH = b'JYTHONPATH'
250 250
251 251 defaults = {
252 252 'jobs': ('HGTEST_JOBS', 1),
253 253 'timeout': ('HGTEST_TIMEOUT', 180),
254 254 'slowtimeout': ('HGTEST_SLOWTIMEOUT', 500),
255 255 'port': ('HGTEST_PORT', 20059),
256 256 'shell': ('HGTEST_SHELL', 'sh'),
257 257 }
258 258
259 259 def canonpath(path):
260 260 return os.path.realpath(os.path.expanduser(path))
261 261
262 262 def parselistfiles(files, listtype, warn=True):
263 263 entries = dict()
264 264 for filename in files:
265 265 try:
266 266 path = os.path.expanduser(os.path.expandvars(filename))
267 267 f = open(path, "rb")
268 268 except IOError as err:
269 269 if err.errno != errno.ENOENT:
270 270 raise
271 271 if warn:
272 272 print("warning: no such %s file: %s" % (listtype, filename))
273 273 continue
274 274
275 275 for line in f.readlines():
276 276 line = line.split(b'#', 1)[0].strip()
277 277 if line:
278 278 entries[line] = filename
279 279
280 280 f.close()
281 281 return entries
282 282
283 283 def parsettestcases(path):
284 284 """read a .t test file, return a set of test case names
285 285
286 286 If path does not exist, return an empty set.
287 287 """
288 288 cases = set()
289 289 try:
290 290 with open(path, 'rb') as f:
291 291 for l in f:
292 292 if l.startswith(b'#testcases '):
293 293 cases.update(l[11:].split())
294 294 except IOError as ex:
295 295 if ex.errno != errno.ENOENT:
296 296 raise
297 297 return cases
298 298
299 299 def getparser():
300 300 """Obtain the OptionParser used by the CLI."""
301 301 parser = argparse.ArgumentParser(usage='%(prog)s [options] [tests]')
302 302
303 303 selection = parser.add_argument_group('Test Selection')
304 304 selection.add_argument('--allow-slow-tests', action='store_true',
305 305 help='allow extremely slow tests')
306 306 selection.add_argument("--blacklist", action="append",
307 307 help="skip tests listed in the specified blacklist file")
308 308 selection.add_argument("--changed",
309 309 help="run tests that are changed in parent rev or working directory")
310 310 selection.add_argument("-k", "--keywords",
311 311 help="run tests matching keywords")
312 312 selection.add_argument("-r", "--retest", action="store_true",
313 313 help = "retest failed tests")
314 314 selection.add_argument("--test-list", action="append",
315 315 help="read tests to run from the specified file")
316 316 selection.add_argument("--whitelist", action="append",
317 317 help="always run tests listed in the specified whitelist file")
318 318 selection.add_argument('tests', metavar='TESTS', nargs='*',
319 319 help='Tests to run')
320 320
321 321 harness = parser.add_argument_group('Test Harness Behavior')
322 322 harness.add_argument('--bisect-repo',
323 323 metavar='bisect_repo',
324 324 help=("Path of a repo to bisect. Use together with "
325 325 "--known-good-rev"))
326 326 harness.add_argument("-d", "--debug", action="store_true",
327 327 help="debug mode: write output of test scripts to console"
328 328 " rather than capturing and diffing it (disables timeout)")
329 329 harness.add_argument("-f", "--first", action="store_true",
330 330 help="exit on the first test failure")
331 331 harness.add_argument("-i", "--interactive", action="store_true",
332 332 help="prompt to accept changed output")
333 333 harness.add_argument("-j", "--jobs", type=int,
334 334 help="number of jobs to run in parallel"
335 335 " (default: $%s or %d)" % defaults['jobs'])
336 336 harness.add_argument("--keep-tmpdir", action="store_true",
337 337 help="keep temporary directory after running tests")
338 338 harness.add_argument('--known-good-rev',
339 339 metavar="known_good_rev",
340 340 help=("Automatically bisect any failures using this "
341 341 "revision as a known-good revision."))
342 342 harness.add_argument("--list-tests", action="store_true",
343 343 help="list tests instead of running them")
344 344 harness.add_argument("--loop", action="store_true",
345 345 help="loop tests repeatedly")
346 346 harness.add_argument('--random', action="store_true",
347 347 help='run tests in random order')
348 harness.add_argument('--order-by-runtime', action="store_true",
349 help='run slowest tests first, according to .testtimes')
348 350 harness.add_argument("-p", "--port", type=int,
349 351 help="port on which servers should listen"
350 352 " (default: $%s or %d)" % defaults['port'])
351 353 harness.add_argument('--profile-runner', action='store_true',
352 354 help='run statprof on run-tests')
353 355 harness.add_argument("-R", "--restart", action="store_true",
354 356 help="restart at last error")
355 357 harness.add_argument("--runs-per-test", type=int, dest="runs_per_test",
356 358 help="run each test N times (default=1)", default=1)
357 359 harness.add_argument("--shell",
358 360 help="shell to use (default: $%s or %s)" % defaults['shell'])
359 361 harness.add_argument('--showchannels', action='store_true',
360 362 help='show scheduling channels')
361 363 harness.add_argument("--slowtimeout", type=int,
362 364 help="kill errant slow tests after SLOWTIMEOUT seconds"
363 365 " (default: $%s or %d)" % defaults['slowtimeout'])
364 366 harness.add_argument("-t", "--timeout", type=int,
365 367 help="kill errant tests after TIMEOUT seconds"
366 368 " (default: $%s or %d)" % defaults['timeout'])
367 369 harness.add_argument("--tmpdir",
368 370 help="run tests in the given temporary directory"
369 371 " (implies --keep-tmpdir)")
370 372 harness.add_argument("-v", "--verbose", action="store_true",
371 373 help="output verbose messages")
372 374
373 375 hgconf = parser.add_argument_group('Mercurial Configuration')
374 376 hgconf.add_argument("--chg", action="store_true",
375 377 help="install and use chg wrapper in place of hg")
376 378 hgconf.add_argument("--compiler",
377 379 help="compiler to build with")
378 380 hgconf.add_argument('--extra-config-opt', action="append", default=[],
379 381 help='set the given config opt in the test hgrc')
380 382 hgconf.add_argument("-l", "--local", action="store_true",
381 383 help="shortcut for --with-hg=<testdir>/../hg, "
382 384 "and --with-chg=<testdir>/../contrib/chg/chg if --chg is set")
383 385 hgconf.add_argument("--ipv6", action="store_true",
384 386 help="prefer IPv6 to IPv4 for network related tests")
385 387 hgconf.add_argument("--pure", action="store_true",
386 388 help="use pure Python code instead of C extensions")
387 389 hgconf.add_argument("-3", "--py3k-warnings", action="store_true",
388 390 help="enable Py3k warnings on Python 2.7+")
389 391 hgconf.add_argument("--with-chg", metavar="CHG",
390 392 help="use specified chg wrapper in place of hg")
391 393 hgconf.add_argument("--with-hg",
392 394 metavar="HG",
393 395 help="test using specified hg script rather than a "
394 396 "temporary installation")
395 397 # This option should be deleted once test-check-py3-compat.t and other
396 398 # Python 3 tests run with Python 3.
397 399 hgconf.add_argument("--with-python3", metavar="PYTHON3",
398 400 help="Python 3 interpreter (if running under Python 2)"
399 401 " (TEMPORARY)")
400 402
401 403 reporting = parser.add_argument_group('Results Reporting')
402 404 reporting.add_argument("-C", "--annotate", action="store_true",
403 405 help="output files annotated with coverage")
404 406 reporting.add_argument("--color", choices=["always", "auto", "never"],
405 407 default=os.environ.get('HGRUNTESTSCOLOR', 'auto'),
406 408 help="colorisation: always|auto|never (default: auto)")
407 409 reporting.add_argument("-c", "--cover", action="store_true",
408 410 help="print a test coverage report")
409 411 reporting.add_argument('--exceptions', action='store_true',
410 412 help='log all exceptions and generate an exception report')
411 413 reporting.add_argument("-H", "--htmlcov", action="store_true",
412 414 help="create an HTML report of the coverage of the files")
413 415 reporting.add_argument("--json", action="store_true",
414 416 help="store test result data in 'report.json' file")
415 417 reporting.add_argument("--outputdir",
416 418 help="directory to write error logs to (default=test directory)")
417 419 reporting.add_argument("-n", "--nodiff", action="store_true",
418 420 help="skip showing test changes")
419 421 reporting.add_argument("-S", "--noskips", action="store_true",
420 422 help="don't report skip tests verbosely")
421 423 reporting.add_argument("--time", action="store_true",
422 424 help="time how long each test takes")
423 425 reporting.add_argument("--view",
424 426 help="external diff viewer")
425 427 reporting.add_argument("--xunit",
426 428 help="record xunit results at specified path")
427 429
428 430 for option, (envvar, default) in defaults.items():
429 431 defaults[option] = type(default)(os.environ.get(envvar, default))
430 432 parser.set_defaults(**defaults)
431 433
432 434 return parser
433 435
434 436 def parseargs(args, parser):
435 437 """Parse arguments with our OptionParser and validate results."""
436 438 options = parser.parse_args(args)
437 439
438 440 # jython is always pure
439 441 if 'java' in sys.platform or '__pypy__' in sys.modules:
440 442 options.pure = True
441 443
442 444 if options.with_hg:
443 445 options.with_hg = canonpath(_bytespath(options.with_hg))
444 446 if not (os.path.isfile(options.with_hg) and
445 447 os.access(options.with_hg, os.X_OK)):
446 448 parser.error('--with-hg must specify an executable hg script')
447 449 if os.path.basename(options.with_hg) not in [b'hg', b'hg.exe']:
448 450 sys.stderr.write('warning: --with-hg should specify an hg script\n')
449 451 if options.local:
450 452 testdir = os.path.dirname(_bytespath(canonpath(sys.argv[0])))
451 453 reporootdir = os.path.dirname(testdir)
452 454 pathandattrs = [(b'hg', 'with_hg')]
453 455 if options.chg:
454 456 pathandattrs.append((b'contrib/chg/chg', 'with_chg'))
455 457 for relpath, attr in pathandattrs:
456 458 binpath = os.path.join(reporootdir, relpath)
457 459 if os.name != 'nt' and not os.access(binpath, os.X_OK):
458 460 parser.error('--local specified, but %r not found or '
459 461 'not executable' % binpath)
460 462 setattr(options, attr, binpath)
461 463
462 464 if (options.chg or options.with_chg) and os.name == 'nt':
463 465 parser.error('chg does not work on %s' % os.name)
464 466 if options.with_chg:
465 467 options.chg = False # no installation to temporary location
466 468 options.with_chg = canonpath(_bytespath(options.with_chg))
467 469 if not (os.path.isfile(options.with_chg) and
468 470 os.access(options.with_chg, os.X_OK)):
469 471 parser.error('--with-chg must specify a chg executable')
470 472 if options.chg and options.with_hg:
471 473 # chg shares installation location with hg
472 474 parser.error('--chg does not work when --with-hg is specified '
473 475 '(use --with-chg instead)')
474 476
475 477 if options.color == 'always' and not pygmentspresent:
476 478 sys.stderr.write('warning: --color=always ignored because '
477 479 'pygments is not installed\n')
478 480
479 481 if options.bisect_repo and not options.known_good_rev:
480 482 parser.error("--bisect-repo cannot be used without --known-good-rev")
481 483
482 484 global useipv6
483 485 if options.ipv6:
484 486 useipv6 = checksocketfamily('AF_INET6')
485 487 else:
486 488 # only use IPv6 if IPv4 is unavailable and IPv6 is available
487 489 useipv6 = ((not checksocketfamily('AF_INET'))
488 490 and checksocketfamily('AF_INET6'))
489 491
490 492 options.anycoverage = options.cover or options.annotate or options.htmlcov
491 493 if options.anycoverage:
492 494 try:
493 495 import coverage
494 496 covver = version.StrictVersion(coverage.__version__).version
495 497 if covver < (3, 3):
496 498 parser.error('coverage options require coverage 3.3 or later')
497 499 except ImportError:
498 500 parser.error('coverage options now require the coverage package')
499 501
500 502 if options.anycoverage and options.local:
501 503 # this needs some path mangling somewhere, I guess
502 504 parser.error("sorry, coverage options do not work when --local "
503 505 "is specified")
504 506
505 507 if options.anycoverage and options.with_hg:
506 508 parser.error("sorry, coverage options do not work when --with-hg "
507 509 "is specified")
508 510
509 511 global verbose
510 512 if options.verbose:
511 513 verbose = ''
512 514
513 515 if options.tmpdir:
514 516 options.tmpdir = canonpath(options.tmpdir)
515 517
516 518 if options.jobs < 1:
517 519 parser.error('--jobs must be positive')
518 520 if options.interactive and options.debug:
519 521 parser.error("-i/--interactive and -d/--debug are incompatible")
520 522 if options.debug:
521 523 if options.timeout != defaults['timeout']:
522 524 sys.stderr.write(
523 525 'warning: --timeout option ignored with --debug\n')
524 526 if options.slowtimeout != defaults['slowtimeout']:
525 527 sys.stderr.write(
526 528 'warning: --slowtimeout option ignored with --debug\n')
527 529 options.timeout = 0
528 530 options.slowtimeout = 0
529 531 if options.py3k_warnings:
530 532 if PYTHON3:
531 533 parser.error(
532 534 '--py3k-warnings can only be used on Python 2.7')
533 535 if options.with_python3:
534 536 if PYTHON3:
535 537 parser.error('--with-python3 cannot be used when executing with '
536 538 'Python 3')
537 539
538 540 options.with_python3 = canonpath(options.with_python3)
539 541 # Verify Python3 executable is acceptable.
540 542 proc = subprocess.Popen([options.with_python3, b'--version'],
541 543 stdout=subprocess.PIPE,
542 544 stderr=subprocess.STDOUT)
543 545 out, _err = proc.communicate()
544 546 ret = proc.wait()
545 547 if ret != 0:
546 548 parser.error('could not determine version of python 3')
547 549 if not out.startswith('Python '):
548 550 parser.error('unexpected output from python3 --version: %s' %
549 551 out)
550 552 vers = version.LooseVersion(out[len('Python '):])
551 553 if vers < version.LooseVersion('3.5.0'):
552 554 parser.error('--with-python3 version must be 3.5.0 or greater; '
553 555 'got %s' % out)
554 556
555 557 if options.blacklist:
556 558 options.blacklist = parselistfiles(options.blacklist, 'blacklist')
557 559 if options.whitelist:
558 560 options.whitelisted = parselistfiles(options.whitelist, 'whitelist')
559 561 else:
560 562 options.whitelisted = {}
561 563
562 564 if options.showchannels:
563 565 options.nodiff = True
564 566
565 567 return options
566 568
567 569 def rename(src, dst):
568 570 """Like os.rename(), trade atomicity and opened files friendliness
569 571 for existing destination support.
570 572 """
571 573 shutil.copy(src, dst)
572 574 os.remove(src)
573 575
574 576 _unified_diff = difflib.unified_diff
575 577 if PYTHON3:
576 578 import functools
577 579 _unified_diff = functools.partial(difflib.diff_bytes, difflib.unified_diff)
578 580
579 581 def getdiff(expected, output, ref, err):
580 582 servefail = False
581 583 lines = []
582 584 for line in _unified_diff(expected, output, ref, err):
583 585 if line.startswith(b'+++') or line.startswith(b'---'):
584 586 line = line.replace(b'\\', b'/')
585 587 if line.endswith(b' \n'):
586 588 line = line[:-2] + b'\n'
587 589 lines.append(line)
588 590 if not servefail and line.startswith(
589 591 b'+ abort: child process failed to start'):
590 592 servefail = True
591 593
592 594 return servefail, lines
593 595
594 596 verbose = False
595 597 def vlog(*msg):
596 598 """Log only when in verbose mode."""
597 599 if verbose is False:
598 600 return
599 601
600 602 return log(*msg)
601 603
602 604 # Bytes that break XML even in a CDATA block: control characters 0-31
603 605 # sans \t, \n and \r
604 606 CDATA_EVIL = re.compile(br"[\000-\010\013\014\016-\037]")
605 607
606 608 # Match feature conditionalized output lines in the form, capturing the feature
607 609 # list in group 2, and the preceeding line output in group 1:
608 610 #
609 611 # output..output (feature !)\n
610 612 optline = re.compile(b'(.*) \((.+?) !\)\n$')
611 613
612 614 def cdatasafe(data):
613 615 """Make a string safe to include in a CDATA block.
614 616
615 617 Certain control characters are illegal in a CDATA block, and
616 618 there's no way to include a ]]> in a CDATA either. This function
617 619 replaces illegal bytes with ? and adds a space between the ]] so
618 620 that it won't break the CDATA block.
619 621 """
620 622 return CDATA_EVIL.sub(b'?', data).replace(b']]>', b'] ]>')
621 623
622 624 def log(*msg):
623 625 """Log something to stdout.
624 626
625 627 Arguments are strings to print.
626 628 """
627 629 with iolock:
628 630 if verbose:
629 631 print(verbose, end=' ')
630 632 for m in msg:
631 633 print(m, end=' ')
632 634 print()
633 635 sys.stdout.flush()
634 636
635 637 def highlightdiff(line, color):
636 638 if not color:
637 639 return line
638 640 assert pygmentspresent
639 641 return pygments.highlight(line.decode('latin1'), difflexer,
640 642 terminal256formatter).encode('latin1')
641 643
642 644 def highlightmsg(msg, color):
643 645 if not color:
644 646 return msg
645 647 assert pygmentspresent
646 648 return pygments.highlight(msg, runnerlexer, runnerformatter)
647 649
648 650 def terminate(proc):
649 651 """Terminate subprocess"""
650 652 vlog('# Terminating process %d' % proc.pid)
651 653 try:
652 654 proc.terminate()
653 655 except OSError:
654 656 pass
655 657
656 658 def killdaemons(pidfile):
657 659 import killdaemons as killmod
658 660 return killmod.killdaemons(pidfile, tryhard=False, remove=True,
659 661 logfn=vlog)
660 662
661 663 class Test(unittest.TestCase):
662 664 """Encapsulates a single, runnable test.
663 665
664 666 While this class conforms to the unittest.TestCase API, it differs in that
665 667 instances need to be instantiated manually. (Typically, unittest.TestCase
666 668 classes are instantiated automatically by scanning modules.)
667 669 """
668 670
669 671 # Status code reserved for skipped tests (used by hghave).
670 672 SKIPPED_STATUS = 80
671 673
672 674 def __init__(self, path, outputdir, tmpdir, keeptmpdir=False,
673 675 debug=False,
674 676 first=False,
675 677 timeout=None,
676 678 startport=None, extraconfigopts=None,
677 679 py3kwarnings=False, shell=None, hgcommand=None,
678 680 slowtimeout=None, usechg=False,
679 681 useipv6=False):
680 682 """Create a test from parameters.
681 683
682 684 path is the full path to the file defining the test.
683 685
684 686 tmpdir is the main temporary directory to use for this test.
685 687
686 688 keeptmpdir determines whether to keep the test's temporary directory
687 689 after execution. It defaults to removal (False).
688 690
689 691 debug mode will make the test execute verbosely, with unfiltered
690 692 output.
691 693
692 694 timeout controls the maximum run time of the test. It is ignored when
693 695 debug is True. See slowtimeout for tests with #require slow.
694 696
695 697 slowtimeout overrides timeout if the test has #require slow.
696 698
697 699 startport controls the starting port number to use for this test. Each
698 700 test will reserve 3 port numbers for execution. It is the caller's
699 701 responsibility to allocate a non-overlapping port range to Test
700 702 instances.
701 703
702 704 extraconfigopts is an iterable of extra hgrc config options. Values
703 705 must have the form "key=value" (something understood by hgrc). Values
704 706 of the form "foo.key=value" will result in "[foo] key=value".
705 707
706 708 py3kwarnings enables Py3k warnings.
707 709
708 710 shell is the shell to execute tests in.
709 711 """
710 712 if timeout is None:
711 713 timeout = defaults['timeout']
712 714 if startport is None:
713 715 startport = defaults['port']
714 716 if slowtimeout is None:
715 717 slowtimeout = defaults['slowtimeout']
716 718 self.path = path
717 719 self.bname = os.path.basename(path)
718 720 self.name = _strpath(self.bname)
719 721 self._testdir = os.path.dirname(path)
720 722 self._outputdir = outputdir
721 723 self._tmpname = os.path.basename(path)
722 724 self.errpath = os.path.join(self._outputdir, b'%s.err' % self.bname)
723 725
724 726 self._threadtmp = tmpdir
725 727 self._keeptmpdir = keeptmpdir
726 728 self._debug = debug
727 729 self._first = first
728 730 self._timeout = timeout
729 731 self._slowtimeout = slowtimeout
730 732 self._startport = startport
731 733 self._extraconfigopts = extraconfigopts or []
732 734 self._py3kwarnings = py3kwarnings
733 735 self._shell = _bytespath(shell)
734 736 self._hgcommand = hgcommand or b'hg'
735 737 self._usechg = usechg
736 738 self._useipv6 = useipv6
737 739
738 740 self._aborted = False
739 741 self._daemonpids = []
740 742 self._finished = None
741 743 self._ret = None
742 744 self._out = None
743 745 self._skipped = None
744 746 self._testtmp = None
745 747 self._chgsockdir = None
746 748
747 749 self._refout = self.readrefout()
748 750
749 751 def readrefout(self):
750 752 """read reference output"""
751 753 # If we're not in --debug mode and reference output file exists,
752 754 # check test output against it.
753 755 if self._debug:
754 756 return None # to match "out is None"
755 757 elif os.path.exists(self.refpath):
756 758 with open(self.refpath, 'rb') as f:
757 759 return f.read().splitlines(True)
758 760 else:
759 761 return []
760 762
761 763 # needed to get base class __repr__ running
762 764 @property
763 765 def _testMethodName(self):
764 766 return self.name
765 767
766 768 def __str__(self):
767 769 return self.name
768 770
769 771 def shortDescription(self):
770 772 return self.name
771 773
772 774 def setUp(self):
773 775 """Tasks to perform before run()."""
774 776 self._finished = False
775 777 self._ret = None
776 778 self._out = None
777 779 self._skipped = None
778 780
779 781 try:
780 782 os.mkdir(self._threadtmp)
781 783 except OSError as e:
782 784 if e.errno != errno.EEXIST:
783 785 raise
784 786
785 787 name = self._tmpname
786 788 self._testtmp = os.path.join(self._threadtmp, name)
787 789 os.mkdir(self._testtmp)
788 790
789 791 # Remove any previous output files.
790 792 if os.path.exists(self.errpath):
791 793 try:
792 794 os.remove(self.errpath)
793 795 except OSError as e:
794 796 # We might have raced another test to clean up a .err
795 797 # file, so ignore ENOENT when removing a previous .err
796 798 # file.
797 799 if e.errno != errno.ENOENT:
798 800 raise
799 801
800 802 if self._usechg:
801 803 self._chgsockdir = os.path.join(self._threadtmp,
802 804 b'%s.chgsock' % name)
803 805 os.mkdir(self._chgsockdir)
804 806
805 807 def run(self, result):
806 808 """Run this test and report results against a TestResult instance."""
807 809 # This function is extremely similar to unittest.TestCase.run(). Once
808 810 # we require Python 2.7 (or at least its version of unittest), this
809 811 # function can largely go away.
810 812 self._result = result
811 813 result.startTest(self)
812 814 try:
813 815 try:
814 816 self.setUp()
815 817 except (KeyboardInterrupt, SystemExit):
816 818 self._aborted = True
817 819 raise
818 820 except Exception:
819 821 result.addError(self, sys.exc_info())
820 822 return
821 823
822 824 success = False
823 825 try:
824 826 self.runTest()
825 827 except KeyboardInterrupt:
826 828 self._aborted = True
827 829 raise
828 830 except unittest.SkipTest as e:
829 831 result.addSkip(self, str(e))
830 832 # The base class will have already counted this as a
831 833 # test we "ran", but we want to exclude skipped tests
832 834 # from those we count towards those run.
833 835 result.testsRun -= 1
834 836 except self.failureException as e:
835 837 # This differs from unittest in that we don't capture
836 838 # the stack trace. This is for historical reasons and
837 839 # this decision could be revisited in the future,
838 840 # especially for PythonTest instances.
839 841 if result.addFailure(self, str(e)):
840 842 success = True
841 843 except Exception:
842 844 result.addError(self, sys.exc_info())
843 845 else:
844 846 success = True
845 847
846 848 try:
847 849 self.tearDown()
848 850 except (KeyboardInterrupt, SystemExit):
849 851 self._aborted = True
850 852 raise
851 853 except Exception:
852 854 result.addError(self, sys.exc_info())
853 855 success = False
854 856
855 857 if success:
856 858 result.addSuccess(self)
857 859 finally:
858 860 result.stopTest(self, interrupted=self._aborted)
859 861
860 862 def runTest(self):
861 863 """Run this test instance.
862 864
863 865 This will return a tuple describing the result of the test.
864 866 """
865 867 env = self._getenv()
866 868 self._genrestoreenv(env)
867 869 self._daemonpids.append(env['DAEMON_PIDS'])
868 870 self._createhgrc(env['HGRCPATH'])
869 871
870 872 vlog('# Test', self.name)
871 873
872 874 ret, out = self._run(env)
873 875 self._finished = True
874 876 self._ret = ret
875 877 self._out = out
876 878
877 879 def describe(ret):
878 880 if ret < 0:
879 881 return 'killed by signal: %d' % -ret
880 882 return 'returned error code %d' % ret
881 883
882 884 self._skipped = False
883 885
884 886 if ret == self.SKIPPED_STATUS:
885 887 if out is None: # Debug mode, nothing to parse.
886 888 missing = ['unknown']
887 889 failed = None
888 890 else:
889 891 missing, failed = TTest.parsehghaveoutput(out)
890 892
891 893 if not missing:
892 894 missing = ['skipped']
893 895
894 896 if failed:
895 897 self.fail('hg have failed checking for %s' % failed[-1])
896 898 else:
897 899 self._skipped = True
898 900 raise unittest.SkipTest(missing[-1])
899 901 elif ret == 'timeout':
900 902 self.fail('timed out')
901 903 elif ret is False:
902 904 self.fail('no result code from test')
903 905 elif out != self._refout:
904 906 # Diff generation may rely on written .err file.
905 907 if (ret != 0 or out != self._refout) and not self._skipped \
906 908 and not self._debug:
907 909 with open(self.errpath, 'wb') as f:
908 910 for line in out:
909 911 f.write(line)
910 912
911 913 # The result object handles diff calculation for us.
912 914 with firstlock:
913 915 if self._result.addOutputMismatch(self, ret, out, self._refout):
914 916 # change was accepted, skip failing
915 917 return
916 918 if self._first:
917 919 global firsterror
918 920 firsterror = True
919 921
920 922 if ret:
921 923 msg = 'output changed and ' + describe(ret)
922 924 else:
923 925 msg = 'output changed'
924 926
925 927 self.fail(msg)
926 928 elif ret:
927 929 self.fail(describe(ret))
928 930
929 931 def tearDown(self):
930 932 """Tasks to perform after run()."""
931 933 for entry in self._daemonpids:
932 934 killdaemons(entry)
933 935 self._daemonpids = []
934 936
935 937 if self._keeptmpdir:
936 938 log('\nKeeping testtmp dir: %s\nKeeping threadtmp dir: %s' %
937 939 (self._testtmp.decode('utf-8'),
938 940 self._threadtmp.decode('utf-8')))
939 941 else:
940 942 shutil.rmtree(self._testtmp, True)
941 943 shutil.rmtree(self._threadtmp, True)
942 944
943 945 if self._usechg:
944 946 # chgservers will stop automatically after they find the socket
945 947 # files are deleted
946 948 shutil.rmtree(self._chgsockdir, True)
947 949
948 950 if (self._ret != 0 or self._out != self._refout) and not self._skipped \
949 951 and not self._debug and self._out:
950 952 with open(self.errpath, 'wb') as f:
951 953 for line in self._out:
952 954 f.write(line)
953 955
954 956 vlog("# Ret was:", self._ret, '(%s)' % self.name)
955 957
956 958 def _run(self, env):
957 959 # This should be implemented in child classes to run tests.
958 960 raise unittest.SkipTest('unknown test type')
959 961
960 962 def abort(self):
961 963 """Terminate execution of this test."""
962 964 self._aborted = True
963 965
964 966 def _portmap(self, i):
965 967 offset = b'' if i == 0 else b'%d' % i
966 968 return (br':%d\b' % (self._startport + i), b':$HGPORT%s' % offset)
967 969
968 970 def _getreplacements(self):
969 971 """Obtain a mapping of text replacements to apply to test output.
970 972
971 973 Test output needs to be normalized so it can be compared to expected
972 974 output. This function defines how some of that normalization will
973 975 occur.
974 976 """
975 977 r = [
976 978 # This list should be parallel to defineport in _getenv
977 979 self._portmap(0),
978 980 self._portmap(1),
979 981 self._portmap(2),
980 982 (br'([^0-9])%s' % re.escape(self._localip()), br'\1$LOCALIP'),
981 983 (br'\bHG_TXNID=TXN:[a-f0-9]{40}\b', br'HG_TXNID=TXN:$ID$'),
982 984 ]
983 985 r.append((self._escapepath(self._testtmp), b'$TESTTMP'))
984 986
985 987 replacementfile = os.path.join(self._testdir, b'common-pattern.py')
986 988
987 989 if os.path.exists(replacementfile):
988 990 data = {}
989 991 with open(replacementfile, mode='rb') as source:
990 992 # the intermediate 'compile' step help with debugging
991 993 code = compile(source.read(), replacementfile, 'exec')
992 994 exec(code, data)
993 995 for value in data.get('substitutions', ()):
994 996 if len(value) != 2:
995 997 msg = 'malformatted substitution in %s: %r'
996 998 msg %= (replacementfile, value)
997 999 raise ValueError(msg)
998 1000 r.append(value)
999 1001 return r
1000 1002
1001 1003 def _escapepath(self, p):
1002 1004 if os.name == 'nt':
1003 1005 return (
1004 1006 (b''.join(c.isalpha() and b'[%s%s]' % (c.lower(), c.upper()) or
1005 1007 c in b'/\\' and br'[/\\]' or c.isdigit() and c or b'\\' + c
1006 1008 for c in p))
1007 1009 )
1008 1010 else:
1009 1011 return re.escape(p)
1010 1012
1011 1013 def _localip(self):
1012 1014 if self._useipv6:
1013 1015 return b'::1'
1014 1016 else:
1015 1017 return b'127.0.0.1'
1016 1018
1017 1019 def _genrestoreenv(self, testenv):
1018 1020 """Generate a script that can be used by tests to restore the original
1019 1021 environment."""
1020 1022 # Put the restoreenv script inside self._threadtmp
1021 1023 scriptpath = os.path.join(self._threadtmp, b'restoreenv.sh')
1022 1024 testenv['HGTEST_RESTOREENV'] = scriptpath
1023 1025
1024 1026 # Only restore environment variable names that the shell allows
1025 1027 # us to export.
1026 1028 name_regex = re.compile('^[a-zA-Z][a-zA-Z0-9_]*$')
1027 1029
1028 1030 # Do not restore these variables; otherwise tests would fail.
1029 1031 reqnames = {'PYTHON', 'TESTDIR', 'TESTTMP'}
1030 1032
1031 1033 with open(scriptpath, 'w') as envf:
1032 1034 for name, value in origenviron.items():
1033 1035 if not name_regex.match(name):
1034 1036 # Skip environment variables with unusual names not
1035 1037 # allowed by most shells.
1036 1038 continue
1037 1039 if name in reqnames:
1038 1040 continue
1039 1041 envf.write('%s=%s\n' % (name, shellquote(value)))
1040 1042
1041 1043 for name in testenv:
1042 1044 if name in origenviron or name in reqnames:
1043 1045 continue
1044 1046 envf.write('unset %s\n' % (name,))
1045 1047
1046 1048 def _getenv(self):
1047 1049 """Obtain environment variables to use during test execution."""
1048 1050 def defineport(i):
1049 1051 offset = '' if i == 0 else '%s' % i
1050 1052 env["HGPORT%s" % offset] = '%s' % (self._startport + i)
1051 1053 env = os.environ.copy()
1052 1054 env['PYTHONUSERBASE'] = sysconfig.get_config_var('userbase') or ''
1053 1055 env['HGEMITWARNINGS'] = '1'
1054 1056 env['TESTTMP'] = self._testtmp
1055 1057 env['TESTNAME'] = self.name
1056 1058 env['HOME'] = self._testtmp
1057 1059 # This number should match portneeded in _getport
1058 1060 for port in xrange(3):
1059 1061 # This list should be parallel to _portmap in _getreplacements
1060 1062 defineport(port)
1061 1063 env["HGRCPATH"] = os.path.join(self._threadtmp, b'.hgrc')
1062 1064 env["DAEMON_PIDS"] = os.path.join(self._threadtmp, b'daemon.pids')
1063 1065 env["HGEDITOR"] = ('"' + sys.executable + '"'
1064 1066 + ' -c "import sys; sys.exit(0)"')
1065 1067 env["HGMERGE"] = "internal:merge"
1066 1068 env["HGUSER"] = "test"
1067 1069 env["HGENCODING"] = "ascii"
1068 1070 env["HGENCODINGMODE"] = "strict"
1069 1071 env['HGIPV6'] = str(int(self._useipv6))
1070 1072
1071 1073 # LOCALIP could be ::1 or 127.0.0.1. Useful for tests that require raw
1072 1074 # IP addresses.
1073 1075 env['LOCALIP'] = self._localip()
1074 1076
1075 1077 # Reset some environment variables to well-known values so that
1076 1078 # the tests produce repeatable output.
1077 1079 env['LANG'] = env['LC_ALL'] = env['LANGUAGE'] = 'C'
1078 1080 env['TZ'] = 'GMT'
1079 1081 env["EMAIL"] = "Foo Bar <foo.bar@example.com>"
1080 1082 env['COLUMNS'] = '80'
1081 1083 env['TERM'] = 'xterm'
1082 1084
1083 1085 for k in ('HG HGPROF CDPATH GREP_OPTIONS http_proxy no_proxy ' +
1084 1086 'HGPLAIN HGPLAINEXCEPT EDITOR VISUAL PAGER ' +
1085 1087 'NO_PROXY CHGDEBUG').split():
1086 1088 if k in env:
1087 1089 del env[k]
1088 1090
1089 1091 # unset env related to hooks
1090 1092 for k in list(env):
1091 1093 if k.startswith('HG_'):
1092 1094 del env[k]
1093 1095
1094 1096 if self._usechg:
1095 1097 env['CHGSOCKNAME'] = os.path.join(self._chgsockdir, b'server')
1096 1098
1097 1099 return env
1098 1100
1099 1101 def _createhgrc(self, path):
1100 1102 """Create an hgrc file for this test."""
1101 1103 with open(path, 'wb') as hgrc:
1102 1104 hgrc.write(b'[ui]\n')
1103 1105 hgrc.write(b'slash = True\n')
1104 1106 hgrc.write(b'interactive = False\n')
1105 1107 hgrc.write(b'mergemarkers = detailed\n')
1106 1108 hgrc.write(b'promptecho = True\n')
1107 1109 hgrc.write(b'[defaults]\n')
1108 1110 hgrc.write(b'[devel]\n')
1109 1111 hgrc.write(b'all-warnings = true\n')
1110 1112 hgrc.write(b'default-date = 0 0\n')
1111 1113 hgrc.write(b'[largefiles]\n')
1112 1114 hgrc.write(b'usercache = %s\n' %
1113 1115 (os.path.join(self._testtmp, b'.cache/largefiles')))
1114 1116 hgrc.write(b'[lfs]\n')
1115 1117 hgrc.write(b'usercache = %s\n' %
1116 1118 (os.path.join(self._testtmp, b'.cache/lfs')))
1117 1119 hgrc.write(b'[web]\n')
1118 1120 hgrc.write(b'address = localhost\n')
1119 1121 hgrc.write(b'ipv6 = %s\n' % str(self._useipv6).encode('ascii'))
1120 1122
1121 1123 for opt in self._extraconfigopts:
1122 1124 section, key = opt.encode('utf-8').split(b'.', 1)
1123 1125 assert b'=' in key, ('extra config opt %s must '
1124 1126 'have an = for assignment' % opt)
1125 1127 hgrc.write(b'[%s]\n%s\n' % (section, key))
1126 1128
1127 1129 def fail(self, msg):
1128 1130 # unittest differentiates between errored and failed.
1129 1131 # Failed is denoted by AssertionError (by default at least).
1130 1132 raise AssertionError(msg)
1131 1133
1132 1134 def _runcommand(self, cmd, env, normalizenewlines=False):
1133 1135 """Run command in a sub-process, capturing the output (stdout and
1134 1136 stderr).
1135 1137
1136 1138 Return a tuple (exitcode, output). output is None in debug mode.
1137 1139 """
1138 1140 if self._debug:
1139 1141 proc = subprocess.Popen(cmd, shell=True, cwd=self._testtmp,
1140 1142 env=env)
1141 1143 ret = proc.wait()
1142 1144 return (ret, None)
1143 1145
1144 1146 proc = Popen4(cmd, self._testtmp, self._timeout, env)
1145 1147 def cleanup():
1146 1148 terminate(proc)
1147 1149 ret = proc.wait()
1148 1150 if ret == 0:
1149 1151 ret = signal.SIGTERM << 8
1150 1152 killdaemons(env['DAEMON_PIDS'])
1151 1153 return ret
1152 1154
1153 1155 output = ''
1154 1156 proc.tochild.close()
1155 1157
1156 1158 try:
1157 1159 output = proc.fromchild.read()
1158 1160 except KeyboardInterrupt:
1159 1161 vlog('# Handling keyboard interrupt')
1160 1162 cleanup()
1161 1163 raise
1162 1164
1163 1165 ret = proc.wait()
1164 1166 if wifexited(ret):
1165 1167 ret = os.WEXITSTATUS(ret)
1166 1168
1167 1169 if proc.timeout:
1168 1170 ret = 'timeout'
1169 1171
1170 1172 if ret:
1171 1173 killdaemons(env['DAEMON_PIDS'])
1172 1174
1173 1175 for s, r in self._getreplacements():
1174 1176 output = re.sub(s, r, output)
1175 1177
1176 1178 if normalizenewlines:
1177 1179 output = output.replace('\r\n', '\n')
1178 1180
1179 1181 return ret, output.splitlines(True)
1180 1182
1181 1183 class PythonTest(Test):
1182 1184 """A Python-based test."""
1183 1185
1184 1186 @property
1185 1187 def refpath(self):
1186 1188 return os.path.join(self._testdir, b'%s.out' % self.bname)
1187 1189
1188 1190 def _run(self, env):
1189 1191 py3kswitch = self._py3kwarnings and b' -3' or b''
1190 1192 cmd = b'%s%s "%s"' % (PYTHON, py3kswitch, self.path)
1191 1193 vlog("# Running", cmd)
1192 1194 normalizenewlines = os.name == 'nt'
1193 1195 result = self._runcommand(cmd, env,
1194 1196 normalizenewlines=normalizenewlines)
1195 1197 if self._aborted:
1196 1198 raise KeyboardInterrupt()
1197 1199
1198 1200 return result
1199 1201
1200 1202 # Some glob patterns apply only in some circumstances, so the script
1201 1203 # might want to remove (glob) annotations that otherwise should be
1202 1204 # retained.
1203 1205 checkcodeglobpats = [
1204 1206 # On Windows it looks like \ doesn't require a (glob), but we know
1205 1207 # better.
1206 1208 re.compile(br'^pushing to \$TESTTMP/.*[^)]$'),
1207 1209 re.compile(br'^moving \S+/.*[^)]$'),
1208 1210 re.compile(br'^pulling from \$TESTTMP/.*[^)]$'),
1209 1211 # Not all platforms have 127.0.0.1 as loopback (though most do),
1210 1212 # so we always glob that too.
1211 1213 re.compile(br'.*\$LOCALIP.*$'),
1212 1214 ]
1213 1215
1214 1216 bchr = chr
1215 1217 if PYTHON3:
1216 1218 bchr = lambda x: bytes([x])
1217 1219
1218 1220 class TTest(Test):
1219 1221 """A "t test" is a test backed by a .t file."""
1220 1222
1221 1223 SKIPPED_PREFIX = b'skipped: '
1222 1224 FAILED_PREFIX = b'hghave check failed: '
1223 1225 NEEDESCAPE = re.compile(br'[\x00-\x08\x0b-\x1f\x7f-\xff]').search
1224 1226
1225 1227 ESCAPESUB = re.compile(br'[\x00-\x08\x0b-\x1f\\\x7f-\xff]').sub
1226 1228 ESCAPEMAP = dict((bchr(i), br'\x%02x' % i) for i in range(256))
1227 1229 ESCAPEMAP.update({b'\\': b'\\\\', b'\r': br'\r'})
1228 1230
1229 1231 def __init__(self, path, *args, **kwds):
1230 1232 # accept an extra "case" parameter
1231 1233 case = kwds.pop('case', None)
1232 1234 self._case = case
1233 1235 self._allcases = parsettestcases(path)
1234 1236 super(TTest, self).__init__(path, *args, **kwds)
1235 1237 if case:
1236 1238 self.name = '%s (case %s)' % (self.name, _strpath(case))
1237 1239 self.errpath = b'%s.%s.err' % (self.errpath[:-4], case)
1238 1240 self._tmpname += b'-%s' % case
1239 1241 self._have = {}
1240 1242
1241 1243 @property
1242 1244 def refpath(self):
1243 1245 return os.path.join(self._testdir, self.bname)
1244 1246
1245 1247 def _run(self, env):
1246 1248 with open(self.path, 'rb') as f:
1247 1249 lines = f.readlines()
1248 1250
1249 1251 # .t file is both reference output and the test input, keep reference
1250 1252 # output updated with the the test input. This avoids some race
1251 1253 # conditions where the reference output does not match the actual test.
1252 1254 if self._refout is not None:
1253 1255 self._refout = lines
1254 1256
1255 1257 salt, script, after, expected = self._parsetest(lines)
1256 1258
1257 1259 # Write out the generated script.
1258 1260 fname = b'%s.sh' % self._testtmp
1259 1261 with open(fname, 'wb') as f:
1260 1262 for l in script:
1261 1263 f.write(l)
1262 1264
1263 1265 cmd = b'%s "%s"' % (self._shell, fname)
1264 1266 vlog("# Running", cmd)
1265 1267
1266 1268 exitcode, output = self._runcommand(cmd, env)
1267 1269
1268 1270 if self._aborted:
1269 1271 raise KeyboardInterrupt()
1270 1272
1271 1273 # Do not merge output if skipped. Return hghave message instead.
1272 1274 # Similarly, with --debug, output is None.
1273 1275 if exitcode == self.SKIPPED_STATUS or output is None:
1274 1276 return exitcode, output
1275 1277
1276 1278 return self._processoutput(exitcode, output, salt, after, expected)
1277 1279
1278 1280 def _hghave(self, reqs):
1279 1281 allreqs = b' '.join(reqs)
1280 1282 if allreqs in self._have:
1281 1283 return self._have.get(allreqs)
1282 1284
1283 1285 # TODO do something smarter when all other uses of hghave are gone.
1284 1286 runtestdir = os.path.abspath(os.path.dirname(_bytespath(__file__)))
1285 1287 tdir = runtestdir.replace(b'\\', b'/')
1286 1288 proc = Popen4(b'%s -c "%s/hghave %s"' %
1287 1289 (self._shell, tdir, allreqs),
1288 1290 self._testtmp, 0, self._getenv())
1289 1291 stdout, stderr = proc.communicate()
1290 1292 ret = proc.wait()
1291 1293 if wifexited(ret):
1292 1294 ret = os.WEXITSTATUS(ret)
1293 1295 if ret == 2:
1294 1296 print(stdout.decode('utf-8'))
1295 1297 sys.exit(1)
1296 1298
1297 1299 if ret != 0:
1298 1300 self._have[allreqs] = (False, stdout)
1299 1301 return False, stdout
1300 1302
1301 1303 if b'slow' in reqs:
1302 1304 self._timeout = self._slowtimeout
1303 1305
1304 1306 self._have[allreqs] = (True, None)
1305 1307 return True, None
1306 1308
1307 1309 def _iftest(self, args):
1308 1310 # implements "#if"
1309 1311 reqs = []
1310 1312 for arg in args:
1311 1313 if arg.startswith(b'no-') and arg[3:] in self._allcases:
1312 1314 if arg[3:] == self._case:
1313 1315 return False
1314 1316 elif arg in self._allcases:
1315 1317 if arg != self._case:
1316 1318 return False
1317 1319 else:
1318 1320 reqs.append(arg)
1319 1321 return self._hghave(reqs)[0]
1320 1322
1321 1323 def _parsetest(self, lines):
1322 1324 # We generate a shell script which outputs unique markers to line
1323 1325 # up script results with our source. These markers include input
1324 1326 # line number and the last return code.
1325 1327 salt = b"SALT%d" % time.time()
1326 1328 def addsalt(line, inpython):
1327 1329 if inpython:
1328 1330 script.append(b'%s %d 0\n' % (salt, line))
1329 1331 else:
1330 1332 script.append(b'echo %s %d $?\n' % (salt, line))
1331 1333
1332 1334 script = []
1333 1335
1334 1336 # After we run the shell script, we re-unify the script output
1335 1337 # with non-active parts of the source, with synchronization by our
1336 1338 # SALT line number markers. The after table contains the non-active
1337 1339 # components, ordered by line number.
1338 1340 after = {}
1339 1341
1340 1342 # Expected shell script output.
1341 1343 expected = {}
1342 1344
1343 1345 pos = prepos = -1
1344 1346
1345 1347 # True or False when in a true or false conditional section
1346 1348 skipping = None
1347 1349
1348 1350 # We keep track of whether or not we're in a Python block so we
1349 1351 # can generate the surrounding doctest magic.
1350 1352 inpython = False
1351 1353
1352 1354 if self._debug:
1353 1355 script.append(b'set -x\n')
1354 1356 if self._hgcommand != b'hg':
1355 1357 script.append(b'alias hg="%s"\n' % self._hgcommand)
1356 1358 if os.getenv('MSYSTEM'):
1357 1359 script.append(b'alias pwd="pwd -W"\n')
1358 1360 if self._case:
1359 1361 if isinstance(self._case, str):
1360 1362 quoted = shellquote(self._case)
1361 1363 else:
1362 1364 quoted = shellquote(self._case.decode('utf8')).encode('utf8')
1363 1365 script.append(b'TESTCASE=%s\n' % quoted)
1364 1366 script.append(b'export TESTCASE\n')
1365 1367
1366 1368 n = 0
1367 1369 for n, l in enumerate(lines):
1368 1370 if not l.endswith(b'\n'):
1369 1371 l += b'\n'
1370 1372 if l.startswith(b'#require'):
1371 1373 lsplit = l.split()
1372 1374 if len(lsplit) < 2 or lsplit[0] != b'#require':
1373 1375 after.setdefault(pos, []).append(' !!! invalid #require\n')
1374 1376 haveresult, message = self._hghave(lsplit[1:])
1375 1377 if not haveresult:
1376 1378 script = [b'echo "%s"\nexit 80\n' % message]
1377 1379 break
1378 1380 after.setdefault(pos, []).append(l)
1379 1381 elif l.startswith(b'#if'):
1380 1382 lsplit = l.split()
1381 1383 if len(lsplit) < 2 or lsplit[0] != b'#if':
1382 1384 after.setdefault(pos, []).append(' !!! invalid #if\n')
1383 1385 if skipping is not None:
1384 1386 after.setdefault(pos, []).append(' !!! nested #if\n')
1385 1387 skipping = not self._iftest(lsplit[1:])
1386 1388 after.setdefault(pos, []).append(l)
1387 1389 elif l.startswith(b'#else'):
1388 1390 if skipping is None:
1389 1391 after.setdefault(pos, []).append(' !!! missing #if\n')
1390 1392 skipping = not skipping
1391 1393 after.setdefault(pos, []).append(l)
1392 1394 elif l.startswith(b'#endif'):
1393 1395 if skipping is None:
1394 1396 after.setdefault(pos, []).append(' !!! missing #if\n')
1395 1397 skipping = None
1396 1398 after.setdefault(pos, []).append(l)
1397 1399 elif skipping:
1398 1400 after.setdefault(pos, []).append(l)
1399 1401 elif l.startswith(b' >>> '): # python inlines
1400 1402 after.setdefault(pos, []).append(l)
1401 1403 prepos = pos
1402 1404 pos = n
1403 1405 if not inpython:
1404 1406 # We've just entered a Python block. Add the header.
1405 1407 inpython = True
1406 1408 addsalt(prepos, False) # Make sure we report the exit code.
1407 1409 script.append(b'%s -m heredoctest <<EOF\n' % PYTHON)
1408 1410 addsalt(n, True)
1409 1411 script.append(l[2:])
1410 1412 elif l.startswith(b' ... '): # python inlines
1411 1413 after.setdefault(prepos, []).append(l)
1412 1414 script.append(l[2:])
1413 1415 elif l.startswith(b' $ '): # commands
1414 1416 if inpython:
1415 1417 script.append(b'EOF\n')
1416 1418 inpython = False
1417 1419 after.setdefault(pos, []).append(l)
1418 1420 prepos = pos
1419 1421 pos = n
1420 1422 addsalt(n, False)
1421 1423 cmd = l[4:].split()
1422 1424 if len(cmd) == 2 and cmd[0] == b'cd':
1423 1425 l = b' $ cd %s || exit 1\n' % cmd[1]
1424 1426 script.append(l[4:])
1425 1427 elif l.startswith(b' > '): # continuations
1426 1428 after.setdefault(prepos, []).append(l)
1427 1429 script.append(l[4:])
1428 1430 elif l.startswith(b' '): # results
1429 1431 # Queue up a list of expected results.
1430 1432 expected.setdefault(pos, []).append(l[2:])
1431 1433 else:
1432 1434 if inpython:
1433 1435 script.append(b'EOF\n')
1434 1436 inpython = False
1435 1437 # Non-command/result. Queue up for merged output.
1436 1438 after.setdefault(pos, []).append(l)
1437 1439
1438 1440 if inpython:
1439 1441 script.append(b'EOF\n')
1440 1442 if skipping is not None:
1441 1443 after.setdefault(pos, []).append(' !!! missing #endif\n')
1442 1444 addsalt(n + 1, False)
1443 1445
1444 1446 return salt, script, after, expected
1445 1447
1446 1448 def _processoutput(self, exitcode, output, salt, after, expected):
1447 1449 # Merge the script output back into a unified test.
1448 1450 warnonly = 1 # 1: not yet; 2: yes; 3: for sure not
1449 1451 if exitcode != 0:
1450 1452 warnonly = 3
1451 1453
1452 1454 pos = -1
1453 1455 postout = []
1454 1456 for l in output:
1455 1457 lout, lcmd = l, None
1456 1458 if salt in l:
1457 1459 lout, lcmd = l.split(salt, 1)
1458 1460
1459 1461 while lout:
1460 1462 if not lout.endswith(b'\n'):
1461 1463 lout += b' (no-eol)\n'
1462 1464
1463 1465 # Find the expected output at the current position.
1464 1466 els = [None]
1465 1467 if expected.get(pos, None):
1466 1468 els = expected[pos]
1467 1469
1468 1470 i = 0
1469 1471 optional = []
1470 1472 while i < len(els):
1471 1473 el = els[i]
1472 1474
1473 1475 r = self.linematch(el, lout)
1474 1476 if isinstance(r, str):
1475 1477 if r == '-glob':
1476 1478 lout = ''.join(el.rsplit(' (glob)', 1))
1477 1479 r = '' # Warn only this line.
1478 1480 elif r == "retry":
1479 1481 postout.append(b' ' + el)
1480 1482 els.pop(i)
1481 1483 break
1482 1484 else:
1483 1485 log('\ninfo, unknown linematch result: %r\n' % r)
1484 1486 r = False
1485 1487 if r:
1486 1488 els.pop(i)
1487 1489 break
1488 1490 if el:
1489 1491 if el.endswith(b" (?)\n"):
1490 1492 optional.append(i)
1491 1493 else:
1492 1494 m = optline.match(el)
1493 1495 if m:
1494 1496 conditions = [
1495 1497 c for c in m.group(2).split(b' ')]
1496 1498
1497 1499 if not self._iftest(conditions):
1498 1500 optional.append(i)
1499 1501
1500 1502 i += 1
1501 1503
1502 1504 if r:
1503 1505 if r == "retry":
1504 1506 continue
1505 1507 # clean up any optional leftovers
1506 1508 for i in optional:
1507 1509 postout.append(b' ' + els[i])
1508 1510 for i in reversed(optional):
1509 1511 del els[i]
1510 1512 postout.append(b' ' + el)
1511 1513 else:
1512 1514 if self.NEEDESCAPE(lout):
1513 1515 lout = TTest._stringescape(b'%s (esc)\n' %
1514 1516 lout.rstrip(b'\n'))
1515 1517 postout.append(b' ' + lout) # Let diff deal with it.
1516 1518 if r != '': # If line failed.
1517 1519 warnonly = 3 # for sure not
1518 1520 elif warnonly == 1: # Is "not yet" and line is warn only.
1519 1521 warnonly = 2 # Yes do warn.
1520 1522 break
1521 1523 else:
1522 1524 # clean up any optional leftovers
1523 1525 while expected.get(pos, None):
1524 1526 el = expected[pos].pop(0)
1525 1527 if el:
1526 1528 if not el.endswith(b" (?)\n"):
1527 1529 m = optline.match(el)
1528 1530 if m:
1529 1531 conditions = [c for c in m.group(2).split(b' ')]
1530 1532
1531 1533 if self._iftest(conditions):
1532 1534 # Don't append as optional line
1533 1535 continue
1534 1536 else:
1535 1537 continue
1536 1538 postout.append(b' ' + el)
1537 1539
1538 1540 if lcmd:
1539 1541 # Add on last return code.
1540 1542 ret = int(lcmd.split()[1])
1541 1543 if ret != 0:
1542 1544 postout.append(b' [%d]\n' % ret)
1543 1545 if pos in after:
1544 1546 # Merge in non-active test bits.
1545 1547 postout += after.pop(pos)
1546 1548 pos = int(lcmd.split()[0])
1547 1549
1548 1550 if pos in after:
1549 1551 postout += after.pop(pos)
1550 1552
1551 1553 if warnonly == 2:
1552 1554 exitcode = False # Set exitcode to warned.
1553 1555
1554 1556 return exitcode, postout
1555 1557
1556 1558 @staticmethod
1557 1559 def rematch(el, l):
1558 1560 try:
1559 1561 el = b'(?:' + el + b')'
1560 1562 # use \Z to ensure that the regex matches to the end of the string
1561 1563 if os.name == 'nt':
1562 1564 return re.match(el + br'\r?\n\Z', l)
1563 1565 return re.match(el + br'\n\Z', l)
1564 1566 except re.error:
1565 1567 # el is an invalid regex
1566 1568 return False
1567 1569
1568 1570 @staticmethod
1569 1571 def globmatch(el, l):
1570 1572 # The only supported special characters are * and ? plus / which also
1571 1573 # matches \ on windows. Escaping of these characters is supported.
1572 1574 if el + b'\n' == l:
1573 1575 if os.altsep:
1574 1576 # matching on "/" is not needed for this line
1575 1577 for pat in checkcodeglobpats:
1576 1578 if pat.match(el):
1577 1579 return True
1578 1580 return b'-glob'
1579 1581 return True
1580 1582 el = el.replace(b'$LOCALIP', b'*')
1581 1583 i, n = 0, len(el)
1582 1584 res = b''
1583 1585 while i < n:
1584 1586 c = el[i:i + 1]
1585 1587 i += 1
1586 1588 if c == b'\\' and i < n and el[i:i + 1] in b'*?\\/':
1587 1589 res += el[i - 1:i + 1]
1588 1590 i += 1
1589 1591 elif c == b'*':
1590 1592 res += b'.*'
1591 1593 elif c == b'?':
1592 1594 res += b'.'
1593 1595 elif c == b'/' and os.altsep:
1594 1596 res += b'[/\\\\]'
1595 1597 else:
1596 1598 res += re.escape(c)
1597 1599 return TTest.rematch(res, l)
1598 1600
1599 1601 def linematch(self, el, l):
1600 1602 retry = False
1601 1603 if el == l: # perfect match (fast)
1602 1604 return True
1603 1605 if el:
1604 1606 if el.endswith(b" (?)\n"):
1605 1607 retry = "retry"
1606 1608 el = el[:-5] + b"\n"
1607 1609 else:
1608 1610 m = optline.match(el)
1609 1611 if m:
1610 1612 conditions = [c for c in m.group(2).split(b' ')]
1611 1613
1612 1614 el = m.group(1) + b"\n"
1613 1615 if not self._iftest(conditions):
1614 1616 retry = "retry" # Not required by listed features
1615 1617
1616 1618 if el.endswith(b" (esc)\n"):
1617 1619 if PYTHON3:
1618 1620 el = el[:-7].decode('unicode_escape') + '\n'
1619 1621 el = el.encode('utf-8')
1620 1622 else:
1621 1623 el = el[:-7].decode('string-escape') + '\n'
1622 1624 if el == l or os.name == 'nt' and el[:-1] + b'\r\n' == l:
1623 1625 return True
1624 1626 if el.endswith(b" (re)\n"):
1625 1627 return TTest.rematch(el[:-6], l) or retry
1626 1628 if el.endswith(b" (glob)\n"):
1627 1629 # ignore '(glob)' added to l by 'replacements'
1628 1630 if l.endswith(b" (glob)\n"):
1629 1631 l = l[:-8] + b"\n"
1630 1632 return TTest.globmatch(el[:-8], l) or retry
1631 1633 if os.altsep:
1632 1634 _l = l.replace(b'\\', b'/')
1633 1635 if el == _l or os.name == 'nt' and el[:-1] + b'\r\n' == _l:
1634 1636 return True
1635 1637 return retry
1636 1638
1637 1639 @staticmethod
1638 1640 def parsehghaveoutput(lines):
1639 1641 '''Parse hghave log lines.
1640 1642
1641 1643 Return tuple of lists (missing, failed):
1642 1644 * the missing/unknown features
1643 1645 * the features for which existence check failed'''
1644 1646 missing = []
1645 1647 failed = []
1646 1648 for line in lines:
1647 1649 if line.startswith(TTest.SKIPPED_PREFIX):
1648 1650 line = line.splitlines()[0]
1649 1651 missing.append(line[len(TTest.SKIPPED_PREFIX):].decode('utf-8'))
1650 1652 elif line.startswith(TTest.FAILED_PREFIX):
1651 1653 line = line.splitlines()[0]
1652 1654 failed.append(line[len(TTest.FAILED_PREFIX):].decode('utf-8'))
1653 1655
1654 1656 return missing, failed
1655 1657
1656 1658 @staticmethod
1657 1659 def _escapef(m):
1658 1660 return TTest.ESCAPEMAP[m.group(0)]
1659 1661
1660 1662 @staticmethod
1661 1663 def _stringescape(s):
1662 1664 return TTest.ESCAPESUB(TTest._escapef, s)
1663 1665
1664 1666 iolock = threading.RLock()
1665 1667 firstlock = threading.RLock()
1666 1668 firsterror = False
1667 1669
1668 1670 class TestResult(unittest._TextTestResult):
1669 1671 """Holds results when executing via unittest."""
1670 1672 # Don't worry too much about accessing the non-public _TextTestResult.
1671 1673 # It is relatively common in Python testing tools.
1672 1674 def __init__(self, options, *args, **kwargs):
1673 1675 super(TestResult, self).__init__(*args, **kwargs)
1674 1676
1675 1677 self._options = options
1676 1678
1677 1679 # unittest.TestResult didn't have skipped until 2.7. We need to
1678 1680 # polyfill it.
1679 1681 self.skipped = []
1680 1682
1681 1683 # We have a custom "ignored" result that isn't present in any Python
1682 1684 # unittest implementation. It is very similar to skipped. It may make
1683 1685 # sense to map it into skip some day.
1684 1686 self.ignored = []
1685 1687
1686 1688 self.times = []
1687 1689 self._firststarttime = None
1688 1690 # Data stored for the benefit of generating xunit reports.
1689 1691 self.successes = []
1690 1692 self.faildata = {}
1691 1693
1692 1694 if options.color == 'auto':
1693 1695 self.color = pygmentspresent and self.stream.isatty()
1694 1696 elif options.color == 'never':
1695 1697 self.color = False
1696 1698 else: # 'always', for testing purposes
1697 1699 self.color = pygmentspresent
1698 1700
1699 1701 def addFailure(self, test, reason):
1700 1702 self.failures.append((test, reason))
1701 1703
1702 1704 if self._options.first:
1703 1705 self.stop()
1704 1706 else:
1705 1707 with iolock:
1706 1708 if reason == "timed out":
1707 1709 self.stream.write('t')
1708 1710 else:
1709 1711 if not self._options.nodiff:
1710 1712 self.stream.write('\n')
1711 1713 # Exclude the '\n' from highlighting to lex correctly
1712 1714 formatted = 'ERROR: %s output changed\n' % test
1713 1715 self.stream.write(highlightmsg(formatted, self.color))
1714 1716 self.stream.write('!')
1715 1717
1716 1718 self.stream.flush()
1717 1719
1718 1720 def addSuccess(self, test):
1719 1721 with iolock:
1720 1722 super(TestResult, self).addSuccess(test)
1721 1723 self.successes.append(test)
1722 1724
1723 1725 def addError(self, test, err):
1724 1726 super(TestResult, self).addError(test, err)
1725 1727 if self._options.first:
1726 1728 self.stop()
1727 1729
1728 1730 # Polyfill.
1729 1731 def addSkip(self, test, reason):
1730 1732 self.skipped.append((test, reason))
1731 1733 with iolock:
1732 1734 if self.showAll:
1733 1735 self.stream.writeln('skipped %s' % reason)
1734 1736 else:
1735 1737 self.stream.write('s')
1736 1738 self.stream.flush()
1737 1739
1738 1740 def addIgnore(self, test, reason):
1739 1741 self.ignored.append((test, reason))
1740 1742 with iolock:
1741 1743 if self.showAll:
1742 1744 self.stream.writeln('ignored %s' % reason)
1743 1745 else:
1744 1746 if reason not in ('not retesting', "doesn't match keyword"):
1745 1747 self.stream.write('i')
1746 1748 else:
1747 1749 self.testsRun += 1
1748 1750 self.stream.flush()
1749 1751
1750 1752 def addOutputMismatch(self, test, ret, got, expected):
1751 1753 """Record a mismatch in test output for a particular test."""
1752 1754 if self.shouldStop or firsterror:
1753 1755 # don't print, some other test case already failed and
1754 1756 # printed, we're just stale and probably failed due to our
1755 1757 # temp dir getting cleaned up.
1756 1758 return
1757 1759
1758 1760 accepted = False
1759 1761 lines = []
1760 1762
1761 1763 with iolock:
1762 1764 if self._options.nodiff:
1763 1765 pass
1764 1766 elif self._options.view:
1765 1767 v = self._options.view
1766 1768 if PYTHON3:
1767 1769 v = _bytespath(v)
1768 1770 os.system(b"%s %s %s" %
1769 1771 (v, test.refpath, test.errpath))
1770 1772 else:
1771 1773 servefail, lines = getdiff(expected, got,
1772 1774 test.refpath, test.errpath)
1773 1775 self.stream.write('\n')
1774 1776 for line in lines:
1775 1777 line = highlightdiff(line, self.color)
1776 1778 if PYTHON3:
1777 1779 self.stream.flush()
1778 1780 self.stream.buffer.write(line)
1779 1781 self.stream.buffer.flush()
1780 1782 else:
1781 1783 self.stream.write(line)
1782 1784 self.stream.flush()
1783 1785
1784 1786 if servefail:
1785 1787 raise test.failureException(
1786 1788 'server failed to start (HGPORT=%s)' % test._startport)
1787 1789
1788 1790 # handle interactive prompt without releasing iolock
1789 1791 if self._options.interactive:
1790 1792 if test.readrefout() != expected:
1791 1793 self.stream.write(
1792 1794 'Reference output has changed (run again to prompt '
1793 1795 'changes)')
1794 1796 else:
1795 1797 self.stream.write('Accept this change? [n] ')
1796 1798 answer = sys.stdin.readline().strip()
1797 1799 if answer.lower() in ('y', 'yes'):
1798 1800 if test.path.endswith(b'.t'):
1799 1801 rename(test.errpath, test.path)
1800 1802 else:
1801 1803 rename(test.errpath, '%s.out' % test.path)
1802 1804 accepted = True
1803 1805 if not accepted:
1804 1806 self.faildata[test.name] = b''.join(lines)
1805 1807
1806 1808 return accepted
1807 1809
1808 1810 def startTest(self, test):
1809 1811 super(TestResult, self).startTest(test)
1810 1812
1811 1813 # os.times module computes the user time and system time spent by
1812 1814 # child's processes along with real elapsed time taken by a process.
1813 1815 # This module has one limitation. It can only work for Linux user
1814 1816 # and not for Windows.
1815 1817 test.started = os.times()
1816 1818 if self._firststarttime is None: # thread racy but irrelevant
1817 1819 self._firststarttime = test.started[4]
1818 1820
1819 1821 def stopTest(self, test, interrupted=False):
1820 1822 super(TestResult, self).stopTest(test)
1821 1823
1822 1824 test.stopped = os.times()
1823 1825
1824 1826 starttime = test.started
1825 1827 endtime = test.stopped
1826 1828 origin = self._firststarttime
1827 1829 self.times.append((test.name,
1828 1830 endtime[2] - starttime[2], # user space CPU time
1829 1831 endtime[3] - starttime[3], # sys space CPU time
1830 1832 endtime[4] - starttime[4], # real time
1831 1833 starttime[4] - origin, # start date in run context
1832 1834 endtime[4] - origin, # end date in run context
1833 1835 ))
1834 1836
1835 1837 if interrupted:
1836 1838 with iolock:
1837 1839 self.stream.writeln('INTERRUPTED: %s (after %d seconds)' % (
1838 1840 test.name, self.times[-1][3]))
1839 1841
1840 1842 class TestSuite(unittest.TestSuite):
1841 1843 """Custom unittest TestSuite that knows how to execute Mercurial tests."""
1842 1844
1843 1845 def __init__(self, testdir, jobs=1, whitelist=None, blacklist=None,
1844 1846 retest=False, keywords=None, loop=False, runs_per_test=1,
1845 1847 loadtest=None, showchannels=False,
1846 1848 *args, **kwargs):
1847 1849 """Create a new instance that can run tests with a configuration.
1848 1850
1849 1851 testdir specifies the directory where tests are executed from. This
1850 1852 is typically the ``tests`` directory from Mercurial's source
1851 1853 repository.
1852 1854
1853 1855 jobs specifies the number of jobs to run concurrently. Each test
1854 1856 executes on its own thread. Tests actually spawn new processes, so
1855 1857 state mutation should not be an issue.
1856 1858
1857 1859 If there is only one job, it will use the main thread.
1858 1860
1859 1861 whitelist and blacklist denote tests that have been whitelisted and
1860 1862 blacklisted, respectively. These arguments don't belong in TestSuite.
1861 1863 Instead, whitelist and blacklist should be handled by the thing that
1862 1864 populates the TestSuite with tests. They are present to preserve
1863 1865 backwards compatible behavior which reports skipped tests as part
1864 1866 of the results.
1865 1867
1866 1868 retest denotes whether to retest failed tests. This arguably belongs
1867 1869 outside of TestSuite.
1868 1870
1869 1871 keywords denotes key words that will be used to filter which tests
1870 1872 to execute. This arguably belongs outside of TestSuite.
1871 1873
1872 1874 loop denotes whether to loop over tests forever.
1873 1875 """
1874 1876 super(TestSuite, self).__init__(*args, **kwargs)
1875 1877
1876 1878 self._jobs = jobs
1877 1879 self._whitelist = whitelist
1878 1880 self._blacklist = blacklist
1879 1881 self._retest = retest
1880 1882 self._keywords = keywords
1881 1883 self._loop = loop
1882 1884 self._runs_per_test = runs_per_test
1883 1885 self._loadtest = loadtest
1884 1886 self._showchannels = showchannels
1885 1887
1886 1888 def run(self, result):
1887 1889 # We have a number of filters that need to be applied. We do this
1888 1890 # here instead of inside Test because it makes the running logic for
1889 1891 # Test simpler.
1890 1892 tests = []
1891 1893 num_tests = [0]
1892 1894 for test in self._tests:
1893 1895 def get():
1894 1896 num_tests[0] += 1
1895 1897 if getattr(test, 'should_reload', False):
1896 1898 return self._loadtest(test, num_tests[0])
1897 1899 return test
1898 1900 if not os.path.exists(test.path):
1899 1901 result.addSkip(test, "Doesn't exist")
1900 1902 continue
1901 1903
1902 1904 if not (self._whitelist and test.bname in self._whitelist):
1903 1905 if self._blacklist and test.bname in self._blacklist:
1904 1906 result.addSkip(test, 'blacklisted')
1905 1907 continue
1906 1908
1907 1909 if self._retest and not os.path.exists(test.errpath):
1908 1910 result.addIgnore(test, 'not retesting')
1909 1911 continue
1910 1912
1911 1913 if self._keywords:
1912 1914 with open(test.path, 'rb') as f:
1913 1915 t = f.read().lower() + test.bname.lower()
1914 1916 ignored = False
1915 1917 for k in self._keywords.lower().split():
1916 1918 if k not in t:
1917 1919 result.addIgnore(test, "doesn't match keyword")
1918 1920 ignored = True
1919 1921 break
1920 1922
1921 1923 if ignored:
1922 1924 continue
1923 1925 for _ in xrange(self._runs_per_test):
1924 1926 tests.append(get())
1925 1927
1926 1928 runtests = list(tests)
1927 1929 done = queue.Queue()
1928 1930 running = 0
1929 1931
1930 1932 channels = [""] * self._jobs
1931 1933
1932 1934 def job(test, result):
1933 1935 for n, v in enumerate(channels):
1934 1936 if not v:
1935 1937 channel = n
1936 1938 break
1937 1939 else:
1938 1940 raise ValueError('Could not find output channel')
1939 1941 channels[channel] = "=" + test.name[5:].split(".")[0]
1940 1942 try:
1941 1943 test(result)
1942 1944 done.put(None)
1943 1945 except KeyboardInterrupt:
1944 1946 pass
1945 1947 except: # re-raises
1946 1948 done.put(('!', test, 'run-test raised an error, see traceback'))
1947 1949 raise
1948 1950 finally:
1949 1951 try:
1950 1952 channels[channel] = ''
1951 1953 except IndexError:
1952 1954 pass
1953 1955
1954 1956 def stat():
1955 1957 count = 0
1956 1958 while channels:
1957 1959 d = '\n%03s ' % count
1958 1960 for n, v in enumerate(channels):
1959 1961 if v:
1960 1962 d += v[0]
1961 1963 channels[n] = v[1:] or '.'
1962 1964 else:
1963 1965 d += ' '
1964 1966 d += ' '
1965 1967 with iolock:
1966 1968 sys.stdout.write(d + ' ')
1967 1969 sys.stdout.flush()
1968 1970 for x in xrange(10):
1969 1971 if channels:
1970 1972 time.sleep(.1)
1971 1973 count += 1
1972 1974
1973 1975 stoppedearly = False
1974 1976
1975 1977 if self._showchannels:
1976 1978 statthread = threading.Thread(target=stat, name="stat")
1977 1979 statthread.start()
1978 1980
1979 1981 try:
1980 1982 while tests or running:
1981 1983 if not done.empty() or running == self._jobs or not tests:
1982 1984 try:
1983 1985 done.get(True, 1)
1984 1986 running -= 1
1985 1987 if result and result.shouldStop:
1986 1988 stoppedearly = True
1987 1989 break
1988 1990 except queue.Empty:
1989 1991 continue
1990 1992 if tests and not running == self._jobs:
1991 1993 test = tests.pop(0)
1992 1994 if self._loop:
1993 1995 if getattr(test, 'should_reload', False):
1994 1996 num_tests[0] += 1
1995 1997 tests.append(
1996 1998 self._loadtest(test, num_tests[0]))
1997 1999 else:
1998 2000 tests.append(test)
1999 2001 if self._jobs == 1:
2000 2002 job(test, result)
2001 2003 else:
2002 2004 t = threading.Thread(target=job, name=test.name,
2003 2005 args=(test, result))
2004 2006 t.start()
2005 2007 running += 1
2006 2008
2007 2009 # If we stop early we still need to wait on started tests to
2008 2010 # finish. Otherwise, there is a race between the test completing
2009 2011 # and the test's cleanup code running. This could result in the
2010 2012 # test reporting incorrect.
2011 2013 if stoppedearly:
2012 2014 while running:
2013 2015 try:
2014 2016 done.get(True, 1)
2015 2017 running -= 1
2016 2018 except queue.Empty:
2017 2019 continue
2018 2020 except KeyboardInterrupt:
2019 2021 for test in runtests:
2020 2022 test.abort()
2021 2023
2022 2024 channels = []
2023 2025
2024 2026 return result
2025 2027
2026 2028 # Save the most recent 5 wall-clock runtimes of each test to a
2027 2029 # human-readable text file named .testtimes. Tests are sorted
2028 2030 # alphabetically, while times for each test are listed from oldest to
2029 2031 # newest.
2030 2032
2031 2033 def loadtimes(outputdir):
2032 2034 times = []
2033 2035 try:
2034 2036 with open(os.path.join(outputdir, b'.testtimes')) as fp:
2035 2037 for line in fp:
2036 2038 m = re.match('(.*?) ([0-9. ]+)', line)
2037 2039 times.append((m.group(1),
2038 2040 [float(t) for t in m.group(2).split()]))
2039 2041 except IOError as err:
2040 2042 if err.errno != errno.ENOENT:
2041 2043 raise
2042 2044 return times
2043 2045
2044 2046 def savetimes(outputdir, result):
2045 2047 saved = dict(loadtimes(outputdir))
2046 2048 maxruns = 5
2047 2049 skipped = set([str(t[0]) for t in result.skipped])
2048 2050 for tdata in result.times:
2049 2051 test, real = tdata[0], tdata[3]
2050 2052 if test not in skipped:
2051 2053 ts = saved.setdefault(test, [])
2052 2054 ts.append(real)
2053 2055 ts[:] = ts[-maxruns:]
2054 2056
2055 2057 fd, tmpname = tempfile.mkstemp(prefix=b'.testtimes',
2056 2058 dir=outputdir, text=True)
2057 2059 with os.fdopen(fd, 'w') as fp:
2058 2060 for name, ts in sorted(saved.items()):
2059 2061 fp.write('%s %s\n' % (name, ' '.join(['%.3f' % (t,) for t in ts])))
2060 2062 timepath = os.path.join(outputdir, b'.testtimes')
2061 2063 try:
2062 2064 os.unlink(timepath)
2063 2065 except OSError:
2064 2066 pass
2065 2067 try:
2066 2068 os.rename(tmpname, timepath)
2067 2069 except OSError:
2068 2070 pass
2069 2071
2070 2072 class TextTestRunner(unittest.TextTestRunner):
2071 2073 """Custom unittest test runner that uses appropriate settings."""
2072 2074
2073 2075 def __init__(self, runner, *args, **kwargs):
2074 2076 super(TextTestRunner, self).__init__(*args, **kwargs)
2075 2077
2076 2078 self._runner = runner
2077 2079
2078 2080 def listtests(self, test):
2079 2081 result = TestResult(self._runner.options, self.stream,
2080 2082 self.descriptions, 0)
2081 2083 test = sorted(test, key=lambda t: t.name)
2082 2084 for t in test:
2083 2085 print(t.name)
2084 2086 result.addSuccess(t)
2085 2087
2086 2088 if self._runner.options.xunit:
2087 2089 with open(self._runner.options.xunit, "wb") as xuf:
2088 2090 self._writexunit(result, xuf)
2089 2091
2090 2092 if self._runner.options.json:
2091 2093 jsonpath = os.path.join(self._runner._outputdir, b'report.json')
2092 2094 with open(jsonpath, 'w') as fp:
2093 2095 self._writejson(result, fp)
2094 2096
2095 2097 return result
2096 2098
2097 2099 def run(self, test):
2098 2100 result = TestResult(self._runner.options, self.stream,
2099 2101 self.descriptions, self.verbosity)
2100 2102
2101 2103 test(result)
2102 2104
2103 2105 failed = len(result.failures)
2104 2106 skipped = len(result.skipped)
2105 2107 ignored = len(result.ignored)
2106 2108
2107 2109 with iolock:
2108 2110 self.stream.writeln('')
2109 2111
2110 2112 if not self._runner.options.noskips:
2111 2113 for test, msg in result.skipped:
2112 2114 formatted = 'Skipped %s: %s\n' % (test.name, msg)
2113 2115 self.stream.write(highlightmsg(formatted, result.color))
2114 2116 for test, msg in result.failures:
2115 2117 formatted = 'Failed %s: %s\n' % (test.name, msg)
2116 2118 self.stream.write(highlightmsg(formatted, result.color))
2117 2119 for test, msg in result.errors:
2118 2120 self.stream.writeln('Errored %s: %s' % (test.name, msg))
2119 2121
2120 2122 if self._runner.options.xunit:
2121 2123 with open(self._runner.options.xunit, "wb") as xuf:
2122 2124 self._writexunit(result, xuf)
2123 2125
2124 2126 if self._runner.options.json:
2125 2127 jsonpath = os.path.join(self._runner._outputdir, b'report.json')
2126 2128 with open(jsonpath, 'w') as fp:
2127 2129 self._writejson(result, fp)
2128 2130
2129 2131 self._runner._checkhglib('Tested')
2130 2132
2131 2133 savetimes(self._runner._outputdir, result)
2132 2134
2133 2135 if failed and self._runner.options.known_good_rev:
2134 2136 self._bisecttests(t for t, m in result.failures)
2135 2137 self.stream.writeln(
2136 2138 '# Ran %d tests, %d skipped, %d failed.'
2137 2139 % (result.testsRun, skipped + ignored, failed))
2138 2140 if failed:
2139 2141 self.stream.writeln('python hash seed: %s' %
2140 2142 os.environ['PYTHONHASHSEED'])
2141 2143 if self._runner.options.time:
2142 2144 self.printtimes(result.times)
2143 2145
2144 2146 if self._runner.options.exceptions:
2145 2147 exceptions = aggregateexceptions(
2146 2148 os.path.join(self._runner._outputdir, b'exceptions'))
2147 2149
2148 2150 self.stream.writeln('Exceptions Report:')
2149 2151 self.stream.writeln('%d total from %d frames' %
2150 2152 (exceptions['total'],
2151 2153 len(exceptions['exceptioncounts'])))
2152 2154 combined = exceptions['combined']
2153 2155 for key in sorted(combined, key=combined.get, reverse=True):
2154 2156 frame, line, exc = key
2155 2157 totalcount, testcount, leastcount, leasttest = combined[key]
2156 2158
2157 2159 self.stream.writeln('%d (%d tests)\t%s: %s (%s - %d total)'
2158 2160 % (totalcount,
2159 2161 testcount,
2160 2162 frame, exc,
2161 2163 leasttest, leastcount))
2162 2164
2163 2165 self.stream.flush()
2164 2166
2165 2167 return result
2166 2168
2167 2169 def _bisecttests(self, tests):
2168 2170 bisectcmd = ['hg', 'bisect']
2169 2171 bisectrepo = self._runner.options.bisect_repo
2170 2172 if bisectrepo:
2171 2173 bisectcmd.extend(['-R', os.path.abspath(bisectrepo)])
2172 2174 def pread(args):
2173 2175 env = os.environ.copy()
2174 2176 env['HGPLAIN'] = '1'
2175 2177 p = subprocess.Popen(args, stderr=subprocess.STDOUT,
2176 2178 stdout=subprocess.PIPE, env=env)
2177 2179 data = p.stdout.read()
2178 2180 p.wait()
2179 2181 return data
2180 2182 for test in tests:
2181 2183 pread(bisectcmd + ['--reset']),
2182 2184 pread(bisectcmd + ['--bad', '.'])
2183 2185 pread(bisectcmd + ['--good', self._runner.options.known_good_rev])
2184 2186 # TODO: we probably need to forward more options
2185 2187 # that alter hg's behavior inside the tests.
2186 2188 opts = ''
2187 2189 withhg = self._runner.options.with_hg
2188 2190 if withhg:
2189 2191 opts += ' --with-hg=%s ' % shellquote(_strpath(withhg))
2190 2192 rtc = '%s %s %s %s' % (sys.executable, sys.argv[0], opts,
2191 2193 test)
2192 2194 data = pread(bisectcmd + ['--command', rtc])
2193 2195 m = re.search(
2194 2196 (br'\nThe first (?P<goodbad>bad|good) revision '
2195 2197 br'is:\nchangeset: +\d+:(?P<node>[a-f0-9]+)\n.*\n'
2196 2198 br'summary: +(?P<summary>[^\n]+)\n'),
2197 2199 data, (re.MULTILINE | re.DOTALL))
2198 2200 if m is None:
2199 2201 self.stream.writeln(
2200 2202 'Failed to identify failure point for %s' % test)
2201 2203 continue
2202 2204 dat = m.groupdict()
2203 2205 verb = 'broken' if dat['goodbad'] == 'bad' else 'fixed'
2204 2206 self.stream.writeln(
2205 2207 '%s %s by %s (%s)' % (
2206 2208 test, verb, dat['node'], dat['summary']))
2207 2209
2208 2210 def printtimes(self, times):
2209 2211 # iolock held by run
2210 2212 self.stream.writeln('# Producing time report')
2211 2213 times.sort(key=lambda t: (t[3]))
2212 2214 cols = '%7.3f %7.3f %7.3f %7.3f %7.3f %s'
2213 2215 self.stream.writeln('%-7s %-7s %-7s %-7s %-7s %s' %
2214 2216 ('start', 'end', 'cuser', 'csys', 'real', 'Test'))
2215 2217 for tdata in times:
2216 2218 test = tdata[0]
2217 2219 cuser, csys, real, start, end = tdata[1:6]
2218 2220 self.stream.writeln(cols % (start, end, cuser, csys, real, test))
2219 2221
2220 2222 @staticmethod
2221 2223 def _writexunit(result, outf):
2222 2224 # See http://llg.cubic.org/docs/junit/ for a reference.
2223 2225 timesd = dict((t[0], t[3]) for t in result.times)
2224 2226 doc = minidom.Document()
2225 2227 s = doc.createElement('testsuite')
2226 2228 s.setAttribute('name', 'run-tests')
2227 2229 s.setAttribute('tests', str(result.testsRun))
2228 2230 s.setAttribute('errors', "0") # TODO
2229 2231 s.setAttribute('failures', str(len(result.failures)))
2230 2232 s.setAttribute('skipped', str(len(result.skipped) +
2231 2233 len(result.ignored)))
2232 2234 doc.appendChild(s)
2233 2235 for tc in result.successes:
2234 2236 t = doc.createElement('testcase')
2235 2237 t.setAttribute('name', tc.name)
2236 2238 tctime = timesd.get(tc.name)
2237 2239 if tctime is not None:
2238 2240 t.setAttribute('time', '%.3f' % tctime)
2239 2241 s.appendChild(t)
2240 2242 for tc, err in sorted(result.faildata.items()):
2241 2243 t = doc.createElement('testcase')
2242 2244 t.setAttribute('name', tc)
2243 2245 tctime = timesd.get(tc)
2244 2246 if tctime is not None:
2245 2247 t.setAttribute('time', '%.3f' % tctime)
2246 2248 # createCDATASection expects a unicode or it will
2247 2249 # convert using default conversion rules, which will
2248 2250 # fail if string isn't ASCII.
2249 2251 err = cdatasafe(err).decode('utf-8', 'replace')
2250 2252 cd = doc.createCDATASection(err)
2251 2253 # Use 'failure' here instead of 'error' to match errors = 0,
2252 2254 # failures = len(result.failures) in the testsuite element.
2253 2255 failelem = doc.createElement('failure')
2254 2256 failelem.setAttribute('message', 'output changed')
2255 2257 failelem.setAttribute('type', 'output-mismatch')
2256 2258 failelem.appendChild(cd)
2257 2259 t.appendChild(failelem)
2258 2260 s.appendChild(t)
2259 2261 for tc, message in result.skipped:
2260 2262 # According to the schema, 'skipped' has no attributes. So store
2261 2263 # the skip message as a text node instead.
2262 2264 t = doc.createElement('testcase')
2263 2265 t.setAttribute('name', tc.name)
2264 2266 binmessage = message.encode('utf-8')
2265 2267 message = cdatasafe(binmessage).decode('utf-8', 'replace')
2266 2268 cd = doc.createCDATASection(message)
2267 2269 skipelem = doc.createElement('skipped')
2268 2270 skipelem.appendChild(cd)
2269 2271 t.appendChild(skipelem)
2270 2272 s.appendChild(t)
2271 2273 outf.write(doc.toprettyxml(indent=' ', encoding='utf-8'))
2272 2274
2273 2275 @staticmethod
2274 2276 def _writejson(result, outf):
2275 2277 timesd = {}
2276 2278 for tdata in result.times:
2277 2279 test = tdata[0]
2278 2280 timesd[test] = tdata[1:]
2279 2281
2280 2282 outcome = {}
2281 2283 groups = [('success', ((tc, None)
2282 2284 for tc in result.successes)),
2283 2285 ('failure', result.failures),
2284 2286 ('skip', result.skipped)]
2285 2287 for res, testcases in groups:
2286 2288 for tc, __ in testcases:
2287 2289 if tc.name in timesd:
2288 2290 diff = result.faildata.get(tc.name, b'')
2289 2291 try:
2290 2292 diff = diff.decode('unicode_escape')
2291 2293 except UnicodeDecodeError as e:
2292 2294 diff = '%r decoding diff, sorry' % e
2293 2295 tres = {'result': res,
2294 2296 'time': ('%0.3f' % timesd[tc.name][2]),
2295 2297 'cuser': ('%0.3f' % timesd[tc.name][0]),
2296 2298 'csys': ('%0.3f' % timesd[tc.name][1]),
2297 2299 'start': ('%0.3f' % timesd[tc.name][3]),
2298 2300 'end': ('%0.3f' % timesd[tc.name][4]),
2299 2301 'diff': diff,
2300 2302 }
2301 2303 else:
2302 2304 # blacklisted test
2303 2305 tres = {'result': res}
2304 2306
2305 2307 outcome[tc.name] = tres
2306 2308 jsonout = json.dumps(outcome, sort_keys=True, indent=4,
2307 2309 separators=(',', ': '))
2308 2310 outf.writelines(("testreport =", jsonout))
2309 2311
2310 def sorttests(testdescs, shuffle=False):
2312 def sorttests(testdescs, previoustimes, shuffle=False):
2311 2313 """Do an in-place sort of tests."""
2312 2314 if shuffle:
2313 2315 random.shuffle(testdescs)
2314 2316 return
2315 2317
2318 if previoustimes:
2319 def sortkey(f):
2320 f = f['path']
2321 if f in previoustimes:
2322 # Use most recent time as estimate
2323 return -previoustimes[f][-1]
2324 else:
2325 # Default to a rather arbitrary value of 1 second for new tests
2326 return -1.0
2327 else:
2316 2328 # keywords for slow tests
2317 2329 slow = {b'svn': 10,
2318 2330 b'cvs': 10,
2319 2331 b'hghave': 10,
2320 2332 b'largefiles-update': 10,
2321 2333 b'run-tests': 10,
2322 2334 b'corruption': 10,
2323 2335 b'race': 10,
2324 2336 b'i18n': 10,
2325 2337 b'check': 100,
2326 2338 b'gendoc': 100,
2327 2339 b'contrib-perf': 200,
2328 2340 }
2329 2341 perf = {}
2330 2342
2331 2343 def sortkey(f):
2332 2344 # run largest tests first, as they tend to take the longest
2333 2345 f = f['path']
2334 2346 try:
2335 2347 return perf[f]
2336 2348 except KeyError:
2337 2349 try:
2338 2350 val = -os.stat(f).st_size
2339 2351 except OSError as e:
2340 2352 if e.errno != errno.ENOENT:
2341 2353 raise
2342 2354 perf[f] = -1e9 # file does not exist, tell early
2343 2355 return -1e9
2344 2356 for kw, mul in slow.items():
2345 2357 if kw in f:
2346 2358 val *= mul
2347 2359 if f.endswith(b'.py'):
2348 2360 val /= 10.0
2349 2361 perf[f] = val / 1000.0
2350 2362 return perf[f]
2351 2363
2352 2364 testdescs.sort(key=sortkey)
2353 2365
2354 2366 class TestRunner(object):
2355 2367 """Holds context for executing tests.
2356 2368
2357 2369 Tests rely on a lot of state. This object holds it for them.
2358 2370 """
2359 2371
2360 2372 # Programs required to run tests.
2361 2373 REQUIREDTOOLS = [
2362 2374 b'diff',
2363 2375 b'grep',
2364 2376 b'unzip',
2365 2377 b'gunzip',
2366 2378 b'bunzip2',
2367 2379 b'sed',
2368 2380 ]
2369 2381
2370 2382 # Maps file extensions to test class.
2371 2383 TESTTYPES = [
2372 2384 (b'.py', PythonTest),
2373 2385 (b'.t', TTest),
2374 2386 ]
2375 2387
2376 2388 def __init__(self):
2377 2389 self.options = None
2378 2390 self._hgroot = None
2379 2391 self._testdir = None
2380 2392 self._outputdir = None
2381 2393 self._hgtmp = None
2382 2394 self._installdir = None
2383 2395 self._bindir = None
2384 2396 self._tmpbinddir = None
2385 2397 self._pythondir = None
2386 2398 self._coveragefile = None
2387 2399 self._createdfiles = []
2388 2400 self._hgcommand = None
2389 2401 self._hgpath = None
2390 2402 self._portoffset = 0
2391 2403 self._ports = {}
2392 2404
2393 2405 def run(self, args, parser=None):
2394 2406 """Run the test suite."""
2395 2407 oldmask = os.umask(0o22)
2396 2408 try:
2397 2409 parser = parser or getparser()
2398 2410 options = parseargs(args, parser)
2399 2411 tests = [_bytespath(a) for a in options.tests]
2400 2412 if options.test_list is not None:
2401 2413 for listfile in options.test_list:
2402 2414 with open(listfile, 'rb') as f:
2403 2415 tests.extend(t for t in f.read().splitlines() if t)
2404 2416 self.options = options
2405 2417
2406 2418 self._checktools()
2407 2419 testdescs = self.findtests(tests)
2408 2420 if options.profile_runner:
2409 2421 import statprof
2410 2422 statprof.start()
2411 2423 result = self._run(testdescs)
2412 2424 if options.profile_runner:
2413 2425 statprof.stop()
2414 2426 statprof.display()
2415 2427 return result
2416 2428
2417 2429 finally:
2418 2430 os.umask(oldmask)
2419 2431
2420 2432 def _run(self, testdescs):
2421 sorttests(testdescs, shuffle=self.options.random)
2422
2423 2433 self._testdir = osenvironb[b'TESTDIR'] = getattr(
2424 2434 os, 'getcwdb', os.getcwd)()
2425 2435 # assume all tests in same folder for now
2426 2436 if testdescs:
2427 2437 pathname = os.path.dirname(testdescs[0]['path'])
2428 2438 if pathname:
2429 2439 osenvironb[b'TESTDIR'] = os.path.join(osenvironb[b'TESTDIR'],
2430 2440 pathname)
2431 2441 if self.options.outputdir:
2432 2442 self._outputdir = canonpath(_bytespath(self.options.outputdir))
2433 2443 else:
2434 2444 self._outputdir = self._testdir
2435 2445 if testdescs and pathname:
2436 2446 self._outputdir = os.path.join(self._outputdir, pathname)
2447 previoustimes = {}
2448 if self.options.order_by_runtime:
2449 previoustimes = dict(loadtimes(self._outputdir))
2450 sorttests(testdescs, previoustimes, shuffle=self.options.random)
2437 2451
2438 2452 if 'PYTHONHASHSEED' not in os.environ:
2439 2453 # use a random python hash seed all the time
2440 2454 # we do the randomness ourself to know what seed is used
2441 2455 os.environ['PYTHONHASHSEED'] = str(random.getrandbits(32))
2442 2456
2443 2457 if self.options.tmpdir:
2444 2458 self.options.keep_tmpdir = True
2445 2459 tmpdir = _bytespath(self.options.tmpdir)
2446 2460 if os.path.exists(tmpdir):
2447 2461 # Meaning of tmpdir has changed since 1.3: we used to create
2448 2462 # HGTMP inside tmpdir; now HGTMP is tmpdir. So fail if
2449 2463 # tmpdir already exists.
2450 2464 print("error: temp dir %r already exists" % tmpdir)
2451 2465 return 1
2452 2466
2453 2467 os.makedirs(tmpdir)
2454 2468 else:
2455 2469 d = None
2456 2470 if os.name == 'nt':
2457 2471 # without this, we get the default temp dir location, but
2458 2472 # in all lowercase, which causes troubles with paths (issue3490)
2459 2473 d = osenvironb.get(b'TMP', None)
2460 2474 tmpdir = tempfile.mkdtemp(b'', b'hgtests.', d)
2461 2475
2462 2476 self._hgtmp = osenvironb[b'HGTMP'] = (
2463 2477 os.path.realpath(tmpdir))
2464 2478
2465 2479 if self.options.with_hg:
2466 2480 self._installdir = None
2467 2481 whg = self.options.with_hg
2468 2482 self._bindir = os.path.dirname(os.path.realpath(whg))
2469 2483 assert isinstance(self._bindir, bytes)
2470 2484 self._hgcommand = os.path.basename(whg)
2471 2485 self._tmpbindir = os.path.join(self._hgtmp, b'install', b'bin')
2472 2486 os.makedirs(self._tmpbindir)
2473 2487
2474 2488 normbin = os.path.normpath(os.path.abspath(whg))
2475 2489 normbin = normbin.replace(os.sep.encode('ascii'), b'/')
2476 2490
2477 2491 # Other Python scripts in the test harness need to
2478 2492 # `import mercurial`. If `hg` is a Python script, we assume
2479 2493 # the Mercurial modules are relative to its path and tell the tests
2480 2494 # to load Python modules from its directory.
2481 2495 with open(whg, 'rb') as fh:
2482 2496 initial = fh.read(1024)
2483 2497
2484 2498 if re.match(b'#!.*python', initial):
2485 2499 self._pythondir = self._bindir
2486 2500 # If it looks like our in-repo Rust binary, use the source root.
2487 2501 # This is a bit hacky. But rhg is still not supported outside the
2488 2502 # source directory. So until it is, do the simple thing.
2489 2503 elif re.search(b'/rust/target/[^/]+/hg', normbin):
2490 2504 self._pythondir = os.path.dirname(self._testdir)
2491 2505 # Fall back to the legacy behavior.
2492 2506 else:
2493 2507 self._pythondir = self._bindir
2494 2508
2495 2509 else:
2496 2510 self._installdir = os.path.join(self._hgtmp, b"install")
2497 2511 self._bindir = os.path.join(self._installdir, b"bin")
2498 2512 self._hgcommand = b'hg'
2499 2513 self._tmpbindir = self._bindir
2500 2514 self._pythondir = os.path.join(self._installdir, b"lib", b"python")
2501 2515
2502 2516 # set CHGHG, then replace "hg" command by "chg"
2503 2517 chgbindir = self._bindir
2504 2518 if self.options.chg or self.options.with_chg:
2505 2519 osenvironb[b'CHGHG'] = os.path.join(self._bindir, self._hgcommand)
2506 2520 else:
2507 2521 osenvironb.pop(b'CHGHG', None) # drop flag for hghave
2508 2522 if self.options.chg:
2509 2523 self._hgcommand = b'chg'
2510 2524 elif self.options.with_chg:
2511 2525 chgbindir = os.path.dirname(os.path.realpath(self.options.with_chg))
2512 2526 self._hgcommand = os.path.basename(self.options.with_chg)
2513 2527
2514 2528 osenvironb[b"BINDIR"] = self._bindir
2515 2529 osenvironb[b"PYTHON"] = PYTHON
2516 2530
2517 2531 if self.options.with_python3:
2518 2532 osenvironb[b'PYTHON3'] = self.options.with_python3
2519 2533
2520 2534 fileb = _bytespath(__file__)
2521 2535 runtestdir = os.path.abspath(os.path.dirname(fileb))
2522 2536 osenvironb[b'RUNTESTDIR'] = runtestdir
2523 2537 if PYTHON3:
2524 2538 sepb = _bytespath(os.pathsep)
2525 2539 else:
2526 2540 sepb = os.pathsep
2527 2541 path = [self._bindir, runtestdir] + osenvironb[b"PATH"].split(sepb)
2528 2542 if os.path.islink(__file__):
2529 2543 # test helper will likely be at the end of the symlink
2530 2544 realfile = os.path.realpath(fileb)
2531 2545 realdir = os.path.abspath(os.path.dirname(realfile))
2532 2546 path.insert(2, realdir)
2533 2547 if chgbindir != self._bindir:
2534 2548 path.insert(1, chgbindir)
2535 2549 if self._testdir != runtestdir:
2536 2550 path = [self._testdir] + path
2537 2551 if self._tmpbindir != self._bindir:
2538 2552 path = [self._tmpbindir] + path
2539 2553 osenvironb[b"PATH"] = sepb.join(path)
2540 2554
2541 2555 # Include TESTDIR in PYTHONPATH so that out-of-tree extensions
2542 2556 # can run .../tests/run-tests.py test-foo where test-foo
2543 2557 # adds an extension to HGRC. Also include run-test.py directory to
2544 2558 # import modules like heredoctest.
2545 2559 pypath = [self._pythondir, self._testdir, runtestdir]
2546 2560 # We have to augment PYTHONPATH, rather than simply replacing
2547 2561 # it, in case external libraries are only available via current
2548 2562 # PYTHONPATH. (In particular, the Subversion bindings on OS X
2549 2563 # are in /opt/subversion.)
2550 2564 oldpypath = osenvironb.get(IMPL_PATH)
2551 2565 if oldpypath:
2552 2566 pypath.append(oldpypath)
2553 2567 osenvironb[IMPL_PATH] = sepb.join(pypath)
2554 2568
2555 2569 if self.options.pure:
2556 2570 os.environ["HGTEST_RUN_TESTS_PURE"] = "--pure"
2557 2571 os.environ["HGMODULEPOLICY"] = "py"
2558 2572
2559 2573 if self.options.allow_slow_tests:
2560 2574 os.environ["HGTEST_SLOW"] = "slow"
2561 2575 elif 'HGTEST_SLOW' in os.environ:
2562 2576 del os.environ['HGTEST_SLOW']
2563 2577
2564 2578 self._coveragefile = os.path.join(self._testdir, b'.coverage')
2565 2579
2566 2580 if self.options.exceptions:
2567 2581 exceptionsdir = os.path.join(self._outputdir, b'exceptions')
2568 2582 try:
2569 2583 os.makedirs(exceptionsdir)
2570 2584 except OSError as e:
2571 2585 if e.errno != errno.EEXIST:
2572 2586 raise
2573 2587
2574 2588 # Remove all existing exception reports.
2575 2589 for f in os.listdir(exceptionsdir):
2576 2590 os.unlink(os.path.join(exceptionsdir, f))
2577 2591
2578 2592 osenvironb[b'HGEXCEPTIONSDIR'] = exceptionsdir
2579 2593 logexceptions = os.path.join(self._testdir, b'logexceptions.py')
2580 2594 self.options.extra_config_opt.append(
2581 2595 'extensions.logexceptions=%s' % logexceptions.decode('utf-8'))
2582 2596
2583 2597 vlog("# Using TESTDIR", self._testdir)
2584 2598 vlog("# Using RUNTESTDIR", osenvironb[b'RUNTESTDIR'])
2585 2599 vlog("# Using HGTMP", self._hgtmp)
2586 2600 vlog("# Using PATH", os.environ["PATH"])
2587 2601 vlog("# Using", IMPL_PATH, osenvironb[IMPL_PATH])
2588 2602 vlog("# Writing to directory", self._outputdir)
2589 2603
2590 2604 try:
2591 2605 return self._runtests(testdescs) or 0
2592 2606 finally:
2593 2607 time.sleep(.1)
2594 2608 self._cleanup()
2595 2609
2596 2610 def findtests(self, args):
2597 2611 """Finds possible test files from arguments.
2598 2612
2599 2613 If you wish to inject custom tests into the test harness, this would
2600 2614 be a good function to monkeypatch or override in a derived class.
2601 2615 """
2602 2616 if not args:
2603 2617 if self.options.changed:
2604 2618 proc = Popen4('hg st --rev "%s" -man0 .' %
2605 2619 self.options.changed, None, 0)
2606 2620 stdout, stderr = proc.communicate()
2607 2621 args = stdout.strip(b'\0').split(b'\0')
2608 2622 else:
2609 2623 args = os.listdir(b'.')
2610 2624
2611 2625 expanded_args = []
2612 2626 for arg in args:
2613 2627 if os.path.isdir(arg):
2614 2628 if not arg.endswith(b'/'):
2615 2629 arg += b'/'
2616 2630 expanded_args.extend([arg + a for a in os.listdir(arg)])
2617 2631 else:
2618 2632 expanded_args.append(arg)
2619 2633 args = expanded_args
2620 2634
2621 2635 tests = []
2622 2636 for t in args:
2623 2637 if not (os.path.basename(t).startswith(b'test-')
2624 2638 and (t.endswith(b'.py') or t.endswith(b'.t'))):
2625 2639 continue
2626 2640 if t.endswith(b'.t'):
2627 2641 # .t file may contain multiple test cases
2628 2642 cases = sorted(parsettestcases(t))
2629 2643 if cases:
2630 2644 tests += [{'path': t, 'case': c} for c in sorted(cases)]
2631 2645 else:
2632 2646 tests.append({'path': t})
2633 2647 else:
2634 2648 tests.append({'path': t})
2635 2649 return tests
2636 2650
2637 2651 def _runtests(self, testdescs):
2638 2652 def _reloadtest(test, i):
2639 2653 # convert a test back to its description dict
2640 2654 desc = {'path': test.path}
2641 2655 case = getattr(test, '_case', None)
2642 2656 if case:
2643 2657 desc['case'] = case
2644 2658 return self._gettest(desc, i)
2645 2659
2646 2660 try:
2647 2661 if self.options.restart:
2648 2662 orig = list(testdescs)
2649 2663 while testdescs:
2650 2664 desc = testdescs[0]
2651 2665 # desc['path'] is a relative path
2652 2666 if 'case' in desc:
2653 2667 errpath = b'%s.%s.err' % (desc['path'], desc['case'])
2654 2668 else:
2655 2669 errpath = b'%s.err' % desc['path']
2656 2670 errpath = os.path.join(self._outputdir, errpath)
2657 2671 if os.path.exists(errpath):
2658 2672 break
2659 2673 testdescs.pop(0)
2660 2674 if not testdescs:
2661 2675 print("running all tests")
2662 2676 testdescs = orig
2663 2677
2664 2678 tests = [self._gettest(d, i) for i, d in enumerate(testdescs)]
2665 2679
2666 2680 failed = False
2667 2681 kws = self.options.keywords
2668 2682 if kws is not None and PYTHON3:
2669 2683 kws = kws.encode('utf-8')
2670 2684
2671 2685 suite = TestSuite(self._testdir,
2672 2686 jobs=self.options.jobs,
2673 2687 whitelist=self.options.whitelisted,
2674 2688 blacklist=self.options.blacklist,
2675 2689 retest=self.options.retest,
2676 2690 keywords=kws,
2677 2691 loop=self.options.loop,
2678 2692 runs_per_test=self.options.runs_per_test,
2679 2693 showchannels=self.options.showchannels,
2680 2694 tests=tests, loadtest=_reloadtest)
2681 2695 verbosity = 1
2682 2696 if self.options.verbose:
2683 2697 verbosity = 2
2684 2698 runner = TextTestRunner(self, verbosity=verbosity)
2685 2699
2686 2700 if self.options.list_tests:
2687 2701 result = runner.listtests(suite)
2688 2702 else:
2689 2703 if self._installdir:
2690 2704 self._installhg()
2691 2705 self._checkhglib("Testing")
2692 2706 else:
2693 2707 self._usecorrectpython()
2694 2708 if self.options.chg:
2695 2709 assert self._installdir
2696 2710 self._installchg()
2697 2711
2698 2712 result = runner.run(suite)
2699 2713
2700 2714 if result.failures:
2701 2715 failed = True
2702 2716
2703 2717 if self.options.anycoverage:
2704 2718 self._outputcoverage()
2705 2719 except KeyboardInterrupt:
2706 2720 failed = True
2707 2721 print("\ninterrupted!")
2708 2722
2709 2723 if failed:
2710 2724 return 1
2711 2725
2712 2726 def _getport(self, count):
2713 2727 port = self._ports.get(count) # do we have a cached entry?
2714 2728 if port is None:
2715 2729 portneeded = 3
2716 2730 # above 100 tries we just give up and let test reports failure
2717 2731 for tries in xrange(100):
2718 2732 allfree = True
2719 2733 port = self.options.port + self._portoffset
2720 2734 for idx in xrange(portneeded):
2721 2735 if not checkportisavailable(port + idx):
2722 2736 allfree = False
2723 2737 break
2724 2738 self._portoffset += portneeded
2725 2739 if allfree:
2726 2740 break
2727 2741 self._ports[count] = port
2728 2742 return port
2729 2743
2730 2744 def _gettest(self, testdesc, count):
2731 2745 """Obtain a Test by looking at its filename.
2732 2746
2733 2747 Returns a Test instance. The Test may not be runnable if it doesn't
2734 2748 map to a known type.
2735 2749 """
2736 2750 path = testdesc['path']
2737 2751 lctest = path.lower()
2738 2752 testcls = Test
2739 2753
2740 2754 for ext, cls in self.TESTTYPES:
2741 2755 if lctest.endswith(ext):
2742 2756 testcls = cls
2743 2757 break
2744 2758
2745 2759 refpath = os.path.join(self._testdir, path)
2746 2760 tmpdir = os.path.join(self._hgtmp, b'child%d' % count)
2747 2761
2748 2762 # extra keyword parameters. 'case' is used by .t tests
2749 2763 kwds = dict((k, testdesc[k]) for k in ['case'] if k in testdesc)
2750 2764
2751 2765 t = testcls(refpath, self._outputdir, tmpdir,
2752 2766 keeptmpdir=self.options.keep_tmpdir,
2753 2767 debug=self.options.debug,
2754 2768 first=self.options.first,
2755 2769 timeout=self.options.timeout,
2756 2770 startport=self._getport(count),
2757 2771 extraconfigopts=self.options.extra_config_opt,
2758 2772 py3kwarnings=self.options.py3k_warnings,
2759 2773 shell=self.options.shell,
2760 2774 hgcommand=self._hgcommand,
2761 2775 usechg=bool(self.options.with_chg or self.options.chg),
2762 2776 useipv6=useipv6, **kwds)
2763 2777 t.should_reload = True
2764 2778 return t
2765 2779
2766 2780 def _cleanup(self):
2767 2781 """Clean up state from this test invocation."""
2768 2782 if self.options.keep_tmpdir:
2769 2783 return
2770 2784
2771 2785 vlog("# Cleaning up HGTMP", self._hgtmp)
2772 2786 shutil.rmtree(self._hgtmp, True)
2773 2787 for f in self._createdfiles:
2774 2788 try:
2775 2789 os.remove(f)
2776 2790 except OSError:
2777 2791 pass
2778 2792
2779 2793 def _usecorrectpython(self):
2780 2794 """Configure the environment to use the appropriate Python in tests."""
2781 2795 # Tests must use the same interpreter as us or bad things will happen.
2782 2796 pyexename = sys.platform == 'win32' and b'python.exe' or b'python'
2783 2797 if getattr(os, 'symlink', None):
2784 2798 vlog("# Making python executable in test path a symlink to '%s'" %
2785 2799 sys.executable)
2786 2800 mypython = os.path.join(self._tmpbindir, pyexename)
2787 2801 try:
2788 2802 if os.readlink(mypython) == sys.executable:
2789 2803 return
2790 2804 os.unlink(mypython)
2791 2805 except OSError as err:
2792 2806 if err.errno != errno.ENOENT:
2793 2807 raise
2794 2808 if self._findprogram(pyexename) != sys.executable:
2795 2809 try:
2796 2810 os.symlink(sys.executable, mypython)
2797 2811 self._createdfiles.append(mypython)
2798 2812 except OSError as err:
2799 2813 # child processes may race, which is harmless
2800 2814 if err.errno != errno.EEXIST:
2801 2815 raise
2802 2816 else:
2803 2817 exedir, exename = os.path.split(sys.executable)
2804 2818 vlog("# Modifying search path to find %s as %s in '%s'" %
2805 2819 (exename, pyexename, exedir))
2806 2820 path = os.environ['PATH'].split(os.pathsep)
2807 2821 while exedir in path:
2808 2822 path.remove(exedir)
2809 2823 os.environ['PATH'] = os.pathsep.join([exedir] + path)
2810 2824 if not self._findprogram(pyexename):
2811 2825 print("WARNING: Cannot find %s in search path" % pyexename)
2812 2826
2813 2827 def _installhg(self):
2814 2828 """Install hg into the test environment.
2815 2829
2816 2830 This will also configure hg with the appropriate testing settings.
2817 2831 """
2818 2832 vlog("# Performing temporary installation of HG")
2819 2833 installerrs = os.path.join(self._hgtmp, b"install.err")
2820 2834 compiler = ''
2821 2835 if self.options.compiler:
2822 2836 compiler = '--compiler ' + self.options.compiler
2823 2837 if self.options.pure:
2824 2838 pure = b"--pure"
2825 2839 else:
2826 2840 pure = b""
2827 2841
2828 2842 # Run installer in hg root
2829 2843 script = os.path.realpath(sys.argv[0])
2830 2844 exe = sys.executable
2831 2845 if PYTHON3:
2832 2846 compiler = _bytespath(compiler)
2833 2847 script = _bytespath(script)
2834 2848 exe = _bytespath(exe)
2835 2849 hgroot = os.path.dirname(os.path.dirname(script))
2836 2850 self._hgroot = hgroot
2837 2851 os.chdir(hgroot)
2838 2852 nohome = b'--home=""'
2839 2853 if os.name == 'nt':
2840 2854 # The --home="" trick works only on OS where os.sep == '/'
2841 2855 # because of a distutils convert_path() fast-path. Avoid it at
2842 2856 # least on Windows for now, deal with .pydistutils.cfg bugs
2843 2857 # when they happen.
2844 2858 nohome = b''
2845 2859 cmd = (b'%(exe)s setup.py %(pure)s clean --all'
2846 2860 b' build %(compiler)s --build-base="%(base)s"'
2847 2861 b' install --force --prefix="%(prefix)s"'
2848 2862 b' --install-lib="%(libdir)s"'
2849 2863 b' --install-scripts="%(bindir)s" %(nohome)s >%(logfile)s 2>&1'
2850 2864 % {b'exe': exe, b'pure': pure,
2851 2865 b'compiler': compiler,
2852 2866 b'base': os.path.join(self._hgtmp, b"build"),
2853 2867 b'prefix': self._installdir, b'libdir': self._pythondir,
2854 2868 b'bindir': self._bindir,
2855 2869 b'nohome': nohome, b'logfile': installerrs})
2856 2870
2857 2871 # setuptools requires install directories to exist.
2858 2872 def makedirs(p):
2859 2873 try:
2860 2874 os.makedirs(p)
2861 2875 except OSError as e:
2862 2876 if e.errno != errno.EEXIST:
2863 2877 raise
2864 2878 makedirs(self._pythondir)
2865 2879 makedirs(self._bindir)
2866 2880
2867 2881 vlog("# Running", cmd)
2868 2882 if os.system(cmd) == 0:
2869 2883 if not self.options.verbose:
2870 2884 try:
2871 2885 os.remove(installerrs)
2872 2886 except OSError as e:
2873 2887 if e.errno != errno.ENOENT:
2874 2888 raise
2875 2889 else:
2876 2890 with open(installerrs, 'rb') as f:
2877 2891 for line in f:
2878 2892 if PYTHON3:
2879 2893 sys.stdout.buffer.write(line)
2880 2894 else:
2881 2895 sys.stdout.write(line)
2882 2896 sys.exit(1)
2883 2897 os.chdir(self._testdir)
2884 2898
2885 2899 self._usecorrectpython()
2886 2900
2887 2901 if self.options.py3k_warnings and not self.options.anycoverage:
2888 2902 vlog("# Updating hg command to enable Py3k Warnings switch")
2889 2903 with open(os.path.join(self._bindir, 'hg'), 'rb') as f:
2890 2904 lines = [line.rstrip() for line in f]
2891 2905 lines[0] += ' -3'
2892 2906 with open(os.path.join(self._bindir, 'hg'), 'wb') as f:
2893 2907 for line in lines:
2894 2908 f.write(line + '\n')
2895 2909
2896 2910 hgbat = os.path.join(self._bindir, b'hg.bat')
2897 2911 if os.path.isfile(hgbat):
2898 2912 # hg.bat expects to be put in bin/scripts while run-tests.py
2899 2913 # installation layout put it in bin/ directly. Fix it
2900 2914 with open(hgbat, 'rb') as f:
2901 2915 data = f.read()
2902 2916 if b'"%~dp0..\python" "%~dp0hg" %*' in data:
2903 2917 data = data.replace(b'"%~dp0..\python" "%~dp0hg" %*',
2904 2918 b'"%~dp0python" "%~dp0hg" %*')
2905 2919 with open(hgbat, 'wb') as f:
2906 2920 f.write(data)
2907 2921 else:
2908 2922 print('WARNING: cannot fix hg.bat reference to python.exe')
2909 2923
2910 2924 if self.options.anycoverage:
2911 2925 custom = os.path.join(self._testdir, 'sitecustomize.py')
2912 2926 target = os.path.join(self._pythondir, 'sitecustomize.py')
2913 2927 vlog('# Installing coverage trigger to %s' % target)
2914 2928 shutil.copyfile(custom, target)
2915 2929 rc = os.path.join(self._testdir, '.coveragerc')
2916 2930 vlog('# Installing coverage rc to %s' % rc)
2917 2931 os.environ['COVERAGE_PROCESS_START'] = rc
2918 2932 covdir = os.path.join(self._installdir, '..', 'coverage')
2919 2933 try:
2920 2934 os.mkdir(covdir)
2921 2935 except OSError as e:
2922 2936 if e.errno != errno.EEXIST:
2923 2937 raise
2924 2938
2925 2939 os.environ['COVERAGE_DIR'] = covdir
2926 2940
2927 2941 def _checkhglib(self, verb):
2928 2942 """Ensure that the 'mercurial' package imported by python is
2929 2943 the one we expect it to be. If not, print a warning to stderr."""
2930 2944 if ((self._bindir == self._pythondir) and
2931 2945 (self._bindir != self._tmpbindir)):
2932 2946 # The pythondir has been inferred from --with-hg flag.
2933 2947 # We cannot expect anything sensible here.
2934 2948 return
2935 2949 expecthg = os.path.join(self._pythondir, b'mercurial')
2936 2950 actualhg = self._gethgpath()
2937 2951 if os.path.abspath(actualhg) != os.path.abspath(expecthg):
2938 2952 sys.stderr.write('warning: %s with unexpected mercurial lib: %s\n'
2939 2953 ' (expected %s)\n'
2940 2954 % (verb, actualhg, expecthg))
2941 2955 def _gethgpath(self):
2942 2956 """Return the path to the mercurial package that is actually found by
2943 2957 the current Python interpreter."""
2944 2958 if self._hgpath is not None:
2945 2959 return self._hgpath
2946 2960
2947 2961 cmd = b'%s -c "import mercurial; print (mercurial.__path__[0])"'
2948 2962 cmd = cmd % PYTHON
2949 2963 if PYTHON3:
2950 2964 cmd = _strpath(cmd)
2951 2965 pipe = os.popen(cmd)
2952 2966 try:
2953 2967 self._hgpath = _bytespath(pipe.read().strip())
2954 2968 finally:
2955 2969 pipe.close()
2956 2970
2957 2971 return self._hgpath
2958 2972
2959 2973 def _installchg(self):
2960 2974 """Install chg into the test environment"""
2961 2975 vlog('# Performing temporary installation of CHG')
2962 2976 assert os.path.dirname(self._bindir) == self._installdir
2963 2977 assert self._hgroot, 'must be called after _installhg()'
2964 2978 cmd = (b'"%(make)s" clean install PREFIX="%(prefix)s"'
2965 2979 % {b'make': 'make', # TODO: switch by option or environment?
2966 2980 b'prefix': self._installdir})
2967 2981 cwd = os.path.join(self._hgroot, b'contrib', b'chg')
2968 2982 vlog("# Running", cmd)
2969 2983 proc = subprocess.Popen(cmd, shell=True, cwd=cwd,
2970 2984 stdin=subprocess.PIPE, stdout=subprocess.PIPE,
2971 2985 stderr=subprocess.STDOUT)
2972 2986 out, _err = proc.communicate()
2973 2987 if proc.returncode != 0:
2974 2988 if PYTHON3:
2975 2989 sys.stdout.buffer.write(out)
2976 2990 else:
2977 2991 sys.stdout.write(out)
2978 2992 sys.exit(1)
2979 2993
2980 2994 def _outputcoverage(self):
2981 2995 """Produce code coverage output."""
2982 2996 import coverage
2983 2997 coverage = coverage.coverage
2984 2998
2985 2999 vlog('# Producing coverage report')
2986 3000 # chdir is the easiest way to get short, relative paths in the
2987 3001 # output.
2988 3002 os.chdir(self._hgroot)
2989 3003 covdir = os.path.join(self._installdir, '..', 'coverage')
2990 3004 cov = coverage(data_file=os.path.join(covdir, 'cov'))
2991 3005
2992 3006 # Map install directory paths back to source directory.
2993 3007 cov.config.paths['srcdir'] = ['.', self._pythondir]
2994 3008
2995 3009 cov.combine()
2996 3010
2997 3011 omit = [os.path.join(x, '*') for x in [self._bindir, self._testdir]]
2998 3012 cov.report(ignore_errors=True, omit=omit)
2999 3013
3000 3014 if self.options.htmlcov:
3001 3015 htmldir = os.path.join(self._outputdir, 'htmlcov')
3002 3016 cov.html_report(directory=htmldir, omit=omit)
3003 3017 if self.options.annotate:
3004 3018 adir = os.path.join(self._outputdir, 'annotated')
3005 3019 if not os.path.isdir(adir):
3006 3020 os.mkdir(adir)
3007 3021 cov.annotate(directory=adir, omit=omit)
3008 3022
3009 3023 def _findprogram(self, program):
3010 3024 """Search PATH for a executable program"""
3011 3025 dpb = _bytespath(os.defpath)
3012 3026 sepb = _bytespath(os.pathsep)
3013 3027 for p in osenvironb.get(b'PATH', dpb).split(sepb):
3014 3028 name = os.path.join(p, program)
3015 3029 if os.name == 'nt' or os.access(name, os.X_OK):
3016 3030 return name
3017 3031 return None
3018 3032
3019 3033 def _checktools(self):
3020 3034 """Ensure tools required to run tests are present."""
3021 3035 for p in self.REQUIREDTOOLS:
3022 3036 if os.name == 'nt' and not p.endswith('.exe'):
3023 3037 p += '.exe'
3024 3038 found = self._findprogram(p)
3025 3039 if found:
3026 3040 vlog("# Found prerequisite", p, "at", found)
3027 3041 else:
3028 3042 print("WARNING: Did not find prerequisite tool: %s " %
3029 3043 p.decode("utf-8"))
3030 3044
3031 3045 def aggregateexceptions(path):
3032 3046 exceptioncounts = collections.Counter()
3033 3047 testsbyfailure = collections.defaultdict(set)
3034 3048 failuresbytest = collections.defaultdict(set)
3035 3049
3036 3050 for f in os.listdir(path):
3037 3051 with open(os.path.join(path, f), 'rb') as fh:
3038 3052 data = fh.read().split(b'\0')
3039 3053 if len(data) != 5:
3040 3054 continue
3041 3055
3042 3056 exc, mainframe, hgframe, hgline, testname = data
3043 3057 exc = exc.decode('utf-8')
3044 3058 mainframe = mainframe.decode('utf-8')
3045 3059 hgframe = hgframe.decode('utf-8')
3046 3060 hgline = hgline.decode('utf-8')
3047 3061 testname = testname.decode('utf-8')
3048 3062
3049 3063 key = (hgframe, hgline, exc)
3050 3064 exceptioncounts[key] += 1
3051 3065 testsbyfailure[key].add(testname)
3052 3066 failuresbytest[testname].add(key)
3053 3067
3054 3068 # Find test having fewest failures for each failure.
3055 3069 leastfailing = {}
3056 3070 for key, tests in testsbyfailure.items():
3057 3071 fewesttest = None
3058 3072 fewestcount = 99999999
3059 3073 for test in sorted(tests):
3060 3074 if len(failuresbytest[test]) < fewestcount:
3061 3075 fewesttest = test
3062 3076 fewestcount = len(failuresbytest[test])
3063 3077
3064 3078 leastfailing[key] = (fewestcount, fewesttest)
3065 3079
3066 3080 # Create a combined counter so we can sort by total occurrences and
3067 3081 # impacted tests.
3068 3082 combined = {}
3069 3083 for key in exceptioncounts:
3070 3084 combined[key] = (exceptioncounts[key],
3071 3085 len(testsbyfailure[key]),
3072 3086 leastfailing[key][0],
3073 3087 leastfailing[key][1])
3074 3088
3075 3089 return {
3076 3090 'exceptioncounts': exceptioncounts,
3077 3091 'total': sum(exceptioncounts.values()),
3078 3092 'combined': combined,
3079 3093 'leastfailing': leastfailing,
3080 3094 'byfailure': testsbyfailure,
3081 3095 'bytest': failuresbytest,
3082 3096 }
3083 3097
3084 3098 if __name__ == '__main__':
3085 3099 runner = TestRunner()
3086 3100
3087 3101 try:
3088 3102 import msvcrt
3089 3103 msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY)
3090 3104 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
3091 3105 msvcrt.setmode(sys.stderr.fileno(), os.O_BINARY)
3092 3106 except ImportError:
3093 3107 pass
3094 3108
3095 3109 sys.exit(runner.run(sys.argv[1:]))
General Comments 0
You need to be logged in to leave comments. Login now