test-run-tests.t
359 lines
| 8.5 KiB
| text/troff
|
Tads3Lexer
/ tests / test-run-tests.t
|
r21732 | This file tests the behavior of run-tests.py itself. | ||
Smoke test | ||||
============ | ||||
$ $TESTDIR/run-tests.py | ||||
# Ran 0 tests, 0 skipped, 0 warned, 0 failed. | ||||
|
r21734 | |||
a succesful test | ||||
======================= | ||||
$ cat > test-success.t << EOF | ||||
> $ echo babar | ||||
> babar | ||||
|
r21995 | > $ echo xyzzy | ||
> xyzzy | ||||
|
r21734 | > EOF | ||
$ $TESTDIR/run-tests.py --with-hg=`which hg` | ||||
. | ||||
# Ran 1 tests, 0 skipped, 0 warned, 0 failed. | ||||
|
r21738 | |||
failing test | ||||
================== | ||||
$ cat > test-failure.t << EOF | ||||
> $ echo babar | ||||
> rataxes | ||||
|
r21995 | > This is a noop statement so that | ||
> this test is still more bytes than success. | ||||
|
r21738 | > EOF | ||
$ $TESTDIR/run-tests.py --with-hg=`which hg` | ||||
--- $TESTTMP/test-failure.t | ||||
+++ $TESTTMP/test-failure.t.err | ||||
|
r21995 | @@ -1,4 +1,4 @@ | ||
|
r21738 | $ echo babar | ||
- rataxes | ||||
+ babar | ||||
|
r21995 | This is a noop statement so that | ||
this test is still more bytes than success. | ||||
|
r21738 | |||
ERROR: test-failure.t output changed | ||||
!. | ||||
Failed test-failure.t: output changed | ||||
# Ran 2 tests, 0 skipped, 0 warned, 1 failed. | ||||
python hash seed: * (glob) | ||||
[1] | ||||
|
r22044 | test --xunit support | ||
$ $TESTDIR/run-tests.py --with-hg=`which hg` --xunit=xunit.xml | ||||
--- $TESTTMP/test-failure.t | ||||
+++ $TESTTMP/test-failure.t.err | ||||
@@ -1,4 +1,4 @@ | ||||
$ echo babar | ||||
- rataxes | ||||
+ babar | ||||
This is a noop statement so that | ||||
this test is still more bytes than success. | ||||
ERROR: test-failure.t output changed | ||||
!. | ||||
Failed test-failure.t: output changed | ||||
# Ran 2 tests, 0 skipped, 0 warned, 1 failed. | ||||
python hash seed: * (glob) | ||||
[1] | ||||
$ cat xunit.xml | ||||
<?xml version="1.0" encoding="utf-8"?> | ||||
<testsuite errors="0" failures="1" name="run-tests" skipped="0" tests="2"> | ||||
<testcase name="test-success.t" time="*"/> (glob) | ||||
<testcase name="test-failure.t" time="*"> (glob) | ||||
<![CDATA[--- $TESTTMP/test-failure.t | ||||
+++ $TESTTMP/test-failure.t.err | ||||
@@ -1,4 +1,4 @@ | ||||
$ echo babar | ||||
- rataxes | ||||
+ babar | ||||
This is a noop statement so that | ||||
this test is still more bytes than success. | ||||
]]> </testcase> | ||||
</testsuite> | ||||
|
r21741 | |||
test for --retest | ||||
==================== | ||||
$ $TESTDIR/run-tests.py --with-hg=`which hg` --retest | ||||
--- $TESTTMP/test-failure.t | ||||
+++ $TESTTMP/test-failure.t.err | ||||
|
r21995 | @@ -1,4 +1,4 @@ | ||
|
r21741 | $ echo babar | ||
- rataxes | ||||
+ babar | ||||
|
r21995 | This is a noop statement so that | ||
this test is still more bytes than success. | ||||
|
r21741 | |||
ERROR: test-failure.t output changed | ||||
! | ||||
Failed test-failure.t: output changed | ||||
|
r21997 | # Ran 2 tests, 1 skipped, 0 warned, 1 failed. | ||
|
r21741 | python hash seed: * (glob) | ||
[1] | ||||
|
r21742 | |||
Selecting Tests To Run | ||||
====================== | ||||
successful | ||||
$ $TESTDIR/run-tests.py --with-hg=`which hg` test-success.t | ||||
. | ||||
# Ran 1 tests, 0 skipped, 0 warned, 0 failed. | ||||
|
r21997 | success w/ keyword | ||
$ $TESTDIR/run-tests.py --with-hg=`which hg` -k xyzzy | ||||
|
r22107 | . | ||
# Ran 2 tests, 1 skipped, 0 warned, 0 failed. | ||||
|
r21997 | |||
|
r21742 | failed | ||
$ $TESTDIR/run-tests.py --with-hg=`which hg` test-failure.t | ||||
--- $TESTTMP/test-failure.t | ||||
+++ $TESTTMP/test-failure.t.err | ||||
|
r21995 | @@ -1,4 +1,4 @@ | ||
|
r21742 | $ echo babar | ||
- rataxes | ||||
+ babar | ||||
|
r21995 | This is a noop statement so that | ||
this test is still more bytes than success. | ||||
|
r21742 | |||
ERROR: test-failure.t output changed | ||||
! | ||||
Failed test-failure.t: output changed | ||||
# Ran 1 tests, 0 skipped, 0 warned, 1 failed. | ||||
python hash seed: * (glob) | ||||
[1] | ||||
|
r21743 | |||
|
r21997 | failure w/ keyword | ||
$ $TESTDIR/run-tests.py --with-hg=`which hg` -k rataxes | ||||
|
r22107 | |||
|
r21997 | --- $TESTTMP/test-failure.t | ||
+++ $TESTTMP/test-failure.t.err | ||||
@@ -1,4 +1,4 @@ | ||||
$ echo babar | ||||
- rataxes | ||||
+ babar | ||||
This is a noop statement so that | ||||
this test is still more bytes than success. | ||||
ERROR: test-failure.t output changed | ||||
! | ||||
Failed test-failure.t: output changed | ||||
|
r22107 | # Ran 2 tests, 1 skipped, 0 warned, 1 failed. | ||
|
r21997 | python hash seed: * (glob) | ||
[1] | ||||
|
r21743 | Running In Debug Mode | ||
====================== | ||||
$ $TESTDIR/run-tests.py --with-hg=`which hg` --debug | ||||
+ echo SALT* 0 0 (glob) | ||||
SALT* 0 0 (glob) | ||||
+ echo babar | ||||
babar | ||||
|
r21995 | + echo SALT* 4 0 (glob) | ||
SALT* 4 0 (glob) | ||||
|
r21743 | .+ echo SALT* 0 0 (glob) | ||
SALT* 0 0 (glob) | ||||
+ echo babar | ||||
babar | ||||
+ echo SALT* 2 0 (glob) | ||||
SALT* 2 0 (glob) | ||||
|
r21995 | + echo xyzzy | ||
xyzzy | ||||
+ echo SALT* 4 0 (glob) | ||||
SALT* 4 0 (glob) | ||||
|
r21743 | . | ||
# Ran 2 tests, 0 skipped, 0 warned, 0 failed. | ||||
|
r21744 | |||
Parallel runs | ||||
============== | ||||
(duplicate the failing test to get predictable output) | ||||
$ cp test-failure.t test-failure-copy.t | ||||
$ $TESTDIR/run-tests.py --with-hg=`which hg` --jobs 2 test-failure*.t | ||||
--- $TESTTMP/test-failure*.t (glob) | ||||
+++ $TESTTMP/test-failure*.t.err (glob) | ||||
|
r21995 | @@ -1,4 +1,4 @@ | ||
|
r21744 | $ echo babar | ||
- rataxes | ||||
+ babar | ||||
|
r21995 | This is a noop statement so that | ||
this test is still more bytes than success. | ||||
|
r21744 | |||
ERROR: test-failure*.t output changed (glob) | ||||
! | ||||
--- $TESTTMP/test-failure*.t (glob) | ||||
+++ $TESTTMP/test-failure*.t.err (glob) | ||||
|
r21995 | @@ -1,4 +1,4 @@ | ||
|
r21744 | $ echo babar | ||
- rataxes | ||||
+ babar | ||||
|
r21995 | This is a noop statement so that | ||
this test is still more bytes than success. | ||||
|
r21744 | |||
ERROR: test-failure*.t output changed (glob) | ||||
! | ||||
Failed test-failure*.t: output changed (glob) | ||||
Failed test-failure*.t: output changed (glob) | ||||
# Ran 2 tests, 0 skipped, 0 warned, 2 failed. | ||||
python hash seed: * (glob) | ||||
[1] | ||||
(delete the duplicated test file) | ||||
$ rm test-failure-copy.t | ||||
|
r21755 | |||
Interactive run | ||||
=============== | ||||
(backup the failing test) | ||||
$ cp test-failure.t backup | ||||
Refuse the fix | ||||
$ echo 'n' | $TESTDIR/run-tests.py --with-hg=`which hg` -i | ||||
--- $TESTTMP/test-failure.t | ||||
+++ $TESTTMP/test-failure.t.err | ||||
|
r21995 | @@ -1,4 +1,4 @@ | ||
|
r21755 | $ echo babar | ||
- rataxes | ||||
+ babar | ||||
|
r21995 | This is a noop statement so that | ||
this test is still more bytes than success. | ||||
|
r21763 | Accept this change? [n] | ||
|
r21755 | ERROR: test-failure.t output changed | ||
|
r21763 | !. | ||
|
r21755 | Failed test-failure.t: output changed | ||
# Ran 2 tests, 0 skipped, 0 warned, 1 failed. | ||||
python hash seed: * (glob) | ||||
[1] | ||||
$ cat test-failure.t | ||||
$ echo babar | ||||
rataxes | ||||
|
r21995 | This is a noop statement so that | ||
this test is still more bytes than success. | ||||
|
r21755 | |||
|
r22108 | View the fix | ||
$ echo 'y' | $TESTDIR/run-tests.py --with-hg=`which hg` --view echo | ||||
$TESTTMP/test-failure.t $TESTTMP/test-failure.t.err | ||||
ERROR: test-failure.t output changed | ||||
!. | ||||
Failed test-failure.t: output changed | ||||
# Ran 2 tests, 0 skipped, 0 warned, 1 failed. | ||||
python hash seed: * (glob) | ||||
[1] | ||||
|
r21755 | Accept the fix | ||
$ echo 'y' | $TESTDIR/run-tests.py --with-hg=`which hg` -i | ||||
--- $TESTTMP/test-failure.t | ||||
+++ $TESTTMP/test-failure.t.err | ||||
|
r21995 | @@ -1,4 +1,4 @@ | ||
|
r21755 | $ echo babar | ||
- rataxes | ||||
+ babar | ||||
|
r21995 | This is a noop statement so that | ||
this test is still more bytes than success. | ||||
|
r21755 | Accept this change? [n] .. | ||
# Ran 2 tests, 0 skipped, 0 warned, 0 failed. | ||||
$ cat test-failure.t | ||||
$ echo babar | ||||
babar | ||||
|
r21995 | This is a noop statement so that | ||
this test is still more bytes than success. | ||||
|
r21755 | |||
(reinstall) | ||||
$ mv backup test-failure.t | ||||
|
r21756 | |||
No Diff | ||||
=============== | ||||
$ $TESTDIR/run-tests.py --with-hg=`which hg` --nodiff | ||||
!. | ||||
Failed test-failure.t: output changed | ||||
# Ran 2 tests, 0 skipped, 0 warned, 1 failed. | ||||
python hash seed: * (glob) | ||||
[1] | ||||
|
r21977 | |||
test for --time | ||||
================== | ||||
$ $TESTDIR/run-tests.py --with-hg=`which hg` test-success.t --time | ||||
. | ||||
# Ran 1 tests, 0 skipped, 0 warned, 0 failed. | ||||
# Producing time report | ||||
cuser csys real Test | ||||
\s*[\d\.]{5} \s*[\d\.]{5} \s*[\d\.]{5} test-success.t (re) | ||||
test for --time with --job enabled | ||||
==================================== | ||||
$ $TESTDIR/run-tests.py --with-hg=`which hg` test-success.t --time --jobs 2 | ||||
. | ||||
# Ran 1 tests, 0 skipped, 0 warned, 0 failed. | ||||
# Producing time report | ||||
cuser csys real Test | ||||
\s*[\d\.]{5} \s*[\d\.]{5} \s*[\d\.]{5} test-success.t (re) | ||||
|
r21996 | |||
Skips | ||||
================ | ||||
$ cat > test-skip.t <<EOF | ||||
> $ echo xyzzy | ||||
|
r22045 | > #require false | ||
|
r21996 | > EOF | ||
$ $TESTDIR/run-tests.py --with-hg=`which hg` --nodiff | ||||
!.s | ||||
Skipped test-skip.t: irrelevant | ||||
Failed test-failure.t: output changed | ||||
# Ran 2 tests, 1 skipped, 0 warned, 1 failed. | ||||
python hash seed: * (glob) | ||||
[1] | ||||
|
r21997 | $ $TESTDIR/run-tests.py --with-hg=`which hg` --keyword xyzzy | ||
|
r22107 | .s | ||
|
r21997 | Skipped test-skip.t: irrelevant | ||
|
r22107 | # Ran 2 tests, 2 skipped, 0 warned, 0 failed. | ||
|
r21997 | |||
|
r22044 | Skips with xml | ||
$ $TESTDIR/run-tests.py --with-hg=`which hg` --keyword xyzzy \ | ||||
> --xunit=xunit.xml | ||||
|
r22107 | .s | ||
|
r22044 | Skipped test-skip.t: irrelevant | ||
|
r22107 | # Ran 2 tests, 2 skipped, 0 warned, 0 failed. | ||
|
r22044 | $ cat xunit.xml | ||
<?xml version="1.0" encoding="utf-8"?> | ||||
|
r22107 | <testsuite errors="0" failures="0" name="run-tests" skipped="2" tests="2"> | ||
|
r22044 | <testcase name="test-success.t" time="*"/> (glob) | ||
</testsuite> | ||||
|
r21997 | Missing skips or blacklisted skips don't count as executed: | ||
$ echo test-failure.t > blacklist | ||||
$ $TESTDIR/run-tests.py --with-hg=`which hg` --blacklist=blacklist \ | ||||
> test-failure.t test-bogus.t | ||||
ss | ||||
Skipped test-bogus.t: Doesn't exist | ||||
Skipped test-failure.t: blacklisted | ||||
# Ran 0 tests, 2 skipped, 0 warned, 0 failed. | ||||