diff --git a/tests/filterpyflakes.py b/tests/filterpyflakes.py
--- a/tests/filterpyflakes.py
+++ b/tests/filterpyflakes.py
@@ -35,7 +35,3 @@ for line in sys.stdin:
 for line in lines:
     sys.stdout.write(line)
 print()
-
-# self test of "undefined name" detection
-if False:
-    print(undefinedname)
diff --git a/tests/test-check-pyflakes.t b/tests/test-check-pyflakes.t
--- a/tests/test-check-pyflakes.t
+++ b/tests/test-check-pyflakes.t
@@ -6,10 +6,16 @@
 run pyflakes on all tracked files ending in .py or without a file ending
 (skipping binary file random-seed)
 
+  $ cat > test.py <<EOF
+  > print(undefinedname)
+  > EOF
+  $ pyflakes test.py 2>/dev/null | "$TESTDIR/filterpyflakes.py"
+  test.py:1: undefined name 'undefinedname'
+  
+
   $ testrepohg locate 'set:**.py or grep("^#!.*python")' \
   > -X hgext/fsmonitor/pywatchman \
   > -X mercurial/pycompat.py -X contrib/python-zstandard \
   > 2>/dev/null \
   > | xargs pyflakes 2>/dev/null | "$TESTDIR/filterpyflakes.py"
-  tests/filterpyflakes.py:41: undefined name 'undefinedname'