Robustness fixes in test suite machinery.

Added a module-level INSTALLED flag, which can be set to false if the test
suite is being run in-place (without ipython having been installed at all).
This is because how we call and import things must be done differently
depending on whether the code is installed or is being run in-place.  The
only ones that can know this reliably are the entry-point scripts, so those
are responsible for setting this flag.

Also made the code that validates ipython in subprocesses report errors
better, by checking stderr for errors before validating stdout output, as
anything on stderr will be likely informative of the real problem.

IPython/core/tests/tclass.py

@@ -1,30 +1,31 b''
 """Simple script to be run *twice*, to check reference counting bugs.
 
 See test_run for details."""
 
 import sys
 
 # We want to ensure that while objects remain available for immediate access,
 # objects from *previous* runs of the same script get collected, to avoid
 # accumulating massive amounts of old references.
 class C(object):
     def __init__(self,name):
         self.name = name
         
     def __del__(self):
         print 'tclass.py: deleting object:',self.name
-
+        sys.stdout.flush()
 
 try:
     name = sys.argv[1]
 except IndexError:
     pass
 else:
     if name.startswith('C'):
         c = C(name)
 
 #print >> sys.stderr, "ARGV:", sys.argv  # dbg
 
 # This next print statement is NOT debugging, we're making the check on a
 # completely separate process so we verify by capturing stdout:
 print 'ARGV 1-:', sys.argv[1:]
+sys.stdout.flush()

IPython/testing/iptest.py

@@ -1,434 +1,450 b''
 # -*- coding: utf-8 -*-
 """IPython Test Suite Runner.
 
 This module provides a main entry point to a user script to test IPython
 itself from the command line. There are two ways of running this script:
 
 1. With the syntax `iptest all`.  This runs our entire test suite by
    calling this script (with different arguments) or trial recursively.  This
    causes modules and package to be tested in different processes, using nose
    or trial where appropriate.
 2. With the regular nose syntax, like `iptest -vvs IPython`.  In this form
    the script simply calls nose, but with special command line flags and
    plugins loaded.
 
 For now, this script requires that both nose and twisted are installed.  This
 will change in the future.
 """
 
 #-----------------------------------------------------------------------------
 # Module imports
 #-----------------------------------------------------------------------------
 
 # Stdlib
 import os
 import os.path as path
 import signal
 import sys
 import subprocess
 import tempfile
 import time
 import warnings
 
 
 # Ugly,  but necessary hack to ensure the test suite finds our version of
 # IPython and not a possibly different one that may exist system-wide.
 # Note that this must be done here, so the imports that come next work
 # correctly even if IPython isn't installed yet.
 p = os.path
 ippath = p.abspath(p.join(p.dirname(__file__),'..','..'))
 sys.path.insert(0, ippath)
 
 # Note: monkeypatch!
 # We need to monkeypatch a small problem in nose itself first, before importing
 # it for actual use.  This should get into nose upstream, but its release cycle
 # is slow and we need it for our parametric tests to work correctly.
 from IPython.testing import nosepatch
 # Now, proceed to import nose itself
 import nose.plugins.builtin
 from nose.core import TestProgram
 
 # Our own imports
 from IPython.utils import genutils
 from IPython.utils.platutils import find_cmd, FindCmdError
 from IPython.testing import globalipapp
 from IPython.testing import tools
 from IPython.testing.plugin.ipdoctest import IPythonDoctest
 
 pjoin = path.join
 
+
+#-----------------------------------------------------------------------------
+# Globals
+#-----------------------------------------------------------------------------
+
+# By default, we assume IPython has been installed.  But if the test suite is
+# being run from a source tree that has NOT been installed yet, this flag can
+# be set to False by the entry point scripts, to let us know that we must call
+# the source tree versions of the scripts which manipulate sys.path instead of
+# assuming that things exist system-wide.
+INSTALLED = True
+
 #-----------------------------------------------------------------------------
 # Warnings control
 #-----------------------------------------------------------------------------
 # Twisted generates annoying warnings with Python 2.6, as will do other code
 # that imports 'sets' as of today
 warnings.filterwarnings('ignore', 'the sets module is deprecated',
                         DeprecationWarning )
 
 # This one also comes from Twisted
 warnings.filterwarnings('ignore', 'the sha module is deprecated',
                         DeprecationWarning)
 
 # Wx on Fedora11 spits these out
 warnings.filterwarnings('ignore', 'wxPython/wxWidgets release number mismatch',
                         UserWarning)
 
 #-----------------------------------------------------------------------------
 # Logic for skipping doctests
 #-----------------------------------------------------------------------------
 
 def test_for(mod):
     """Test to see if mod is importable."""
     try:
         __import__(mod)
     except (ImportError, RuntimeError):
         # GTK reports Runtime error if it can't be initialized even if  it's
         # importable.
         return False
     else:
         return True
 
-    
 have_curses = test_for('_curses')
 have_wx = test_for('wx')
 have_wx_aui = test_for('wx.aui')
 have_zi = test_for('zope.interface')
 have_twisted = test_for('twisted')
 have_foolscap = test_for('foolscap')
 have_objc = test_for('objc')
 have_pexpect = test_for('pexpect')
 have_gtk = test_for('gtk')
 have_gobject = test_for('gobject')
 
+#-----------------------------------------------------------------------------
+# Functions and classes
+#-----------------------------------------------------------------------------
 
 def make_exclude():
     """Make patterns of modules and packages to exclude from testing.
     
     For the IPythonDoctest plugin, we need to exclude certain patterns that
     cause testing problems.  We should strive to minimize the number of
     skipped modules, since this means untested code.  As the testing
     machinery solidifies, this list should eventually become empty.
     These modules and packages will NOT get scanned by nose at all for tests.
     """
     # Simple utility to make IPython paths more readably, we need a lot of
     # these below
     ipjoin = lambda *paths: pjoin('IPython', *paths)
     
     exclusions = [ipjoin('external'),
                   ipjoin('frontend', 'process', 'winprocess.py'),
                   # Deprecated old Shell and iplib modules, skip to avoid
                   # warnings
                   ipjoin('Shell'),
                   ipjoin('iplib'),
                   pjoin('IPython_doctest_plugin'),
                   ipjoin('quarantine'),
                   ipjoin('deathrow'),
                   ipjoin('testing', 'attic'),
                   # This guy is probably attic material
                   ipjoin('testing', 'mkdoctests'),
                   # Testing inputhook will need a lot of thought, to figure out
                   # how to have tests that don't lock up with the gui event
                   # loops in the picture
                   ipjoin('lib', 'inputhook'),
                   # Config files aren't really importable stand-alone
                   ipjoin('config', 'default'),
                   ipjoin('config', 'profile'),
                   ]
 
     if not have_wx:
         exclusions.append(ipjoin('gui'))
         exclusions.append(ipjoin('frontend', 'wx'))
         exclusions.append(ipjoin('lib', 'inputhookwx'))
 
     if not have_gtk or not have_gobject:
         exclusions.append(ipjoin('lib', 'inputhookgtk'))
 
     if not have_wx_aui:
         exclusions.append(ipjoin('gui', 'wx', 'wxIPython'))
 
     if not have_objc:
         exclusions.append(ipjoin('frontend', 'cocoa'))
 
     if not sys.platform == 'win32':
         exclusions.append(ipjoin('utils', 'platutils_win32'))
 
     # These have to be skipped on win32 because the use echo, rm, cd, etc.
     # See ticket https://bugs.launchpad.net/bugs/366982
     if sys.platform == 'win32':
         exclusions.append(ipjoin('testing', 'plugin', 'test_exampleip'))
         exclusions.append(ipjoin('testing', 'plugin', 'dtexample'))
 
     if not os.name == 'posix':
         exclusions.append(ipjoin('utils', 'platutils_posix'))
 
     if not have_pexpect:
         exclusions.extend([ipjoin('scripts', 'irunner'),
                            ipjoin('lib', 'irunner')])
 
     # This is scary.  We still have things in frontend and testing that
     # are being tested by nose that use twisted.  We need to rethink
     # how we are isolating dependencies in testing.
     if not (have_twisted and have_zi and have_foolscap):
         exclusions.extend(
             [ipjoin('frontend', 'asyncfrontendbase'),
              ipjoin('frontend', 'prefilterfrontend'),
              ipjoin('frontend', 'frontendbase'),
              ipjoin('frontend', 'linefrontendbase'),
              ipjoin('frontend', 'tests', 'test_linefrontend'),
              ipjoin('frontend', 'tests', 'test_frontendbase'),
              ipjoin('frontend', 'tests', 'test_prefilterfrontend'),
              ipjoin('frontend', 'tests', 'test_asyncfrontendbase'),
              ipjoin('testing', 'parametric'),
              ipjoin('testing', 'util'),
              ipjoin('testing', 'tests', 'test_decorators_trial'),
              ] )
 
     # This is needed for the reg-exp to match on win32 in the ipdoctest plugin.
     if sys.platform == 'win32':
         exclusions = [s.replace('\\','\\\\') for s in exclusions]
 
     return exclusions
 
 
-#-----------------------------------------------------------------------------
-# Functions and classes
-#-----------------------------------------------------------------------------
-
 class IPTester(object):
     """Call that calls iptest or trial in a subprocess.
     """
     #: string, name of test runner that will be called
     runner = None
     #: list, parameters for test runner
     params = None
     #: list, arguments of system call to be made to call test runner
     call_args = None
     #: list, process ids of subprocesses we start (for cleanup)
     pids = None
     
     def __init__(self, runner='iptest', params=None):
         """Create new test runner."""
+        p = os.path
         if runner == 'iptest':
-            # Find our own 'iptest' script OS-level entry point.  Don't look
-            # system-wide, so we are sure we pick up *this one*.  And pass
-            # through to subprocess call our own sys.argv
-            self.runner = tools.cmd2argv(os.path.abspath(__file__)) + \
-                          sys.argv[1:]
+            if INSTALLED:
+                self.runner = tools.cmd2argv(
+                    p.abspath(find_cmd('iptest'))) + sys.argv[1:]
+            else:
+                # Find our own 'iptest' script OS-level entry point.  Don't
+                # look system-wide, so we are sure we pick up *this one*.  And
+                # pass through to subprocess call our own sys.argv
+                ippath = p.abspath(p.join(p.dirname(__file__),'..','..'))
+                script = p.join(ippath, 'iptest.py')
+                self.runner = tools.cmd2argv(script) + sys.argv[1:]
+                
         else:
-            self.runner = tools.cmd2argv(os.path.abspath(find_cmd('trial')))
+            # For trial, it needs to be installed system-wide
+            self.runner = tools.cmd2argv(p.abspath(find_cmd('trial')))
         if params is None:
             params = []
         if isinstance(params, str):
             params = [params]
         self.params = params
 
         # Assemble call
         self.call_args = self.runner+self.params
 
         # Store pids of anything we start to clean up on deletion, if possible
         # (on posix only, since win32 has no os.kill)
         self.pids = []
 
     if sys.platform == 'win32':
         def _run_cmd(self):
             # On Windows, use os.system instead of subprocess.call, because I
             # was having problems with subprocess and I just don't know enough
             # about win32 to debug this reliably.  Os.system may be the 'old
             # fashioned' way to do it, but it works just fine.  If someone
             # later can clean this up that's fine, as long as the tests run
             # reliably in win32.
             return os.system(' '.join(self.call_args))
     else:
         def _run_cmd(self):
             #print >> sys.stderr, '*** CMD:', ' '.join(self.call_args) # dbg
             subp = subprocess.Popen(self.call_args)
             self.pids.append(subp.pid)
             # If this fails, the pid will be left in self.pids and cleaned up
             # later, but if the wait call succeeds, then we can clear the
             # stored pid.
             retcode = subp.wait()
             self.pids.pop()
             return retcode
         
     def run(self):
         """Run the stored commands"""
         try:
             return self._run_cmd()
         except:
             import traceback
             traceback.print_exc()
             return 1  # signal failure
 
     def __del__(self):
         """Cleanup on exit by killing any leftover processes."""
 
         if not hasattr(os, 'kill'):
             return
         
         for pid in self.pids:
             try:
                 print 'Cleaning stale PID:', pid
                 os.kill(pid, signal.SIGKILL)
             except OSError:
                 # This is just a best effort, if we fail or the process was
                 # really gone, ignore it.
                 pass     
 
 
 def make_runners():
     """Define the top-level packages that need to be tested.
     """
 
     # Packages to be tested via nose, that only depend on the stdlib
     nose_pkg_names = ['config', 'core', 'extensions', 'frontend', 'lib',
                      'scripts', 'testing', 'utils' ]
     # The machinery in kernel needs twisted for real testing
     trial_pkg_names = []
 
     if have_wx:
         nose_pkg_names.append('gui')
 
     # And add twisted ones if conditions are met
     if have_zi and have_twisted and have_foolscap:
         # Note that we list the kernel here, though the bulk of it is
         # twisted-based, because nose picks up doctests that twisted doesn't.
         nose_pkg_names.append('kernel')
         trial_pkg_names.append('kernel')
 
     # For debugging this code, only load quick stuff
-    #nose_pkg_names = ['core']  # dbg
+    #nose_pkg_names = ['core', 'extensions']  # dbg
     #trial_pkg_names = []  # dbg 
 
     # Make fully qualified package names prepending 'IPython.' to our name lists
     nose_packages = ['IPython.%s' % m for m in nose_pkg_names ]
     trial_packages = ['IPython.%s' % m for m in trial_pkg_names ]
 
     # Make runners
     runners = [ (v, IPTester('iptest', params=v)) for v in nose_packages ]
     runners.extend([ (v, IPTester('trial', params=v)) for v in trial_packages ])
     
     return runners
 
 
 def run_iptest():
     """Run the IPython test suite using nose.
     
     This function is called when this script is **not** called with the form
     `iptest all`.  It simply calls nose with appropriate command line flags
     and accepts all of the standard nose arguments.
     """
 
     warnings.filterwarnings('ignore', 
         'This will be removed soon.  Use IPython.testing.util instead')
 
     argv = sys.argv + [ '--detailed-errors',  # extra info in tracebacks
-                        
-                        # I don't fully understand why we need this one, but
-                        # depending on what directory the test suite is run
-                        # from, if we don't give it, 0 tests get run.
-                        # Specifically, if the test suite is run from the
-                        # source dir with an argument (like 'iptest.py
-                        # IPython.core', 0 tests are run, even if the same call
-                        # done in this directory works fine).  It appears that
-                        # if  the requested package is in the current dir,
-                        # nose bails early by default.  Since it's otherwise
-                        # harmless, leave it in by default. 
-                        '--traverse-namespace',
-                        
+                                                
                         # Loading ipdoctest causes problems with Twisted, but
                         # our test suite runner now separates things and runs
                         # all Twisted tests with trial.
                         '--with-ipdoctest',
                         '--ipdoctest-tests','--ipdoctest-extension=txt',
                         
                         # We add --exe because of setuptools' imbecility (it
                         # blindly does chmod +x on ALL files).  Nose does the
                         # right thing and it tries to avoid executables,
                         # setuptools unfortunately forces our hand here.  This
                         # has been discussed on the distutils list and the
                         # setuptools devs refuse to fix this problem!
                         '--exe',
                         ]
 
+    if nose.__version__ >= '0.11':
+        # I don't fully understand why we need this one, but depending on what
+        # directory the test suite is run from, if we don't give it, 0 tests
+        # get run.  Specifically, if the test suite is run from the source dir
+        # with an argument (like 'iptest.py IPython.core', 0 tests are run,
+        # even if the same call done in this directory works fine).  It appears
+        # that if the requested package is in the current dir, nose bails early
+        # by default.  Since it's otherwise harmless, leave it in by default
+        # for nose >= 0.11, though unfortunately nose 0.10 doesn't support it.
+        argv.append('--traverse-namespace')
 
     # Construct list of plugins, omitting the existing doctest plugin, which
     # ours replaces (and extends).
     plugins = [IPythonDoctest(make_exclude())]
     for p in nose.plugins.builtin.plugins:
         plug = p()
         if plug.name == 'doctest':
             continue
         plugins.append(plug)
 
     # We need a global ipython running in this process
     globalipapp.start_ipython()
     # Now nose can run
     TestProgram(argv=argv, plugins=plugins)
 
 
 def run_iptestall():
     """Run the entire IPython test suite by calling nose and trial.
     
     This function constructs :class:`IPTester` instances for all IPython
     modules and package and then runs each of them.  This causes the modules
     and packages of IPython to be tested each in their own subprocess using
     nose or twisted.trial appropriately.
     """
 
     runners = make_runners()
 
     # Run the test runners in a temporary dir so we can nuke it when finished
     # to clean up any junk files left over by accident.  This also makes it
     # robust against being run in non-writeable directories by mistake, as the
     # temp dir will always be user-writeable.
     curdir = os.getcwd()
     testdir = tempfile.gettempdir()
     os.chdir(testdir)
 
     # Run all test runners, tracking execution time
     failed = []
     t_start = time.time()
     try:
         for (name, runner) in runners:
             print '*'*70
             print 'IPython test group:',name
             res = runner.run()
             if res:
                 failed.append( (name, runner) )
     finally:
         os.chdir(curdir)
     t_end = time.time()
     t_tests = t_end - t_start
     nrunners = len(runners)
     nfail = len(failed)
     # summarize results
     print
     print '*'*70
     print 'Ran %s test groups in %.3fs' % (nrunners, t_tests)
     print
     if not failed:
         print 'OK'
     else:
         # If anything went wrong, point out what command to rerun manually to
         # see the actual errors and individual summary
         print 'ERROR - %s out of %s test groups failed.' % (nfail, nrunners)
         for name, failed_runner in failed:
             print '-'*40
             print 'Runner failed:',name
             print 'You may wish to rerun this one individually, with:'
             print ' '.join(failed_runner.call_args)
             print
 
 
 def main():
     for arg in sys.argv[1:]:
         if arg.startswith('IPython'):
             # This is in-process
             run_iptest()
     else:
         # This starts subprocesses
         run_iptestall()
 
 
 if __name__ == '__main__':
     main()

IPython/testing/tools.py

@@ -1,317 +1,337 b''
 """Generic testing tools that do NOT depend on Twisted.
 
 In particular, this module exposes a set of top-level assert* functions that
 can be used in place of nose.tools.assert* in method generators (the ones in
 nose can not, at least as of nose 0.10.4).
 
 Note: our testing package contains testing.util, which does depend on Twisted
 and provides utilities for tests that manage Deferreds.  All testing support
 tools that only depend on nose, IPython or the standard library should go here
 instead.
 
 
 Authors
 -------
 - Fernando Perez <[e-mail]>
 """
 
 #*****************************************************************************
 #       Copyright (C) 2009 The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 #-----------------------------------------------------------------------------
 # Required modules and packages
 #-----------------------------------------------------------------------------
 from __future__ import absolute_import
 
 import os
 import re
 import sys
 import tempfile
 
 try:
     # These tools are used by parts of the runtime, so we make the nose
     # dependency optional at this point.  Nose is a hard dependency to run the
     # test suite, but NOT to use ipython itself.
     import nose.tools as nt
     has_nose = True
 except ImportError:
     has_nose = False
 
 from IPython.utils import genutils, platutils
 
 from . import decorators as dec
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
+# By default, we assume IPython has been installed.  But if the test suite is
+# being run from a source tree that has NOT been installed yet, this flag can
+# be set to False by the entry point scripts, to let us know that we must call
+# the source tree versions of the scripts which manipulate sys.path instead of
+# assuming that things exist system-wide.
+INSTALLED = True
+
 # Make a bunch of nose.tools assert wrappers that can be used in test
 # generators.  This will expose an assert* function for each one in nose.tools.
 
 _tpl = """
 def %(name)s(*a,**kw):
     return nt.%(name)s(*a,**kw)
 """
 
 if has_nose:
     for _x in [a for a in dir(nt) if a.startswith('assert')]:
         exec _tpl % dict(name=_x)
 
 #-----------------------------------------------------------------------------
 # Functions and classes
 #-----------------------------------------------------------------------------
 
 # The docstring for full_path doctests differently on win32 (different path
 # separator) so just skip the doctest there.  The example remains informative.
 doctest_deco = dec.skip_doctest if sys.platform == 'win32' else dec.null_deco
 
 @doctest_deco
 def full_path(startPath,files):
     """Make full paths for all the listed files, based on startPath.
 
     Only the base part of startPath is kept, since this routine is typically
     used with a script's __file__ variable as startPath.  The base of startPath
     is then prepended to all the listed files, forming the output list.
 
     Parameters
     ----------
       startPath : string
         Initial path to use as the base for the results.  This path is split
       using os.path.split() and only its first component is kept.
 
       files : string or list
         One or more files.
 
     Examples
     --------
 
     >>> full_path('/foo/bar.py',['a.txt','b.txt'])
     ['/foo/a.txt', '/foo/b.txt']
 
     >>> full_path('/foo',['a.txt','b.txt'])
     ['/a.txt', '/b.txt']
 
     If a single file is given, the output is still a list:
     >>> full_path('/foo','a.txt')
     ['/a.txt']
     """
 
     files = genutils.list_strings(files)
     base = os.path.split(startPath)[0]
     return [ os.path.join(base,f) for f in files ]
 
 
 def parse_test_output(txt):
     """Parse the output of a test run and return errors, failures.
 
     Parameters
     ----------
     txt : str
       Text output of a test run, assumed to contain a line of one of the
       following forms::
         'FAILED (errors=1)'
         'FAILED (failures=1)'
         'FAILED (errors=1, failures=1)'
 
     Returns
     -------
     nerr, nfail: number of errors and failures.
     """
 
     err_m = re.search(r'^FAILED \(errors=(\d+)\)', txt, re.MULTILINE)
     if err_m:
         nerr = int(err_m.group(1))
         nfail = 0
         return  nerr, nfail
     
     fail_m = re.search(r'^FAILED \(failures=(\d+)\)', txt, re.MULTILINE)
     if fail_m:
         nerr = 0
         nfail = int(fail_m.group(1))
         return  nerr, nfail
 
     both_m = re.search(r'^FAILED \(errors=(\d+), failures=(\d+)\)', txt,
                        re.MULTILINE)
     if both_m:
         nerr = int(both_m.group(1))
         nfail = int(both_m.group(2))
         return  nerr, nfail
     
     # If the input didn't match any of these forms, assume no error/failures
     return 0, 0
 
 
 # So nose doesn't think this is a test
 parse_test_output.__test__ = False
 
 
 def cmd2argv(cmd):
     r"""Take the path of a command and return a list (argv-style).
 
     For a given path ``cmd``, this returns [cmd] if cmd's extension is .exe,
     .com or .bat, and ['python', cmd] otherwise.
 
     This is mostly a Windows utility, to deal with the fact that the scripts in
     Windows get wrapped in .exe entry points, so we have to call them
     differently.
 
     Parameters
     ----------
     cmd : string
       The path of the command.
 
     Returns
     -------
     argv-style list.
 
     Examples
     --------
     In [2]: cmd2argv('/usr/bin/ipython')
     Out[2]: ['python', '/usr/bin/ipython']
 
     In [3]: cmd2argv(r'C:\Python26\Scripts\ipython.exe')
     Out[3]: ['C:\\Python26\\Scripts\\ipython.exe']
     """
     ext = os.path.splitext(cmd)[1]
     if ext in ['.exe', '.com', '.bat']:
         return [cmd]
     else:
         return ['python', cmd]
     
 
 def temp_pyfile(src, ext='.py'):
     """Make a temporary python file, return filename and filehandle.
 
     Parameters
     ----------
     src : string or list of strings (no need for ending newlines if list)
       Source code to be written to the file.
 
     ext : optional, string
       Extension for the generated file.
 
     Returns
     -------
     (filename, open filehandle)
       It is the caller's responsibility to close the open file and unlink it.
     """
     fname = tempfile.mkstemp(ext)[1]
     f = open(fname,'w')
     f.write(src)
     f.flush()
     return fname, f
 
 
 def default_argv():
     """Return a valid default argv for creating testing instances of ipython"""
 
     return ['--quick', # so no config file is loaded
             # Other defaults to minimize side effects on stdout
             '--colors=NoColor', '--no-term-title','--no-banner',
             '--autocall=0']
     
 
 def ipexec(fname, options=None):
     """Utility to call 'ipython filename'.
 
     Starts IPython witha minimal and safe configuration to make startup as fast
     as possible.
 
     Note that this starts IPython in a subprocess!
 
     Parameters
     ----------
     fname : str
       Name of file to be executed (should have .py or .ipy extension).
 
     options : optional, list
       Extra command-line flags to be passed to IPython.
 
     Returns
     -------
     (stdout, stderr) of ipython subprocess.
     """
     if options is None: options = []
     
     # For these subprocess calls, eliminate all prompt printing so we only see
     # output from script execution
     prompt_opts = ['--prompt-in1=""', '--prompt-in2=""', '--prompt-out=""']
     cmdargs = ' '.join(default_argv() + prompt_opts + options)
     
     _ip = get_ipython()
     test_dir = os.path.dirname(__file__)
 
     # Find the ipython script from the package we're using, so that the test
     # suite can be run from the source tree without an installed IPython
     p = os.path
-    ippath = p.abspath(p.join(p.dirname(__file__),'..','..'))
-    ipython_script = p.join(ippath, 'ipython.py')
-    ipython_cmd = 'python "%s"' % ipython_script
+    if INSTALLED:
+        ipython_cmd = platutils.find_cmd('ipython')
+    else:
+        ippath = p.abspath(p.join(p.dirname(__file__),'..','..'))
+        ipython_script = p.join(ippath, 'ipython.py')
+        ipython_cmd = 'python "%s"' % ipython_script
     # Absolute path for filename
     full_fname = p.join(test_dir, fname)
-    full_cmd = '%s %s "%s"' % (ipython_cmd, cmdargs, full_fname)
+    full_cmd = '%s %s %s' % (ipython_cmd, cmdargs, full_fname)
+    #print >> sys.stderr, 'FULL CMD:', full_cmd # dbg
     return genutils.getoutputerror(full_cmd)
 
 
-def ipexec_validate(fname, expected_out, expected_err=None,
+def ipexec_validate(fname, expected_out, expected_err='',
                     options=None):
     """Utility to call 'ipython filename' and validate output/error.
 
     This function raises an AssertionError if the validation fails.
 
     Note that this starts IPython in a subprocess!
 
     Parameters
     ----------
     fname : str
       Name of the file to be executed (should have .py or .ipy extension).
 
     expected_out : str
       Expected stdout of the process.
 
     expected_err : optional, str
       Expected stderr of the process.
 
     options : optional, list
       Extra command-line flags to be passed to IPython.
 
     Returns
     -------
     None
     """
 
     import nose.tools as nt
     
     out, err = ipexec(fname)
+    #print 'OUT', out  # dbg
+    #print 'ERR', err  # dbg
+    # If there are any errors, we must check those befor stdout, as they may be
+    # more informative than simply having an empty stdout.
+    if err:
+        if expected_err:
+            nt.assert_equals(err.strip(), expected_err.strip())
+        else:
+            raise ValueError('Running file %r produced error: %r' %
+                             (fname, err))
+    # If no errors or output on stderr was expected, match stdout
     nt.assert_equals(out.strip(), expected_out.strip())
-    if expected_err:
-        nt.assert_equals(err.strip(), expected_err.strip())
 
 
 class TempFileMixin(object):
     """Utility class to create temporary Python/IPython files.
 
     Meant as a mixin class for test cases."""
     
     def mktmp(self, src, ext='.py'):
         """Make a valid python temp file."""
         fname, f = temp_pyfile(src, ext)
         self.tmpfile = f
         self.fname = fname
 
     def teardown(self):
         if hasattr(self, 'tmpfile'):
             # If the tmpfile wasn't made because of skipped tests, like in
             # win32, there's nothing to cleanup.
             self.tmpfile.close()
             try:
                 os.unlink(self.fname)
             except:
                 # On Windows, even though we close the file, we still can't
                 # delete it.  I have no clue why
                 pass
 

iptest.py

@@ -1,21 +1,26 b''
 #!/usr/bin/env python
 """Test script for IPython.
 
 The actual ipython test script to be installed with 'python setup.py install'
 is in './scripts' directory, and will test IPython from an importable
 location.
 
 This file is here (ipython source root directory) to facilitate non-root
 'zero-installation testing and development' (just copy the source tree
 somewhere and run iptest.py).
 
 You can run this script directly, type -h to see all options."""
 
 # Ensure that the imported IPython packages come from *THIS* IPython, not some
 # other one that may exist system-wide
 import os, sys
 this_dir = os.path.dirname(os.path.abspath(__file__))
 sys.path.insert(0, this_dir)
 
+import IPython.testing.tools as t
+import IPython.testing.iptest as ipt
+t.INSTALLED = False
+ipt.INSTALLED = False
+
 # Now proceed with execution
 execfile(os.path.join(this_dir, 'IPython', 'scripts', 'iptest'))