Robustness fixes in test suite machinery.

Added a module-level INSTALLED flag, which can be set to false if the test
suite is being run in-place (without ipython having been installed at all).
This is because how we call and import things must be done differently
depending on whether the code is installed or is being run in-place.  The
only ones that can know this reliably are the entry-point scripts, so those
are responsible for setting this flag.

Also made the code that validates ipython in subprocesses report errors
better, by checking stderr for errors before validating stdout output, as
anything on stderr will be likely informative of the real problem.

IPython/core/tests/tclass.py

@@ -13,7 +13,7 b' class C(object):'
         
     def __del__(self):
         print 'tclass.py: deleting object:',self.name
-
+        sys.stdout.flush()
 
 try:
     name = sys.argv[1]
@@ -28,3 +28,4 b' else:'
 # This next print statement is NOT debugging, we're making the check on a
 # completely separate process so we verify by capturing stdout:
 print 'ARGV 1-:', sys.argv[1:]
+sys.stdout.flush()

IPython/testing/iptest.py

@@ -57,6 +57,18 b' from IPython.testing.plugin.ipdoctest import IPythonDoctest'
 
 pjoin = path.join
 
+
+#-----------------------------------------------------------------------------
+# Globals
+#-----------------------------------------------------------------------------
+
+# By default, we assume IPython has been installed.  But if the test suite is
+# being run from a source tree that has NOT been installed yet, this flag can
+# be set to False by the entry point scripts, to let us know that we must call
+# the source tree versions of the scripts which manipulate sys.path instead of
+# assuming that things exist system-wide.
+INSTALLED = True
+
 #-----------------------------------------------------------------------------
 # Warnings control
 #-----------------------------------------------------------------------------
@@ -88,7 +100,6 b' def test_for(mod):'
     else:
         return True
 
-    
 have_curses = test_for('_curses')
 have_wx = test_for('wx')
 have_wx_aui = test_for('wx.aui')
@@ -100,6 +111,9 b" have_pexpect = test_for('pexpect')"
 have_gtk = test_for('gtk')
 have_gobject = test_for('gobject')
 
+#-----------------------------------------------------------------------------
+# Functions and classes
+#-----------------------------------------------------------------------------
 
 def make_exclude():
     """Make patterns of modules and packages to exclude from testing.
@@ -190,10 +204,6 b' def make_exclude():'
     return exclusions
 
 
-#-----------------------------------------------------------------------------
-# Functions and classes
-#-----------------------------------------------------------------------------
-
 class IPTester(object):
     """Call that calls iptest or trial in a subprocess.
     """
@@ -208,14 +218,22 b' class IPTester(object):'
     
     def __init__(self, runner='iptest', params=None):
         """Create new test runner."""
+        p = os.path
         if runner == 'iptest':
-            # Find our own 'iptest' script OS-level entry point.  Don't look
-            # system-wide, so we are sure we pick up *this one*.  And pass
-            # through to subprocess call our own sys.argv
-            self.runner = tools.cmd2argv(os.path.abspath(__file__)) + \
-                          sys.argv[1:]
+            if INSTALLED:
+                self.runner = tools.cmd2argv(
+                    p.abspath(find_cmd('iptest'))) + sys.argv[1:]
+            else:
+                # Find our own 'iptest' script OS-level entry point.  Don't
+                # look system-wide, so we are sure we pick up *this one*.  And
+                # pass through to subprocess call our own sys.argv
+                ippath = p.abspath(p.join(p.dirname(__file__),'..','..'))
+                script = p.join(ippath, 'iptest.py')
+                self.runner = tools.cmd2argv(script) + sys.argv[1:]
+                
         else:
-            self.runner = tools.cmd2argv(os.path.abspath(find_cmd('trial')))
+            # For trial, it needs to be installed system-wide
+            self.runner = tools.cmd2argv(p.abspath(find_cmd('trial')))
         if params is None:
             params = []
         if isinstance(params, str):
@@ -296,7 +314,7 b' def make_runners():'
         trial_pkg_names.append('kernel')
 
     # For debugging this code, only load quick stuff
-    #nose_pkg_names = ['core']  # dbg
+    #nose_pkg_names = ['core', 'extensions']  # dbg
     #trial_pkg_names = []  # dbg 
 
     # Make fully qualified package names prepending 'IPython.' to our name lists
@@ -322,19 +340,7 b' def run_iptest():'
         'This will be removed soon.  Use IPython.testing.util instead')
 
     argv = sys.argv + [ '--detailed-errors',  # extra info in tracebacks
-                        
-                        # I don't fully understand why we need this one, but
-                        # depending on what directory the test suite is run
-                        # from, if we don't give it, 0 tests get run.
-                        # Specifically, if the test suite is run from the
-                        # source dir with an argument (like 'iptest.py
-                        # IPython.core', 0 tests are run, even if the same call
-                        # done in this directory works fine).  It appears that
-                        # if  the requested package is in the current dir,
-                        # nose bails early by default.  Since it's otherwise
-                        # harmless, leave it in by default. 
-                        '--traverse-namespace',
-                        
+                                                
                         # Loading ipdoctest causes problems with Twisted, but
                         # our test suite runner now separates things and runs
                         # all Twisted tests with trial.
@@ -350,6 +356,16 b' def run_iptest():'
                         '--exe',
                         ]
 
+    if nose.__version__ >= '0.11':
+        # I don't fully understand why we need this one, but depending on what
+        # directory the test suite is run from, if we don't give it, 0 tests
+        # get run.  Specifically, if the test suite is run from the source dir
+        # with an argument (like 'iptest.py IPython.core', 0 tests are run,
+        # even if the same call done in this directory works fine).  It appears
+        # that if the requested package is in the current dir, nose bails early
+        # by default.  Since it's otherwise harmless, leave it in by default
+        # for nose >= 0.11, though unfortunately nose 0.10 doesn't support it.
+        argv.append('--traverse-namespace')
 
     # Construct list of plugins, omitting the existing doctest plugin, which
     # ours replaces (and extends).

IPython/testing/tools.py

@@ -49,6 +49,13 b' from . import decorators as dec'
 # Globals
 #-----------------------------------------------------------------------------
 
+# By default, we assume IPython has been installed.  But if the test suite is
+# being run from a source tree that has NOT been installed yet, this flag can
+# be set to False by the entry point scripts, to let us know that we must call
+# the source tree versions of the scripts which manipulate sys.path instead of
+# assuming that things exist system-wide.
+INSTALLED = True
+
 # Make a bunch of nose.tools assert wrappers that can be used in test
 # generators.  This will expose an assert* function for each one in nose.tools.
 
@@ -248,16 +255,20 b' def ipexec(fname, options=None):'
     # Find the ipython script from the package we're using, so that the test
     # suite can be run from the source tree without an installed IPython
     p = os.path
-    ippath = p.abspath(p.join(p.dirname(__file__),'..','..'))
-    ipython_script = p.join(ippath, 'ipython.py')
-    ipython_cmd = 'python "%s"' % ipython_script
+    if INSTALLED:
+        ipython_cmd = platutils.find_cmd('ipython')
+    else:
+        ippath = p.abspath(p.join(p.dirname(__file__),'..','..'))
+        ipython_script = p.join(ippath, 'ipython.py')
+        ipython_cmd = 'python "%s"' % ipython_script
     # Absolute path for filename
     full_fname = p.join(test_dir, fname)
-    full_cmd = '%s %s "%s"' % (ipython_cmd, cmdargs, full_fname)
+    full_cmd = '%s %s %s' % (ipython_cmd, cmdargs, full_fname)
+    #print >> sys.stderr, 'FULL CMD:', full_cmd # dbg
     return genutils.getoutputerror(full_cmd)
 
 
-def ipexec_validate(fname, expected_out, expected_err=None,
+def ipexec_validate(fname, expected_out, expected_err='',
                     options=None):
     """Utility to call 'ipython filename' and validate output/error.
 
@@ -287,9 +298,18 b' def ipexec_validate(fname, expected_out, expected_err=None,'
     import nose.tools as nt
     
     out, err = ipexec(fname)
+    #print 'OUT', out  # dbg
+    #print 'ERR', err  # dbg
+    # If there are any errors, we must check those befor stdout, as they may be
+    # more informative than simply having an empty stdout.
+    if err:
+        if expected_err:
+            nt.assert_equals(err.strip(), expected_err.strip())
+        else:
+            raise ValueError('Running file %r produced error: %r' %
+                             (fname, err))
+    # If no errors or output on stderr was expected, match stdout
     nt.assert_equals(out.strip(), expected_out.strip())
-    if expected_err:
-        nt.assert_equals(err.strip(), expected_err.strip())
 
 
 class TempFileMixin(object):

iptest.py

@@ -17,5 +17,10 b' import os, sys'
 this_dir = os.path.dirname(os.path.abspath(__file__))
 sys.path.insert(0, this_dir)
 
+import IPython.testing.tools as t
+import IPython.testing.iptest as ipt
+t.INSTALLED = False
+ipt.INSTALLED = False
+
 # Now proceed with execution
 execfile(os.path.join(this_dir, 'IPython', 'scripts', 'iptest'))