Refactor TestController API to allow it to display extra info.

Add a setup() method to be called when we know we're going to use a test
group, for creating temporary dirs etc., and a print_extra_info() method
to display extra information.

IPython/testing/iptestcontroller.py

@@ -50,21 +50,27 b' class TestController(object):'
     process = None
     #: str, process stdout+stderr
     stdout = None
-    #: bool, whether to capture process stdout & stderr
-    buffer_output = False
 
     def __init__(self):
         self.cmd = []
         self.env = {}
         self.dirs = []
 
-    def launch(self):
+    def setup(self):
+        """Create temporary directories etc.
+        
+        This is only called when we know the test group will be run. Things
+        created here may be cleaned up by self.cleanup().
+        """
+        pass
+
+    def launch(self, buffer_output=False):
         # print('*** ENV:', self.env)  # dbg
         # print('*** CMD:', self.cmd)  # dbg
         env = os.environ.copy()
         env.update(self.env)
-        output = subprocess.PIPE if self.buffer_output else None
-        stdout = subprocess.STDOUT if self.buffer_output else None
+        output = subprocess.PIPE if buffer_output else None
+        stdout = subprocess.STDOUT if buffer_output else None
         self.process = subprocess.Popen(self.cmd, stdout=output,
                 stderr=stdout, env=env)
 
@@ -72,14 +78,17 b' class TestController(object):'
         self.stdout, _ = self.process.communicate()
         return self.process.returncode
 
-    def dump_failure(self):
-        """Print buffered results of a test failure.
+    def print_extra_info(self):
+        """Print extra information about this test run.
+        
+        If we're running in parallel and showing the concise view, this is only
+        called if the test group fails. Otherwise, it's called before the test
+        group is started.
         
-        Called after tests fail while running in parallel. The base
-        implementation just prints the output from the test subprocess, but
-        subclasses can override it to add extra information.
+        The base implementation does nothing, but it can be overridden by
+        subclasses.
         """
-        print(self.stdout)
+        return
 
     def cleanup_process(self):
         """Cleanup on exit by killing any leftover processes."""
@@ -125,6 +134,8 b' class PyTestController(TestController):'
         # pycmd is put into cmd[2] in PyTestController.launch()
         self.cmd = [sys.executable, '-c', None, section]
         self.pycmd = "from IPython.testing.iptest import run_iptest; run_iptest()"
+
+    def setup(self):
         ipydir = TemporaryDirectory()
         self.dirs.append(ipydir)
         self.env['IPYTHONDIR'] = ipydir.name
@@ -164,9 +175,9 b' class PyTestController(TestController):'
         self.env['COVERAGE_PROCESS_START'] = config_file
         self.pycmd = "import coverage; coverage.process_startup(); " + self.pycmd
 
-    def launch(self):
+    def launch(self, buffer_output=False):
         self.cmd[2] = self.pycmd
-        super(PyTestController, self).launch()
+        super(PyTestController, self).launch(buffer_output=buffer_output)
 
 js_prefix = 'js/'
 
@@ -187,14 +198,13 b' class JSController(TestController):'
         TestController.__init__(self)
         self.section = section
 
-
-    def launch(self):
+    def setup(self):
         self.ipydir = TemporaryDirectory()
         self.nbdir = TemporaryDirectory()
-        os.makedirs(os.path.join(self.nbdir.name, os.path.join(u'sub ∂ir1', u'sub ∂ir 1a')))
-        os.makedirs(os.path.join(self.nbdir.name, os.path.join(u'sub ∂ir2', u'sub ∂ir 1b')))
         self.dirs.append(self.ipydir)
         self.dirs.append(self.nbdir)
+        os.makedirs(os.path.join(self.nbdir.name, os.path.join(u'sub ∂ir1', u'sub ∂ir 1a')))
+        os.makedirs(os.path.join(self.nbdir.name, os.path.join(u'sub ∂ir2', u'sub ∂ir 1b')))
         
         # start the ipython notebook, so we get the port number
         self._init_server()
@@ -203,7 +213,9 b' class JSController(TestController):'
         test_cases = os.path.join(js_test_dir, self.section[len(js_prefix):])
         port = '--port=' + str(self.server_port)
         self.cmd = ['casperjs', 'test', port, includes, test_cases]
-        super(JSController, self).launch()
+
+    def print_extra_info(self):
+        print("Running tests with notebook directory %r" % self.nbdir.name)
 
     @property
     def will_run(self):
@@ -216,10 +228,6 b' class JSController(TestController):'
         self.server.start()
         self.server_port = q.get()
 
-    def dump_failure(self):
-        print("Ran tests with notebook directory %r" % self.nbdir.name)
-        super(JSController, self).dump_failure()
-
     def cleanup(self):
         self.server.terminate()
         self.server.join()
@@ -287,10 +295,27 b' def configure_py_controllers(controllers, xunit=False, coverage=False,'
         controller.env['IPTEST_SUBPROC_STREAMS'] = subproc_streams
         controller.cmd.extend(extra_args)
 
-def do_run(controller):
+def do_run(controller, buffer_output=True):
+    """Setup and run a test controller.
+    
+    If buffer_output is True, no output is displayed, to avoid it appearing
+    interleaved. In this case, the caller is responsible for displaying test
+    output on failure.
+    
+    Returns
+    -------
+    controller : TestController
+      The same controller as passed in, as a convenience for using map() type
+      APIs.
+    exitcode : int
+      The exit code of the test subprocess. Non-zero indicates failure.
+    """
     try:
         try:
-            controller.launch()
+            controller.setup()
+            if not buffer_output:
+                controller.print_extra_info()
+            controller.launch(buffer_output=buffer_output)
         except Exception:
             import traceback
             traceback.print_exc()
@@ -377,10 +402,6 b' def run_iptestall(options):'
     extra_args : list
       Extra arguments to pass to the test subprocesses, e.g. '-v'
     """
-    if options.fast != 1:
-        # If running in parallel, capture output so it doesn't get interleaved
-        TestController.buffer_output = True
-
     to_run, not_run = prepare_controllers(options)
 
     def justify(ltext, rtext, width=70, fill='-'):
@@ -398,7 +419,7 b' def run_iptestall(options):'
         for controller in to_run:
             print('IPython test group:', controller.section)
             sys.stdout.flush()  # Show in correct order when output is piped
-            controller, res = do_run(controller)
+            controller, res = do_run(controller, buffer_output=False)
             if res:
                 failed.append(controller)
                 if res == -signal.SIGINT:
@@ -414,7 +435,8 b' def run_iptestall(options):'
                 res_string = 'OK' if res == 0 else 'FAILED'
                 print(justify('Test group: ' + controller.section, res_string))
                 if res:
-                    controller.dump_failure()
+                    controller.print_extra_info()
+                    print(bytes_to_str(controller.stdout))
                     failed.append(controller)
                     if res == -signal.SIGINT:
                         print("Interrupted")