From f173ff8e0d8cfd8c67974466057fafd3b80b974a 2010-01-08 00:04:08 From: Fernando Perez Date: 2010-01-08 00:04:08 Subject: [PATCH] Work in multiple places to improve state of the test suite. With these changes, on my system now all the test sub-suites pass except for the Twisted one (see https://bugs.launchpad.net/ipython/+bug/504515 for details on that one). --- diff --git a/IPython/core/iplib.py b/IPython/core/iplib.py index 9af3ad4..282694e 100644 --- a/IPython/core/iplib.py +++ b/IPython/core/iplib.py @@ -964,26 +964,43 @@ class InteractiveShell(Component, Magic): method. If they were not empty before, data will simply be added to therm. """ - # Store myself as the public api!!! - self.user_ns['get_ipython'] = self.get_ipython + # This function works in two parts: first we put a few things in + # user_ns, and we sync that contents into user_config_ns so that these + # initial variables aren't shown by %who. After the sync, we add the + # rest of what we *do* want the user to see with %who even on a new + # session. + ns = {} + + # Put 'help' in the user namespace + try: + from site import _Helper + ns['help'] = _Helper() + except ImportError: + warn('help() not available - check site.py') # make global variables for user access to the histories - self.user_ns['_ih'] = self.input_hist - self.user_ns['_oh'] = self.output_hist - self.user_ns['_dh'] = self.dir_hist + ns['_ih'] = self.input_hist + ns['_oh'] = self.output_hist + ns['_dh'] = self.dir_hist + + ns['_sh'] = shadowns + + # Sync what we've added so far to user_config_ns so these aren't seen + # by %who + self.user_config_ns.update(ns) + + # Now, continue adding more contents # user aliases to input and output histories - self.user_ns['In'] = self.input_hist - self.user_ns['Out'] = self.output_hist + ns['In'] = self.input_hist + ns['Out'] = self.output_hist - self.user_ns['_sh'] = shadowns + # Store myself as the public api!!! + ns['get_ipython'] = self.get_ipython + + # And update the real user's namespace + self.user_ns.update(ns) - # Put 'help' in the user namespace - try: - from site import _Helper - self.user_ns['help'] = _Helper() - except ImportError: - warn('help() not available - check site.py') def reset(self): """Clear all internal namespaces. diff --git a/IPython/core/tests/test_iplib.py b/IPython/core/tests/test_iplib.py index 8a972ae..e41cf11 100644 --- a/IPython/core/tests/test_iplib.py +++ b/IPython/core/tests/test_iplib.py @@ -15,7 +15,7 @@ import nose.tools as nt # our own packages from IPython.core import iplib from IPython.core import ipapi - +from IPython.testing import decorators as dec #----------------------------------------------------------------------------- # Globals @@ -43,15 +43,33 @@ if ip is None: # Test functions #----------------------------------------------------------------------------- +@dec.parametric def test_reset(): """reset must clear most namespaces.""" - ip.reset() # first, it should run without error - # Then, check that most namespaces end up empty + # The number of variables in the private user_config_ns is not zero, but it + # should be constant regardless of what we do + nvars_config_ns = len(ip.user_config_ns) + + # Check that reset runs without error + ip.reset() + + # Once we've reset it (to clear of any junk that might have been there from + # other tests, we can count how many variables are in the user's namespace + nvars_user_ns = len(ip.user_ns) + + # Now add a few variables to user_ns, and check that reset clears them + ip.user_ns['x'] = 1 + ip.user_ns['y'] = 1 + ip.reset() + + # Finally, check that all namespaces have only as many variables as we + # expect to find in them: for ns in ip.ns_refs_table: if ns is ip.user_ns: - # The user namespace is reset with some data, so we can't check for - # it being empty - continue - nt.assert_equals(len(ns),0) - - \ No newline at end of file + nvars_expected = nvars_user_ns + elif ns is ip.user_config_ns: + nvars_expected = nvars_config_ns + else: + nvars_expected = 0 + + yield nt.assert_equals(len(ns), nvars_expected) diff --git a/IPython/frontend/prefilterfrontend.py b/IPython/frontend/prefilterfrontend.py index 257669d..6d35d74 100644 --- a/IPython/frontend/prefilterfrontend.py +++ b/IPython/frontend/prefilterfrontend.py @@ -9,7 +9,6 @@ functionnality is abstracted out of ipython0 in reusable functions and is added on the interpreter. This class can be a used to guide this refactoring. """ -__docformat__ = "restructuredtext en" #------------------------------------------------------------------------------- # Copyright (C) 2008 The IPython Development Team @@ -27,7 +26,7 @@ import os import re import __builtin__ -from IPython.core.ipmaker import make_IPython +from IPython.core.ipapp import IPythonApp from IPython.kernel.core.redirector_output_trap import RedirectorOutputTrap from IPython.kernel.core.sync_traceback_trap import SyncTracebackTrap @@ -36,6 +35,9 @@ from IPython.utils.genutils import Term from linefrontendbase import LineFrontEndBase, common_prefix +#----------------------------------------------------------------------------- +# Utility functions +#----------------------------------------------------------------------------- def mk_system_call(system_call_function, command): """ given a os.system replacement, and a leading string command, @@ -74,7 +76,7 @@ class PrefilterFrontEnd(LineFrontEndBase): Used as the instance's argv value. If not given, [] is used. """ if argv is None: - argv = [] + argv = ['--no-banner'] # This is a hack to avoid the IPython exception hook to trigger # on exceptions (https://bugs.launchpad.net/bugs/337105) # XXX: This is horrible: module-leve monkey patching -> side @@ -101,12 +103,15 @@ class PrefilterFrontEnd(LineFrontEndBase): return '\n' old_rawinput = __builtin__.raw_input __builtin__.raw_input = my_rawinput - # XXX: argv=[] is a bit bold. - ipython0 = make_IPython(argv=argv, - user_ns=self.shell.user_ns, - user_global_ns=self.shell.user_global_ns) + ipython0 = IPythonApp(argv=argv, + user_ns=self.shell.user_ns, + user_global_ns=self.shell.user_global_ns) + ipython0.initialize() __builtin__.raw_input = old_rawinput - self.ipython0 = ipython0 + # XXX This will need to be updated as we refactor things, but for now, + # the .shell attribute of the ipythonapp instance conforms to the old + # api. + self.ipython0 = ipython0.shell # Set the pager: self.ipython0.set_hook('show_in_pager', lambda s, string: self.write("\n" + string)) @@ -202,8 +207,7 @@ class PrefilterFrontEnd(LineFrontEndBase): if completions: prefix = common_prefix(completions) line = line[:-len(word)] + prefix - return line, completions - + return line, completions #-------------------------------------------------------------------------- # LineFrontEndBase interface @@ -220,23 +224,11 @@ class PrefilterFrontEnd(LineFrontEndBase): self.capture_output() self.last_result = dict(number=self.prompt_number) - ## try: - ## for line in input_string.split('\n'): - ## filtered_lines.append( - ## self.ipython0.prefilter(line, False).rstrip()) - ## except: - ## # XXX: probably not the right thing to do. - ## self.ipython0.showsyntaxerror() - ## self.after_execute() - ## finally: - ## self.release_output() - - try: try: for line in input_string.split('\n'): - filtered_lines.append( - self.ipython0.prefilter(line, False).rstrip()) + pf = self.ipython0.prefilter_manager.prefilter_lines + filtered_lines.append(pf(line, False).rstrip()) except: # XXX: probably not the right thing to do. self.ipython0.showsyntaxerror() @@ -244,13 +236,10 @@ class PrefilterFrontEnd(LineFrontEndBase): finally: self.release_output() - - # Clean up the trailing whitespace, to avoid indentation errors filtered_string = '\n'.join(filtered_lines) return filtered_string - #-------------------------------------------------------------------------- # PrefilterFrontEnd interface #-------------------------------------------------------------------------- @@ -261,13 +250,11 @@ class PrefilterFrontEnd(LineFrontEndBase): """ return os.system(command_string) - def do_exit(self): """ Exit the shell, cleanup and save the history. """ self.ipython0.atexit_operations() - def _get_completion_text(self, line): """ Returns the text to be completed by breaking the line at specified delimiters. @@ -281,4 +268,3 @@ class PrefilterFrontEnd(LineFrontEndBase): complete_sep = re.compile(expression) text = complete_sep.split(line)[-1] return text - diff --git a/IPython/frontend/tests/test_prefilterfrontend.py b/IPython/frontend/tests/test_prefilterfrontend.py index 653b3e6..ebe100c 100644 --- a/IPython/frontend/tests/test_prefilterfrontend.py +++ b/IPython/frontend/tests/test_prefilterfrontend.py @@ -23,6 +23,9 @@ from IPython.frontend.prefilterfrontend import PrefilterFrontEnd from IPython.core.ipapi import get as get_ipython0 from IPython.testing.plugin.ipdoctest import default_argv +#----------------------------------------------------------------------------- +# Support utilities +#----------------------------------------------------------------------------- class TestPrefilterFrontEnd(PrefilterFrontEnd): @@ -93,6 +96,9 @@ def isolate_ipython0(func): my_func.__name__ = func.__name__ return my_func +#----------------------------------------------------------------------------- +# Tests +#----------------------------------------------------------------------------- @isolate_ipython0 def test_execution(): @@ -166,7 +172,7 @@ def test_magic(): f.input_buffer += '%who' f._on_enter() out_value = f.out.getvalue() - assert_equal(out_value, 'Interactive namespace is empty.\n') + assert_equal(out_value, 'In\tOut\tget_ipython\t\n') @isolate_ipython0 diff --git a/IPython/testing/iptest.py b/IPython/testing/iptest.py index 89a46a0..6c64a9f 100644 --- a/IPython/testing/iptest.py +++ b/IPython/testing/iptest.py @@ -31,12 +31,20 @@ import warnings import nose.plugins.builtin from nose.core import TestProgram -from IPython.utils.platutils import find_cmd -# from IPython.testing.plugin.ipdoctest import IPythonDoctest +from IPython.utils import genutils +from IPython.utils.platutils import find_cmd, FindCmdError pjoin = path.join #----------------------------------------------------------------------------- +# Warnings control +#----------------------------------------------------------------------------- +# Twisted generates annoying warnings with Python 2.6, as will do other code +# that imports 'sets' as of today +warnings.filterwarnings('ignore', 'the sets module is deprecated', + DeprecationWarning ) + +#----------------------------------------------------------------------------- # Logic for skipping doctests #----------------------------------------------------------------------------- @@ -63,10 +71,10 @@ have_gobject = test_for('gobject') def make_exclude(): - # For the IPythonDoctest plugin, we need to exclude certain patterns that cause - # testing problems. We should strive to minimize the number of skipped - # modules, since this means untested code. As the testing machinery - # solidifies, this list should eventually become empty. + # For the IPythonDoctest plugin, we need to exclude certain patterns that + # cause testing problems. We should strive to minimize the number of + # skipped modules, since this means untested code. As the testing + # machinery solidifies, this list should eventually become empty. EXCLUDE = [pjoin('IPython', 'external'), pjoin('IPython', 'frontend', 'process', 'winprocess.py'), pjoin('IPython_doctest_plugin'), @@ -137,6 +145,82 @@ def make_exclude(): # Functions and classes #----------------------------------------------------------------------------- +class IPTester(object): + """Call that calls iptest or trial in a subprocess. + """ + def __init__(self,runner='iptest',params=None): + """ """ + if runner == 'iptest': + # Find our own 'iptest' script OS-level entry point + try: + iptest_path = find_cmd('iptest') + except FindCmdError: + # Script not installed (may be the case for testing situations + # that are running from a source tree only), pull from internal + # path: + iptest_path = pjoin(genutils.get_ipython_package_dir(), + 'scripts','iptest') + self.runner = [iptest_path,'-v'] + else: + self.runner = [find_cmd('trial')] + if params is None: + params = [] + if isinstance(params,str): + params = [params] + self.params = params + + # Assemble call + self.call_args = self.runner+self.params + + if sys.platform == 'win32': + def _run_cmd(self): + # On Windows, use os.system instead of subprocess.call, because I + # was having problems with subprocess and I just don't know enough + # about win32 to debug this reliably. Os.system may be the 'old + # fashioned' way to do it, but it works just fine. If someone + # later can clean this up that's fine, as long as the tests run + # reliably in win32. + return os.system(' '.join(self.call_args)) + else: + def _run_cmd(self): + return subprocess.call(self.call_args) + + def run(self): + """Run the stored commands""" + try: + return self._run_cmd() + except: + import traceback + traceback.print_exc() + return 1 # signal failure + + +def make_runners(): + """Define the top-level packages that need to be tested. + """ + + nose_packages = ['config', 'core', 'extensions', 'frontend', 'lib', + 'scripts', 'testing', 'utils'] + trial_packages = ['kernel'] + #trial_packages = [] # dbg + + if have_wx: + nose_packages.append('gui') + + nose_packages = ['IPython.%s' % m for m in nose_packages ] + trial_packages = ['IPython.%s' % m for m in trial_packages ] + + # Make runners, most with nose + nose_testers = [IPTester(params=v) for v in nose_packages] + runners = dict(zip(nose_packages, nose_testers)) + # And add twisted ones if conditions are met + if have_zi and have_twisted and have_foolscap: + trial_testers = [IPTester('trial',params=v) for v in trial_packages] + runners.update(dict(zip(trial_packages,trial_testers))) + + return runners + + def run_iptest(): """Run the IPython test suite using nose. @@ -194,81 +278,6 @@ def run_iptest(): TestProgram(argv=argv,plugins=plugins) -class IPTester(object): - """Call that calls iptest or trial in a subprocess. - """ - def __init__(self,runner='iptest',params=None): - """ """ - if runner == 'iptest': - self.runner = ['iptest','-v'] - else: - self.runner = [find_cmd('trial')] - if params is None: - params = [] - if isinstance(params,str): - params = [params] - self.params = params - - # Assemble call - self.call_args = self.runner+self.params - - if sys.platform == 'win32': - def run(self): - """Run the stored commands""" - # On Windows, cd to temporary directory to run tests. Otherwise, - # Twisted's trial may not be able to execute 'trial IPython', since - # it will confuse the IPython module name with the ipython - # execution scripts, because the windows file system isn't case - # sensitive. - # We also use os.system instead of subprocess.call, because I was - # having problems with subprocess and I just don't know enough - # about win32 to debug this reliably. Os.system may be the 'old - # fashioned' way to do it, but it works just fine. If someone - # later can clean this up that's fine, as long as the tests run - # reliably in win32. - curdir = os.getcwd() - os.chdir(tempfile.gettempdir()) - stat = os.system(' '.join(self.call_args)) - os.chdir(curdir) - return stat - else: - def run(self): - """Run the stored commands""" - try: - return subprocess.call(self.call_args) - except: - import traceback - traceback.print_exc() - return 1 # signal failure - - -def make_runners(): - """Define the top-level packages that need to be tested. - """ - - nose_packages = ['config', 'core', 'extensions', - 'frontend', 'lib', - 'scripts', 'testing', 'utils'] - trial_packages = ['kernel'] - - if have_wx: - nose_packages.append('gui') - - nose_packages = ['IPython.%s' % m for m in nose_packages ] - trial_packages = ['IPython.%s' % m for m in trial_packages ] - - # Make runners - runners = dict() - - nose_runners = dict(zip(nose_packages, [IPTester(params=v) for v in nose_packages])) - if have_zi and have_twisted and have_foolscap: - trial_runners = dict(zip(trial_packages, [IPTester('trial',params=v) for v in trial_packages])) - runners.update(nose_runners) - runners.update(trial_runners) - - return runners - - def run_iptestall(): """Run the entire IPython test suite by calling nose and trial. @@ -280,15 +289,26 @@ def run_iptestall(): runners = make_runners() + # Run the test runners in a temporary dir so we can nuke it when finished + # to clean up any junk files left over by accident. This also makes it + # robust against being run in non-writeable directories by mistake, as the + # temp dir will always be user-writeable. + curdir = os.getcwd() + testdir = tempfile.gettempdir() + os.chdir(testdir) + # Run all test runners, tracking execution time failed = {} t_start = time.time() - for name,runner in runners.iteritems(): - print '*'*77 - print 'IPython test group:',name - res = runner.run() - if res: - failed[name] = res + try: + for name,runner in runners.iteritems(): + print '*'*77 + print 'IPython test group:',name + res = runner.run() + if res: + failed[name] = res + finally: + os.chdir(curdir) t_end = time.time() t_tests = t_end - t_start nrunners = len(runners) diff --git a/IPython/testing/plugin/ipdoctest.py b/IPython/testing/plugin/ipdoctest.py index 016d757..06d3c2f 100644 --- a/IPython/testing/plugin/ipdoctest.py +++ b/IPython/testing/plugin/ipdoctest.py @@ -68,7 +68,8 @@ def default_argv(): ipcdir = os.path.dirname(default.__file__) ipconf = os.path.join(ipcdir,'ipython_config.py') #print 'conf:',ipconf # dbg - return ['--colors=NoColor','--no-term-title','--config-file=%s' % ipconf] + return ['--colors=NoColor', '--no-term-title','--no-banner', + '--config-file=%s' % ipconf] # Hack to modify the %run command so we can sync the user's namespace with the