Merging with upstream

IPython/frontend/tests/test_linefrontend.py

@@ -0,0 +1,37 b''
+# encoding: utf-8
+"""
+Test the LineFrontEnd 
+"""
+
+__docformat__ = "restructuredtext en"
+
+#-------------------------------------------------------------------------------
+#  Copyright (C) 2008  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is
+#  in the file COPYING, distributed as part of this software.
+#-------------------------------------------------------------------------------
+
+from IPython.frontend.linefrontendbase import LineFrontEndBase
+from copy import deepcopy
+import nose.tools as nt
+
+class ConcreteLineFrontEnd(LineFrontEndBase):
+    """ A concrete class to test the LineFrontEndBase.
+    """
+    def capture_output(self):
+        pass
+
+    def release_output(self):
+        pass
+
+
+def test_is_complete():
+    """ Tests line completion heuristic.
+    """
+    frontend = ConcreteLineFrontEnd()
+    yield nt.assert_true, not frontend.is_complete('for x in \\')
+    yield nt.assert_true, not frontend.is_complete('for x in (1, ):')
+    yield nt.assert_true, frontend.is_complete('for x in (1, ):\n  pass')
+
+

IPython/Extensions/ipy_greedycompleter.py

@@ -1,75 +1,75 b''
 """ Greedy completer extension for IPython
 
 Normal tab completer refuses to evaluate nonsafe stuff. This will evaluate
 everything, so you need to consider the consequences of pressing tab 
 yourself!
 
 Note that this extension simplifies readline interaction by setting
 only whitespace as completer delimiter. If this works well, we will
 do the same in default completer.
 
 """
 from IPython import generics,ipapi
 from IPython.genutils import dir2
 
 def attr_matches(self, text):
     """Compute matches when text contains a dot.
 
     MONKEYPATCHED VERSION (ipy_greedycompleter.py)
     
     Assuming the text is of the form NAME.NAME....[NAME], and is
     evaluatable in self.namespace or self.global_namespace, it will be
     evaluated and its attributes (as revealed by dir()) are used as
     possible completions.  (For class instances, class members are are
     also considered.)
 
     WARNING: this can still invoke arbitrary C code, if an object
     with a __getattr__ hook is evaluated.
 
     """
     import re
 
     force_complete = 1
     # Another option, seems to work great. Catches things like ''.<tab>
     m = re.match(r"(\S+(\.\w+)*)\.(\w*)$", text)
 
     if m:
         expr, attr = m.group(1, 3)
     else:
         # force match - eval anything that ends with colon
         if not force_complete:
             return []
                 
         m2 = re.match(r"(.+)\.(\w*)$", self.lbuf)
         if not m2:
             return []
         expr, attr = m2.group(1,2)
         
             
     try:        
         obj = eval(expr, self.namespace)
     except:
         try:
             obj = eval(expr, self.global_namespace)
         except:
             return []
 
     words = dir2(obj)
     
     try:
         words = generics.complete_object(obj, words)
     except ipapi.TryNext:
         pass
     # Build match list to return
     n = len(attr)
     res = ["%s.%s" % (expr, w) for w in words if w[:n] == attr ]
     return res
 
 def main():
-    import readline
+    import IPython.rlineimpl as readline
     readline.set_completer_delims(" \n\t")
     # monkeypatch - the code will be folded to normal completer later on
     import IPython.completer
     IPython.completer.Completer.attr_matches = attr_matches
 
 main()
  No newline at end of file

IPython/UserConfig/ipy_user_conf.py

@@ -1,116 +1,114 b''
 """ User configuration file for IPython
 
 This is a more flexible and safe way to configure ipython than *rc files
 (ipythonrc, ipythonrc-pysh etc.)
 
 This file is always imported on ipython startup. You can import the
 ipython extensions you need here (see IPython/Extensions directory).
 
 Feel free to edit this file to customize your ipython experience.
 
 Note that as such this file does nothing, for backwards compatibility.
 Consult e.g. file 'ipy_profile_sh.py' for an example of the things 
 you can do here.
 
 See http://ipython.scipy.org/moin/IpythonExtensionApi for detailed
 description on what you could do here.
 """
 
 # Most of your config files and extensions will probably start with this import
 
 import IPython.ipapi
 ip = IPython.ipapi.get()
 
 # You probably want to uncomment this if you did %upgrade -nolegacy
 # import ipy_defaults    
 
 import os   
 
 def main():   
 
     # uncomment if you want to get ipython -p sh behaviour
     # without having to use command line switches  
     # import ipy_profile_sh
 
     # Configure your favourite editor?
     # Good idea e.g. for %edit os.path.isfile
 
     #import ipy_editors
     
     # Choose one of these:
     
     #ipy_editors.scite()
     #ipy_editors.scite('c:/opt/scite/scite.exe')
     #ipy_editors.komodo()
     #ipy_editors.idle()
     # ... or many others, try 'ipy_editors??' after import to see them
     
     # Or roll your own:
     #ipy_editors.install_editor("c:/opt/jed +$line $file")
     
     
     o = ip.options
     # An example on how to set options
     #o.autocall = 1
     o.system_verbose = 0
     
     #import_all("os sys")
     #execf('~/_ipython/ns.py')
 
 
     # -- prompt
     # A different, more compact set of prompts from the default ones, that
     # always show your current location in the filesystem:
 
     #o.prompt_in1 = r'\C_LightBlue[\C_LightCyan\Y2\C_LightBlue]\C_Normal\n\C_Green|\#>'
     #o.prompt_in2 = r'.\D: '
     #o.prompt_out = r'[\#] '
     
     # Try one of these color settings if you can't read the text easily
     # autoexec is a list of IPython commands to execute on startup
     #o.autoexec.append('%colors LightBG')
     #o.autoexec.append('%colors NoColor')
     #o.autoexec.append('%colors Linux')
     
     # for sane integer division that converts to float (1/2 == 0.5)
     #o.autoexec.append('from __future__ import division')
     
     # For %tasks and %kill
     #import jobctrl 
     
     # For autoreloading of modules (%autoreload, %aimport)    
     #import ipy_autoreload
     
     # For winpdb support (%wdb)
     #import ipy_winpdb
     
     # For bzr completer, requires bzrlib (the python installation of bzr)
     #ip.load('ipy_bzr')
     
     # Tab completer that is not quite so picky (i.e. 
     # "foo".<TAB> and str(2).<TAB> will work). Complete 
     # at your own risk!
     #import ipy_greedycompleter
     
     # If you are on Linux, you may be annoyed by
     # "Display all N possibilities? (y or n)" on tab completion,
     # as well as the paging through "more". Uncomment the following
     # lines to disable that behaviour
     #import readline
     #readline.parse_and_bind('set completion-query-items 1000')
     #readline.parse_and_bind('set page-completions no')
-    
-    
-    
-    
+
+
 # some config helper functions you can use 
 def import_all(modules):
     """ Usage: import_all("os sys") """ 
     for m in modules.split():
         ip.ex("from %s import *" % m)
-        
+
 def execf(fname):
     """ Execute a file in user namespace """
     ip.ex('execfile("%s")' % os.path.expanduser(fname))
 
 main()

IPython/frontend/asyncfrontendbase.py

@@ -1,76 +1,77 b''
 """
 Base front end class for all async frontends.
 """
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
-from IPython.external import guid
 
+from IPython.external import guid
 
 from zope.interface import Interface, Attribute, implements, classProvides
 from twisted.python.failure import Failure
-from IPython.frontend.frontendbase import FrontEndBase, IFrontEnd, IFrontEndFactory
+from IPython.frontend.frontendbase import (
+    FrontEndBase, IFrontEnd, IFrontEndFactory)
 from IPython.kernel.core.history import FrontEndHistory
 from IPython.kernel.engineservice import IEngineCore
 
 
 class AsyncFrontEndBase(FrontEndBase):
     """
     Overrides FrontEndBase to wrap execute in a deferred result.
     All callbacks are made as callbacks on the deferred result.
     """
     
     implements(IFrontEnd)
     classProvides(IFrontEndFactory)
     
     def __init__(self, engine=None, history=None):
         assert(engine==None or IEngineCore.providedBy(engine))
         self.engine = IEngineCore(engine)
         if history is None:
                 self.history = FrontEndHistory(input_cache=[''])
         else:
             self.history = history
     
     
     def execute(self, block, blockID=None):
         """Execute the block and return the deferred result.
         
         Parameters:
             block : {str, AST}
             blockID : any
                 Caller may provide an ID to identify this block. 
                 result['blockID'] := blockID
         
         Result:
             Deferred result of self.interpreter.execute
         """
         
         if(not self.is_complete(block)):
             return Failure(Exception("Block is not compilable"))
         
         if(blockID == None):
             blockID = guid.generate() 
         
         d = self.engine.execute(block)
         d.addCallback(self._add_history, block=block)
         d.addCallbacks(self._add_block_id_for_result,
                 errback=self._add_block_id_for_failure,
                 callbackArgs=(blockID,),
                 errbackArgs=(blockID,))
         d.addBoth(self.update_cell_prompt, blockID=blockID)
         d.addCallbacks(self.render_result, 
             errback=self.render_error)
         
         return d
     
 

IPython/frontend/cocoa/tests/test_cocoa_frontend.py

@@ -1,94 +1,100 b''
 # encoding: utf-8
 """This file contains unittests for the 
 IPython.frontend.cocoa.cocoa_frontend module.
 """
 __docformat__ = "restructuredtext en"
                                                                             
 #---------------------------------------------------------------------------
 #       Copyright (C) 2005  The IPython Development Team    
 #                                                                           
 #  Distributed under the terms of the BSD License.  The full license is in  
 #  the file COPYING, distributed as part of this software.                  
 #---------------------------------------------------------------------------
                                                                             
 #---------------------------------------------------------------------------
 # Imports                                                                   
 #---------------------------------------------------------------------------
 
+# Tell nose to skip this module
+__test__ = {}
+
+from twisted.trial import unittest
+from twisted.internet.defer import succeed
+
+from IPython.kernel.core.interpreter import Interpreter
+import IPython.kernel.engineservice as es
+
 try:
-    from IPython.kernel.core.interpreter import Interpreter
-    import IPython.kernel.engineservice as es
-    from IPython.testing.util import DeferredTestCase
-    from twisted.internet.defer import succeed
-    from IPython.frontend.cocoa.cocoa_frontend import IPythonCocoaController                                                    
+    from IPython.frontend.cocoa.cocoa_frontend import IPythonCocoaController
     from Foundation import NSMakeRect
-    from AppKit import NSTextView, NSScrollView                                                
+    from AppKit import NSTextView, NSScrollView
 except ImportError:
-    import nose
-    raise nose.SkipTest("This test requires zope.interface, Twisted, Foolscap and PyObjC")
+    # This tells twisted.trial to skip this module if PyObjC is not found
+    skip = True
 
-class TestIPythonCocoaControler(DeferredTestCase):
+#---------------------------------------------------------------------------
+# Tests
+#---------------------------------------------------------------------------
+class TestIPythonCocoaControler(unittest.TestCase):
     """Tests for IPythonCocoaController"""
-   
+
     def setUp(self):
         self.controller = IPythonCocoaController.alloc().init()
         self.engine = es.EngineService()
         self.engine.startService()
-   
+
     def tearDown(self):
         self.controller = None
         self.engine.stopService()
-   
+
     def testControllerExecutesCode(self):
         code ="""5+5"""
         expected = Interpreter().execute(code)
         del expected['number']
         def removeNumberAndID(result):
             del result['number']
             del result['id']
             return result
-        self.assertDeferredEquals(
-            self.controller.execute(code).addCallback(removeNumberAndID), 
-            expected)
-   
+        d = self.controller.execute(code)
+        d.addCallback(removeNumberAndID)
+        d.addCallback(lambda r: self.assertEquals(r, expected))
+
     def testControllerMirrorsUserNSWithValuesAsStrings(self):
         code = """userns1=1;userns2=2"""
         def testControllerUserNS(result):
             self.assertEquals(self.controller.userNS['userns1'], 1)
             self.assertEquals(self.controller.userNS['userns2'], 2)
-       
         self.controller.execute(code).addCallback(testControllerUserNS)
-   
-   
+
     def testControllerInstantiatesIEngine(self):
         self.assert_(es.IEngineBase.providedBy(self.controller.engine))
-   
+
     def testControllerCompletesToken(self):
         code = """longNameVariable=10"""
         def testCompletes(result):
             self.assert_("longNameVariable" in result)
-       
+
         def testCompleteToken(result):
             self.controller.complete("longNa").addCallback(testCompletes)
-       
+
         self.controller.execute(code).addCallback(testCompletes)
-   
-   
+
+
     def testCurrentIndent(self):
         """test that current_indent_string returns current indent or None.
         Uses _indent_for_block for direct unit testing.
         """
-       
+        
         self.controller.tabUsesSpaces = True
         self.assert_(self.controller._indent_for_block("""a=3""") == None)
         self.assert_(self.controller._indent_for_block("") == None)
         block = """def test():\n    a=3"""
         self.assert_(self.controller._indent_for_block(block) == \
                     ' ' * self.controller.tabSpaces)
-       
+        
         block = """if(True):\n%sif(False):\n%spass""" % \
                     (' '*self.controller.tabSpaces,
                      2*' '*self.controller.tabSpaces)
         self.assert_(self.controller._indent_for_block(block) == \
                     2*(' '*self.controller.tabSpaces))
-   
+

IPython/frontend/linefrontendbase.py

@@ -1,333 +1,372 b''
 """
 Base front end class for all line-oriented frontends, rather than
 block-oriented.
 
 Currently this focuses on synchronous frontends.
 """
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 import re
 
-import IPython
 import sys
 import codeop
-import traceback
 
 from frontendbase import FrontEndBase
 from IPython.kernel.core.interpreter import Interpreter
 
 def common_prefix(strings):
     """ Given a list of strings, return the common prefix between all
         these strings.
     """
     ref = strings[0]
     prefix = ''
     for size in range(len(ref)):
         test_prefix = ref[:size+1]
         for string in strings[1:]:
             if not string.startswith(test_prefix):
                 return prefix
         prefix = test_prefix
 
     return prefix
 
 #-------------------------------------------------------------------------------
 # Base class for the line-oriented front ends
 #-------------------------------------------------------------------------------
 class LineFrontEndBase(FrontEndBase):
     """ Concrete implementation of the FrontEndBase class. This is meant
     to be the base class behind all the frontend that are line-oriented,
     rather than block-oriented.
     """
     
     # We need to keep the prompt number, to be able to increment
     # it when there is an exception.
     prompt_number = 1
 
     # We keep a reference to the last result: it helps testing and
     # programatic control of the frontend. 
     last_result = dict(number=0)
 
+    # The last prompt displayed. Useful for continuation prompts.
+    last_prompt = ''
+
     # The input buffer being edited
     input_buffer = ''
 
     # Set to true for debug output
     debug = False
 
     # A banner to print at startup
     banner = None
 
     #--------------------------------------------------------------------------
     # FrontEndBase interface
     #--------------------------------------------------------------------------
 
     def __init__(self, shell=None, history=None, banner=None, *args, **kwargs):
         if shell is None:
             shell = Interpreter()
         FrontEndBase.__init__(self, shell=shell, history=history)
 
         if banner is not None:
             self.banner = banner
 
     def start(self):
         """ Put the frontend in a state where it is ready for user
             interaction.
         """
         if self.banner is not None:
             self.write(self.banner, refresh=False)
 
         self.new_prompt(self.input_prompt_template.substitute(number=1))
 
 
     def complete(self, line):
         """Complete line in engine's user_ns
         
         Parameters
         ----------
         line : string
         
         Result
         ------
         The replacement for the line and the list of possible completions.
         """
         completions = self.shell.complete(line)
         complete_sep =  re.compile('[\s\{\}\[\]\(\)\=]')
         if completions:
             prefix = common_prefix(completions) 
             residual = complete_sep.split(line)[:-1]
             line = line[:-len(residual)] + prefix
         return line, completions 
  
 
     def render_result(self, result):
         """ Frontend-specific rendering of the result of a calculation
         that has been sent to an engine.
         """
         if 'stdout' in result and result['stdout']:
             self.write('\n' + result['stdout'])
         if 'display' in result and result['display']:
             self.write("%s%s\n" % ( 
                             self.output_prompt_template.substitute(
                                     number=result['number']),
                             result['display']['pprint']
                             ) )
        
 
     def render_error(self, failure):
         """ Frontend-specific rendering of error. 
         """
         self.write('\n\n'+str(failure)+'\n\n')
         return failure
 
 
     def is_complete(self, string):
         """ Check if a string forms a complete, executable set of
         commands.
 
         For the line-oriented frontend, multi-line code is not executed
         as soon as it is complete: the users has to enter two line
         returns.
         """
         if string in ('', '\n'):
             # Prefiltering, eg through ipython0, may return an empty
             # string although some operations have been accomplished. We
             # thus want to consider an empty string as a complete
             # statement.
             return True
         elif ( len(self.input_buffer.split('\n'))>2 
                         and not re.findall(r"\n[\t ]*\n[\t ]*$", string)):
             return False
         else:
             self.capture_output()
             try:
                 # Add line returns here, to make sure that the statement is
-                # complete.
-                is_complete = codeop.compile_command(string.rstrip() + '\n\n',
+                # complete (except if '\' was used).
+                # This should probably be done in a different place (like
+                # maybe 'prefilter_input' method? For now, this works.
+                clean_string = string.rstrip('\n')
+                if not clean_string.endswith('\\'): clean_string +='\n\n' 
+                is_complete = codeop.compile_command(clean_string,
                             "<string>", "exec")
                 self.release_output()
             except Exception, e:
                 # XXX: Hack: return True so that the
                 # code gets executed and the error captured.
                 is_complete = True
             return is_complete
 
 
     def write(self, string, refresh=True):
         """ Write some characters to the display.
 
             Subclass should overide this method.
 
             The refresh keyword argument is used in frontends with an
             event loop, to choose whether the write should trigget an UI
             refresh, and thus be syncrhonous, or not.
         """
         print >>sys.__stderr__, string
 
 
     def execute(self, python_string, raw_string=None):
         """ Stores the raw_string in the history, and sends the
         python string to the interpreter.
         """
         if raw_string is None:
             raw_string = python_string
         # Create a false result, in case there is an exception
         self.last_result = dict(number=self.prompt_number)
 
-        ## try:
-        ##     self.history.input_cache[-1] = raw_string.rstrip()
-        ##     result = self.shell.execute(python_string)
-        ##     self.last_result = result
-        ##     self.render_result(result)
-        ## except:
-        ##     self.show_traceback()
-        ## finally:
-        ##     self.after_execute()
-
         try:
             try:
                 self.history.input_cache[-1] = raw_string.rstrip()
                 result = self.shell.execute(python_string)
                 self.last_result = result
                 self.render_result(result)
             except:
                 self.show_traceback()
         finally:
             self.after_execute()
 
 
     #--------------------------------------------------------------------------
     # LineFrontEndBase interface
     #--------------------------------------------------------------------------
 
     def prefilter_input(self, string):
         """ Prefilter the input to turn it in valid python.
         """
         string = string.replace('\r\n', '\n')
         string = string.replace('\t', 4*' ')
         # Clean the trailing whitespace
         string = '\n'.join(l.rstrip()  for l in string.split('\n'))
         return string
 
 
     def after_execute(self):
         """ All the operations required after an execution to put the
             terminal back in a shape where it is usable.
         """
         self.prompt_number += 1
         self.new_prompt(self.input_prompt_template.substitute(
                             number=(self.last_result['number'] + 1)))
         # Start a new empty history entry
         self._add_history(None, '')
         self.history_cursor = len(self.history.input_cache) - 1
 
 
     def complete_current_input(self):
         """ Do code completion on current line. 
         """
         if self.debug:
             print >>sys.__stdout__, "complete_current_input", 
         line = self.input_buffer
         new_line, completions = self.complete(line)
         if len(completions)>1:
             self.write_completion(completions, new_line=new_line)
         elif not line == new_line:
             self.input_buffer = new_line
         if self.debug:
             print >>sys.__stdout__, 'line', line
             print >>sys.__stdout__, 'new_line', new_line
             print >>sys.__stdout__, completions 
 
 
     def get_line_width(self):
         """ Return the width of the line in characters.
         """
         return 80
 
 
     def write_completion(self, possibilities, new_line=None):
         """ Write the list of possible completions.
 
             new_line is the completed input line that should be displayed
             after the completion are writen. If None, the input_buffer
             before the completion is used.
         """
         if new_line is None:
             new_line = self.input_buffer
         
         self.write('\n')
         max_len = len(max(possibilities, key=len)) + 1
         
         # Now we check how much symbol we can put on a line...
         chars_per_line = self.get_line_width()
         symbols_per_line = max(1, chars_per_line/max_len)
 
         pos = 1
-        buf = []
+        completion_string = []
         for symbol in possibilities:
             if pos < symbols_per_line:
-                buf.append(symbol.ljust(max_len))
+                completion_string.append(symbol.ljust(max_len))
                 pos += 1
             else:
-                buf.append(symbol.rstrip() + '\n')
+                completion_string.append(symbol.rstrip() + '\n')
                 pos = 1
-        self.write(''.join(buf))
+        self.write(''.join(completion_string))
         self.new_prompt(self.input_prompt_template.substitute(
                             number=self.last_result['number'] + 1))
         self.input_buffer = new_line
 
 
     def new_prompt(self, prompt):
         """ Prints a prompt and starts a new editing buffer. 
 
             Subclasses should use this method to make sure that the
             terminal is put in a state favorable for a new line
             input.
         """
         self.input_buffer = ''
         self.write(prompt)
 
 
+    def continuation_prompt(self):
+        """Returns the current continuation prompt.
+        """
+        return ("."*(len(self.last_prompt)-2) + ': ')
+ 
+
+    def execute_command(self, command, hidden=False):
+        """ Execute a command, not only in the model, but also in the
+            view, if any.
+        """
+        return self.shell.execute(command)
+
     #--------------------------------------------------------------------------
     # Private API
     #--------------------------------------------------------------------------
  
-    def _on_enter(self):
+    def _on_enter(self, new_line_pos=0):
         """ Called when the return key is pressed in a line editing
             buffer.
+
+            Parameters
+            ----------
+            new_line_pos : integer, optional
+                Position of the new line to add, starting from the
+                end (0 adds a new line after the last line, -1 before
+                the last line...)
+
+            Returns
+            -------
+            True if execution is triggered
         """
         current_buffer = self.input_buffer
-        cleaned_buffer = self.prefilter_input(current_buffer)
+        # XXX: This string replace is ugly, but there should be no way it
+        # fails.
+        prompt_less_buffer = re.sub('^' + self.continuation_prompt(),
+                '', current_buffer).replace('\n' + self.continuation_prompt(),
+                                            '\n')
+        cleaned_buffer = self.prefilter_input(prompt_less_buffer)
         if self.is_complete(cleaned_buffer):
             self.execute(cleaned_buffer, raw_string=current_buffer)
+            return True
         else:
-            self.input_buffer += self._get_indent_string(
-                                                current_buffer[:-1])
-            if len(current_buffer.split('\n')) == 2:
-                self.input_buffer += '\t\t'
-            if current_buffer[:-1].split('\n')[-1].rstrip().endswith(':'):
-                self.input_buffer += '\t'
-
+            # Start a new line.
+            new_line_pos = -new_line_pos
+            lines = current_buffer.split('\n')[:-1]
+            prompt_less_lines = prompt_less_buffer.split('\n')
+            # Create the new line, with the continuation prompt, and the 
+            # same amount of indent than the line above it.
+            new_line = self.continuation_prompt() + \
+                  self._get_indent_string('\n'.join(
+                                    prompt_less_lines[:new_line_pos-1]))
+            if len(lines) == 1:
+                # We are starting a first continuation line. Indent it.
+                new_line += '\t'
+            elif current_buffer[:-1].split('\n')[-1].rstrip().endswith(':'):
+                # The last line ends with ":", autoindent the new line.
+                new_line += '\t'
+
+            if new_line_pos == 0:
+                lines.append(new_line)
+            else:
+                lines.insert(new_line_pos, new_line)
+            self.input_buffer = '\n'.join(lines)
+            
 
     def _get_indent_string(self, string):
         """ Return the string of whitespace that prefixes a line. Used to
         add the right amount of indendation when creating a new line.
         """
         string = string.replace('\t', ' '*4)
         string = string.split('\n')[-1]
         indent_chars = len(string) - len(string.lstrip())
         indent_string = '\t'*(indent_chars // 4) + \
                             ' '*(indent_chars % 4)
 
         return indent_string
  
     

IPython/frontend/prefilterfrontend.py

@@ -1,246 +1,285 b''
 """
 Frontend class that uses IPython0 to prefilter the inputs.
 
 Using the IPython0 mechanism gives us access to the magics.
 
 This is a transitory class, used here to do the transition between
 ipython0 and ipython1. This class is meant to be short-lived as more
 functionnality is abstracted out of ipython0 in reusable functions and
 is added on the interpreter. This class can be a used to guide this
 refactoring.
 """
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 import sys
-
-from linefrontendbase import LineFrontEndBase, common_prefix
-from frontendbase import FrontEndBase
+import pydoc
+import os
+import re
+import __builtin__
 
 from IPython.ipmaker import make_IPython
 from IPython.ipapi import IPApi
 from IPython.kernel.core.redirector_output_trap import RedirectorOutputTrap
 
 from IPython.kernel.core.sync_traceback_trap import SyncTracebackTrap
 
 from IPython.genutils import Term
-import pydoc
-import os
-import sys
+
+from linefrontendbase import LineFrontEndBase, common_prefix
 
 
 def mk_system_call(system_call_function, command):
     """ given a os.system replacement, and a leading string command,
         returns a function that will execute the command with the given
         argument string.
     """
     def my_system_call(args):
         system_call_function("%s %s" % (command, args))
+
+    my_system_call.__doc__ = "Calls %s" % command
     return my_system_call
 
 #-------------------------------------------------------------------------------
 # Frontend class using ipython0 to do the prefiltering. 
 #-------------------------------------------------------------------------------
 class PrefilterFrontEnd(LineFrontEndBase):
     """ Class that uses ipython0 to do prefilter the input, do the
     completion and the magics.
 
     The core trick is to use an ipython0 instance to prefilter the
     input, and share the namespace between the interpreter instance used
     to execute the statements and the ipython0 used for code
     completion...
     """
 
     debug = False
     
-    def __init__(self, ipython0=None, *args, **kwargs):
+    def __init__(self, ipython0=None, argv=None, *args, **kwargs):
         """ Parameters:
             -----------
 
             ipython0: an optional ipython0 instance to use for command
             prefiltering and completion.
+
+            argv : list, optional
+              Used as the instance's argv value.  If not given, [] is used.
         """
+        if argv is None:
+            argv = []
+        # This is a hack to avoid the IPython exception hook to trigger
+        # on exceptions (https://bugs.launchpad.net/bugs/337105)
+        # XXX: This is horrible: module-leve monkey patching -> side
+        # effects.
+        from IPython import iplib
+        iplib.InteractiveShell.isthreaded = True
+
         LineFrontEndBase.__init__(self, *args, **kwargs)
         self.shell.output_trap = RedirectorOutputTrap(
                             out_callback=self.write,
                             err_callback=self.write,
                                             )
         self.shell.traceback_trap = SyncTracebackTrap(
                         formatters=self.shell.traceback_trap.formatters,
                             )
 
         # Start the ipython0 instance:
         self.save_output_hooks()
         if ipython0 is None:
             # Instanciate an IPython0 interpreter to be able to use the
             # prefiltering.
+            # Suppress all key input, to avoid waiting
+            def my_rawinput(x=None):
+                return '\n'
+            old_rawinput = __builtin__.raw_input
+            __builtin__.raw_input = my_rawinput
             # XXX: argv=[] is a bit bold.
-            ipython0 = make_IPython(argv=[], 
+            ipython0 = make_IPython(argv=argv, 
                                     user_ns=self.shell.user_ns,
                                     user_global_ns=self.shell.user_global_ns)
+            __builtin__.raw_input = old_rawinput
         self.ipython0 = ipython0
         # Set the pager:
         self.ipython0.set_hook('show_in_pager', 
                     lambda s, string: self.write("\n" + string))
         self.ipython0.write = self.write
         self._ip = _ip = IPApi(self.ipython0)
         # Make sure the raw system call doesn't get called, as we don't
         # have a stdin accessible.
         self._ip.system = self.system_call
         # XXX: Muck around with magics so that they work better
         # in our environment
-        self.ipython0.magic_ls = mk_system_call(self.system_call, 
-                                                            'ls -CF')
+        if not sys.platform.startswith('win'):
+            self.ipython0.magic_ls = mk_system_call(self.system_call, 
+                                                                'ls -CF')
         # And now clean up the mess created by ipython0
         self.release_output()
 
 
         if not 'banner' in kwargs and self.banner is None:
-            self.banner = self.ipython0.BANNER + """
-This is the wx frontend, by Gael Varoquaux. This is EXPERIMENTAL code."""
+            self.banner = self.ipython0.BANNER
 
+        # FIXME: __init__ and start should be two different steps
         self.start()
 
     #--------------------------------------------------------------------------
     # FrontEndBase interface 
     #--------------------------------------------------------------------------
 
     def show_traceback(self):
         """ Use ipython0 to capture the last traceback and display it.
         """
-        self.capture_output()
+        # Don't do the capture; the except_hook has already done some
+        # modifications to the IO streams, if we store them, we'll be
+        # storing the wrong ones.
+        #self.capture_output()
         self.ipython0.showtraceback(tb_offset=-1)
         self.release_output()
 
 
     def execute(self, python_string, raw_string=None):
         if self.debug:
             print 'Executing Python code:', repr(python_string)
         self.capture_output()
         LineFrontEndBase.execute(self, python_string,
                                     raw_string=raw_string)
         self.release_output()
 
 
     def save_output_hooks(self):
         """ Store all the output hooks we can think of, to be able to
         restore them. 
         
         We need to do this early, as starting the ipython0 instance will
         screw ouput hooks.
         """
         self.__old_cout_write = Term.cout.write
         self.__old_cerr_write = Term.cerr.write
         self.__old_stdout = sys.stdout
         self.__old_stderr= sys.stderr
         self.__old_help_output = pydoc.help.output
         self.__old_display_hook = sys.displayhook
 
 
     def capture_output(self):
         """ Capture all the output mechanisms we can think of.
         """
         self.save_output_hooks()
         Term.cout.write = self.write
         Term.cerr.write = self.write
         sys.stdout = Term.cout
         sys.stderr = Term.cerr
         pydoc.help.output = self.shell.output_trap.out
 
 
     def release_output(self):
         """ Release all the different captures we have made.
         """
         Term.cout.write = self.__old_cout_write
         Term.cerr.write = self.__old_cerr_write
         sys.stdout = self.__old_stdout
         sys.stderr = self.__old_stderr
         pydoc.help.output = self.__old_help_output 
         sys.displayhook = self.__old_display_hook 
 
 
     def complete(self, line):
         # FIXME: This should be factored out in the linefrontendbase
         # method.
-        word = line.split('\n')[-1].split(' ')[-1]
+        word = self._get_completion_text(line)
         completions = self.ipython0.complete(word)
         # FIXME: The proper sort should be done in the complete method.
         key = lambda x: x.replace('_', '')
         completions.sort(key=key)
         if completions:
             prefix = common_prefix(completions) 
             line = line[:-len(word)] + prefix
         return line, completions 
  
     
     #--------------------------------------------------------------------------
     # LineFrontEndBase interface 
     #--------------------------------------------------------------------------
 
     def prefilter_input(self, input_string):
         """ Using IPython0 to prefilter the commands to turn them
         in executable statements that are valid Python strings.
         """
         input_string = LineFrontEndBase.prefilter_input(self, input_string)
         filtered_lines = []
         # The IPython0 prefilters sometime produce output. We need to
         # capture it.
         self.capture_output()
         self.last_result = dict(number=self.prompt_number)
         
         ## try:
         ##     for line in input_string.split('\n'):
         ##         filtered_lines.append(
         ##                 self.ipython0.prefilter(line, False).rstrip())
         ## except:
         ##     # XXX: probably not the right thing to do.
         ##     self.ipython0.showsyntaxerror()
         ##     self.after_execute()
         ## finally:
         ##     self.release_output()
 
 
         try:
             try:
                 for line in input_string.split('\n'):
                     filtered_lines.append(
                             self.ipython0.prefilter(line, False).rstrip())
             except:
                 # XXX: probably not the right thing to do.
                 self.ipython0.showsyntaxerror()
                 self.after_execute()
         finally:
             self.release_output()
 
 
 
         # Clean up the trailing whitespace, to avoid indentation errors
         filtered_string = '\n'.join(filtered_lines)
         return filtered_string
 
 
     #--------------------------------------------------------------------------
     # PrefilterFrontEnd interface 
     #--------------------------------------------------------------------------
  
     def system_call(self, command_string):
         """ Allows for frontend to define their own system call, to be
             able capture output and redirect input.
         """
         return os.system(command_string)
 
 
     def do_exit(self):
         """ Exit the shell, cleanup and save the history.
         """
         self.ipython0.atexit_operations()
 
+
+    def _get_completion_text(self, line):
+        """ Returns the text to be completed by breaking the line at specified
+        delimiters.
+        """
+        # Break at: spaces, '=', all parentheses (except if balanced).
+        # FIXME2: In the future, we need to make the implementation similar to
+        # that in the 'pyreadline' module (modes/basemode.py) where we break at
+        # each delimiter and try to complete the residual line, until we get a
+        # successful list of completions.
+        expression = '\s|=|,|:|\((?!.*\))|\[(?!.*\])|\{(?!.*\})' 
+        complete_sep = re.compile(expression)
+        text = complete_sep.split(line)[-1]
+        return text
+

IPython/frontend/process/killableprocess.py

@@ -1,179 +1,184 b''
 # Addapted from killableprocess.py.  
 #______________________________________________________________________________
 #
 # killableprocess - subprocesses which can be reliably killed
 #
 # Parts of this module are copied from the subprocess.py file contained
 # in the Python distribution.
 #
 # Copyright (c) 2003-2004 by Peter Astrand <[e-mail]>
 #
 # Additions and modifications written by Benjamin Smedberg
 # <[e-mail]> are Copyright (c) 2006 by the Mozilla Foundation
 # <http://www.mozilla.org/>
 #
 # By obtaining, using, and/or copying this software and/or its
 # associated documentation, you agree that you have read, understood,
 # and will comply with the following terms and conditions:
 #
 # Permission to use, copy, modify, and distribute this software and
 # its associated documentation for any purpose and without fee is
 # hereby granted, provided that the above copyright notice appears in
 # all copies, and that both that copyright notice and this permission
 # notice appear in supporting documentation, and that the name of the
 # author not be used in advertising or publicity pertaining to
 # distribution of the software without specific, written prior
 # permission.
 #
 # THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
 # INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS.
 # IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, INDIRECT OR
 # CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
 # OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
 # NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
 # WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
 r"""killableprocess - Subprocesses which can be reliably killed
 
 This module is a subclass of the builtin "subprocess" module. It allows
 processes that launch subprocesses to be reliably killed on Windows (via the Popen.kill() method.
 
 It also adds a timeout argument to Wait() for a limited period of time before
 forcefully killing the process.
 
 Note: On Windows, this module requires Windows 2000 or higher (no support for
 Windows 95, 98, or NT 4.0). It also requires ctypes, which is bundled with
 Python 2.5+ or available from http://python.net/crew/theller/ctypes/
 """
 
 import subprocess
 from subprocess import PIPE
 import sys
 import os
 import types
 
 try:
     from subprocess import CalledProcessError
 except ImportError:
     # Python 2.4 doesn't implement CalledProcessError
     class CalledProcessError(Exception):
         """This exception is raised when a process run by check_call() returns
         a non-zero exit status. The exit status will be stored in the
         returncode attribute."""
         def __init__(self, returncode, cmd):
             self.returncode = returncode
             self.cmd = cmd
         def __str__(self):
             return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
 
 mswindows = (sys.platform == "win32")
 
 skip = False
 
 if mswindows:
     import platform
     if platform.uname()[3] == '' or platform.uname()[3] > '6.0.6000':
         # Killable process does not work under vista when starting for
         # something else than cmd.
         skip = True
     else:
         import winprocess
 else:
     import signal
 
 if not mswindows:
     def DoNothing(*args):
         pass
 
 
 if skip:
  Popen = subprocess.Popen
 else:
  class Popen(subprocess.Popen):
     if not mswindows:
         # Override __init__ to set a preexec_fn
         def __init__(self, *args, **kwargs):
             if len(args) >= 7:
                 raise Exception("Arguments preexec_fn and after must be passed by keyword.")
 
             real_preexec_fn = kwargs.pop("preexec_fn", None)
             def setpgid_preexec_fn():
                 os.setpgid(0, 0)
                 if real_preexec_fn:
                     apply(real_preexec_fn)
 
             kwargs['preexec_fn'] = setpgid_preexec_fn
 
             subprocess.Popen.__init__(self, *args, **kwargs)
 
     if mswindows:
         def _execute_child(self, args, executable, preexec_fn, close_fds,
                            cwd, env, universal_newlines, startupinfo,
                            creationflags, shell,
                            p2cread, p2cwrite,
                            c2pread, c2pwrite,
                            errread, errwrite):
             if not isinstance(args, types.StringTypes):
                 args = subprocess.list2cmdline(args)
 
             if startupinfo is None:
                 startupinfo = winprocess.STARTUPINFO()
             
             if None not in (p2cread, c2pwrite, errwrite):
                 startupinfo.dwFlags |= winprocess.STARTF_USESTDHANDLES
                 
                 startupinfo.hStdInput = int(p2cread)
                 startupinfo.hStdOutput = int(c2pwrite)
                 startupinfo.hStdError = int(errwrite)
             if shell:
                 startupinfo.dwFlags |= winprocess.STARTF_USESHOWWINDOW
                 startupinfo.wShowWindow = winprocess.SW_HIDE
                 comspec = os.environ.get("COMSPEC", "cmd.exe")
                 args = comspec + " /c " + args
 
             # We create a new job for this process, so that we can kill
             # the process and any sub-processes 
             self._job = winprocess.CreateJobObject()
 
             creationflags |= winprocess.CREATE_SUSPENDED
             creationflags |= winprocess.CREATE_UNICODE_ENVIRONMENT
 
             hp, ht, pid, tid = winprocess.CreateProcess(
                 executable, args,
                 None, None, # No special security
                 1, # Must inherit handles!
                 creationflags,
                 winprocess.EnvironmentBlock(env),
                 cwd, startupinfo)
             
             self._child_created = True
             self._handle = hp
             self._thread = ht
             self.pid = pid
 
-            winprocess.AssignProcessToJobObject(self._job, hp)
+            # XXX: A try/except to fix UAC-related problems under
+            # Windows Vista, when reparenting jobs.
+            try:
+                winprocess.AssignProcessToJobObject(self._job, hp)
+            except WindowsError:
+                pass
             winprocess.ResumeThread(ht)
 
             if p2cread is not None:
                 p2cread.Close()
             if c2pwrite is not None:
                 c2pwrite.Close()
             if errwrite is not None:
                 errwrite.Close()
 
     def kill(self, group=True):
         """Kill the process. If group=True, all sub-processes will also be killed."""
         if mswindows:
             if group:
                 winprocess.TerminateJobObject(self._job, 127)
             else:
                 winprocess.TerminateProcess(self._handle, 127)
             self.returncode = 127    
         else:
             if group:
                 os.killpg(self.pid, signal.SIGKILL)
             else:
                 os.kill(self.pid, signal.SIGKILL)
             self.returncode = -9
 
 

IPython/frontend/tests/test_asyncfrontendbase.py

@@ -1,155 +1,109 b''
 # encoding: utf-8
 
-"""This file contains unittests for the frontendbase module."""
+"""This file contains unittests for the asyncfrontendbase module."""
 
 __docformat__ = "restructuredtext en"
 
 #---------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team                         
 #                                                                           
 #  Distributed under the terms of the BSD License.  The full license is in  
 #  the file COPYING, distributed as part of this software.                  
 #---------------------------------------------------------------------------
                                                                             
 #---------------------------------------------------------------------------
 # Imports                                                                   
 #---------------------------------------------------------------------------
 
-import unittest
+# Tell nose to skip this module
+__test__ = {}
 
-try:
-    from IPython.frontend.asyncfrontendbase import AsyncFrontEndBase
-    from IPython.frontend import frontendbase 
-    from IPython.kernel.engineservice import EngineService
-except ImportError:
-    import nose
-    raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")
+from twisted.trial import unittest
+from IPython.frontend.asyncfrontendbase import AsyncFrontEndBase
+from IPython.frontend import frontendbase 
+from IPython.kernel.engineservice import EngineService
+from IPython.testing.parametric import Parametric, parametric
 
-from IPython.testing.decorators import skip
 
 class FrontEndCallbackChecker(AsyncFrontEndBase):
     """FrontEndBase subclass for checking callbacks"""
     def __init__(self, engine=None, history=None):
         super(FrontEndCallbackChecker, self).__init__(engine=engine, 
                                                     history=history)
         self.updateCalled = False
         self.renderResultCalled = False
         self.renderErrorCalled = False
    
     def update_cell_prompt(self, result, blockID=None):
         self.updateCalled = True
         return result
    
     def render_result(self, result):
         self.renderResultCalled = True
         return result
     
-    
     def render_error(self, failure):
         self.renderErrorCalled = True
         return failure
-    
 
 
-    
 class TestAsyncFrontendBase(unittest.TestCase):
     def setUp(self):
         """Setup the EngineService and FrontEndBase"""
         
         self.fb = FrontEndCallbackChecker(engine=EngineService())
     
     def test_implements_IFrontEnd(self):
-        assert(frontendbase.IFrontEnd.implementedBy(
+        self.assert_(frontendbase.IFrontEnd.implementedBy(
                                     AsyncFrontEndBase))
     
     def test_is_complete_returns_False_for_incomplete_block(self):
-        """"""
-        
         block = """def test(a):"""
-        
-        assert(self.fb.is_complete(block) == False)
+        self.assert_(self.fb.is_complete(block) == False)
     
     def test_is_complete_returns_True_for_complete_block(self):
-        """"""
-        
         block = """def test(a): pass"""
-            
-        assert(self.fb.is_complete(block))
-        
+        self.assert_(self.fb.is_complete(block))
         block = """a=3"""
-        
-        assert(self.fb.is_complete(block))
+        self.assert_(self.fb.is_complete(block))
     
     def test_blockID_added_to_result(self):
         block = """3+3"""
-        
         d = self.fb.execute(block, blockID='TEST_ID')
-        
-        d.addCallback(self.checkBlockID, expected='TEST_ID')
+        d.addCallback(lambda r: self.assert_(r['blockID']=='TEST_ID'))
+        return d
     
     def test_blockID_added_to_failure(self):
         block = "raise Exception()"
-        
         d = self.fb.execute(block,blockID='TEST_ID')
-        d.addErrback(self.checkFailureID, expected='TEST_ID')
-    
-    def checkBlockID(self, result, expected=""):
-        assert(result['blockID'] == expected)
-    
-    
-    def checkFailureID(self, failure, expected=""):
-        assert(failure.blockID == expected)
-    
+        d.addErrback(lambda f: self.assert_(f.blockID=='TEST_ID'))
+        return d
     
     def test_callbacks_added_to_execute(self):
-        """test that
-        update_cell_prompt
-        render_result
-        
-        are added to execute request
-        """
-        
         d = self.fb.execute("10+10")
-        d.addCallback(self.checkCallbacks)
+        d.addCallback(lambda r: self.assert_(self.fb.updateCalled and self.fb.renderResultCalled))
+        return d
     
-    def checkCallbacks(self, result):
-        assert(self.fb.updateCalled)
-        assert(self.fb.renderResultCalled)
-    
-    @skip("This test fails and lead to an unhandled error in a Deferred.")
     def test_error_callback_added_to_execute(self):
-        """test that render_error called on execution error"""
+        """Test that render_error called on execution error."""
         
         d = self.fb.execute("raise Exception()")
-        d.addCallback(self.checkRenderError)
-    
-    def checkRenderError(self, result):
-        assert(self.fb.renderErrorCalled)
+        d.addErrback(lambda f: self.assert_(self.fb.renderErrorCalled))
+        return d
     
     def test_history_returns_expected_block(self):
-        """Make sure history browsing doesn't fail"""
+        """Make sure history browsing doesn't fail."""
         
         blocks = ["a=1","a=2","a=3"]
-        for b in blocks:
-            d = self.fb.execute(b)
-        
-        # d is now the deferred for the last executed block
-        d.addCallback(self.historyTests, blocks)
-        
-    
-    def historyTests(self, result, blocks):
-        """historyTests"""
-        
-        assert(len(blocks) >= 3)
-        assert(self.fb.get_history_previous("") == blocks[-2])
-        assert(self.fb.get_history_previous("") == blocks[-3])
-        assert(self.fb.get_history_next() == blocks[-2])
-    
-    
-    def test_history_returns_none_at_startup(self):
-        """test_history_returns_none_at_startup"""
-        
-        assert(self.fb.get_history_previous("")==None)
-        assert(self.fb.get_history_next()==None)
-    
-    
+        d = self.fb.execute(blocks[0])
+        d.addCallback(lambda _: self.fb.execute(blocks[1]))
+        d.addCallback(lambda _: self.fb.execute(blocks[2]))
+        d.addCallback(lambda _: self.assert_(self.fb.get_history_previous("")==blocks[-2]))
+        d.addCallback(lambda _: self.assert_(self.fb.get_history_previous("")==blocks[-3]))
+        d.addCallback(lambda _: self.assert_(self.fb.get_history_next()==blocks[-2]))
+        return d
+    
+    def test_history_returns_none_at_startup(self):        
+        self.assert_(self.fb.get_history_previous("")==None)
+        self.assert_(self.fb.get_history_next()==None)
+

IPython/frontend/tests/test_prefilterfrontend.py

@@ -1,180 +1,252 b''
 # encoding: utf-8
 """
 Test process execution and IO redirection.
 """
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is
 #  in the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
+from copy import copy, deepcopy
 from cStringIO import StringIO
 import string
 
-from IPython.ipapi import get as get_ipython0
+from nose.tools import assert_equal
+
 from IPython.frontend.prefilterfrontend import PrefilterFrontEnd
-from copy import deepcopy
+from IPython.ipapi import get as get_ipython0
+from IPython.testing.plugin.ipdoctest import default_argv
+
+
+def safe_deepcopy(d):
+    """ Deep copy every key of the given dict, when possible. Elsewhere
+        do a copy.
+    """
+    copied_d = dict()
+    for key, value in d.iteritems():
+        try:
+            copied_d[key] = deepcopy(value)
+        except:
+            try:
+                copied_d[key] = copy(value)
+            except:
+                copied_d[key] = value
+    return copied_d
+
 
 class TestPrefilterFrontEnd(PrefilterFrontEnd):
     
     input_prompt_template = string.Template('')
     output_prompt_template = string.Template('')
     banner = ''
 
     def __init__(self):
-        ipython0 = get_ipython0().IP
         self.out = StringIO()
-        PrefilterFrontEnd.__init__(self, ipython0=ipython0)
-        # Clean up the namespace for isolation between tests
-        user_ns = self.ipython0.user_ns
-        # We need to keep references to things so that they don't
-        # get garbage collected (this stinks).
-        self.shadow_ns = dict()
-        for i in self.ipython0.magic_who_ls():
-            self.shadow_ns[i] = user_ns.pop(i)
+        PrefilterFrontEnd.__init__(self,argv=default_argv())
         # Some more code for isolation (yeah, crazy)
         self._on_enter()
         self.out.flush()
         self.out.reset()
         self.out.truncate()
 
     def write(self, string, *args, **kwargs):
        self.out.write(string) 
 
     def _on_enter(self):
         self.input_buffer += '\n'
         PrefilterFrontEnd._on_enter(self)
 
 
 def isolate_ipython0(func):
     """ Decorator to isolate execution that involves an iptyhon0.
+
+        Notes
+        -----
+
+        Apply only to functions with no arguments. Nose skips functions
+        with arguments.
     """
-    def my_func(*args, **kwargs):
-        ipython0 = get_ipython0().IP
-        user_ns = deepcopy(ipython0.user_ns)
-        global_ns = deepcopy(ipython0.global_ns)
+    def my_func():
+        iplib = get_ipython0()
+        if iplib is None:
+            return func()
+        ipython0 = iplib.IP
+        global_ns = safe_deepcopy(ipython0.user_global_ns)
+        user_ns = safe_deepcopy(ipython0.user_ns)
         try:
-            func(*args, **kwargs)
+            out = func()
         finally:
             ipython0.user_ns = user_ns
-            ipython0.global_ns = global_ns
+            ipython0.user_global_ns = global_ns
+        # Undo the hack at creation of PrefilterFrontEnd
+        from IPython import iplib
+        iplib.InteractiveShell.isthreaded = False
+        return out
 
+    my_func.__name__ = func.__name__
     return my_func
 
 
 @isolate_ipython0
 def test_execution():
     """ Test execution of a command.
     """
     f = TestPrefilterFrontEnd()
     f.input_buffer = 'print 1'
     f._on_enter()
     out_value = f.out.getvalue()
-    assert out_value  == '1\n'
+    assert_equal(out_value, '1\n')
 
 
 @isolate_ipython0
 def test_multiline():
     """ Test execution of a multiline command.
     """
     f = TestPrefilterFrontEnd()
     f.input_buffer = 'if True:'
     f._on_enter()
     f.input_buffer += 'print 1'
     f._on_enter()
     out_value = f.out.getvalue()
-    assert out_value == ''
+    yield assert_equal, out_value, ''
     f._on_enter()
     out_value = f.out.getvalue()
-    assert out_value == '1\n'
+    yield assert_equal, out_value, '1\n'
     f = TestPrefilterFrontEnd()
     f.input_buffer='(1 +'
     f._on_enter()
     f.input_buffer += '0)'
     f._on_enter()
     out_value = f.out.getvalue()
-    assert out_value == ''
+    yield assert_equal, out_value, ''
     f._on_enter()
     out_value = f.out.getvalue()
-    assert out_value == '1\n'
+    yield assert_equal, out_value, '1\n'
 
 
 @isolate_ipython0
 def test_capture():
     """ Test the capture of output in different channels.
     """
     # Test on the OS-level stdout, stderr.
     f = TestPrefilterFrontEnd()
     f.input_buffer = \
             'import os; out=os.fdopen(1, "w"); out.write("1") ; out.flush()'
     f._on_enter()
     out_value = f.out.getvalue()
-    assert out_value == '1'
+    yield assert_equal, out_value, '1'
     f = TestPrefilterFrontEnd()
     f.input_buffer = \
             'import os; out=os.fdopen(2, "w"); out.write("1") ; out.flush()'
     f._on_enter()
     out_value = f.out.getvalue()
-    assert out_value == '1'
+    yield assert_equal, out_value, '1'
 
      
 @isolate_ipython0
 def test_magic():
     """ Test the magic expansion and history.
     
         This test is fairly fragile and will break when magics change.
     """
     f = TestPrefilterFrontEnd()
+    # Before checking the interactive namespace, make sure it's clear (it can
+    # otherwise pick up things stored in the user's local db)
+    f.input_buffer += '%reset -f'
+    f._on_enter()
+    f.complete_current_input()
+    # Now, run the %who magic and check output
     f.input_buffer += '%who'
     f._on_enter()
     out_value = f.out.getvalue()
-    assert out_value == 'Interactive namespace is empty.\n'
+    assert_equal(out_value, 'Interactive namespace is empty.\n')
 
 
 @isolate_ipython0
 def test_help():
     """ Test object inspection.
     """
     f = TestPrefilterFrontEnd()
     f.input_buffer += "def f():"
     f._on_enter()
     f.input_buffer += "'foobar'"
     f._on_enter()
     f.input_buffer += "pass"
     f._on_enter()
     f._on_enter()
     f.input_buffer += "f?"
     f._on_enter()
     assert 'traceback' not in f.last_result
     ## XXX: ipython doctest magic breaks this. I have no clue why
     #out_value = f.out.getvalue()
     #assert out_value.split()[-1] == 'foobar' 
 
 
 @isolate_ipython0
-def test_completion():
-    """ Test command-line completion.
+def test_completion_simple():
+    """ Test command-line completion on trivial examples.
     """
     f = TestPrefilterFrontEnd()
     f.input_buffer = 'zzza = 1'
     f._on_enter()
     f.input_buffer = 'zzzb = 2'
     f._on_enter()
     f.input_buffer = 'zz'
     f.complete_current_input()
     out_value = f.out.getvalue()
-    assert out_value == '\nzzza zzzb '
-    assert f.input_buffer == 'zzz'
+    yield assert_equal, out_value, '\nzzza zzzb '
+    yield assert_equal, f.input_buffer, 'zzz'
+
+
+@isolate_ipython0
+def test_completion_parenthesis():
+    """ Test command-line completion when a parenthesis is open.
+    """
+    f = TestPrefilterFrontEnd()
+    f.input_buffer = 'zzza = 1'
+    f._on_enter()
+    f.input_buffer = 'zzzb = 2'
+    f._on_enter()
+    f.input_buffer = 'map(zz'
+    f.complete_current_input()
+    out_value = f.out.getvalue()
+    yield assert_equal, out_value, '\nzzza zzzb '
+    yield assert_equal, f.input_buffer, 'map(zzz'
+
+
+@isolate_ipython0
+def test_completion_indexing():
+    """ Test command-line completion when indexing on objects.
+    """
+    f = TestPrefilterFrontEnd()
+    f.input_buffer = 'a = [0]'
+    f._on_enter()
+    f.input_buffer = 'a[0].'
+    f.complete_current_input()
+    assert_equal(f.input_buffer, 'a[0].__')
+
+
+@isolate_ipython0
+def test_completion_equal():
+    """ Test command-line completion when the delimiter is "=", not " ".
+    """
+    f = TestPrefilterFrontEnd()
+    f.input_buffer = 'a=1.'
+    f.complete_current_input()
+    assert_equal(f.input_buffer, 'a=1.__')
+
 
 
 if __name__ == '__main__':
     test_magic()
     test_help()
     test_execution()
     test_multiline()
     test_capture()
-    test_completion()
+    test_completion_simple()
+    test_completion_complex()

IPython/frontend/tests/test_process.py

@@ -1,67 +1,67 b''
 # encoding: utf-8
 """
 Test process execution and IO redirection.
 """
 
 __docformat__ = "restructuredtext en"
 
-#-------------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is
 #  in the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
 from cStringIO import StringIO
 from time import sleep
 import sys
 
-from IPython.frontend._process import PipedProcess
+from IPython.frontend.process import PipedProcess
 from IPython.testing import decorators as testdec
 
 
 def test_capture_out():
     """ A simple test to see if we can execute a process and get the output.
     """
     s = StringIO()
     p = PipedProcess('echo 1', out_callback=s.write, )
     p.start()
     p.join()
     result = s.getvalue().rstrip()
     assert result == '1'
 
 
 def test_io():
     """ Checks that we can send characters on stdin to the process.
     """
     s = StringIO()
     p = PipedProcess(sys.executable + ' -c "a = raw_input(); print a"', 
                             out_callback=s.write, )
     p.start()
     test_string = '12345\n'
     while not hasattr(p, 'process'):
         sleep(0.1)
     p.process.stdin.write(test_string)
     p.join()
     result = s.getvalue()
     assert result == test_string
 
 
 def test_kill():
     """ Check that we can kill a process, and its subprocess.
     """
     s = StringIO()
     p = PipedProcess(sys.executable + ' -c "a = raw_input();"', 
                             out_callback=s.write, )
     p.start()
     while not hasattr(p, 'process'):
         sleep(0.1)
     p.process.kill() 
     assert p.process.poll() is not None
 
 
 if __name__ == '__main__':
     test_capture_out()
     test_io()
     test_kill()
 

IPython/frontend/wx/console_widget.py

@@ -1,436 +1,625 b''
 # encoding: utf-8
 """
 A Wx widget to act as a console and input commands.
 
 This widget deals with prompts and provides an edit buffer
 restricted to after the last prompt.
 """
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is
 #  in the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 import wx
 import wx.stc  as  stc
 
 from wx.py import editwindow
 import time
 import sys
+import string
+
 LINESEP = '\n'
 if sys.platform == 'win32':
     LINESEP = '\n\r'
 
 import re
 
 # FIXME: Need to provide an API for non user-generated display on the
 # screen: this should not be editable by the user.
+#-------------------------------------------------------------------------------
+# Constants 
+#-------------------------------------------------------------------------------
+_COMPLETE_BUFFER_MARKER = 31
+_ERROR_MARKER = 30
+_INPUT_MARKER = 29
 
 _DEFAULT_SIZE = 10
 if sys.platform == 'darwin':
     _DEFAULT_SIZE = 12
 
 _DEFAULT_STYLE = {
-    'stdout'      : 'fore:#0000FF',
-    'stderr'      : 'fore:#007f00',
-    'trace'       : 'fore:#FF0000',
-
+    #background definition
     'default'     : 'size:%d' % _DEFAULT_SIZE,
     'bracegood'   : 'fore:#00AA00,back:#000000,bold',
     'bracebad'    : 'fore:#FF0000,back:#000000,bold',
 
+    # Edge column: a number of None
+    'edge_column' : -1,
+
     # properties for the various Python lexer styles
     'comment'       : 'fore:#007F00',
     'number'        : 'fore:#007F7F',
     'string'        : 'fore:#7F007F,italic',
     'char'          : 'fore:#7F007F,italic',
     'keyword'       : 'fore:#00007F,bold',
     'triple'        : 'fore:#7F0000',
     'tripledouble'  : 'fore:#7F0000',
     'class'         : 'fore:#0000FF,bold,underline',
     'def'           : 'fore:#007F7F,bold',
-    'operator'      : 'bold'
+    'operator'      : 'bold',
+
+    # Default colors
+    'trace'         : '#FAFAF1', # Nice green
+    'stdout'        : '#FDFFD3', # Nice yellow
+    'stderr'        : '#FFF1F1', # Nice red
+
+    # Default scintilla settings
+    'antialiasing'  : True,
+    'carret_color'  : 'BLACK',
+    'background_color' :'WHITE', 
+
+    #prompt definition
+    'prompt_in1'    : \
+        '\n\x01\x1b[0;34m\x02In [\x01\x1b[1;34m\x02$number\x01\x1b[0;34m\x02]: \x01\x1b[0m\x02',
+
+    'prompt_out': \
+        '\x01\x1b[0;31m\x02Out[\x01\x1b[1;31m\x02$number\x01\x1b[0;31m\x02]: \x01\x1b[0m\x02',
     }
 
 # new style numbers
 _STDOUT_STYLE = 15
 _STDERR_STYLE = 16
 _TRACE_STYLE  = 17
 
 
 # system colors
 #SYS_COLOUR_BACKGROUND = wx.SystemSettings.GetColour(wx.SYS_COLOUR_BACKGROUND)
 
+# Translation table from ANSI escape sequences to color. 
+ANSI_STYLES = {'0;30': [0, 'BLACK'],            '0;31': [1, 'RED'],
+               '0;32': [2, 'GREEN'],           '0;33': [3, 'BROWN'],
+               '0;34': [4, 'BLUE'],            '0;35': [5, 'PURPLE'],
+               '0;36': [6, 'CYAN'],            '0;37': [7, 'LIGHT GREY'],
+               '1;30': [8, 'DARK GREY'],       '1;31': [9, 'RED'],
+               '1;32': [10, 'SEA GREEN'],      '1;33': [11, 'YELLOW'],
+               '1;34': [12, 'LIGHT BLUE'],     '1;35': 
+                                                 [13, 'MEDIUM VIOLET RED'],
+               '1;36': [14, 'LIGHT STEEL BLUE'], '1;37': [15, 'YELLOW']}
+
+# XXX: Maybe one day we should factor this code with ColorANSI. Right now
+# ColorANSI is hard to reuse and makes our code more complex.
+
+#we define platform specific fonts
+if wx.Platform == '__WXMSW__':
+    FACES = { 'times': 'Times New Roman',
+                'mono' : 'Courier New',
+                'helv' : 'Arial',
+                'other': 'Comic Sans MS',
+                'size' : 10,
+                'size2': 8,
+                }
+elif wx.Platform == '__WXMAC__':
+    FACES = { 'times': 'Times New Roman',
+                'mono' : 'Monaco',
+                'helv' : 'Arial',
+                'other': 'Comic Sans MS',
+                'size' : 10,
+                'size2': 8,
+                }
+else:
+    FACES = { 'times': 'Times',
+                'mono' : 'Courier',
+                'helv' : 'Helvetica',
+                'other': 'new century schoolbook',
+                'size' : 10,
+                'size2': 8,
+                }
+ 
+
 #-------------------------------------------------------------------------------
 # The console widget class
 #-------------------------------------------------------------------------------
 class ConsoleWidget(editwindow.EditWindow):
     """ Specialized styled text control view for console-like workflow.
 
         This widget is mainly interested in dealing with the prompt and
         keeping the cursor inside the editing line.
     """
 
     # This is where the title captured from the ANSI escape sequences are
     # stored.
     title = 'Console'
 
+    # Last prompt printed
+    last_prompt = ''
+
     # The buffer being edited.
     def _set_input_buffer(self, string):
         self.SetSelection(self.current_prompt_pos, self.GetLength())
         self.ReplaceSelection(string)
         self.GotoPos(self.GetLength())
 
     def _get_input_buffer(self):
         """ Returns the text in current edit buffer.
         """
         input_buffer = self.GetTextRange(self.current_prompt_pos,
                                                 self.GetLength())
         input_buffer = input_buffer.replace(LINESEP, '\n')
         return input_buffer
 
     input_buffer = property(_get_input_buffer, _set_input_buffer)
 
     style = _DEFAULT_STYLE.copy()
 
     # Translation table from ANSI escape sequences to color. Override
     # this to specify your colors.
-    ANSI_STYLES = {'0;30': [0, 'BLACK'],            '0;31': [1, 'RED'],
-                   '0;32': [2, 'GREEN'],            '0;33': [3, 'BROWN'],
-                   '0;34': [4, 'BLUE'],             '0;35': [5, 'PURPLE'],
-                   '0;36': [6, 'CYAN'],             '0;37': [7, 'LIGHT GREY'],
-                   '1;30': [8, 'DARK GREY'],        '1;31': [9, 'RED'],
-                   '1;32': [10, 'SEA GREEN'],       '1;33': [11, 'YELLOW'],
-                   '1;34': [12, 'LIGHT BLUE'],      '1;35':
-                                                     [13, 'MEDIUM VIOLET RED'],
-                   '1;36': [14, 'LIGHT STEEL BLUE'], '1;37': [15, 'YELLOW']}
-
-    # The color of the carret (call _apply_style() after setting)
-    carret_color = 'BLACK'
+    ANSI_STYLES = ANSI_STYLES.copy()
     
+    # Font faces
+    faces = FACES.copy()
+                 
     # Store the last time a refresh was done
     _last_refresh_time = 0
 
     #--------------------------------------------------------------------------
     # Public API
     #--------------------------------------------------------------------------
     
     def __init__(self, parent, id=wx.ID_ANY, pos=wx.DefaultPosition, 
                         size=wx.DefaultSize, style=wx.WANTS_CHARS, ):
         editwindow.EditWindow.__init__(self, parent, id, pos, size, style)
-        self._configure_scintilla()
+        self.configure_scintilla()
+        # Track if 'enter' key as ever been processed
+        # This variable will only be reallowed until key goes up
+        self.enter_catched = False 
+        self.current_prompt_pos = 0
 
         self.Bind(wx.EVT_KEY_DOWN, self._on_key_down)
         self.Bind(wx.EVT_KEY_UP, self._on_key_up)
     
 
     def write(self, text, refresh=True):
         """ Write given text to buffer, while translating the ansi escape
             sequences.
         """
         # XXX: do not put print statements to sys.stdout/sys.stderr in 
         # this method, the print statements will call this method, as 
         # you will end up with an infinit loop
         title = self.title_pat.split(text)
         if len(title)>1:
             self.title = title[-2]
 
         text = self.title_pat.sub('', text)
         segments = self.color_pat.split(text)
         segment = segments.pop(0)
         self.GotoPos(self.GetLength())
         self.StartStyling(self.GetLength(), 0xFF)
         try:
             self.AppendText(segment)
         except UnicodeDecodeError:
             # XXX: Do I really want to skip the exception?
             pass
         
         if segments:
             for ansi_tag, text in zip(segments[::2], segments[1::2]):
                 self.StartStyling(self.GetLength(), 0xFF)
                 try:
                     self.AppendText(text)
                 except UnicodeDecodeError:
                     # XXX: Do I really want to skip the exception?
                     pass
 
                 if ansi_tag not in self.ANSI_STYLES:
                     style = 0
                 else:
                     style = self.ANSI_STYLES[ansi_tag][0]
 
                 self.SetStyling(len(text), style) 
                 
         self.GotoPos(self.GetLength())
         if refresh:
             current_time = time.time()
             if current_time - self._last_refresh_time > 0.03:
                 if sys.platform == 'win32':
                     wx.SafeYield()
                 else:
                     wx.Yield()
                 #    self.ProcessEvent(wx.PaintEvent())
                 self._last_refresh_time = current_time 
 
    
     def new_prompt(self, prompt):
         """ Prints a prompt at start of line, and move the start of the
             current block there.
 
             The prompt can be given with ascii escape sequences.
         """
         self.write(prompt, refresh=False)
         # now we update our cursor giving end of prompt
         self.current_prompt_pos = self.GetLength()
         self.current_prompt_line = self.GetCurrentLine()
         self.EnsureCaretVisible()
+        self.last_prompt = prompt
 
 
+    def continuation_prompt(self):
+        """ Returns the current continuation prompt.
+            We need to implement this method here to deal with the
+            ascii escape sequences cleaning up.
+        """
+        # ASCII-less prompt
+        ascii_less = ''.join(self.color_pat.split(self.last_prompt)[2::2])
+        return "."*(len(ascii_less)-2) + ': '
+ 
+
     def scroll_to_bottom(self):
         maxrange = self.GetScrollRange(wx.VERTICAL)
         self.ScrollLines(maxrange)
 
 
     def pop_completion(self, possibilities, offset=0):
         """ Pops up an autocompletion menu. Offset is the offset
             in characters of the position at which the menu should
             appear, relativ to the cursor.
         """
         self.AutoCompSetIgnoreCase(False)
         self.AutoCompSetAutoHide(False)
         self.AutoCompSetMaxHeight(len(possibilities))
         self.AutoCompShow(offset, " ".join(possibilities))
 
 
     def get_line_width(self):
         """ Return the width of the line in characters.
         """
         return self.GetSize()[0]/self.GetCharWidth()
 
-    #--------------------------------------------------------------------------
-    # EditWindow API
-    #--------------------------------------------------------------------------
 
-    def OnUpdateUI(self, event):
-        """ Override the OnUpdateUI of the EditWindow class, to prevent 
-            syntax highlighting both for faster redraw, and for more
-            consistent look and feel.
+    def configure_scintilla(self):
+        """ Set up all the styling option of the embedded scintilla
+            widget.
         """
+        p = self.style.copy()
+        
+        # Marker for complete buffer.
+        self.MarkerDefine(_COMPLETE_BUFFER_MARKER, stc.STC_MARK_BACKGROUND,
+                                background=p['trace'])
 
-    #--------------------------------------------------------------------------
-    # Private API
-    #--------------------------------------------------------------------------
-    
-    def _apply_style(self):
-        """ Applies the colors for the different text elements and the
-            carret.
-        """
-        self.SetCaretForeground(self.carret_color)
-
-        #self.StyleClearAll()
-        self.StyleSetSpec(stc.STC_STYLE_BRACELIGHT,  
-                          "fore:#FF0000,back:#0000FF,bold")
-        self.StyleSetSpec(stc.STC_STYLE_BRACEBAD,
-                          "fore:#000000,back:#FF0000,bold")
-
-        for style in self.ANSI_STYLES.values():
-            self.StyleSetSpec(style[0], "bold,fore:%s" % style[1])
+        # Marker for current input buffer.
+        self.MarkerDefine(_INPUT_MARKER, stc.STC_MARK_BACKGROUND,
+                                background=p['stdout'])
+        # Marker for tracebacks.
+        self.MarkerDefine(_ERROR_MARKER, stc.STC_MARK_BACKGROUND,
+                                background=p['stderr'])
 
-   
-    def _configure_scintilla(self):
         self.SetEOLMode(stc.STC_EOL_LF)
 
         # Ctrl"+" or Ctrl "-" can be used to zoomin/zoomout the text inside 
         # the widget
         self.CmdKeyAssign(ord('+'), stc.STC_SCMOD_CTRL, stc.STC_CMD_ZOOMIN)
         self.CmdKeyAssign(ord('-'), stc.STC_SCMOD_CTRL, stc.STC_CMD_ZOOMOUT)
         # Also allow Ctrl Shift "=" for poor non US keyboard users. 
         self.CmdKeyAssign(ord('='), stc.STC_SCMOD_CTRL|stc.STC_SCMOD_SHIFT, 
                                             stc.STC_CMD_ZOOMIN)
         
         # Keys: we need to clear some of the keys the that don't play
         # well with a console.
         self.CmdKeyClear(ord('D'), stc.STC_SCMOD_CTRL)
         self.CmdKeyClear(ord('L'), stc.STC_SCMOD_CTRL)
         self.CmdKeyClear(ord('T'), stc.STC_SCMOD_CTRL)
         self.CmdKeyClear(ord('A'), stc.STC_SCMOD_CTRL)
 
         self.SetEOLMode(stc.STC_EOL_CRLF)
         self.SetWrapMode(stc.STC_WRAP_CHAR)
         self.SetWrapMode(stc.STC_WRAP_WORD)
         self.SetBufferedDraw(True)
-        self.SetUseAntiAliasing(True)
+
+        self.SetUseAntiAliasing(p['antialiasing'])
+
         self.SetLayoutCache(stc.STC_CACHE_PAGE)
         self.SetUndoCollection(False)
         self.SetUseTabs(True)
         self.SetIndent(4)
         self.SetTabWidth(4)
 
         # we don't want scintilla's autocompletion to choose 
         # automaticaly out of a single choice list, as we pop it up
         # automaticaly
         self.AutoCompSetChooseSingle(False)
         self.AutoCompSetMaxHeight(10)
         # XXX: this doesn't seem to have an effect.
         self.AutoCompSetFillUps('\n')
 
         self.SetMargins(3, 3) #text is moved away from border with 3px
         # Suppressing Scintilla margins
         self.SetMarginWidth(0, 0)
         self.SetMarginWidth(1, 0)
         self.SetMarginWidth(2, 0)
 
-        self._apply_style()
-
         # Xterm escape sequences
         self.color_pat = re.compile('\x01?\x1b\[(.*?)m\x02?')
         self.title_pat = re.compile('\x1b]0;(.*?)\x07')
 
-        #self.SetEdgeMode(stc.STC_EDGE_LINE)
-        #self.SetEdgeColumn(80)
-
         # styles
-        p = self.style
-        self.StyleSetSpec(stc.STC_STYLE_DEFAULT, p['default'])
+        
+        self.SetCaretForeground(p['carret_color'])
+        
+        background_color = p['background_color']
+            
+        if 'default' in p:
+            if 'back' not in p['default']:
+                p['default'] += ',back:%s' % background_color
+            if 'size' not in p['default']:
+                p['default'] += ',size:%s' % self.faces['size']
+            if 'face' not in p['default']:
+                p['default'] += ',face:%s' % self.faces['mono']
+            
+            self.StyleSetSpec(stc.STC_STYLE_DEFAULT, p['default'])
+        else:
+            self.StyleSetSpec(stc.STC_STYLE_DEFAULT, 
+                            "fore:%s,back:%s,size:%d,face:%s" 
+                            % (self.ANSI_STYLES['0;30'][1], 
+                               background_color,
+                               self.faces['size'], self.faces['mono']))
+        
         self.StyleClearAll()
+        
+        # XXX: two lines below are usefull if not using the lexer        
+        #for style in self.ANSI_STYLES.values():
+        #    self.StyleSetSpec(style[0], "bold,fore:%s" % style[1])        
+
+        # prompt definition
+        self.prompt_in1 = p['prompt_in1']
+        self.prompt_out = p['prompt_out']
+
+        self.output_prompt_template = string.Template(self.prompt_out)
+        self.input_prompt_template = string.Template(self.prompt_in1)
+
         self.StyleSetSpec(_STDOUT_STYLE, p['stdout'])
         self.StyleSetSpec(_STDERR_STYLE, p['stderr'])
         self.StyleSetSpec(_TRACE_STYLE, p['trace'])
-
         self.StyleSetSpec(stc.STC_STYLE_BRACELIGHT, p['bracegood'])
         self.StyleSetSpec(stc.STC_STYLE_BRACEBAD, p['bracebad'])
         self.StyleSetSpec(stc.STC_P_COMMENTLINE, p['comment'])
         self.StyleSetSpec(stc.STC_P_NUMBER, p['number'])
         self.StyleSetSpec(stc.STC_P_STRING, p['string'])
         self.StyleSetSpec(stc.STC_P_CHARACTER, p['char'])
         self.StyleSetSpec(stc.STC_P_WORD, p['keyword'])
         self.StyleSetSpec(stc.STC_P_WORD2, p['keyword'])
         self.StyleSetSpec(stc.STC_P_TRIPLE, p['triple'])
         self.StyleSetSpec(stc.STC_P_TRIPLEDOUBLE, p['tripledouble'])
         self.StyleSetSpec(stc.STC_P_CLASSNAME, p['class'])
         self.StyleSetSpec(stc.STC_P_DEFNAME, p['def'])
         self.StyleSetSpec(stc.STC_P_OPERATOR, p['operator'])
         self.StyleSetSpec(stc.STC_P_COMMENTBLOCK, p['comment'])
 
+        edge_column = p['edge_column']
+        if edge_column is not None and edge_column > 0:
+            #we add a vertical line to console widget
+            self.SetEdgeMode(stc.STC_EDGE_LINE)
+            self.SetEdgeColumn(edge_column)
+ 
+ 
+    #--------------------------------------------------------------------------
+    # EditWindow API
+    #--------------------------------------------------------------------------
+
+    def OnUpdateUI(self, event):
+        """ Override the OnUpdateUI of the EditWindow class, to prevent 
+            syntax highlighting both for faster redraw, and for more
+            consistent look and feel.
+        """
+
+       
+    #--------------------------------------------------------------------------
+    # Private API
+    #--------------------------------------------------------------------------
+   
     def _on_key_down(self, event, skip=True):
         """ Key press callback used for correcting behavior for 
             console-like interfaces: the cursor is constraint to be after
             the last prompt.
 
             Return True if event as been catched.
         """
         catched = True
+        # XXX: Would the right way to do this be to have a
+        #  dictionary at the instance level associating keys with
+        #  callbacks? How would we deal with inheritance? And Do the
+        #  different callbacks share local variables?
+
         # Intercept some specific keys.
         if event.KeyCode == ord('L') and event.ControlDown() :
             self.scroll_to_bottom()
         elif event.KeyCode == ord('K') and event.ControlDown() :
             self.input_buffer = ''
         elif event.KeyCode == ord('A') and event.ControlDown() :
             self.GotoPos(self.GetLength())
             self.SetSelectionStart(self.current_prompt_pos)
             self.SetSelectionEnd(self.GetCurrentPos())
             catched = True
         elif event.KeyCode == ord('E') and event.ControlDown() :
             self.GotoPos(self.GetLength())
             catched = True
         elif event.KeyCode == wx.WXK_PAGEUP:
             self.ScrollPages(-1)
         elif event.KeyCode == wx.WXK_PAGEDOWN:
             self.ScrollPages(1)
+        elif event.KeyCode == wx.WXK_HOME:
+            self.GotoPos(self.GetLength())
+        elif event.KeyCode == wx.WXK_END:
+            self.GotoPos(self.GetLength())
         elif event.KeyCode == wx.WXK_UP and event.ShiftDown():
             self.ScrollLines(-1)
         elif event.KeyCode == wx.WXK_DOWN and event.ShiftDown():
             self.ScrollLines(1)
         else:
             catched = False
 
         if self.AutoCompActive():
             event.Skip()
         else:
             if event.KeyCode in (13, wx.WXK_NUMPAD_ENTER) and \
-                        event.Modifiers in (wx.MOD_NONE, wx.MOD_WIN):
+                                event.Modifiers in (wx.MOD_NONE, wx.MOD_WIN,
+                                                    wx.MOD_SHIFT):
                 catched = True
-                self.CallTipCancel()
-                self.write('\n', refresh=False)
-                # Under windows scintilla seems to be doing funny stuff to the 
-                # line returns here, but the getter for input_buffer filters 
-                # this out.
-                if sys.platform == 'win32':
-                    self.input_buffer = self.input_buffer
-                self._on_enter()
+                if not self.enter_catched:
+                    self.CallTipCancel()
+                    if event.Modifiers == wx.MOD_SHIFT:
+                        # Try to force execution
+                        self.GotoPos(self.GetLength())
+                        self.write('\n' + self.continuation_prompt(), 
+                                        refresh=False)
+                        self._on_enter()
+                    else:
+                        self._on_enter()
+                    self.enter_catched = True
 
             elif event.KeyCode == wx.WXK_HOME:
                 if event.Modifiers in (wx.MOD_NONE, wx.MOD_WIN):
                     self.GotoPos(self.current_prompt_pos)
                     catched = True
 
                 elif event.Modifiers == wx.MOD_SHIFT:
                     # FIXME: This behavior is not ideal: if the selection
                     # is already started, it will jump.
                     self.SetSelectionStart(self.current_prompt_pos) 
                     self.SetSelectionEnd(self.GetCurrentPos())
                     catched = True
 
             elif event.KeyCode == wx.WXK_UP:
                 if self.GetCurrentLine() > self.current_prompt_line:
                     if self.GetCurrentLine() == self.current_prompt_line + 1 \
                             and self.GetColumn(self.GetCurrentPos()) < \
                                 self.GetColumn(self.current_prompt_pos):
                         self.GotoPos(self.current_prompt_pos)
                     else:
                         event.Skip()
                 catched = True
 
             elif event.KeyCode in (wx.WXK_LEFT, wx.WXK_BACK):
-                if self.GetCurrentPos() > self.current_prompt_pos:
+                if not self._keep_cursor_in_buffer(self.GetCurrentPos() - 1):
+                    event.Skip()
+                catched = True
+
+            elif event.KeyCode == wx.WXK_RIGHT:
+                if not self._keep_cursor_in_buffer(self.GetCurrentPos() + 1):
+                    event.Skip()
+                catched = True
+
+
+            elif event.KeyCode == wx.WXK_DELETE:
+                if not self._keep_cursor_in_buffer(self.GetCurrentPos() - 1):
                     event.Skip()
                 catched = True
 
             if skip and not catched:
                 # Put the cursor back in the edit region
-                if self.GetCurrentPos() < self.current_prompt_pos:
-                    self.GotoPos(self.current_prompt_pos)
-                else:
-                    event.Skip()
+                if not self._keep_cursor_in_buffer():
+                    if not (self.GetCurrentPos() == self.GetLength()
+                                and event.KeyCode == wx.WXK_DELETE):
+                        event.Skip()
+                    catched = True
 
         return catched
 
 
     def _on_key_up(self, event, skip=True):
         """ If cursor is outside the editing region, put it back.
         """
-        event.Skip()
-        if self.GetCurrentPos() < self.current_prompt_pos:
-            self.GotoPos(self.current_prompt_pos)
+        if skip:
+            event.Skip()
+        self._keep_cursor_in_buffer()
+
+
+    # XXX:  I need to avoid the problem of having an empty glass;
+    def _keep_cursor_in_buffer(self, pos=None):
+        """ Checks if the cursor is where it is allowed to be. If not,
+            put it back.
 
+            Returns
+            -------
+            cursor_moved: Boolean
+                whether or not the cursor was moved by this routine.
+
+            Notes
+            ------
+                WARNING: This does proper checks only for horizontal
+                movements.
+        """
+        if pos is None:
+            current_pos = self.GetCurrentPos()
+        else:
+            current_pos = pos
+        if  current_pos < self.current_prompt_pos:
+            self.GotoPos(self.current_prompt_pos)
+            return True
+        line_num = self.LineFromPosition(current_pos)
+        if not current_pos > self.GetLength():
+            line_pos = self.GetColumn(current_pos)
+        else:
+            line_pos = self.GetColumn(self.GetLength())
+        line = self.GetLine(line_num)
+        # Jump the continuation prompt
+        continuation_prompt = self.continuation_prompt()
+        if ( line.startswith(continuation_prompt)
+                     and line_pos < len(continuation_prompt)):
+            if line_pos < 2:
+                # We are at the beginning of the line, trying to move
+                # forward: jump forward.
+                self.GotoPos(current_pos + 1 +
+                                    len(continuation_prompt) - line_pos)
+            else:
+                # Jump back up
+                self.GotoPos(self.GetLineEndPosition(line_num-1))
+            return True
+        elif ( current_pos > self.GetLineEndPosition(line_num) 
+                        and not current_pos == self.GetLength()): 
+            # Jump to next line
+            self.GotoPos(current_pos + 1 +
+                                    len(continuation_prompt))
+            return True
+
+        # We re-allow enter event processing
+        self.enter_catched = False 
+        return False
 
 
 if __name__ == '__main__':
     # Some simple code to test the console widget.
     class MainWindow(wx.Frame):
         def __init__(self, parent, id, title):
-            wx.Frame.__init__(self, parent, id, title, size=(300,250))
+            wx.Frame.__init__(self, parent, id, title, size=(300, 250))
             self._sizer = wx.BoxSizer(wx.VERTICAL)
             self.console_widget = ConsoleWidget(self)
             self._sizer.Add(self.console_widget, 1, wx.EXPAND)
             self.SetSizer(self._sizer)
             self.SetAutoLayout(1)
             self.Show(True)
 
     app = wx.PySimpleApp()
     w = MainWindow(None, wx.ID_ANY, 'ConsoleWidget')
     w.SetSize((780, 460))
     w.Show()
 
     app.MainLoop()
 
 

IPython/frontend/wx/ipythonx.py

@@ -1,110 +1,119 b''
 """
 Entry point for a simple application giving a graphical frontend to
 ipython.
 """
 
 try:
     import wx
 except ImportError, e:
     e.message = """%s
 ________________________________________________________________________________
 You need wxPython to run this application.
 """ % e.message
     e.args = (e.message, ) + e.args[1:]
     raise e
 
 from wx_frontend import WxController
 import __builtin__
 
 
 class IPythonXController(WxController):
     """ Sub class of WxController that adds some application-specific
         bindings.
     """
 
     debug = False
 
     def __init__(self, *args, **kwargs):
         WxController.__init__(self, *args, **kwargs)
         self.ipython0.ask_exit = self.do_exit
         # Scroll to top
         maxrange = self.GetScrollRange(wx.VERTICAL)
         self.ScrollLines(-maxrange)
 
 
     def _on_key_down(self, event, skip=True):
         # Intercept Ctrl-D to quit
         if event.KeyCode == ord('D') and event.ControlDown() and \
                 self.input_buffer == '' and \
                 self._input_state == 'readline':
             wx.CallAfter(self.ask_exit)
         else:
             WxController._on_key_down(self, event, skip=skip) 
 
 
     def ask_exit(self):
         """ Ask the user whether to exit.
         """
         self._input_state = 'subprocess'
         self.write('\n', refresh=False)
         self.capture_output()
         self.ipython0.shell.exit()
         self.release_output()
         if not self.ipython0.exit_now:
             wx.CallAfter(self.new_prompt,
                          self.input_prompt_template.substitute(
                                 number=self.last_result['number'] + 1))
         else:
             wx.CallAfter(wx.GetApp().Exit)
         self.write('Exiting ...', refresh=False)
  
 
     def do_exit(self):
         """ Exits the interpreter, kills the windows.
         """
         WxController.do_exit(self)
         self.release_output()
         wx.CallAfter(wx.Exit)
 
 
 
 class IPythonX(wx.Frame):
     """ Main frame of the IPythonX app.
     """
 
     def __init__(self, parent, id, title, debug=False):
         wx.Frame.__init__(self, parent, id, title, size=(300,250))
         self._sizer = wx.BoxSizer(wx.VERTICAL)
         self.shell = IPythonXController(self, debug=debug)
         self._sizer.Add(self.shell, 1, wx.EXPAND)
         self.SetSizer(self._sizer)
         self.SetAutoLayout(1)
         self.Show(True)
+        wx.EVT_CLOSE(self, self.on_close)
+
+        
+    def on_close(self, event):
+        """ Called on closing the windows. 
+            
+            Stops the event loop, to close all the child windows.
+        """
+        wx.CallAfter(wx.Exit)
 
 
 def main():
     from optparse import OptionParser
     usage = """usage: %prog [options]
 
 Simple graphical frontend to IPython, using WxWidgets."""
     parser = OptionParser(usage=usage)
     parser.add_option("-d", "--debug",
                     action="store_true", dest="debug", default=False,
                     help="Enable debug message for the wx frontend.")
 
     options, args = parser.parse_args()
 
     # Clear the options, to avoid having the ipython0 instance complain
     import sys
     sys.argv = sys.argv[:1]
 
     app = wx.PySimpleApp()
     frame = IPythonX(None, wx.ID_ANY, 'IPythonX', debug=options.debug)
     frame.shell.SetFocus()
     frame.shell.app = app
     frame.SetSize((680, 460))
 
     app.MainLoop()
 
 if __name__ == '__main__':
     main()

IPython/frontend/wx/wx_frontend.py

@@ -1,526 +1,601 b''
 # encoding: utf-8 -*- test-case-name:
 # FIXME: Need to add tests.
 # ipython1.frontend.wx.tests.test_wx_frontend -*-
 
 """Classes to provide a Wx frontend to the
 IPython.kernel.core.interpreter.
 
 This class inherits from ConsoleWidget, that provides a console-like
 widget to provide a text-rendering widget suitable for a terminal.
 """
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #       Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 # Major library imports
 import re
 import __builtin__
-from time import sleep
 import sys
 from threading import Lock
-import string
 
 import wx
 from wx import stc
 
 # Ipython-specific imports.
-from IPython.frontend._process import PipedProcess
-from console_widget import ConsoleWidget
+from IPython.frontend.process import PipedProcess
+from console_widget import ConsoleWidget, _COMPLETE_BUFFER_MARKER, \
+    _ERROR_MARKER, _INPUT_MARKER
 from IPython.frontend.prefilterfrontend import PrefilterFrontEnd
 
 #-------------------------------------------------------------------------------
-# Constants 
-#-------------------------------------------------------------------------------
-
-_COMPLETE_BUFFER_BG = '#FAFAF1' # Nice green
-_INPUT_BUFFER_BG = '#FDFFD3' # Nice yellow
-_ERROR_BG = '#FFF1F1' # Nice red
-
-_COMPLETE_BUFFER_MARKER = 31
-_ERROR_MARKER = 30
-_INPUT_MARKER = 29
-
-prompt_in1 = \
-        '\n\x01\x1b[0;34m\x02In [\x01\x1b[1;34m\x02$number\x01\x1b[0;34m\x02]: \x01\x1b[0m\x02'
-
-prompt_out = \
-    '\x01\x1b[0;31m\x02Out[\x01\x1b[1;31m\x02$number\x01\x1b[0;31m\x02]: \x01\x1b[0m\x02'
-
-#-------------------------------------------------------------------------------
 # Classes to implement the Wx frontend
 #-------------------------------------------------------------------------------
 class WxController(ConsoleWidget, PrefilterFrontEnd):
     """Classes to provide a Wx frontend to the
     IPython.kernel.core.interpreter.
 
     This class inherits from ConsoleWidget, that provides a console-like
     widget to provide a text-rendering widget suitable for a terminal.
     """
-
-    output_prompt_template = string.Template(prompt_out)
-
-    input_prompt_template = string.Template(prompt_in1)
-
+    
     # Print debug info on what is happening to the console.
     debug = False
 
     # The title of the terminal, as captured through the ANSI escape
     # sequences.
     def _set_title(self, title):
             return self.Parent.SetTitle(title)
 
     def _get_title(self):
             return self.Parent.GetTitle()
 
     title = property(_get_title, _set_title)
 
 
     # The buffer being edited.
     # We are duplicating the definition here because of multiple
     # inheritence
     def _set_input_buffer(self, string):
         ConsoleWidget._set_input_buffer(self, string)
         self._colorize_input_buffer()
 
     def _get_input_buffer(self):
         """ Returns the text in current edit buffer.
         """
         return ConsoleWidget._get_input_buffer(self)
 
     input_buffer = property(_get_input_buffer, _set_input_buffer)
 
 
     #--------------------------------------------------------------------------
     # Private Attributes
     #--------------------------------------------------------------------------
 
     # A flag governing the behavior of the input. Can be:
     #
     #       'readline' for readline-like behavior with a prompt 
     #            and an edit buffer.
     #       'raw_input' similar to readline, but triggered by a raw-input
     #           call. Can be used by subclasses to act differently.
     #       'subprocess' for sending the raw input directly to a
     #           subprocess.
     #       'buffering' for buffering of the input, that will be used
     #           when the input state switches back to another state.
     _input_state = 'readline'
 
     # Attribute to store reference to the pipes of a subprocess, if we
     # are running any.
     _running_process = False
 
     # A queue for writing fast streams to the screen without flooding the
     # event loop
     _out_buffer = []
 
     # A lock to lock the _out_buffer to make sure we don't empty it
     # while it is being swapped
     _out_buffer_lock = Lock()
 
     # The different line markers used to higlight the prompts.
     _markers = dict()
 
     #--------------------------------------------------------------------------
     # Public API
     #--------------------------------------------------------------------------
  
     def __init__(self, parent, id=wx.ID_ANY, pos=wx.DefaultPosition,
                  size=wx.DefaultSize,
                  style=wx.CLIP_CHILDREN|wx.WANTS_CHARS,
+                 styledef=None,
                  *args, **kwds):
         """ Create Shell instance.
+
+            Parameters
+            -----------
+            styledef : dict, optional
+                styledef is the dictionary of options used to define the
+                style.
         """
+        if styledef is not None:
+            self.style = styledef
         ConsoleWidget.__init__(self, parent, id, pos, size, style)
         PrefilterFrontEnd.__init__(self, **kwds)
         
         # Stick in our own raw_input:
         self.ipython0.raw_input = self.raw_input
 
-        # Marker for complete buffer.
-        self.MarkerDefine(_COMPLETE_BUFFER_MARKER, stc.STC_MARK_BACKGROUND,
-                                background=_COMPLETE_BUFFER_BG)
-        # Marker for current input buffer.
-        self.MarkerDefine(_INPUT_MARKER, stc.STC_MARK_BACKGROUND,
-                                background=_INPUT_BUFFER_BG)
-        # Marker for tracebacks.
-        self.MarkerDefine(_ERROR_MARKER, stc.STC_MARK_BACKGROUND,
-                                background=_ERROR_BG)
-
         # A time for flushing the write buffer
         BUFFER_FLUSH_TIMER_ID = 100
         self._buffer_flush_timer = wx.Timer(self, BUFFER_FLUSH_TIMER_ID)
         wx.EVT_TIMER(self, BUFFER_FLUSH_TIMER_ID, self._buffer_flush)
 
         if 'debug' in kwds:
             self.debug = kwds['debug']
             kwds.pop('debug')
 
         # Inject self in namespace, for debug
         if self.debug:
             self.shell.user_ns['self'] = self
         # Inject our own raw_input in namespace
         self.shell.user_ns['raw_input'] = self.raw_input
-
-
+        
     def raw_input(self, prompt=''):
         """ A replacement from python's raw_input.
         """
         self.new_prompt(prompt)
         self._input_state = 'raw_input'
         if hasattr(self, '_cursor'):
             del self._cursor 
         self.SetCursor(wx.StockCursor(wx.CURSOR_CROSS))
         self.__old_on_enter = self._on_enter
         event_loop = wx.EventLoop()
         def my_on_enter():
             event_loop.Exit()
         self._on_enter = my_on_enter
         # XXX: Running a separate event_loop. Ugly.
         event_loop.Run() 
         self._on_enter = self.__old_on_enter
         self._input_state = 'buffering'
         self._cursor = wx.BusyCursor()
         return self.input_buffer.rstrip('\n')
 
 
     def system_call(self, command_string):
         self._input_state = 'subprocess'
         event_loop = wx.EventLoop()
         def _end_system_call():
             self._input_state = 'buffering'
             self._running_process = False
             event_loop.Exit()
 
         self._running_process = PipedProcess(command_string, 
                     out_callback=self.buffered_write,
                     end_callback = _end_system_call)
         self._running_process.start()
         # XXX: Running a separate event_loop. Ugly.
         event_loop.Run() 
         # Be sure to flush the buffer.
         self._buffer_flush(event=None)
 
 
     def do_calltip(self):
         """ Analyse current and displays useful calltip for it.
         """
         if self.debug:
             print >>sys.__stdout__, "do_calltip" 
         separators =  re.compile('[\s\{\}\[\]\(\)\= ,:]')
         symbol = self.input_buffer
         symbol_string = separators.split(symbol)[-1]
         base_symbol_string = symbol_string.split('.')[0]
         if base_symbol_string in self.shell.user_ns:
             symbol = self.shell.user_ns[base_symbol_string]
         elif base_symbol_string in self.shell.user_global_ns:
             symbol = self.shell.user_global_ns[base_symbol_string]
         elif base_symbol_string in __builtin__.__dict__:
             symbol = __builtin__.__dict__[base_symbol_string]
         else:
             return False
         try:
             for name in symbol_string.split('.')[1:] + ['__doc__']:
                 symbol = getattr(symbol, name)
             self.AutoCompCancel()
             # Check that the symbol can indeed be converted to a string:
             symbol += ''
             wx.CallAfter(self.CallTipShow, self.GetCurrentPos(), symbol)
         except:
             # The retrieve symbol couldn't be converted to a string
             pass
 
 
     def _popup_completion(self, create=False):
         """ Updates the popup completion menu if it exists. If create is 
             true, open the menu.
         """
         if self.debug:
             print >>sys.__stdout__, "_popup_completion" 
         line = self.input_buffer
         if (self.AutoCompActive() and line and not line[-1] == '.') \
                     or create==True:
             suggestion, completions = self.complete(line)
-            offset=0
             if completions:
-                complete_sep =  re.compile('[\s\{\}\[\]\(\)\= ,:]')
-                residual = complete_sep.split(line)[-1]
-                offset = len(residual)
+                offset = len(self._get_completion_text(line))
                 self.pop_completion(completions, offset=offset)
                 if self.debug:
                     print >>sys.__stdout__, completions 
 
 
     def buffered_write(self, text):
         """ A write method for streams, that caches the stream in order
             to avoid flooding the event loop.
 
             This can be called outside of the main loop, in separate
             threads.
         """
         self._out_buffer_lock.acquire()
         self._out_buffer.append(text)
         self._out_buffer_lock.release()
         if not self._buffer_flush_timer.IsRunning():
             wx.CallAfter(self._buffer_flush_timer.Start, 
                                         milliseconds=100, oneShot=True)
 
 
+    def clear_screen(self):
+        """ Empty completely the widget.
+        """
+        self.ClearAll()
+        self.new_prompt(self.input_prompt_template.substitute(
+                                number=(self.last_result['number'] + 1)))
+
+
     #--------------------------------------------------------------------------
     # LineFrontEnd interface 
     #--------------------------------------------------------------------------
  
     def execute(self, python_string, raw_string=None):
         self._input_state = 'buffering'
         self.CallTipCancel()
         self._cursor = wx.BusyCursor()
         if raw_string is None:
             raw_string = python_string
         end_line = self.current_prompt_line \
                         + max(1,  len(raw_string.split('\n'))-1)
         for i in range(self.current_prompt_line, end_line):
             if i in self._markers:
                 self.MarkerDeleteHandle(self._markers[i])
             self._markers[i] = self.MarkerAdd(i, _COMPLETE_BUFFER_MARKER)
         # Use a callafter to update the display robustly under windows
         def callback():
             self.GotoPos(self.GetLength())
             PrefilterFrontEnd.execute(self, python_string, 
                                             raw_string=raw_string)
         wx.CallAfter(callback)
 
+
+    def execute_command(self, command, hidden=False):
+        """ Execute a command, not only in the model, but also in the
+            view.
+        """
+        # XXX: This method needs to be integrated in the base fronted
+        # interface
+        if hidden:
+            return self.shell.execute(command)
+        else:
+            # XXX: we are not storing the input buffer previous to the
+            # execution, as this forces us to run the execution
+            # input_buffer a yield, which is not good.
+            ##current_buffer = self.shell.control.input_buffer
+            command = command.rstrip()
+            if len(command.split('\n')) > 1:
+                # The input command is several lines long, we need to
+                # force the execution to happen
+                command += '\n'
+            cleaned_command = self.prefilter_input(command)
+            self.input_buffer = command
+            # Do not use wx.Yield() (aka GUI.process_events()) to avoid
+            # recursive yields.
+            self.ProcessEvent(wx.PaintEvent())
+            self.write('\n')
+            if not self.is_complete(cleaned_command + '\n'):
+                self._colorize_input_buffer()
+                self.render_error('Incomplete or invalid input')
+                self.new_prompt(self.input_prompt_template.substitute(
+                                number=(self.last_result['number'] + 1)))
+                return False
+            self._on_enter()
+            return True
+
+
     def save_output_hooks(self):    
         self.__old_raw_input = __builtin__.raw_input
         PrefilterFrontEnd.save_output_hooks(self)
 
     def capture_output(self):
         self.SetLexer(stc.STC_LEX_NULL)
         PrefilterFrontEnd.capture_output(self)
         __builtin__.raw_input = self.raw_input
         
     
     def release_output(self):
         __builtin__.raw_input = self.__old_raw_input
         PrefilterFrontEnd.release_output(self)
         self.SetLexer(stc.STC_LEX_PYTHON)
 
 
     def after_execute(self):
         PrefilterFrontEnd.after_execute(self)
         # Clear the wait cursor
         if hasattr(self, '_cursor'):
             del self._cursor
         self.SetCursor(wx.StockCursor(wx.CURSOR_CHAR))
 
 
     def show_traceback(self):
         start_line = self.GetCurrentLine()
         PrefilterFrontEnd.show_traceback(self)
         self.ProcessEvent(wx.PaintEvent())
         #wx.Yield()
         for i in range(start_line, self.GetCurrentLine()):
             self._markers[i] = self.MarkerAdd(i, _ERROR_MARKER)
 
     
     #--------------------------------------------------------------------------
     # FrontEndBase interface 
     #--------------------------------------------------------------------------
     
     def render_error(self, e):
         start_line = self.GetCurrentLine()
         self.write('\n' + e + '\n')
         for i in range(start_line, self.GetCurrentLine()):
             self._markers[i] = self.MarkerAdd(i, _ERROR_MARKER)
 
 
     #--------------------------------------------------------------------------
     # ConsoleWidget interface 
     #--------------------------------------------------------------------------
 
     def new_prompt(self, prompt):
         """ Display a new prompt, and start a new input buffer.
         """
         self._input_state = 'readline'
         ConsoleWidget.new_prompt(self, prompt)
         i = self.current_prompt_line
         self._markers[i] = self.MarkerAdd(i, _INPUT_MARKER)
 
 
+    def continuation_prompt(self, *args, **kwargs):
+        # Avoid multiple inheritence, be explicit about which
+        # parent method class gets called
+        return ConsoleWidget.continuation_prompt(self, *args, **kwargs)
+
+
     def write(self, *args, **kwargs):
         # Avoid multiple inheritence, be explicit about which
         # parent method class gets called
-        ConsoleWidget.write(self, *args, **kwargs)
+        return ConsoleWidget.write(self, *args, **kwargs)
 
 
     def _on_key_down(self, event, skip=True):
         """ Capture the character events, let the parent
             widget handle them, and put our logic afterward.
         """
         # FIXME: This method needs to be broken down in smaller ones.
-        current_line_number = self.GetCurrentLine()
+        current_line_num = self.GetCurrentLine()
         if event.KeyCode in (ord('c'), ord('C')) and event.ControlDown():
             # Capture Control-C
             if self._input_state == 'subprocess':
                 if self.debug:
                     print >>sys.__stderr__, 'Killing running process'
                 if hasattr(self._running_process, 'process'):
                     self._running_process.process.kill()
             elif self._input_state == 'buffering':
                 if self.debug:
                     print >>sys.__stderr__, 'Raising KeyboardInterrupt'
                 raise KeyboardInterrupt
                 # XXX: We need to make really sure we
                 # get back to a prompt.
         elif self._input_state == 'subprocess' and (
                 ( event.KeyCode<256 and
                         not event.ControlDown() )
                     or 
                 ( event.KeyCode in (ord('d'), ord('D')) and
                   event.ControlDown())):
             #  We are running a process, we redirect keys.
             ConsoleWidget._on_key_down(self, event, skip=skip)
             char = chr(event.KeyCode)
             # Deal with some inconsistency in wx keycodes:
             if char == '\r':
                 char = '\n'
             elif not event.ShiftDown():
                 char = char.lower()
             if event.ControlDown() and event.KeyCode in (ord('d'), ord('D')):
                 char = '\04'
             self._running_process.process.stdin.write(char)
             self._running_process.process.stdin.flush()
         elif event.KeyCode in (ord('('), 57, 53):
             # Calltips
             event.Skip()
             self.do_calltip()
         elif self.AutoCompActive() and not event.KeyCode == ord('\t'):
             event.Skip()
             if event.KeyCode in (wx.WXK_BACK, wx.WXK_DELETE): 
                 wx.CallAfter(self._popup_completion, create=True)
             elif not event.KeyCode in (wx.WXK_UP, wx.WXK_DOWN, wx.WXK_LEFT,
                             wx.WXK_RIGHT, wx.WXK_ESCAPE):
                 wx.CallAfter(self._popup_completion)
         else:
             # Up history
             if event.KeyCode == wx.WXK_UP and (
-                    ( current_line_number == self.current_prompt_line and
+                    ( current_line_num == self.current_prompt_line and
                         event.Modifiers in (wx.MOD_NONE, wx.MOD_WIN) ) 
                     or event.ControlDown() ):
                 new_buffer = self.get_history_previous(
                                             self.input_buffer)
                 if new_buffer is not None:
                     self.input_buffer = new_buffer
                     if self.GetCurrentLine() > self.current_prompt_line:
                         # Go to first line, for seemless history up.
                         self.GotoPos(self.current_prompt_pos)
             # Down history
             elif event.KeyCode == wx.WXK_DOWN and (
-                    ( current_line_number == self.LineCount -1 and
+                    ( current_line_num == self.LineCount -1 and
                         event.Modifiers in (wx.MOD_NONE, wx.MOD_WIN) ) 
                     or event.ControlDown() ):
                 new_buffer = self.get_history_next()
                 if new_buffer is not None:
                     self.input_buffer = new_buffer
             # Tab-completion
             elif event.KeyCode == ord('\t'):
-                current_line, current_line_number = self.CurLine
+                current_line, current_line_num = self.CurLine
                 if not re.match(r'^\s*$', current_line):
                     self.complete_current_input()
                     if self.AutoCompActive():
                         wx.CallAfter(self._popup_completion, create=True)
                 else:
                     event.Skip()
+            elif event.KeyCode == wx.WXK_BACK:
+                # If characters where erased, check if we have to
+                # remove a line.
+                # XXX: What about DEL?
+                # FIXME: This logics should be in ConsoleWidget, as it is
+                # independant of IPython
+                current_line, _ = self.CurLine
+                current_pos = self.GetCurrentPos()
+                current_line_num = self.LineFromPosition(current_pos)
+                current_col = self.GetColumn(current_pos)
+                len_prompt = len(self.continuation_prompt())
+                if ( current_line.startswith(self.continuation_prompt())
+                                            and current_col == len_prompt):
+                    new_lines = []
+                    for line_num, line in enumerate(
+                                    self.input_buffer.split('\n')):
+                        if (line_num + self.current_prompt_line ==
+                                            current_line_num):
+                            new_lines.append(line[len_prompt:])
+                        else:
+                            new_lines.append('\n'+line)
+                    # The first character is '\n', due to the above
+                    # code:
+                    self.input_buffer = ''.join(new_lines)[1:]
+                    self.GotoPos(current_pos - 1 - len_prompt)
+                else:
+                    ConsoleWidget._on_key_down(self, event, skip=skip)
             else:
                 ConsoleWidget._on_key_down(self, event, skip=skip)
+                        
 
 
     def _on_key_up(self, event, skip=True):
         """ Called when any key is released.
         """
         if event.KeyCode in (59, ord('.')):
             # Intercepting '.'
             event.Skip()
             wx.CallAfter(self._popup_completion, create=True)
         else:
             ConsoleWidget._on_key_up(self, event, skip=skip)
+        # Make sure the continuation_prompts are always followed by a 
+        # whitespace
+        new_lines = []
+        if self._input_state == 'readline':
+            position = self.GetCurrentPos()
+            continuation_prompt = self.continuation_prompt()[:-1]
+            for line in self.input_buffer.split('\n'):
+                if not line == continuation_prompt:
+                    new_lines.append(line)
+            self.input_buffer = '\n'.join(new_lines)
+            self.GotoPos(position)
 
 
     def _on_enter(self):
         """ Called on return key down, in readline input_state.
         """
+        last_line_num = self.LineFromPosition(self.GetLength())
+        current_line_num = self.LineFromPosition(self.GetCurrentPos())
+        new_line_pos = (last_line_num - current_line_num)
         if self.debug:
             print >>sys.__stdout__, repr(self.input_buffer)
-        PrefilterFrontEnd._on_enter(self)
+        self.write('\n', refresh=False)
+        # Under windows scintilla seems to be doing funny
+        # stuff to the line returns here, but the getter for
+        # input_buffer filters this out.
+        if sys.platform == 'win32':
+            self.input_buffer = self.input_buffer
+        old_prompt_num = self.current_prompt_pos
+        has_executed = PrefilterFrontEnd._on_enter(self, 
+                                            new_line_pos=new_line_pos)
+        if old_prompt_num == self.current_prompt_pos:
+            # No execution has happened 
+            self.GotoPos(self.GetLineEndPosition(current_line_num + 1))
+        return has_executed
 
 
     #--------------------------------------------------------------------------
     # EditWindow API
     #--------------------------------------------------------------------------
 
     def OnUpdateUI(self, event):
         """ Override the OnUpdateUI of the EditWindow class, to prevent 
             syntax highlighting both for faster redraw, and for more
             consistent look and feel.
         """
         if not self._input_state == 'readline':
             ConsoleWidget.OnUpdateUI(self, event)
 
     #--------------------------------------------------------------------------
     # Private API
     #--------------------------------------------------------------------------
  
     def _buffer_flush(self, event):
         """ Called by the timer to flush the write buffer.
             
             This is always called in the mainloop, by the wx timer.
         """
         self._out_buffer_lock.acquire()
         _out_buffer = self._out_buffer
         self._out_buffer = []
         self._out_buffer_lock.release()
         self.write(''.join(_out_buffer), refresh=False)
 
 
     def _colorize_input_buffer(self):
         """ Keep the input buffer lines at a bright color.
         """
         if not self._input_state in ('readline', 'raw_input'):
             return
         end_line = self.GetCurrentLine()
         if not sys.platform == 'win32':
             end_line += 1
         for i in range(self.current_prompt_line, end_line):
             if i in self._markers:
                 self.MarkerDeleteHandle(self._markers[i])
             self._markers[i] = self.MarkerAdd(i, _INPUT_MARKER)
 
 
 if __name__ == '__main__':
     class MainWindow(wx.Frame):
         def __init__(self, parent, id, title):
             wx.Frame.__init__(self, parent, id, title, size=(300,250))
             self._sizer = wx.BoxSizer(wx.VERTICAL)
             self.shell = WxController(self)
             self._sizer.Add(self.shell, 1, wx.EXPAND)
             self.SetSizer(self._sizer)
             self.SetAutoLayout(1)
             self.Show(True)
 
     app = wx.PySimpleApp()
     frame = MainWindow(None, wx.ID_ANY, 'Ipython')
     frame.shell.SetFocus()
     frame.SetSize((680, 460))
     self = frame.shell
 
     app.MainLoop()
 

IPython/genutils.py

@@ -1,2161 +1,2171 b''
 # -*- coding: utf-8 -*-
 """General purpose utilities.
 
 This is a grab-bag of stuff I find useful in most programs I write. Some of
 these things are also convenient when working at the command line.
 """
 
 #*****************************************************************************
 #       Copyright (C) 2001-2006 Fernando Perez. <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 #****************************************************************************
 # required modules from the Python standard library
 import __main__
 import commands
 try:
     import doctest
 except ImportError:
     pass
 import os
 import platform
 import re
 import shlex
 import shutil
 import subprocess
 import sys
 import tempfile
 import time
 import types
 import warnings
 
 # Curses and termios are Unix-only modules
 try:
     import curses
     # We need termios as well, so if its import happens to raise, we bail on
     # using curses altogether.
     import termios
 except ImportError:
     USE_CURSES = False
 else:
     # Curses on Solaris may not be complete, so we can't use it there
     USE_CURSES = hasattr(curses,'initscr')
 
 # Other IPython utilities
 import IPython
 from IPython.Itpl import Itpl,itpl,printpl
 from IPython import DPyGetOpt, platutils
 from IPython.generics import result_display
 import IPython.ipapi
 from IPython.external.path import path
 if os.name == "nt":
     from IPython.winconsole import get_console_size
 
 try:
     set
 except:
     from sets import Set as set
 
 
 #****************************************************************************
 # Exceptions
 class Error(Exception):
     """Base class for exceptions in this module."""
     pass
 
 #----------------------------------------------------------------------------
 class IOStream:
     def __init__(self,stream,fallback):
         if not hasattr(stream,'write') or not hasattr(stream,'flush'):
             stream = fallback
         self.stream = stream
         self._swrite = stream.write
         self.flush = stream.flush
 
     def write(self,data):
         try:
             self._swrite(data)
         except:
             try:
                 # print handles some unicode issues which may trip a plain
                 # write() call.  Attempt to emulate write() by using a
                 # trailing comma
                 print >> self.stream, data,
             except:
                 # if we get here, something is seriously broken.
                 print >> sys.stderr, \
                       'ERROR - failed to write data to stream:', self.stream
 
     def close(self):
         pass
 
 
 class IOTerm:
     """ Term holds the file or file-like objects for handling I/O operations.
 
     These are normally just sys.stdin, sys.stdout and sys.stderr but for
     Windows they can can replaced to allow editing the strings before they are
     displayed."""
 
     # In the future, having IPython channel all its I/O operations through
     # this class will make it easier to embed it into other environments which
     # are not a normal terminal (such as a GUI-based shell)
     def __init__(self,cin=None,cout=None,cerr=None):
         self.cin  = IOStream(cin,sys.stdin)
         self.cout = IOStream(cout,sys.stdout)
         self.cerr = IOStream(cerr,sys.stderr)
 
 # Global variable to be used for all I/O
 Term = IOTerm()
 
 import IPython.rlineimpl as readline
 # Remake Term to use the readline i/o facilities
 if sys.platform == 'win32' and readline.have_readline:
 
     Term = IOTerm(cout=readline._outputfile,cerr=readline._outputfile)
 
 
 #****************************************************************************
 # Generic warning/error printer, used by everything else
 def warn(msg,level=2,exit_val=1):
     """Standard warning printer. Gives formatting consistency.
 
     Output is sent to Term.cerr (sys.stderr by default).
 
     Options:
 
     -level(2): allows finer control:
       0 -> Do nothing, dummy function.
       1 -> Print message.
       2 -> Print 'WARNING:' + message. (Default level).
       3 -> Print 'ERROR:' + message.
       4 -> Print 'FATAL ERROR:' + message and trigger a sys.exit(exit_val).
 
     -exit_val (1): exit value returned by sys.exit() for a level 4
     warning. Ignored for all other levels."""
 
     if level>0:
         header = ['','','WARNING: ','ERROR: ','FATAL ERROR: ']
         print >> Term.cerr, '%s%s' % (header[level],msg)
         if level == 4:
             print >> Term.cerr,'Exiting.\n'
             sys.exit(exit_val)
 
 def info(msg):
     """Equivalent to warn(msg,level=1)."""
 
     warn(msg,level=1)
 
 def error(msg):
     """Equivalent to warn(msg,level=3)."""
 
     warn(msg,level=3)
 
 def fatal(msg,exit_val=1):
     """Equivalent to warn(msg,exit_val=exit_val,level=4)."""
 
     warn(msg,exit_val=exit_val,level=4)
 
 #---------------------------------------------------------------------------
 # Debugging routines
 #
 def debugx(expr,pre_msg=''):
     """Print the value of an expression from the caller's frame.
 
     Takes an expression, evaluates it in the caller's frame and prints both
     the given expression and the resulting value (as well as a debug mark
     indicating the name of the calling function.  The input must be of a form
     suitable for eval().
 
     An optional message can be passed, which will be prepended to the printed
     expr->value pair."""
 
     cf = sys._getframe(1)
     print '[DBG:%s] %s%s -> %r' % (cf.f_code.co_name,pre_msg,expr,
                                    eval(expr,cf.f_globals,cf.f_locals))
 
 # deactivate it by uncommenting the following line, which makes it a no-op
 #def debugx(expr,pre_msg=''): pass
 
 #----------------------------------------------------------------------------
 StringTypes = types.StringTypes
 
 # Basic timing functionality
 
 # If possible (Unix), use the resource module instead of time.clock()
 try:
     import resource
     def clocku():
         """clocku() -> floating point number
 
         Return the *USER* CPU time in seconds since the start of the process.
         This is done via a call to resource.getrusage, so it avoids the
         wraparound problems in time.clock()."""
 
         return resource.getrusage(resource.RUSAGE_SELF)[0]
 
     def clocks():
         """clocks() -> floating point number
 
         Return the *SYSTEM* CPU time in seconds since the start of the process.
         This is done via a call to resource.getrusage, so it avoids the
         wraparound problems in time.clock()."""
 
         return resource.getrusage(resource.RUSAGE_SELF)[1]
 
     def clock():
         """clock() -> floating point number
 
         Return the *TOTAL USER+SYSTEM* CPU time in seconds since the start of
         the process.  This is done via a call to resource.getrusage, so it
         avoids the wraparound problems in time.clock()."""
 
         u,s = resource.getrusage(resource.RUSAGE_SELF)[:2]
         return u+s
 
     def clock2():
         """clock2() -> (t_user,t_system)
 
         Similar to clock(), but return a tuple of user/system times."""
         return resource.getrusage(resource.RUSAGE_SELF)[:2]
 
 except ImportError:
     # There is no distinction of user/system time under windows, so we just use
     # time.clock() for everything...
     clocku = clocks = clock = time.clock
     def clock2():
         """Under windows, system CPU time can't be measured.
 
         This just returns clock() and zero."""
         return time.clock(),0.0
 
 def timings_out(reps,func,*args,**kw):
     """timings_out(reps,func,*args,**kw) -> (t_total,t_per_call,output)
 
     Execute a function reps times, return a tuple with the elapsed total
     CPU time in seconds, the time per call and the function's output.
 
     Under Unix, the return value is the sum of user+system time consumed by
     the process, computed via the resource module.  This prevents problems
     related to the wraparound effect which the time.clock() function has.
 
     Under Windows the return value is in wall clock seconds. See the
     documentation for the time module for more details."""
 
     reps = int(reps)
     assert reps >=1, 'reps must be >= 1'
     if reps==1:
         start = clock()
         out = func(*args,**kw)
         tot_time = clock()-start
     else:
         rng = xrange(reps-1) # the last time is executed separately to store output
         start = clock()
         for dummy in rng: func(*args,**kw)
         out = func(*args,**kw)  # one last time
         tot_time = clock()-start
     av_time = tot_time / reps
     return tot_time,av_time,out
 
 def timings(reps,func,*args,**kw):
     """timings(reps,func,*args,**kw) -> (t_total,t_per_call)
 
     Execute a function reps times, return a tuple with the elapsed total CPU
     time in seconds and the time per call. These are just the first two values
     in timings_out()."""
 
     return timings_out(reps,func,*args,**kw)[0:2]
 
 def timing(func,*args,**kw):
     """timing(func,*args,**kw) -> t_total
 
     Execute a function once, return the elapsed total CPU time in
     seconds. This is just the first value in timings_out()."""
 
     return timings_out(1,func,*args,**kw)[0]
 
 #****************************************************************************
 # file and system
 
 def arg_split(s,posix=False):
     """Split a command line's arguments in a shell-like manner.
 
     This is a modified version of the standard library's shlex.split()
     function, but with a default of posix=False for splitting, so that quotes
     in inputs are respected."""
 
     # XXX - there may be unicode-related problems here!!!  I'm not sure that
     # shlex is truly unicode-safe, so it might be necessary to do
     #
     # s = s.encode(sys.stdin.encoding)
     #
     # first, to ensure that shlex gets a normal string.  Input from anyone who
     # knows more about unicode and shlex than I would be good to have here...
     lex = shlex.shlex(s, posix=posix)
     lex.whitespace_split = True
     return list(lex)
 
 def system(cmd,verbose=0,debug=0,header=''):
     """Execute a system command, return its exit status.
 
     Options:
 
     - verbose (0): print the command to be executed.
 
     - debug (0): only print, do not actually execute.
 
     - header (''): Header to print on screen prior to the executed command (it
     is only prepended to the command, no newlines are added).
 
     Note: a stateful version of this function is available through the
     SystemExec class."""
 
     stat = 0
     if verbose or debug: print header+cmd
     sys.stdout.flush()
     if not debug: stat = os.system(cmd)
     return stat
 
 def abbrev_cwd():
     """ Return abbreviated version of cwd, e.g. d:mydir """
     cwd = os.getcwd().replace('\\','/')
     drivepart = ''
     tail = cwd
     if sys.platform == 'win32':
         if len(cwd) < 4:
             return cwd
         drivepart,tail = os.path.splitdrive(cwd)
 
 
     parts = tail.split('/')
     if len(parts) > 2:
         tail = '/'.join(parts[-2:])
 
     return (drivepart + (
         cwd == '/' and '/' or tail))
 
 
 # This function is used by ipython in a lot of places to make system calls.
 # We need it to be slightly different under win32, due to the vagaries of
 # 'network shares'.  A win32 override is below.
 
 def shell(cmd,verbose=0,debug=0,header=''):
     """Execute a command in the system shell, always return None.
 
     Options:
 
     - verbose (0): print the command to be executed.
 
     - debug (0): only print, do not actually execute.
 
     - header (''): Header to print on screen prior to the executed command (it
     is only prepended to the command, no newlines are added).
 
     Note: this is similar to genutils.system(), but it returns None so it can
     be conveniently used in interactive loops without getting the return value
     (typically 0) printed many times."""
 
     stat = 0
     if verbose or debug: print header+cmd
     # flush stdout so we don't mangle python's buffering
     sys.stdout.flush()
 
     if not debug:
         platutils.set_term_title("IPy " + cmd)
         os.system(cmd)
         platutils.set_term_title("IPy " + abbrev_cwd())
 
 # override shell() for win32 to deal with network shares
 if os.name in ('nt','dos'):
 
     shell_ori = shell
 
     def shell(cmd,verbose=0,debug=0,header=''):
         if os.getcwd().startswith(r"\\"):
             path = os.getcwd()
             # change to c drive (cannot be on UNC-share when issuing os.system,
             # as cmd.exe cannot handle UNC addresses)
             os.chdir("c:")
             # issue pushd to the UNC-share and then run the command
             try:
                 shell_ori('"pushd %s&&"'%path+cmd,verbose,debug,header)
             finally:
                 os.chdir(path)
         else:
             shell_ori(cmd,verbose,debug,header)
 
     shell.__doc__ = shell_ori.__doc__
 
 def getoutput(cmd,verbose=0,debug=0,header='',split=0):
     """Dummy substitute for perl's backquotes.
 
     Executes a command and returns the output.
 
     Accepts the same arguments as system(), plus:
 
     - split(0): if true, the output is returned as a list split on newlines.
 
     Note: a stateful version of this function is available through the
     SystemExec class.
 
     This is pretty much deprecated and rarely used,
     genutils.getoutputerror may be what you need.
 
     """
 
     if verbose or debug: print header+cmd
     if not debug:
         output = os.popen(cmd).read()
         # stipping last \n is here for backwards compat.
         if output.endswith('\n'):
             output = output[:-1]
         if split:
             return output.split('\n')
         else:
             return output
 
 def getoutputerror(cmd,verbose=0,debug=0,header='',split=0):
     """Return (standard output,standard error) of executing cmd in a shell.
 
     Accepts the same arguments as system(), plus:
 
     - split(0): if true, each of stdout/err is returned as a list split on
     newlines.
 
     Note: a stateful version of this function is available through the
     SystemExec class."""
 
     if verbose or debug: print header+cmd
     if not cmd:
         if split:
             return [],[]
         else:
             return '',''
     if not debug:
         pin,pout,perr = os.popen3(cmd)
         tout = pout.read().rstrip()
         terr = perr.read().rstrip()
         pin.close()
         pout.close()
         perr.close()
         if split:
             return tout.split('\n'),terr.split('\n')
         else:
             return tout,terr
 
 # for compatibility with older naming conventions
 xsys = system
 bq = getoutput
 
 class SystemExec:
     """Access the system and getoutput functions through a stateful interface.
 
     Note: here we refer to the system and getoutput functions from this
     library, not the ones from the standard python library.
 
     This class offers the system and getoutput functions as methods, but the
     verbose, debug and header parameters can be set for the instance (at
     creation time or later) so that they don't need to be specified on each
     call.
 
     For efficiency reasons, there's no way to override the parameters on a
     per-call basis other than by setting instance attributes. If you need
     local overrides, it's best to directly call system() or getoutput().
 
     The following names are provided as alternate options:
      - xsys: alias to system
      - bq: alias to getoutput
 
     An instance can then be created as:
     >>> sysexec = SystemExec(verbose=1,debug=0,header='Calling: ')
     """
 
     def __init__(self,verbose=0,debug=0,header='',split=0):
         """Specify the instance's values for verbose, debug and header."""
         setattr_list(self,'verbose debug header split')
 
     def system(self,cmd):
         """Stateful interface to system(), with the same keyword parameters."""
 
         system(cmd,self.verbose,self.debug,self.header)
 
     def shell(self,cmd):
         """Stateful interface to shell(), with the same keyword parameters."""
 
         shell(cmd,self.verbose,self.debug,self.header)
 
     xsys = system  # alias
 
     def getoutput(self,cmd):
         """Stateful interface to getoutput()."""
 
         return getoutput(cmd,self.verbose,self.debug,self.header,self.split)
 
     def getoutputerror(self,cmd):
         """Stateful interface to getoutputerror()."""
 
         return getoutputerror(cmd,self.verbose,self.debug,self.header,self.split)
 
     bq = getoutput  # alias
 
 #-----------------------------------------------------------------------------
 def mutex_opts(dict,ex_op):
     """Check for presence of mutually exclusive keys in a dict.
 
     Call: mutex_opts(dict,[[op1a,op1b],[op2a,op2b]...]"""
     for op1,op2 in ex_op:
         if op1 in dict and op2 in dict:
             raise ValueError,'\n*** ERROR in Arguments *** '\
                   'Options '+op1+' and '+op2+' are mutually exclusive.'
 
 #-----------------------------------------------------------------------------
 def get_py_filename(name):
     """Return a valid python filename in the current directory.
 
     If the given name is not a file, it adds '.py' and searches again.
     Raises IOError with an informative message if the file isn't found."""
 
     name = os.path.expanduser(name)
     if not os.path.isfile(name) and not name.endswith('.py'):
         name += '.py'
     if os.path.isfile(name):
         return name
     else:
         raise IOError,'File `%s` not found.' % name
 
 #-----------------------------------------------------------------------------
 def filefind(fname,alt_dirs = None):
     """Return the given filename either in the current directory, if it
     exists, or in a specified list of directories.
 
     ~ expansion is done on all file and directory names.
 
     Upon an unsuccessful search, raise an IOError exception."""
 
     if alt_dirs is None:
         try:
             alt_dirs = get_home_dir()
         except HomeDirError:
             alt_dirs = os.getcwd()
     search = [fname] + list_strings(alt_dirs)
     search = map(os.path.expanduser,search)
     #print 'search list for',fname,'list:',search  # dbg
     fname = search[0]
     if os.path.isfile(fname):
         return fname
     for direc in search[1:]:
         testname = os.path.join(direc,fname)
         #print 'testname',testname  # dbg
         if os.path.isfile(testname):
             return testname
     raise IOError,'File' + `fname` + \
           ' not found in current or supplied directories:' + `alt_dirs`
 
 #----------------------------------------------------------------------------
 def file_read(filename):
     """Read a file and close it.  Returns the file source."""
     fobj = open(filename,'r');
     source = fobj.read();
     fobj.close()
     return source
 
 def file_readlines(filename):
     """Read a file and close it.  Returns the file source using readlines()."""
     fobj = open(filename,'r');
     lines = fobj.readlines();
     fobj.close()
     return lines
 
 #----------------------------------------------------------------------------
 def target_outdated(target,deps):
     """Determine whether a target is out of date.
 
     target_outdated(target,deps) -> 1/0
 
     deps: list of filenames which MUST exist.
     target: single filename which may or may not exist.
 
     If target doesn't exist or is older than any file listed in deps, return
     true, otherwise return false.
     """
     try:
         target_time = os.path.getmtime(target)
     except os.error:
         return 1
     for dep in deps:
         dep_time = os.path.getmtime(dep)
         if dep_time > target_time:
             #print "For target",target,"Dep failed:",dep # dbg
             #print "times (dep,tar):",dep_time,target_time # dbg
             return 1
     return 0
 
 #-----------------------------------------------------------------------------
 def target_update(target,deps,cmd):
     """Update a target with a given command given a list of dependencies.
 
     target_update(target,deps,cmd) -> runs cmd if target is outdated.
 
     This is just a wrapper around target_outdated() which calls the given
     command if target is outdated."""
 
     if target_outdated(target,deps):
         xsys(cmd)
 
 #----------------------------------------------------------------------------
 def unquote_ends(istr):
     """Remove a single pair of quotes from the endpoints of a string."""
 
     if not istr:
         return istr
     if (istr[0]=="'" and istr[-1]=="'") or \
        (istr[0]=='"' and istr[-1]=='"'):
         return istr[1:-1]
     else:
         return istr
 
 #----------------------------------------------------------------------------
 def process_cmdline(argv,names=[],defaults={},usage=''):
     """ Process command-line options and arguments.
 
     Arguments:
 
     - argv: list of arguments, typically sys.argv.
 
     - names: list of option names. See DPyGetOpt docs for details on options
     syntax.
 
     - defaults: dict of default values.
 
     - usage: optional usage notice to print if a wrong argument is passed.
 
     Return a dict of options and a list of free arguments."""
 
     getopt = DPyGetOpt.DPyGetOpt()
     getopt.setIgnoreCase(0)
     getopt.parseConfiguration(names)
 
     try:
         getopt.processArguments(argv)
     except DPyGetOpt.ArgumentError, exc:
         print usage
         warn('"%s"' % exc,level=4)
 
     defaults.update(getopt.optionValues)
     args = getopt.freeValues
 
     return defaults,args
 
 #----------------------------------------------------------------------------
 def optstr2types(ostr):
     """Convert a string of option names to a dict of type mappings.
 
     optstr2types(str) -> {None:'string_opts',int:'int_opts',float:'float_opts'}
 
     This is used to get the types of all the options in a string formatted
     with the conventions of DPyGetOpt. The 'type' None is used for options
     which are strings (they need no further conversion). This function's main
     use is to get a typemap for use with read_dict().
     """
 
     typeconv = {None:'',int:'',float:''}
     typemap = {'s':None,'i':int,'f':float}
     opt_re = re.compile(r'([\w]*)([^:=]*:?=?)([sif]?)')
 
     for w in ostr.split():
         oname,alias,otype = opt_re.match(w).groups()
         if otype == '' or alias == '!':   # simple switches are integers too
             otype = 'i'
         typeconv[typemap[otype]] += oname + ' '
     return typeconv
 
 #----------------------------------------------------------------------------
 def read_dict(filename,type_conv=None,**opt):
     r"""Read a dictionary of key=value pairs from an input file, optionally
     performing conversions on the resulting values.
 
     read_dict(filename,type_conv,**opt) -> dict
 
     Only one value per line is accepted, the format should be
      # optional comments are ignored
      key value\n
 
     Args:
 
       - type_conv: A dictionary specifying which keys need to be converted to
       which types. By default all keys are read as strings. This dictionary
       should have as its keys valid conversion functions for strings
       (int,long,float,complex, or your own).  The value for each key
       (converter) should be a whitespace separated string containing the names
       of all the entries in the file to be converted using that function. For
       keys to be left alone, use None as the conversion function (only needed
       with purge=1, see below).
 
       - opt: dictionary with extra options as below (default in parens)
 
         purge(0): if set to 1, all keys *not* listed in type_conv are purged out
         of the dictionary to be returned. If purge is going to be used, the
         set of keys to be left as strings also has to be explicitly specified
         using the (non-existent) conversion function None.
 
         fs(None): field separator. This is the key/value separator to be used
         when parsing the file. The None default means any whitespace [behavior
         of string.split()].
 
         strip(0): if 1, strip string values of leading/trailinig whitespace.
 
         warn(1): warning level if requested keys are not found in file.
           - 0: silently ignore.
           - 1: inform but proceed.
           - 2: raise KeyError exception.
 
         no_empty(0): if 1, remove keys with whitespace strings as a value.
 
         unique([]): list of keys (or space separated string) which can't be
         repeated. If one such key is found in the file, each new instance
         overwrites the previous one. For keys not listed here, the behavior is
         to make a list of all appearances.
 
     Example:
 
     If the input file test.ini contains (we put it in a string to keep the test
     self-contained):
 
     >>> test_ini = '''\
     ... i 3
     ... x 4.5
     ... y 5.5
     ... s hi ho'''
 
     Then we can use it as follows:
     >>> type_conv={int:'i',float:'x',None:'s'}
 
     >>> d = read_dict(test_ini)
 
     >>> sorted(d.items())
     [('i', '3'), ('s', 'hi ho'), ('x', '4.5'), ('y', '5.5')]
 
     >>> d = read_dict(test_ini,type_conv)
 
     >>> sorted(d.items())
     [('i', 3), ('s', 'hi ho'), ('x', 4.5), ('y', '5.5')]
 
     >>> d = read_dict(test_ini,type_conv,purge=True)
 
     >>> sorted(d.items())
     [('i', 3), ('s', 'hi ho'), ('x', 4.5)]
     """
 
     # starting config
     opt.setdefault('purge',0)
     opt.setdefault('fs',None)  # field sep defaults to any whitespace
     opt.setdefault('strip',0)
     opt.setdefault('warn',1)
     opt.setdefault('no_empty',0)
     opt.setdefault('unique','')
     if type(opt['unique']) in StringTypes:
         unique_keys = qw(opt['unique'])
     elif type(opt['unique']) in (types.TupleType,types.ListType):
         unique_keys = opt['unique']
     else:
         raise ValueError, 'Unique keys must be given as a string, List or Tuple'
 
     dict = {}
 
     # first read in table of values as strings
     if '\n' in filename:
         lines = filename.splitlines()
         file = None
     else:
         file = open(filename,'r')
         lines = file.readlines()
     for line in lines:
         line = line.strip()
         if len(line) and line[0]=='#': continue
         if len(line)>0:
             lsplit = line.split(opt['fs'],1)
             try:
                 key,val = lsplit
             except ValueError:
                 key,val = lsplit[0],''
             key = key.strip()
             if opt['strip']: val = val.strip()
             if val == "''" or val == '""': val = ''
             if opt['no_empty'] and (val=='' or val.isspace()):
                 continue
             # if a key is found more than once in the file, build a list
             # unless it's in the 'unique' list. In that case, last found in file
             # takes precedence. User beware.
             try:
                 if dict[key] and key in unique_keys:
                     dict[key] = val
                 elif type(dict[key]) is types.ListType:
                     dict[key].append(val)
                 else:
                     dict[key] = [dict[key],val]
             except KeyError:
                 dict[key] = val
     # purge if requested
     if opt['purge']:
         accepted_keys = qwflat(type_conv.values())
         for key in dict.keys():
             if key in accepted_keys: continue
             del(dict[key])
     # now convert if requested
     if type_conv==None: return dict
     conversions = type_conv.keys()
     try: conversions.remove(None)
     except: pass
     for convert in conversions:
         for val in qw(type_conv[convert]):
             try:
                 dict[val] = convert(dict[val])
             except KeyError,e:
                 if opt['warn'] == 0:
                     pass
                 elif opt['warn'] == 1:
                     print >>sys.stderr, 'Warning: key',val,\
                           'not found in file',filename
                 elif opt['warn'] == 2:
                     raise KeyError,e
                 else:
                     raise ValueError,'Warning level must be 0,1 or 2'
 
     return dict
 
 #----------------------------------------------------------------------------
 def flag_calls(func):
     """Wrap a function to detect and flag when it gets called.
 
     This is a decorator which takes a function and wraps it in a function with
     a 'called' attribute. wrapper.called is initialized to False.
 
     The wrapper.called attribute is set to False right before each call to the
     wrapped function, so if the call fails it remains False.  After the call
     completes, wrapper.called is set to True and the output is returned.
 
     Testing for truth in wrapper.called allows you to determine if a call to
     func() was attempted and succeeded."""
 
     def wrapper(*args,**kw):
         wrapper.called = False
         out = func(*args,**kw)
         wrapper.called = True
         return out
 
     wrapper.called = False
     wrapper.__doc__ = func.__doc__
     return wrapper
 
 #----------------------------------------------------------------------------
 def dhook_wrap(func,*a,**k):
     """Wrap a function call in a sys.displayhook controller.
 
     Returns a wrapper around func which calls func, with all its arguments and
     keywords unmodified, using the default sys.displayhook.  Since IPython
     modifies sys.displayhook, it breaks the behavior of certain systems that
     rely on the default behavior, notably doctest.
     """
 
     def f(*a,**k):
 
         dhook_s = sys.displayhook
         sys.displayhook = sys.__displayhook__
         try:
             out = func(*a,**k)
         finally:
             sys.displayhook = dhook_s
 
         return out
 
     f.__doc__ = func.__doc__
     return f
 
 #----------------------------------------------------------------------------
 def doctest_reload():
     """Properly reload doctest to reuse it interactively.
 
     This routine:
 
       - reloads doctest
 
       - resets its global 'master' attribute to None, so that multiple uses of
       the module interactively don't produce cumulative reports.
 
       - Monkeypatches its core test runner method to protect it from IPython's
       modified displayhook.  Doctest expects the default displayhook behavior
       deep down, so our modification breaks it completely.  For this reason, a
       hard monkeypatch seems like a reasonable solution rather than asking
       users to manually use a different doctest runner when under IPython."""
 
     import doctest
     reload(doctest)
     doctest.master=None
 
     try:
         doctest.DocTestRunner
     except AttributeError:
         # This is only for python 2.3 compatibility, remove once we move to
         # 2.4 only.
         pass
     else:
         doctest.DocTestRunner.run = dhook_wrap(doctest.DocTestRunner.run)
 
 #----------------------------------------------------------------------------
 class HomeDirError(Error):
     pass
 
 def get_home_dir():
     """Return the closest possible equivalent to a 'home' directory.
 
     We first try $HOME.  Absent that, on NT it's $HOMEDRIVE\$HOMEPATH.
 
     Currently only Posix and NT are implemented, a HomeDirError exception is
     raised for all other OSes. """
 
     isdir = os.path.isdir
     env = os.environ
 
     # first, check py2exe distribution root directory for _ipython.
     # This overrides all. Normally does not exist.
 
     if hasattr(sys, "frozen"): #Is frozen by py2exe
         if '\\library.zip\\' in IPython.__file__.lower():#libraries compressed to zip-file
             root, rest = IPython.__file__.lower().split('library.zip')
         else: 
             root=os.path.join(os.path.split(IPython.__file__)[0],"../../")
         root=os.path.abspath(root).rstrip('\\')
         if isdir(os.path.join(root, '_ipython')):
             os.environ["IPYKITROOT"] = root
         return root
     try:
         homedir = env['HOME']
         if not isdir(homedir):
             # in case a user stuck some string which does NOT resolve to a
             # valid path, it's as good as if we hadn't foud it
             raise KeyError
         return homedir
     except KeyError:
         if os.name == 'posix':
             raise HomeDirError,'undefined $HOME, IPython can not proceed.'
         elif os.name == 'nt':
             # For some strange reason, win9x returns 'nt' for os.name.
             try:
                 homedir = os.path.join(env['HOMEDRIVE'],env['HOMEPATH'])
                 if not isdir(homedir):
                     homedir = os.path.join(env['USERPROFILE'])
                     if not isdir(homedir):
                         raise HomeDirError
                 return homedir
             except KeyError:
                 try:
                     # Use the registry to get the 'My Documents' folder.
                     import _winreg as wreg
                     key = wreg.OpenKey(wreg.HKEY_CURRENT_USER,
                                        "Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders")
                     homedir = wreg.QueryValueEx(key,'Personal')[0]
                     key.Close()
                     if not isdir(homedir):
                         e = ('Invalid "Personal" folder registry key '
                              'typically "My Documents".\n'
                              'Value: %s\n'
                              'This is not a valid directory on your system.' %
                              homedir)
                         raise HomeDirError(e)
                     return homedir
                 except HomeDirError:
                     raise
                 except:
                     return 'C:\\'
         elif os.name == 'dos':
             # Desperate, may do absurd things in classic MacOS. May work under DOS.
             return 'C:\\'
         else:
             raise HomeDirError,'support for your operating system not implemented.'
 
 
 def get_ipython_dir():
     """Get the IPython directory for this platform and user.
     
     This uses the logic in `get_home_dir` to find the home directory
     and the adds either .ipython or _ipython to the end of the path.
     """
     if os.name == 'posix':
          ipdir_def = '.ipython'
     else:
          ipdir_def = '_ipython'
     home_dir = get_home_dir()
     ipdir = os.path.abspath(os.environ.get('IPYTHONDIR',
                                            os.path.join(home_dir, ipdir_def)))
     return ipdir.decode(sys.getfilesystemencoding())
 
 def get_security_dir():
     """Get the IPython security directory.
     
     This directory is the default location for all security related files,
     including SSL/TLS certificates and FURL files.
     
     If the directory does not exist, it is created with 0700 permissions.
     If it exists, permissions are set to 0700.
     """
     security_dir = os.path.join(get_ipython_dir(), 'security')
     if not os.path.isdir(security_dir):
         os.mkdir(security_dir, 0700)
     else:
         os.chmod(security_dir, 0700)
     return security_dir
-        
+
+def get_log_dir():
+    """Get the IPython log directory.
+    
+    If the log directory does not exist, it is created.
+    """
+    log_dir = os.path.join(get_ipython_dir(), 'log')
+    if not os.path.isdir(log_dir):
+        os.mkdir(log_dir, 0777)
+    return log_dir
+
 #****************************************************************************
 # strings and text
 
 class LSString(str):
     """String derivative with a special access attributes.
 
     These are normal strings, but with the special attributes:
 
         .l (or .list) : value as list (split on newlines).
         .n (or .nlstr): original value (the string itself).
         .s (or .spstr): value as whitespace-separated string.
         .p (or .paths): list of path objects
 
     Any values which require transformations are computed only once and
     cached.
 
     Such strings are very useful to efficiently interact with the shell, which
     typically only understands whitespace-separated options for commands."""
 
     def get_list(self):
         try:
             return self.__list
         except AttributeError:
             self.__list = self.split('\n')
             return self.__list
 
     l = list = property(get_list)
 
     def get_spstr(self):
         try:
             return self.__spstr
         except AttributeError:
             self.__spstr = self.replace('\n',' ')
             return self.__spstr
 
     s = spstr = property(get_spstr)
 
     def get_nlstr(self):
         return self
 
     n = nlstr = property(get_nlstr)
 
     def get_paths(self):
         try:
             return self.__paths
         except AttributeError:
             self.__paths = [path(p) for p in self.split('\n') if os.path.exists(p)]
             return self.__paths
 
     p = paths = property(get_paths)
 
 def print_lsstring(arg):
     """ Prettier (non-repr-like) and more informative printer for LSString """
     print "LSString (.p, .n, .l, .s available). Value:"
     print arg
 
 print_lsstring = result_display.when_type(LSString)(print_lsstring)
 
 #----------------------------------------------------------------------------
 class SList(list):
     """List derivative with a special access attributes.
 
     These are normal lists, but with the special attributes:
 
         .l (or .list) : value as list (the list itself).
         .n (or .nlstr): value as a string, joined on newlines.
         .s (or .spstr): value as a string, joined on spaces.
         .p (or .paths): list of path objects
 
     Any values which require transformations are computed only once and
     cached."""
 
     def get_list(self):
         return self
 
     l = list = property(get_list)
 
     def get_spstr(self):
         try:
             return self.__spstr
         except AttributeError:
             self.__spstr = ' '.join(self)
             return self.__spstr
 
     s = spstr = property(get_spstr)
 
     def get_nlstr(self):
         try:
             return self.__nlstr
         except AttributeError:
             self.__nlstr = '\n'.join(self)
             return self.__nlstr
 
     n = nlstr = property(get_nlstr)
 
     def get_paths(self):
         try:
             return self.__paths
         except AttributeError:
             self.__paths = [path(p) for p in self if os.path.exists(p)]
             return self.__paths
 
     p = paths = property(get_paths)
 
     def grep(self, pattern, prune = False, field = None):
         """ Return all strings matching 'pattern' (a regex or callable)
 
         This is case-insensitive. If prune is true, return all items
         NOT matching the pattern.
 
         If field is specified, the match must occur in the specified
         whitespace-separated field.
 
         Examples::
 
             a.grep( lambda x: x.startswith('C') )
             a.grep('Cha.*log', prune=1)
             a.grep('chm', field=-1)
         """
 
         def match_target(s):
             if field is None:
                 return s
             parts = s.split()
             try:
                 tgt = parts[field]
                 return tgt
             except IndexError:
                 return ""
 
         if isinstance(pattern, basestring):
             pred = lambda x : re.search(pattern, x, re.IGNORECASE)
         else:
             pred = pattern
         if not prune:
             return SList([el for el in self if pred(match_target(el))])
         else:
             return SList([el for el in self if not pred(match_target(el))])
     def fields(self, *fields):
         """ Collect whitespace-separated fields from string list
 
         Allows quick awk-like usage of string lists.
 
         Example data (in var a, created by 'a = !ls -l')::
             -rwxrwxrwx  1 ville None      18 Dec 14  2006 ChangeLog
             drwxrwxrwx+ 6 ville None       0 Oct 24 18:05 IPython
 
         a.fields(0) is ['-rwxrwxrwx', 'drwxrwxrwx+']
         a.fields(1,0) is ['1 -rwxrwxrwx', '6 drwxrwxrwx+']
         (note the joining by space).
         a.fields(-1) is ['ChangeLog', 'IPython']
 
         IndexErrors are ignored.
 
         Without args, fields() just split()'s the strings.
         """
         if len(fields) == 0:
             return [el.split() for el in self]
 
         res = SList()
         for el in [f.split() for f in self]:
             lineparts = []
 
             for fd in fields:
                 try:
                     lineparts.append(el[fd])
                 except IndexError:
                     pass
             if lineparts:
                 res.append(" ".join(lineparts))
 
         return res
     def sort(self,field= None,  nums = False):
         """ sort by specified fields (see fields())
 
         Example::
             a.sort(1, nums = True)
 
         Sorts a by second field, in numerical order (so that 21 > 3)
 
         """
 
         #decorate, sort, undecorate
         if field is not None:
             dsu = [[SList([line]).fields(field),  line] for line in self]
         else:
             dsu = [[line,  line] for line in self]
         if nums:
             for i in range(len(dsu)):
                 numstr = "".join([ch for ch in dsu[i][0] if ch.isdigit()])
                 try:
                     n = int(numstr)
                 except ValueError:
                     n = 0;
                 dsu[i][0] = n
 
 
         dsu.sort()
         return SList([t[1] for t in dsu])
 
 def print_slist(arg):
     """ Prettier (non-repr-like) and more informative printer for SList """
     print "SList (.p, .n, .l, .s, .grep(), .fields(), sort() available):"
     if hasattr(arg,  'hideonce') and arg.hideonce:
         arg.hideonce = False
         return
 
     nlprint(arg)
 
 print_slist = result_display.when_type(SList)(print_slist)
 
 
 
 #----------------------------------------------------------------------------
 def esc_quotes(strng):
     """Return the input string with single and double quotes escaped out"""
 
     return strng.replace('"','\\"').replace("'","\\'")
 
 #----------------------------------------------------------------------------
 def make_quoted_expr(s):
     """Return string s in appropriate quotes, using raw string if possible.
 
     XXX - example removed because it caused encoding errors in documentation
     generation.  We need a new example that doesn't contain invalid chars.
 
     Note the use of raw string and padding at the end to allow trailing
     backslash.
     """
 
     tail = ''
     tailpadding = ''
     raw  = ''
     if "\\" in s:
         raw = 'r'
         if s.endswith('\\'):
             tail = '[:-1]'
             tailpadding = '_'
     if '"' not in s:
         quote = '"'
     elif "'" not in s:
         quote = "'"
     elif '"""' not in s and not s.endswith('"'):
         quote = '"""'
     elif "'''" not in s and not s.endswith("'"):
         quote = "'''"
     else:
         # give up, backslash-escaped string will do
         return '"%s"' % esc_quotes(s)
     res = raw + quote + s + tailpadding + quote + tail
     return res
 
 
 #----------------------------------------------------------------------------
 def raw_input_multi(header='', ps1='==> ', ps2='..> ',terminate_str = '.'):
     """Take multiple lines of input.
 
     A list with each line of input as a separate element is returned when a
     termination string is entered (defaults to a single '.'). Input can also
     terminate via EOF (^D in Unix, ^Z-RET in Windows).
 
     Lines of input which end in \\ are joined into single entries (and a
     secondary continuation prompt is issued as long as the user terminates
     lines with \\). This allows entering very long strings which are still
     meant to be treated as single entities.
     """
 
     try:
         if header:
             header += '\n'
         lines = [raw_input(header + ps1)]
     except EOFError:
         return []
     terminate = [terminate_str]
     try:
         while lines[-1:] != terminate:
             new_line = raw_input(ps1)
             while new_line.endswith('\\'):
                 new_line = new_line[:-1] + raw_input(ps2)
             lines.append(new_line)
 
         return lines[:-1]  # don't return the termination command
     except EOFError:
         print
         return lines
 
 #----------------------------------------------------------------------------
 def raw_input_ext(prompt='',  ps2='... '):
     """Similar to raw_input(), but accepts extended lines if input ends with \\."""
 
     line = raw_input(prompt)
     while line.endswith('\\'):
         line = line[:-1] + raw_input(ps2)
     return line
 
 #----------------------------------------------------------------------------
 def ask_yes_no(prompt,default=None):
     """Asks a question and returns a boolean (y/n) answer.
 
     If default is given (one of 'y','n'), it is used if the user input is
     empty. Otherwise the question is repeated until an answer is given.
 
     An EOF is treated as the default answer.  If there is no default, an
     exception is raised to prevent infinite loops.
 
     Valid answers are: y/yes/n/no (match is not case sensitive)."""
 
     answers = {'y':True,'n':False,'yes':True,'no':False}
     ans = None
     while ans not in answers.keys():
         try:
             ans = raw_input(prompt+' ').lower()
             if not ans:  # response was an empty string
                 ans = default
         except KeyboardInterrupt:
             pass
         except EOFError:
             if default in answers.keys():
                 ans = default
                 print
             else:
                 raise
 
     return answers[ans]
 
 #----------------------------------------------------------------------------
 def marquee(txt='',width=78,mark='*'):
     """Return the input string centered in a 'marquee'."""
     if not txt:
         return (mark*width)[:width]
     nmark = (width-len(txt)-2)/len(mark)/2
     if nmark < 0: nmark =0
     marks = mark*nmark
     return '%s %s %s' % (marks,txt,marks)
 
 #----------------------------------------------------------------------------
 class EvalDict:
     """
     Emulate a dict which evaluates its contents in the caller's frame.
 
     Usage:
     >>> number = 19
 
     >>> text = "python"
 
     >>> print "%(text.capitalize())s %(number/9.0).1f rules!" % EvalDict()
     Python 2.1 rules!
     """
 
     # This version is due to [e-mail] on c.l.py, and is basically a
     # modified (shorter) version of:
     # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/66018 by
     # Skip Montanaro ([e-mail]).
 
     def __getitem__(self, name):
         frame = sys._getframe(1)
         return eval(name, frame.f_globals, frame.f_locals)
 
 EvalString = EvalDict  # for backwards compatibility
 #----------------------------------------------------------------------------
 def qw(words,flat=0,sep=None,maxsplit=-1):
     """Similar to Perl's qw() operator, but with some more options.
 
     qw(words,flat=0,sep=' ',maxsplit=-1) -> words.split(sep,maxsplit)
 
     words can also be a list itself, and with flat=1, the output will be
     recursively flattened.
 
     Examples:
 
     >>> qw('1 2')
     ['1', '2']
 
     >>> qw(['a b','1 2',['m n','p q']])
     [['a', 'b'], ['1', '2'], [['m', 'n'], ['p', 'q']]]
 
     >>> qw(['a b','1 2',['m n','p q']],flat=1)
     ['a', 'b', '1', '2', 'm', 'n', 'p', 'q']
     """
 
     if type(words) in StringTypes:
         return [word.strip() for word in words.split(sep,maxsplit)
                 if word and not word.isspace() ]
     if flat:
         return flatten(map(qw,words,[1]*len(words)))
     return map(qw,words)
 
 #----------------------------------------------------------------------------
 def qwflat(words,sep=None,maxsplit=-1):
     """Calls qw(words) in flat mode. It's just a convenient shorthand."""
     return qw(words,1,sep,maxsplit)
 
 #----------------------------------------------------------------------------
 def qw_lol(indata):
     """qw_lol('a b') -> [['a','b']],
     otherwise it's just a call to qw().
 
     We need this to make sure the modules_some keys *always* end up as a
     list of lists."""
 
     if type(indata) in StringTypes:
         return [qw(indata)]
     else:
         return qw(indata)
 
 #-----------------------------------------------------------------------------
 def list_strings(arg):
     """Always return a list of strings, given a string or list of strings
     as input."""
 
     if type(arg) in StringTypes: return [arg]
     else: return arg
 
 #----------------------------------------------------------------------------
 def grep(pat,list,case=1):
     """Simple minded grep-like function.
     grep(pat,list) returns occurrences of pat in list, None on failure.
 
     It only does simple string matching, with no support for regexps. Use the
     option case=0 for case-insensitive matching."""
 
     # This is pretty crude. At least it should implement copying only references
     # to the original data in case it's big. Now it copies the data for output.
     out=[]
     if case:
         for term in list:
             if term.find(pat)>-1: out.append(term)
     else:
         lpat=pat.lower()
         for term in list:
             if term.lower().find(lpat)>-1: out.append(term)
 
     if len(out): return out
     else: return None
 
 #----------------------------------------------------------------------------
 def dgrep(pat,*opts):
     """Return grep() on dir()+dir(__builtins__).
 
     A very common use of grep() when working interactively."""
 
     return grep(pat,dir(__main__)+dir(__main__.__builtins__),*opts)
 
 #----------------------------------------------------------------------------
 def idgrep(pat):
     """Case-insensitive dgrep()"""
 
     return dgrep(pat,0)
 
 #----------------------------------------------------------------------------
 def igrep(pat,list):
     """Synonym for case-insensitive grep."""
 
     return grep(pat,list,case=0)
 
 #----------------------------------------------------------------------------
 def indent(str,nspaces=4,ntabs=0):
     """Indent a string a given number of spaces or tabstops.
 
     indent(str,nspaces=4,ntabs=0) -> indent str by ntabs+nspaces.
     """
     if str is None:
         return
     ind = '\t'*ntabs+' '*nspaces
     outstr = '%s%s' % (ind,str.replace(os.linesep,os.linesep+ind))
     if outstr.endswith(os.linesep+ind):
         return outstr[:-len(ind)]
     else:
         return outstr
 
 #-----------------------------------------------------------------------------
 def native_line_ends(filename,backup=1):
     """Convert (in-place) a file to line-ends native to the current OS.
 
     If the optional backup argument is given as false, no backup of the
     original file is left.  """
 
     backup_suffixes = {'posix':'~','dos':'.bak','nt':'.bak','mac':'.bak'}
 
     bak_filename = filename + backup_suffixes[os.name]
 
     original = open(filename).read()
     shutil.copy2(filename,bak_filename)
     try:
         new = open(filename,'wb')
         new.write(os.linesep.join(original.splitlines()))
         new.write(os.linesep) # ALWAYS put an eol at the end of the file
         new.close()
     except:
         os.rename(bak_filename,filename)
     if not backup:
         try:
             os.remove(bak_filename)
         except:
             pass
 
 #----------------------------------------------------------------------------
 def get_pager_cmd(pager_cmd = None):
     """Return a pager command.
 
     Makes some attempts at finding an OS-correct one."""
 
     if os.name == 'posix':
         default_pager_cmd = 'less -r'  # -r for color control sequences
     elif os.name in ['nt','dos']:
         default_pager_cmd = 'type'
 
     if pager_cmd is None:
         try:
             pager_cmd = os.environ['PAGER']
         except:
             pager_cmd = default_pager_cmd
     return pager_cmd
 
 #-----------------------------------------------------------------------------
 def get_pager_start(pager,start):
     """Return the string for paging files with an offset.
 
     This is the '+N' argument which less and more (under Unix) accept.
     """
 
     if pager in ['less','more']:
         if start:
             start_string = '+' + str(start)
         else:
             start_string = ''
     else:
         start_string = ''
     return start_string
 
 #----------------------------------------------------------------------------
 # (X)emacs on W32 doesn't like to be bypassed with msvcrt.getch()
 if os.name == 'nt' and os.environ.get('TERM','dumb') != 'emacs':
     import msvcrt
     def page_more():
         """ Smart pausing between pages
 
         @return:    True if need print more lines, False if quit
         """
         Term.cout.write('---Return to continue, q to quit--- ')
         ans = msvcrt.getch()
         if ans in ("q", "Q"):
             result = False
         else:
             result = True
         Term.cout.write("\b"*37 + " "*37 + "\b"*37)
         return result
 else:
     def page_more():
         ans = raw_input('---Return to continue, q to quit--- ')
         if ans.lower().startswith('q'):
             return False
         else:
             return True
 
 esc_re = re.compile(r"(\x1b[^m]+m)")
 
 def page_dumb(strng,start=0,screen_lines=25):
     """Very dumb 'pager' in Python, for when nothing else works.
 
     Only moves forward, same interface as page(), except for pager_cmd and
     mode."""
 
     out_ln  = strng.splitlines()[start:]
     screens = chop(out_ln,screen_lines-1)
     if len(screens) == 1:
         print >>Term.cout, os.linesep.join(screens[0])
     else:
         last_escape = ""
         for scr in screens[0:-1]:
             hunk = os.linesep.join(scr)
             print >>Term.cout, last_escape + hunk
             if not page_more():
                 return
             esc_list = esc_re.findall(hunk)
             if len(esc_list) > 0:
                 last_escape = esc_list[-1]
         print >>Term.cout, last_escape + os.linesep.join(screens[-1])
 
 #----------------------------------------------------------------------------
 def page(strng,start=0,screen_lines=0,pager_cmd = None):
     """Print a string, piping through a pager after a certain length.
 
     The screen_lines parameter specifies the number of *usable* lines of your
     terminal screen (total lines minus lines you need to reserve to show other
     information).
 
     If you set screen_lines to a number <=0, page() will try to auto-determine
     your screen size and will only use up to (screen_size+screen_lines) for
     printing, paging after that. That is, if you want auto-detection but need
     to reserve the bottom 3 lines of the screen, use screen_lines = -3, and for
     auto-detection without any lines reserved simply use screen_lines = 0.
 
     If a string won't fit in the allowed lines, it is sent through the
     specified pager command. If none given, look for PAGER in the environment,
     and ultimately default to less.
 
     If no system pager works, the string is sent through a 'dumb pager'
     written in python, very simplistic.
     """
 
     # Some routines may auto-compute start offsets incorrectly and pass a
     # negative value.  Offset to 0 for robustness.
     start = max(0,start)
 
     # first, try the hook
     ip = IPython.ipapi.get()
     if ip:
         try:
             ip.IP.hooks.show_in_pager(strng)
             return
         except IPython.ipapi.TryNext:
             pass
 
     # Ugly kludge, but calling curses.initscr() flat out crashes in emacs
     TERM = os.environ.get('TERM','dumb')
     if TERM in ['dumb','emacs'] and os.name != 'nt':
         print strng
         return
     # chop off the topmost part of the string we don't want to see
     str_lines = strng.split(os.linesep)[start:]
     str_toprint = os.linesep.join(str_lines)
     num_newlines = len(str_lines)
     len_str = len(str_toprint)
 
     # Dumb heuristics to guesstimate number of on-screen lines the string
     # takes.  Very basic, but good enough for docstrings in reasonable
     # terminals. If someone later feels like refining it, it's not hard.
     numlines = max(num_newlines,int(len_str/80)+1)
 
     if os.name == "nt":
         screen_lines_def = get_console_size(defaulty=25)[1]
     else:
         screen_lines_def = 25 # default value if we can't auto-determine
 
     # auto-determine screen size
     if screen_lines <= 0:
         if TERM=='xterm':
             use_curses = USE_CURSES
         else:
             # curses causes problems on many terminals other than xterm.
             use_curses = False
         if use_curses:
             # There is a bug in curses, where *sometimes* it fails to properly
             # initialize, and then after the endwin() call is made, the
             # terminal is left in an unusable state.  Rather than trying to
             # check everytime for this (by requesting and comparing termios
             # flags each time), we just save the initial terminal state and
             # unconditionally reset it every time.  It's cheaper than making
             # the checks.
             term_flags = termios.tcgetattr(sys.stdout)
             scr = curses.initscr()
             screen_lines_real,screen_cols = scr.getmaxyx()
             curses.endwin()
             # Restore terminal state in case endwin() didn't.
             termios.tcsetattr(sys.stdout,termios.TCSANOW,term_flags)
             # Now we have what we needed: the screen size in rows/columns
             screen_lines += screen_lines_real
             #print '***Screen size:',screen_lines_real,'lines x',\
             #screen_cols,'columns.' # dbg
         else:
             screen_lines += screen_lines_def
 
     #print 'numlines',numlines,'screenlines',screen_lines  # dbg
     if numlines <= screen_lines :
         #print '*** normal print'  # dbg
         print >>Term.cout, str_toprint
     else:
         # Try to open pager and default to internal one if that fails.
         # All failure modes are tagged as 'retval=1', to match the return
         # value of a failed system command.  If any intermediate attempt
         # sets retval to 1, at the end we resort to our own page_dumb() pager.
         pager_cmd = get_pager_cmd(pager_cmd)
         pager_cmd += ' ' + get_pager_start(pager_cmd,start)
         if os.name == 'nt':
             if pager_cmd.startswith('type'):
                 # The default WinXP 'type' command is failing on complex strings.
                 retval = 1
             else:
                 tmpname = tempfile.mktemp('.txt')
                 tmpfile = file(tmpname,'wt')
                 tmpfile.write(strng)
                 tmpfile.close()
                 cmd = "%s < %s" % (pager_cmd,tmpname)
                 if os.system(cmd):
                   retval = 1
                 else:
                   retval = None
                 os.remove(tmpname)
         else:
             try:
                 retval = None
                 # if I use popen4, things hang. No idea why.
                 #pager,shell_out = os.popen4(pager_cmd)
                 pager = os.popen(pager_cmd,'w')
                 pager.write(strng)
                 pager.close()
                 retval = pager.close()  # success returns None
             except IOError,msg:  # broken pipe when user quits
                 if msg.args == (32,'Broken pipe'):
                     retval = None
                 else:
                     retval = 1
             except OSError:
                 # Other strange problems, sometimes seen in Win2k/cygwin
                 retval = 1
         if retval is not None:
             page_dumb(strng,screen_lines=screen_lines)
 
 #----------------------------------------------------------------------------
 def page_file(fname,start = 0, pager_cmd = None):
     """Page a file, using an optional pager command and starting line.
     """
 
     pager_cmd = get_pager_cmd(pager_cmd)
     pager_cmd += ' ' + get_pager_start(pager_cmd,start)
 
     try:
         if os.environ['TERM'] in ['emacs','dumb']:
             raise EnvironmentError
         xsys(pager_cmd + ' ' + fname)
     except:
         try:
             if start > 0:
                 start -= 1
             page(open(fname).read(),start)
         except:
             print 'Unable to show file',`fname`
 
 
 #----------------------------------------------------------------------------
 def snip_print(str,width = 75,print_full = 0,header = ''):
     """Print a string snipping the midsection to fit in width.
 
     print_full: mode control:
       - 0: only snip long strings
       - 1: send to page() directly.
       - 2: snip long strings and ask for full length viewing with page()
     Return 1 if snipping was necessary, 0 otherwise."""
 
     if print_full == 1:
         page(header+str)
         return 0
 
     print header,
     if len(str) < width:
         print str
         snip = 0
     else:
         whalf = int((width -5)/2)
         print str[:whalf] + ' <...> ' + str[-whalf:]
         snip = 1
     if snip and print_full == 2:
         if raw_input(header+' Snipped. View (y/n)? [N]').lower() == 'y':
             page(str)
     return snip
 
 #****************************************************************************
 # lists, dicts and structures
 
 def belong(candidates,checklist):
     """Check whether a list of items appear in a given list of options.
 
     Returns a list of 1 and 0, one for each candidate given."""
 
     return [x in checklist for x in candidates]
 
 #----------------------------------------------------------------------------
 def uniq_stable(elems):
     """uniq_stable(elems) -> list
 
     Return from an iterable, a list of all the unique elements in the input,
     but maintaining the order in which they first appear.
 
     A naive solution to this problem which just makes a dictionary with the
     elements as keys fails to respect the stability condition, since
     dictionaries are unsorted by nature.
 
     Note: All elements in the input must be valid dictionary keys for this
     routine to work, as it internally uses a dictionary for efficiency
     reasons."""
 
     unique = []
     unique_dict = {}
     for nn in elems:
         if nn not in unique_dict:
             unique.append(nn)
             unique_dict[nn] = None
     return unique
 
 #----------------------------------------------------------------------------
 class NLprinter:
     """Print an arbitrarily nested list, indicating index numbers.
 
     An instance of this class called nlprint is available and callable as a
     function.
 
     nlprint(list,indent=' ',sep=': ') -> prints indenting each level by 'indent'
     and using 'sep' to separate the index from the value. """
 
     def __init__(self):
         self.depth = 0
 
     def __call__(self,lst,pos='',**kw):
         """Prints the nested list numbering levels."""
         kw.setdefault('indent',' ')
         kw.setdefault('sep',': ')
         kw.setdefault('start',0)
         kw.setdefault('stop',len(lst))
         # we need to remove start and stop from kw so they don't propagate
         # into a recursive call for a nested list.
         start = kw['start']; del kw['start']
         stop = kw['stop']; del kw['stop']
         if self.depth == 0 and 'header' in kw.keys():
             print kw['header']
 
         for idx in range(start,stop):
             elem = lst[idx]
             if type(elem)==type([]):
                 self.depth += 1
                 self.__call__(elem,itpl('$pos$idx,'),**kw)
                 self.depth -= 1
             else:
                 printpl(kw['indent']*self.depth+'$pos$idx$kw["sep"]$elem')
 
 nlprint = NLprinter()
 #----------------------------------------------------------------------------
 def all_belong(candidates,checklist):
     """Check whether a list of items ALL appear in a given list of options.
 
     Returns a single 1 or 0 value."""
 
     return 1-(0 in [x in checklist for x in candidates])
 
 #----------------------------------------------------------------------------
 def sort_compare(lst1,lst2,inplace = 1):
     """Sort and compare two lists.
 
     By default it does it in place, thus modifying the lists. Use inplace = 0
     to avoid that (at the cost of temporary copy creation)."""
     if not inplace:
         lst1 = lst1[:]
         lst2 = lst2[:]
     lst1.sort(); lst2.sort()
     return lst1 == lst2
 
 #----------------------------------------------------------------------------
 def list2dict(lst):
     """Takes a list of (key,value) pairs and turns it into a dict."""
 
     dic = {}
     for k,v in lst: dic[k] = v
     return dic
 
 #----------------------------------------------------------------------------
 def list2dict2(lst,default=''):
     """Takes a list and turns it into a dict.
     Much slower than list2dict, but more versatile. This version can take
     lists with sublists of arbitrary length (including sclars)."""
 
     dic = {}
     for elem in lst:
         if type(elem) in (types.ListType,types.TupleType):
             size = len(elem)
             if  size == 0:
                 pass
             elif size == 1:
                 dic[elem] = default
             else:
                 k,v = elem[0], elem[1:]
                 if len(v) == 1: v = v[0]
                 dic[k] = v
         else:
             dic[elem] = default
     return dic
 
 #----------------------------------------------------------------------------
 def flatten(seq):
     """Flatten a list of lists (NOT recursive, only works for 2d lists)."""
 
     return [x for subseq in seq for x in subseq]
 
 #----------------------------------------------------------------------------
 def get_slice(seq,start=0,stop=None,step=1):
     """Get a slice of a sequence with variable step. Specify start,stop,step."""
     if stop == None:
         stop = len(seq)
     item = lambda i: seq[i]
     return map(item,xrange(start,stop,step))
 
 #----------------------------------------------------------------------------
 def chop(seq,size):
     """Chop a sequence into chunks of the given size."""
     chunk = lambda i: seq[i:i+size]
     return map(chunk,xrange(0,len(seq),size))
 
 #----------------------------------------------------------------------------
 # with is a keyword as of python 2.5, so this function is renamed to withobj
 # from its old 'with' name.
 def with_obj(object, **args):
     """Set multiple attributes for an object, similar to Pascal's with.
 
     Example:
     with_obj(jim,
              born = 1960,
              haircolour = 'Brown',
              eyecolour = 'Green')
 
     Credit: Greg Ewing, in
     http://mail.python.org/pipermail/python-list/2001-May/040703.html.
 
     NOTE: up until IPython 0.7.2, this was called simply 'with', but 'with'
     has become a keyword for Python 2.5, so we had to rename it."""
 
     object.__dict__.update(args)
 
 #----------------------------------------------------------------------------
 def setattr_list(obj,alist,nspace = None):
     """Set a list of attributes for an object taken from a namespace.
 
     setattr_list(obj,alist,nspace) -> sets in obj all the attributes listed in
     alist with their values taken from nspace, which must be a dict (something
     like locals() will often do) If nspace isn't given, locals() of the
     *caller* is used, so in most cases you can omit it.
 
     Note that alist can be given as a string, which will be automatically
     split into a list on whitespace. If given as a list, it must be a list of
     *strings* (the variable names themselves), not of variables."""
 
     # this grabs the local variables from the *previous* call frame -- that is
     # the locals from the function that called setattr_list().
     # - snipped from weave.inline()
     if nspace is None:
         call_frame = sys._getframe().f_back
         nspace = call_frame.f_locals
 
     if type(alist) in StringTypes:
         alist = alist.split()
     for attr in alist:
         val = eval(attr,nspace)
         setattr(obj,attr,val)
 
 #----------------------------------------------------------------------------
 def getattr_list(obj,alist,*args):
     """getattr_list(obj,alist[, default]) -> attribute list.
 
     Get a list of named attributes for an object. When a default argument is
     given, it is returned when the attribute doesn't exist; without it, an
     exception is raised in that case.
 
     Note that alist can be given as a string, which will be automatically
     split into a list on whitespace. If given as a list, it must be a list of
     *strings* (the variable names themselves), not of variables."""
 
     if type(alist) in StringTypes:
         alist = alist.split()
     if args:
         if len(args)==1:
             default = args[0]
             return map(lambda attr: getattr(obj,attr,default),alist)
         else:
             raise ValueError,'getattr_list() takes only one optional argument'
     else:
         return map(lambda attr: getattr(obj,attr),alist)
 
 #----------------------------------------------------------------------------
 def map_method(method,object_list,*argseq,**kw):
     """map_method(method,object_list,*args,**kw) -> list
 
     Return a list of the results of applying the methods to the items of the
     argument sequence(s).  If more than one sequence is given, the method is
     called with an argument list consisting of the corresponding item of each
     sequence. All sequences must be of the same length.
 
     Keyword arguments are passed verbatim to all objects called.
 
     This is Python code, so it's not nearly as fast as the builtin map()."""
 
     out_list = []
     idx = 0
     for object in object_list:
         try:
             handler = getattr(object, method)
         except AttributeError:
             out_list.append(None)
         else:
             if argseq:
                 args = map(lambda lst:lst[idx],argseq)
                 #print 'ob',object,'hand',handler,'ar',args # dbg
                 out_list.append(handler(args,**kw))
             else:
                 out_list.append(handler(**kw))
         idx += 1
     return out_list
 
 #----------------------------------------------------------------------------
 def get_class_members(cls):
     ret = dir(cls)
     if hasattr(cls,'__bases__'):
         for base in cls.__bases__:
             ret.extend(get_class_members(base))
     return ret
 
 #----------------------------------------------------------------------------
 def dir2(obj):
     """dir2(obj) -> list of strings
 
     Extended version of the Python builtin dir(), which does a few extra
     checks, and supports common objects with unusual internals that confuse
     dir(), such as Traits and PyCrust.
 
     This version is guaranteed to return only a list of true strings, whereas
     dir() returns anything that objects inject into themselves, even if they
     are later not really valid for attribute access (many extension libraries
     have such bugs).
     """
 
     # Start building the attribute list via dir(), and then complete it
     # with a few extra special-purpose calls.
     words = dir(obj)
 
     if hasattr(obj,'__class__'):
         words.append('__class__')
         words.extend(get_class_members(obj.__class__))
     #if '__base__' in words: 1/0
 
     # Some libraries (such as traits) may introduce duplicates, we want to
     # track and clean this up if it happens
     may_have_dupes = False
 
     # this is the 'dir' function for objects with Enthought's traits
     if hasattr(obj, 'trait_names'):
         try:
             words.extend(obj.trait_names())
             may_have_dupes = True
         except TypeError:
             # This will happen if `obj` is a class and not an instance.
             pass
 
     # Support for PyCrust-style _getAttributeNames magic method.
     if hasattr(obj, '_getAttributeNames'):
         try:
             words.extend(obj._getAttributeNames())
             may_have_dupes = True
         except TypeError:
             # `obj` is a class and not an instance.  Ignore
             # this error.
             pass
 
     if may_have_dupes:
         # eliminate possible duplicates, as some traits may also
         # appear as normal attributes in the dir() call.
         words = list(set(words))
         words.sort()
 
     # filter out non-string attributes which may be stuffed by dir() calls
     # and poor coding in third-party modules
     return [w for w in words if isinstance(w, basestring)]
 
 #----------------------------------------------------------------------------
 def import_fail_info(mod_name,fns=None):
     """Inform load failure for a module."""
 
     if fns == None:
         warn("Loading of %s failed.\n" % (mod_name,))
     else:
         warn("Loading of %s from %s failed.\n" % (fns,mod_name))
 
 #----------------------------------------------------------------------------
 # Proposed popitem() extension, written as a method
 
 
 class NotGiven: pass
 
 def popkey(dct,key,default=NotGiven):
     """Return dct[key] and delete dct[key].
 
     If default is given, return it if dct[key] doesn't exist, otherwise raise
     KeyError.  """
 
     try:
         val = dct[key]
     except KeyError:
         if default is NotGiven:
             raise
         else:
             return default
     else:
         del dct[key]
         return val
 
 def wrap_deprecated(func, suggest = '<nothing>'):
     def newFunc(*args, **kwargs):
         warnings.warn("Call to deprecated function %s, use %s instead" %
                       ( func.__name__, suggest),
                       category=DeprecationWarning,
                       stacklevel = 2)
         return func(*args, **kwargs)
     return newFunc
 
 
 def _num_cpus_unix():
     """Return the number of active CPUs on a Unix system."""
     return os.sysconf("SC_NPROCESSORS_ONLN")
 
 
 def _num_cpus_darwin():
     """Return the number of active CPUs on a Darwin system."""
     p = subprocess.Popen(['sysctl','-n','hw.ncpu'],stdout=subprocess.PIPE)
     return p.stdout.read()
 
 
 def _num_cpus_windows():
     """Return the number of active CPUs on a Windows system."""
     return os.environ.get("NUMBER_OF_PROCESSORS")
 
 
 def num_cpus():
    """Return the effective number of CPUs in the system as an integer.
 
    This cross-platform function makes an attempt at finding the total number of
    available CPUs in the system, as returned by various underlying system and
    python calls.
 
    If it can't find a sensible answer, it returns 1 (though an error *may* make
    it return a large positive number that's actually incorrect).
    """
 
    # Many thanks to the Parallel Python project (http://www.parallelpython.com)
    # for the names of the keys we needed to look up for this function.  This
    # code was inspired by their equivalent function.
 
    ncpufuncs = {'Linux':_num_cpus_unix,
                 'Darwin':_num_cpus_darwin,
                 'Windows':_num_cpus_windows,
                 # On Vista, python < 2.5.2 has a bug and returns 'Microsoft'
                 # See http://bugs.python.org/issue1082 for details.
                 'Microsoft':_num_cpus_windows,
                 }
 
    ncpufunc = ncpufuncs.get(platform.system(),
                             # default to unix version (Solaris, AIX, etc)
                             _num_cpus_unix)
 
    try:
        ncpus = max(1,int(ncpufunc()))
    except:
        ncpus = 1
    return ncpus
 
 #*************************** end of file <genutils.py> **********************

IPython/iplib.py

@@ -1,2865 +1,2870 b''
 # -*- coding: utf-8 -*-
 """
 IPython -- An enhanced Interactive Python
 
 Requires Python 2.4 or newer.
 
 This file contains all the classes and helper functions specific to IPython.
 """
 
 #*****************************************************************************
 #       Copyright (C) 2001 Janko Hauser <[e-mail]> and
 #       Copyright (C) 2001-2006 Fernando Perez. <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #
 # Note: this code originally subclassed code.InteractiveConsole from the
 # Python standard library.  Over time, all of that class has been copied
 # verbatim here for modifications which could not be accomplished by
 # subclassing.  At this point, there are no dependencies at all on the code
 # module anymore (it is not even imported).  The Python License (sec. 2)
 # allows for this, but it's always nice to acknowledge credit where credit is
 # due.
 #*****************************************************************************
 
 #****************************************************************************
 # Modules and globals
 
 # Python standard modules
 import __main__
 import __builtin__
 import StringIO
 import bdb
 import cPickle as pickle
 import codeop
 import exceptions
 import glob
 import inspect
 import keyword
 import new
 import os
 import pydoc
 import re
 import shutil
 import string
 import sys
 import tempfile
 import traceback
 import types
 from pprint import pprint, pformat
 
 # IPython's own modules
 #import IPython
 from IPython import Debugger,OInspect,PyColorize,ultraTB
 from IPython.ColorANSI import ColorScheme,ColorSchemeTable  # too long names
 from IPython.Extensions import pickleshare
 from IPython.FakeModule import FakeModule, init_fakemod_dict
 from IPython.Itpl import Itpl,itpl,printpl,ItplNS,itplns
 from IPython.Logger import Logger
 from IPython.Magic import Magic
 from IPython.Prompts import CachedOutput
 from IPython.ipstruct import Struct
 from IPython.background_jobs import BackgroundJobManager
 from IPython.usage import cmd_line_usage,interactive_usage
 from IPython.genutils import *
 from IPython.strdispatch import StrDispatch
 import IPython.ipapi
 import IPython.history
 import IPython.prefilter as prefilter
 import IPython.shadowns
 # Globals
 
 # store the builtin raw_input globally, and use this always, in case user code
 # overwrites it (like wx.py.PyShell does)
 raw_input_original = raw_input
 
 # compiled regexps for autoindent management
 dedent_re = re.compile(r'^\s+raise|^\s+return|^\s+pass')
 
 
 #****************************************************************************
 # Some utility function definitions
 
 ini_spaces_re = re.compile(r'^(\s+)')
 
 def num_ini_spaces(strng):
     """Return the number of initial spaces in a string"""
 
     ini_spaces = ini_spaces_re.match(strng)
     if ini_spaces:
         return ini_spaces.end()
     else:
         return 0
 
 def softspace(file, newvalue):
     """Copied from code.py, to remove the dependency"""
 
     oldvalue = 0
     try:
         oldvalue = file.softspace
     except AttributeError:
         pass
     try:
         file.softspace = newvalue
     except (AttributeError, TypeError):
         # "attribute-less object" or "read-only attributes"
         pass
     return oldvalue
 
 
 def user_setup(ipythondir,rc_suffix,mode='install',interactive=True):
     """Install or upgrade the user configuration directory.
 
     Can be called when running for the first time or to upgrade the user's
     .ipython/ directory.
 
     Parameters
     ----------
       ipythondir : path
         The directory to be used for installation/upgrade.  In 'install' mode,
         if this path already exists, the function exits immediately.
 
       rc_suffix : str
         Extension for the config files.  On *nix platforms it is typically the
         empty string, while Windows normally uses '.ini'.
 
       mode : str, optional
         Valid modes are 'install' and 'upgrade'.
 
       interactive : bool, optional
         If False, do not wait for user input on any errors.  Normally after
         printing its status information, this function waits for the user to
         hit Return before proceeding.  This is because the default use case is
         when first installing the IPython configuration, so we want the user to
         acknowledge the initial message, which contains some useful
         information.
         """
 
     # For automatic use, deactivate all i/o
     if interactive:
         def wait():
             try:
                 raw_input("Please press <RETURN> to start IPython.")
             except EOFError:
                 print >> Term.cout
             print '*'*70
 
         def printf(s):
             print s
     else:
         wait = lambda : None
         printf = lambda s : None
 
     # Install mode should be re-entrant: if the install dir already exists,
-    # bail out cleanly
+    # bail out cleanly.
+    # XXX.  This is too hasty to return.  We need to check to make sure that
+    # all the expected config files and directories are actually there. We
+    # currently have a failure mode if someone deletes a needed config file
+    # but still has the ipythondir.
     if mode == 'install' and os.path.isdir(ipythondir):
         return
 
     cwd = os.getcwd()  # remember where we started
     glb = glob.glob
 
     printf('*'*70)
     if mode == 'install':
         printf(
 """Welcome to IPython. I will try to create a personal configuration directory
 where you can customize many aspects of IPython's functionality in:\n""")
     else:
         printf('I am going to upgrade your configuration in:')
 
     printf(ipythondir)
 
     rcdirend = os.path.join('IPython','UserConfig')
     cfg = lambda d: os.path.join(d,rcdirend)
     try:
         rcdir = filter(os.path.isdir,map(cfg,sys.path))[0]
         printf("Initializing from configuration: %s" % rcdir)
     except IndexError:
         warning = """
 Installation error. IPython's directory was not found.
 
 Check the following:
 
 The ipython/IPython directory should be in a directory belonging to your
 PYTHONPATH environment variable (that is, it should be in a directory
 belonging to sys.path). You can copy it explicitly there or just link to it.
 
 IPython will create a minimal default configuration for you.
 
 """
         warn(warning)
         wait()
 
         if sys.platform =='win32':
             inif = 'ipythonrc.ini'
         else:
             inif = 'ipythonrc'
         minimal_setup = {'ipy_user_conf.py' : 'import ipy_defaults',
                          inif : '# intentionally left blank' }
         os.makedirs(ipythondir, mode = 0777)
         for f, cont in minimal_setup.items():
             # In 2.5, this can be more cleanly done using 'with'
             fobj = file(ipythondir + '/' + f,'w')
             fobj.write(cont)
             fobj.close()
 
         return
 
     if mode == 'install':
         try:
             shutil.copytree(rcdir,ipythondir)
             os.chdir(ipythondir)
             rc_files = glb("ipythonrc*")
             for rc_file in rc_files:
                 os.rename(rc_file,rc_file+rc_suffix)
         except:
             warning = """
 
 There was a problem with the installation:
 %s
 Try to correct it or contact the developers if you think it's a bug.
 IPython will proceed with builtin defaults.""" % sys.exc_info()[1]
             warn(warning)
             wait()
             return
 
     elif mode == 'upgrade':
         try:
             os.chdir(ipythondir)
         except:
             printf("""
 Can not upgrade: changing to directory %s failed. Details:
 %s
 """ % (ipythondir,sys.exc_info()[1]) )
             wait()
             return
         else:
             sources = glb(os.path.join(rcdir,'[A-Za-z]*'))
             for new_full_path in sources:
                 new_filename = os.path.basename(new_full_path)
                 if new_filename.startswith('ipythonrc'):
                     new_filename = new_filename + rc_suffix
                 # The config directory should only contain files, skip any
                 # directories which may be there (like CVS)
                 if os.path.isdir(new_full_path):
                     continue
                 if os.path.exists(new_filename):
                     old_file = new_filename+'.old'
                     if os.path.exists(old_file):
                         os.remove(old_file)
                     os.rename(new_filename,old_file)
                 shutil.copy(new_full_path,new_filename)
     else:
         raise ValueError('unrecognized mode for install: %r' % mode)
 
     # Fix line-endings to those native to each platform in the config
     # directory.
     try:
         os.chdir(ipythondir)
     except:
         printf("""
 Problem: changing to directory %s failed.
 Details:
 %s
 
 Some configuration files may have incorrect line endings.  This should not
 cause any problems during execution.  """ % (ipythondir,sys.exc_info()[1]) )
         wait()
     else:
         for fname in glb('ipythonrc*'):
             try:
                 native_line_ends(fname,backup=0)
             except IOError:
                 pass
 
     if mode == 'install':
         printf("""
 Successful installation!
 
 Please read the sections 'Initial Configuration' and 'Quick Tips' in the
 IPython manual (there are both HTML and PDF versions supplied with the
 distribution) to make sure that your system environment is properly configured
 to take advantage of IPython's features.
 
 Important note: the configuration system has changed! The old system is
 still in place, but its setting may be partly overridden by the settings in 
 "~/.ipython/ipy_user_conf.py" config file. Please take a look at the file 
 if some of the new settings bother you. 
 
 """)
     else:
         printf("""
 Successful upgrade!
 
 All files in your directory:
 %(ipythondir)s
 which would have been overwritten by the upgrade were backed up with a .old
 extension.  If you had made particular customizations in those files you may
 want to merge them back into the new files.""" % locals() )
     wait()
     os.chdir(cwd)
 
 #****************************************************************************
 # Local use exceptions
 class SpaceInInput(exceptions.Exception): pass
 
 
 #****************************************************************************
 # Local use classes
 class Bunch: pass
 
 class Undefined: pass
 
 class Quitter(object):
     """Simple class to handle exit, similar to Python 2.5's.
 
     It handles exiting in an ipython-safe manner, which the one in Python 2.5
     doesn't do (obviously, since it doesn't know about ipython)."""
     
     def __init__(self,shell,name):
         self.shell = shell
         self.name = name
         
     def __repr__(self):
         return 'Type %s() to exit.' % self.name
     __str__ = __repr__
 
     def __call__(self):
         self.shell.exit()
 
 class InputList(list):
     """Class to store user input.
 
     It's basically a list, but slices return a string instead of a list, thus
     allowing things like (assuming 'In' is an instance):
 
     exec In[4:7]
 
     or
 
     exec In[5:9] + In[14] + In[21:25]"""
 
     def __getslice__(self,i,j):
         return ''.join(list.__getslice__(self,i,j))
 
 class SyntaxTB(ultraTB.ListTB):
     """Extension which holds some state: the last exception value"""
 
     def __init__(self,color_scheme = 'NoColor'):
         ultraTB.ListTB.__init__(self,color_scheme)
         self.last_syntax_error = None
 
     def __call__(self, etype, value, elist):
         self.last_syntax_error = value
         ultraTB.ListTB.__call__(self,etype,value,elist)
 
     def clear_err_state(self):
         """Return the current error state and clear it"""
         e = self.last_syntax_error
         self.last_syntax_error = None
         return e
 
 #****************************************************************************
 # Main IPython class
 
 # FIXME: the Magic class is a mixin for now, and will unfortunately remain so
 # until a full rewrite is made.  I've cleaned all cross-class uses of
 # attributes and methods, but too much user code out there relies on the
 # equlity %foo == __IP.magic_foo, so I can't actually remove the mixin usage.
 #
 # But at least now, all the pieces have been separated and we could, in
 # principle, stop using the mixin.  This will ease the transition to the
 # chainsaw branch.
 
 # For reference, the following is the list of 'self.foo' uses in the Magic
 # class as of 2005-12-28.  These are names we CAN'T use in the main ipython
 # class, to prevent clashes.
 
 # ['self.__class__', 'self.__dict__', 'self._inspect', 'self._ofind',
 #  'self.arg_err', 'self.extract_input', 'self.format_', 'self.lsmagic',
 #  'self.magic_', 'self.options_table', 'self.parse', 'self.shell',
 #  'self.value']
 
 class InteractiveShell(object,Magic):
     """An enhanced console for Python."""
 
     # class attribute to indicate whether the class supports threads or not.
     # Subclasses with thread support should override this as needed.
     isthreaded = False
 
     def __init__(self,name,usage=None,rc=Struct(opts=None,args=None),
                  user_ns=None,user_global_ns=None,banner2='',
                  custom_exceptions=((),None),embedded=False):
 
         # log system
         self.logger = Logger(self,logfname='ipython_log.py',logmode='rotate')
             
         # Job manager (for jobs run as background threads)
         self.jobs = BackgroundJobManager()
 
         # Store the actual shell's name
         self.name = name
         self.more = False
 
         # We need to know whether the instance is meant for embedding, since
         # global/local namespaces need to be handled differently in that case
         self.embedded = embedded
         if embedded:
             # Control variable so users can, from within the embedded instance,
             # permanently deactivate it.
             self.embedded_active = True
 
         # command compiler
         self.compile = codeop.CommandCompiler()
 
         # User input buffer
         self.buffer = []
 
         # Default name given in compilation of code
         self.filename = '<ipython console>'
 
         # Install our own quitter instead of the builtins.  For python2.3-2.4,
         # this brings in behavior like 2.5, and for 2.5 it's identical.
         __builtin__.exit = Quitter(self,'exit')
         __builtin__.quit = Quitter(self,'quit')
         
         # Make an empty namespace, which extension writers can rely on both
         # existing and NEVER being used by ipython itself.  This gives them a
         # convenient location for storing additional information and state
         # their extensions may require, without fear of collisions with other
         # ipython names that may develop later.
         self.meta = Struct()
 
         # Create the namespace where the user will operate.  user_ns is
         # normally the only one used, and it is passed to the exec calls as
         # the locals argument.  But we do carry a user_global_ns namespace
         # given as the exec 'globals' argument,  This is useful in embedding
         # situations where the ipython shell opens in a context where the
         # distinction between locals and globals is meaningful.  For
         # non-embedded contexts, it is just the same object as the user_ns dict.
 
         # FIXME. For some strange reason, __builtins__ is showing up at user
         # level as a dict instead of a module. This is a manual fix, but I
         # should really track down where the problem is coming from. Alex
         # Schmolck reported this problem first.
 
         # A useful post by Alex Martelli on this topic:
         # Re: inconsistent value from __builtins__
         # Von: Alex Martelli <[e-mail]>
         # Datum: Freitag 01 Oktober 2004 04:45:34 nachmittags/abends
         # Gruppen: comp.lang.python
 
         # Michael Hohn <[e-mail]> wrote:
         # > >>> print type(builtin_check.get_global_binding('__builtins__'))
         # > <type 'dict'>
         # > >>> print type(__builtins__)
         # > <type 'module'>
         # > Is this difference in return value intentional?
 
         # Well, it's documented that '__builtins__' can be either a dictionary
         # or a module, and it's been that way for a long time. Whether it's
         # intentional (or sensible), I don't know. In any case, the idea is
         # that if you need to access the built-in namespace directly, you
         # should start with "import __builtin__" (note, no 's') which will
         # definitely give you a module. Yeah, it's somewhat confusing:-(.
 
         # These routines return properly built dicts as needed by the rest of
         # the code, and can also be used by extension writers to generate
         # properly initialized namespaces.
         user_ns, user_global_ns = IPython.ipapi.make_user_namespaces(user_ns,
             user_global_ns)
 
         # Assign namespaces
         # This is the namespace where all normal user variables live
         self.user_ns = user_ns
         self.user_global_ns = user_global_ns
 
         # An auxiliary namespace that checks what parts of the user_ns were
         # loaded at startup, so we can list later only variables defined in
         # actual interactive use.  Since it is always a subset of user_ns, it
         # doesn't need to be seaparately tracked in the ns_table
         self.user_config_ns = {}
 
         # A namespace to keep track of internal data structures to prevent
         # them from cluttering user-visible stuff.  Will be updated later
         self.internal_ns = {}
 
         # Namespace of system aliases.  Each entry in the alias
         # table must be a 2-tuple of the form (N,name), where N is the number
         # of positional arguments of the alias.
         self.alias_table = {}
 
         # Now that FakeModule produces a real module, we've run into a nasty
         # problem: after script execution (via %run), the module where the user
         # code ran is deleted.  Now that this object is a true module (needed
         # so docetst and other tools work correctly), the Python module
         # teardown mechanism runs over it, and sets to None every variable
         # present in that module.  Top-level references to objects from the
         # script survive, because the user_ns is updated with them.  However,
         # calling functions defined in the script that use other things from
         # the script will fail, because the function's closure had references
         # to the original objects, which are now all None.  So we must protect
         # these modules from deletion by keeping a cache.
         # 
         # To avoid keeping stale modules around (we only need the one from the
         # last run), we use a dict keyed with the full path to the script, so
         # only the last version of the module is held in the cache.  Note,
         # however, that we must cache the module *namespace contents* (their
         # __dict__).  Because if we try to cache the actual modules, old ones
         # (uncached) could be destroyed while still holding references (such as
         # those held by GUI objects that tend to be long-lived)>
         # 
         # The %reset command will flush this cache.  See the cache_main_mod()
         # and clear_main_mod_cache() methods for details on use.
 
         # This is the cache used for 'main' namespaces
         self._main_ns_cache = {}
         # And this is the single instance of FakeModule whose __dict__ we keep
         # copying and clearing for reuse on each %run
         self._user_main_module = FakeModule()
 
         # A table holding all the namespaces IPython deals with, so that
         # introspection facilities can search easily.
         self.ns_table = {'user':user_ns,
                          'user_global':user_global_ns,
                          'alias':self.alias_table,
                          'internal':self.internal_ns,
                          'builtin':__builtin__.__dict__
                          }
 
         # Similarly, track all namespaces where references can be held and that
         # we can safely clear (so it can NOT include builtin).  This one can be
         # a simple list.
         self.ns_refs_table = [ user_ns, user_global_ns, self.user_config_ns,
                                self.alias_table, self.internal_ns,
                                self._main_ns_cache ]
         
         # We need to insert into sys.modules something that looks like a
         # module but which accesses the IPython namespace, for shelve and
         # pickle to work interactively. Normally they rely on getting
         # everything out of __main__, but for embedding purposes each IPython
         # instance has its own private namespace, so we can't go shoving
         # everything into __main__.
 
         # note, however, that we should only do this for non-embedded
         # ipythons, which really mimic the __main__.__dict__ with their own
         # namespace.  Embedded instances, on the other hand, should not do
         # this because they need to manage the user local/global namespaces
         # only, but they live within a 'normal' __main__ (meaning, they
         # shouldn't overtake the execution environment of the script they're
         # embedded in).
 
         if not embedded:
             try:
                 main_name = self.user_ns['__name__']
             except KeyError:
                 raise KeyError,'user_ns dictionary MUST have a "__name__" key'
             else:
                 #print "pickle hack in place"  # dbg
                 #print 'main_name:',main_name # dbg
                 sys.modules[main_name] = FakeModule(self.user_ns)
         
         # List of input with multi-line handling.
         self.input_hist = InputList()
         # This one will hold the 'raw' input history, without any
         # pre-processing.  This will allow users to retrieve the input just as
         # it was exactly typed in by the user, with %hist -r.
         self.input_hist_raw = InputList()
 
         # list of visited directories
         try:
             self.dir_hist = [os.getcwd()]
         except OSError:
             self.dir_hist = []
 
         # dict of output history
         self.output_hist = {}
 
         # Get system encoding at startup time.  Certain terminals (like Emacs
         # under Win32 have it set to None, and we need to have a known valid
         # encoding to use in the raw_input() method
         try:
             self.stdin_encoding = sys.stdin.encoding or 'ascii'
         except AttributeError:
             self.stdin_encoding = 'ascii'
 
         # dict of things NOT to alias (keywords, builtins and some magics)
         no_alias = {}
         no_alias_magics = ['cd','popd','pushd','dhist','alias','unalias']
         for key in keyword.kwlist + no_alias_magics:
             no_alias[key] = 1
         no_alias.update(__builtin__.__dict__)
         self.no_alias = no_alias
 
         # Object variable to store code object waiting execution.  This is
         # used mainly by the multithreaded shells, but it can come in handy in
         # other situations.  No need to use a Queue here, since it's a single
         # item which gets cleared once run.
         self.code_to_run = None
         
         # escapes for automatic behavior on the command line
         self.ESC_SHELL  = '!'
         self.ESC_SH_CAP = '!!'
         self.ESC_HELP   = '?'
         self.ESC_MAGIC  = '%'
         self.ESC_QUOTE  = ','
         self.ESC_QUOTE2 = ';'
         self.ESC_PAREN  = '/'
 
         # And their associated handlers
         self.esc_handlers = {self.ESC_PAREN  : self.handle_auto,
                              self.ESC_QUOTE  : self.handle_auto,
                              self.ESC_QUOTE2 : self.handle_auto,
                              self.ESC_MAGIC  : self.handle_magic,
                              self.ESC_HELP   : self.handle_help,
                              self.ESC_SHELL  : self.handle_shell_escape,
                              self.ESC_SH_CAP : self.handle_shell_escape,
                              }
 
         # class initializations
         Magic.__init__(self,self)
 
         # Python source parser/formatter for syntax highlighting
         pyformat = PyColorize.Parser().format
         self.pycolorize = lambda src: pyformat(src,'str',self.rc['colors'])
 
         # hooks holds pointers used for user-side customizations
         self.hooks = Struct()
         
         self.strdispatchers = {}
         
         # Set all default hooks, defined in the IPython.hooks module.
         hooks = IPython.hooks
         for hook_name in hooks.__all__:
             # default hooks have priority 100, i.e. low; user hooks should have
             # 0-100 priority
             self.set_hook(hook_name,getattr(hooks,hook_name), 100)
             #print "bound hook",hook_name
 
         # Flag to mark unconditional exit
         self.exit_now = False
 
         self.usage_min =  """\
         An enhanced console for Python.
         Some of its features are:
         - Readline support if the readline library is present.
         - Tab completion in the local namespace.
         - Logging of input, see command-line options.
         - System shell escape via ! , eg !ls.
         - Magic commands, starting with a % (like %ls, %pwd, %cd, etc.)
         - Keeps track of locally defined variables via %who, %whos.
         - Show object information with a ? eg ?x or x? (use ?? for more info).
         """
         if usage: self.usage = usage
         else: self.usage = self.usage_min
 
         # Storage
         self.rc = rc   # This will hold all configuration information
         self.pager = 'less'
         # temporary files used for various purposes.  Deleted at exit.
         self.tempfiles = []
 
         # Keep track of readline usage (later set by init_readline)
         self.has_readline = False
 
         # template for logfile headers.  It gets resolved at runtime by the
         # logstart method.
         self.loghead_tpl = \
 """#log# Automatic Logger file. *** THIS MUST BE THE FIRST LINE ***
 #log# DO NOT CHANGE THIS LINE OR THE TWO BELOW
 #log# opts = %s
 #log# args = %s
 #log# It is safe to make manual edits below here.
 #log#-----------------------------------------------------------------------
 """
         # for pushd/popd management
         try:
             self.home_dir = get_home_dir()
         except HomeDirError,msg:
             fatal(msg)
 
         self.dir_stack = []
 
         # Functions to call the underlying shell.
 
         # The first is similar to os.system, but it doesn't return a value,
         # and it allows interpolation of variables in the user's namespace.
         self.system = lambda cmd: \
                       self.hooks.shell_hook(self.var_expand(cmd,depth=2))
 
         # These are for getoutput and getoutputerror:
         self.getoutput = lambda cmd: \
                          getoutput(self.var_expand(cmd,depth=2),
                                    header=self.rc.system_header,
                                    verbose=self.rc.system_verbose)
 
         self.getoutputerror = lambda cmd: \
                               getoutputerror(self.var_expand(cmd,depth=2),
                                              header=self.rc.system_header,
                                              verbose=self.rc.system_verbose)
  
 
         # keep track of where we started running (mainly for crash post-mortem)
         self.starting_dir = os.getcwd()
 
         # Various switches which can be set
         self.CACHELENGTH = 5000  # this is cheap, it's just text
         self.BANNER = "Python %(version)s on %(platform)s\n" % sys.__dict__
         self.banner2 = banner2
 
         # TraceBack handlers:
 
         # Syntax error handler.
         self.SyntaxTB = SyntaxTB(color_scheme='NoColor')
         
         # The interactive one is initialized with an offset, meaning we always
         # want to remove the topmost item in the traceback, which is our own
         # internal code. Valid modes: ['Plain','Context','Verbose']
         self.InteractiveTB = ultraTB.AutoFormattedTB(mode = 'Plain',
                                                      color_scheme='NoColor',
                                                      tb_offset = 1)
 
         # IPython itself shouldn't crash. This will produce a detailed
         # post-mortem if it does.  But we only install the crash handler for
         # non-threaded shells, the threaded ones use a normal verbose reporter
         # and lose the crash handler.  This is because exceptions in the main
         # thread (such as in GUI code) propagate directly to sys.excepthook,
         # and there's no point in printing crash dumps for every user exception.
         if self.isthreaded:
             ipCrashHandler = ultraTB.FormattedTB()
         else:
             from IPython import CrashHandler
             ipCrashHandler = CrashHandler.IPythonCrashHandler(self)
         self.set_crash_handler(ipCrashHandler)
 
         # and add any custom exception handlers the user may have specified
         self.set_custom_exc(*custom_exceptions)
 
         # indentation management
         self.autoindent = False
         self.indent_current_nsp = 0
 
         # Make some aliases automatically
         # Prepare list of shell aliases to auto-define
         if os.name == 'posix':
             auto_alias = ('mkdir mkdir', 'rmdir rmdir',
                           'mv mv -i','rm rm -i','cp cp -i',
                           'cat cat','less less','clear clear',
                           # a better ls
                           'ls ls -F',
                           # long ls
                           'll ls -lF')
             # Extra ls aliases with color, which need special treatment on BSD
             # variants
             ls_extra = ( # color ls
                          'lc ls -F -o --color',
                          # ls normal files only
                          'lf ls -F -o --color %l | grep ^-',
                          # ls symbolic links
                          'lk ls -F -o --color %l | grep ^l',
                          # directories or links to directories,
                          'ldir ls -F -o --color %l | grep /$',
                          # things which are executable
                          'lx ls -F -o --color %l | grep ^-..x',
                          )
             # The BSDs don't ship GNU ls, so they don't understand the
             # --color switch out of the box
             if 'bsd' in sys.platform:
                 ls_extra = ( # ls normal files only
                              'lf ls -lF | grep ^-',
                              # ls symbolic links
                              'lk ls -lF | grep ^l',
                              # directories or links to directories,
                              'ldir ls -lF | grep /$',
                              # things which are executable
                              'lx ls -lF | grep ^-..x',
                              )
             auto_alias = auto_alias + ls_extra
         elif os.name in ['nt','dos']:
             auto_alias = ('ls dir /on',
                           'ddir dir /ad /on', 'ldir dir /ad /on',
                           'mkdir mkdir','rmdir rmdir','echo echo',
                           'ren ren','cls cls','copy copy')
         else:
             auto_alias = ()
         self.auto_alias = [s.split(None,1) for s in auto_alias]
         
         # Produce a public API instance
         self.api = IPython.ipapi.IPApi(self)
 
         # Initialize all user-visible namespaces
         self.init_namespaces()
         
         # Call the actual (public) initializer
         self.init_auto_alias()
 
         # track which builtins we add, so we can clean up later
         self.builtins_added = {}
         # This method will add the necessary builtins for operation, but
         # tracking what it did via the builtins_added dict.
         
         #TODO: remove this, redundant
         self.add_builtins()
     # end __init__
 
     def var_expand(self,cmd,depth=0):
         """Expand python variables in a string.
 
         The depth argument indicates how many frames above the caller should
         be walked to look for the local namespace where to expand variables.
 
         The global namespace for expansion is always the user's interactive
         namespace.
         """
 
         return str(ItplNS(cmd,
                           self.user_ns,  # globals
                           # Skip our own frame in searching for locals:
                           sys._getframe(depth+1).f_locals # locals
                           ))
 
     def pre_config_initialization(self):
         """Pre-configuration init method
 
         This is called before the configuration files are processed to
         prepare the services the config files might need.
         
         self.rc already has reasonable default values at this point.
         """
         rc = self.rc
         try:
             self.db = pickleshare.PickleShareDB(rc.ipythondir + "/db")
         except exceptions.UnicodeDecodeError:
             print "Your ipythondir can't be decoded to unicode!"
             print "Please set HOME environment variable to something that"
             print r"only has ASCII characters, e.g. c:\home"
             print "Now it is",rc.ipythondir
             sys.exit()
         self.shadowhist = IPython.history.ShadowHist(self.db)
 
     def post_config_initialization(self):
         """Post configuration init method
 
         This is called after the configuration files have been processed to
         'finalize' the initialization."""
 
         rc = self.rc
 
         # Object inspector
         self.inspector = OInspect.Inspector(OInspect.InspectColors,
                                             PyColorize.ANSICodeColors,
                                             'NoColor',
                                             rc.object_info_string_level)
         
         self.rl_next_input = None
         self.rl_do_indent = False
         # Load readline proper
         if rc.readline:
             self.init_readline()
         
         # local shortcut, this is used a LOT
         self.log = self.logger.log
 
         # Initialize cache, set in/out prompts and printing system
         self.outputcache = CachedOutput(self,
                                         rc.cache_size,
                                         rc.pprint,
                                         input_sep = rc.separate_in,
                                         output_sep = rc.separate_out,
                                         output_sep2 = rc.separate_out2,
                                         ps1 = rc.prompt_in1,
                                         ps2 = rc.prompt_in2,
                                         ps_out = rc.prompt_out,
                                         pad_left = rc.prompts_pad_left)
 
         # user may have over-ridden the default print hook:
         try:
             self.outputcache.__class__.display = self.hooks.display
         except AttributeError:
             pass
 
         # I don't like assigning globally to sys, because it means when
         # embedding instances, each embedded instance overrides the previous
         # choice. But sys.displayhook seems to be called internally by exec,
         # so I don't see a way around it.  We first save the original and then
         # overwrite it.
         self.sys_displayhook = sys.displayhook
         sys.displayhook = self.outputcache
 
         # Do a proper resetting of doctest, including the necessary displayhook
         # monkeypatching
         try:
             doctest_reload()
         except ImportError:
             warn("doctest module does not exist.")
         
         # Set user colors (don't do it in the constructor above so that it
         # doesn't crash if colors option is invalid)
         self.magic_colors(rc.colors)
 
         # Set calling of pdb on exceptions
         self.call_pdb = rc.pdb
 
         # Load user aliases
         for alias in rc.alias:
             self.magic_alias(alias)
         
         self.hooks.late_startup_hook()
         
         for cmd in self.rc.autoexec:
             #print "autoexec>",cmd #dbg
             self.api.runlines(cmd)
             
         batchrun = False
         for batchfile in [path(arg) for arg in self.rc.args 
             if arg.lower().endswith('.ipy')]:
             if not batchfile.isfile():
                 print "No such batch file:", batchfile
                 continue
             self.api.runlines(batchfile.text())
             batchrun = True
         # without -i option, exit after running the batch file
         if batchrun and not self.rc.interact:
             self.ask_exit()            
 
     def init_namespaces(self):
         """Initialize all user-visible namespaces to their minimum defaults.
 
         Certain history lists are also initialized here, as they effectively
         act as user namespaces.
 
         Note
         ----
         All data structures here are only filled in, they are NOT reset by this
         method.  If they were not empty before, data will simply be added to
         therm.
         """
         # The user namespace MUST have a pointer to the shell itself.
         self.user_ns[self.name] = self
 
         # Store the public api instance
         self.user_ns['_ip'] = self.api
 
         # make global variables for user access to the histories
         self.user_ns['_ih'] = self.input_hist
         self.user_ns['_oh'] = self.output_hist
         self.user_ns['_dh'] = self.dir_hist
 
         # user aliases to input and output histories
         self.user_ns['In']  = self.input_hist
         self.user_ns['Out'] = self.output_hist
 
         self.user_ns['_sh'] = IPython.shadowns
 
         # Fill the history zero entry, user counter starts at 1
         self.input_hist.append('\n')
         self.input_hist_raw.append('\n')
 
     def add_builtins(self):
         """Store ipython references into the builtin namespace.
 
         Some parts of ipython operate via builtins injected here, which hold a
         reference to IPython itself."""
 
         # TODO: deprecate all of these, they are unsafe
         builtins_new  = dict(__IPYTHON__ = self,
              ip_set_hook = self.set_hook, 
              jobs = self.jobs,
              ipmagic = wrap_deprecated(self.ipmagic,'_ip.magic()'),  
              ipalias = wrap_deprecated(self.ipalias),  
              ipsystem = wrap_deprecated(self.ipsystem,'_ip.system()'),
              #_ip = self.api
              )
         for biname,bival in builtins_new.items():
             try:
                 # store the orignal value so we can restore it
                 self.builtins_added[biname] =  __builtin__.__dict__[biname]
             except KeyError:
                 # or mark that it wasn't defined, and we'll just delete it at
                 # cleanup
                 self.builtins_added[biname] = Undefined
             __builtin__.__dict__[biname] = bival
             
         # Keep in the builtins a flag for when IPython is active.  We set it
         # with setdefault so that multiple nested IPythons don't clobber one
         # another.  Each will increase its value by one upon being activated,
         # which also gives us a way to determine the nesting level.
         __builtin__.__dict__.setdefault('__IPYTHON__active',0)
 
     def clean_builtins(self):
         """Remove any builtins which might have been added by add_builtins, or
         restore overwritten ones to their previous values."""
         for biname,bival in self.builtins_added.items():
             if bival is Undefined:
                 del __builtin__.__dict__[biname]
             else:
                 __builtin__.__dict__[biname] = bival
         self.builtins_added.clear()
     
     def set_hook(self,name,hook, priority = 50, str_key = None, re_key = None):
         """set_hook(name,hook) -> sets an internal IPython hook.
 
         IPython exposes some of its internal API as user-modifiable hooks.  By
         adding your function to one of these hooks, you can modify IPython's 
         behavior to call at runtime your own routines."""
 
         # At some point in the future, this should validate the hook before it
         # accepts it.  Probably at least check that the hook takes the number
         # of args it's supposed to.
         
         f = new.instancemethod(hook,self,self.__class__)
 
         # check if the hook is for strdispatcher first
         if str_key is not None:
             sdp = self.strdispatchers.get(name, StrDispatch())
             sdp.add_s(str_key, f, priority )
             self.strdispatchers[name] = sdp
             return
         if re_key is not None:
             sdp = self.strdispatchers.get(name, StrDispatch())
             sdp.add_re(re.compile(re_key), f, priority )
             self.strdispatchers[name] = sdp
             return
             
         dp = getattr(self.hooks, name, None)
         if name not in IPython.hooks.__all__:
             print "Warning! Hook '%s' is not one of %s" % (name, IPython.hooks.__all__ )
         if not dp:
             dp = IPython.hooks.CommandChainDispatcher()
         
         try:
             dp.add(f,priority)
         except AttributeError:
             # it was not commandchain, plain old func - replace
             dp = f
 
         setattr(self.hooks,name, dp)
         
         
         #setattr(self.hooks,name,new.instancemethod(hook,self,self.__class__))
 
     def set_crash_handler(self,crashHandler):
         """Set the IPython crash handler.
 
         This must be a callable with a signature suitable for use as
         sys.excepthook."""
 
         # Install the given crash handler as the Python exception hook
         sys.excepthook = crashHandler
         
         # The instance will store a pointer to this, so that runtime code
         # (such as magics) can access it.  This is because during the
         # read-eval loop, it gets temporarily overwritten (to deal with GUI
         # frameworks).
         self.sys_excepthook = sys.excepthook
 
 
     def set_custom_exc(self,exc_tuple,handler):
         """set_custom_exc(exc_tuple,handler)
 
         Set a custom exception handler, which will be called if any of the
         exceptions in exc_tuple occur in the mainloop (specifically, in the
         runcode() method.
 
         Inputs:
 
           - exc_tuple: a *tuple* of valid exceptions to call the defined
           handler for.  It is very important that you use a tuple, and NOT A
           LIST here, because of the way Python's except statement works.  If
           you only want to trap a single exception, use a singleton tuple:
 
             exc_tuple == (MyCustomException,)
 
           - handler: this must be defined as a function with the following
           basic interface: def my_handler(self,etype,value,tb).
 
           This will be made into an instance method (via new.instancemethod)
           of IPython itself, and it will be called if any of the exceptions
           listed in the exc_tuple are caught.  If the handler is None, an
           internal basic one is used, which just prints basic info.
 
         WARNING: by putting in your own exception handler into IPython's main
         execution loop, you run a very good chance of nasty crashes.  This
         facility should only be used if you really know what you are doing."""
 
         assert type(exc_tuple)==type(()) , \
                "The custom exceptions must be given AS A TUPLE."
 
         def dummy_handler(self,etype,value,tb):
             print '*** Simple custom exception handler ***'
             print 'Exception type :',etype
             print 'Exception value:',value
             print 'Traceback      :',tb
             print 'Source code    :','\n'.join(self.buffer)
 
         if handler is None: handler = dummy_handler
 
         self.CustomTB = new.instancemethod(handler,self,self.__class__)
         self.custom_exceptions = exc_tuple
 
     def set_custom_completer(self,completer,pos=0):
         """set_custom_completer(completer,pos=0)
 
         Adds a new custom completer function.
 
         The position argument (defaults to 0) is the index in the completers
         list where you want the completer to be inserted."""
 
         newcomp = new.instancemethod(completer,self.Completer,
                                      self.Completer.__class__)
         self.Completer.matchers.insert(pos,newcomp)
 
     def set_completer(self):
         """reset readline's completer to be our own."""
         self.readline.set_completer(self.Completer.complete)
         
     def _get_call_pdb(self):
         return self._call_pdb
 
     def _set_call_pdb(self,val):
 
         if val not in (0,1,False,True):
             raise ValueError,'new call_pdb value must be boolean'
 
         # store value in instance
         self._call_pdb = val
 
         # notify the actual exception handlers
         self.InteractiveTB.call_pdb = val
         if self.isthreaded:
             try:
                 self.sys_excepthook.call_pdb = val
             except:
                 warn('Failed to activate pdb for threaded exception handler')
 
     call_pdb = property(_get_call_pdb,_set_call_pdb,None,
                         'Control auto-activation of pdb at exceptions')
  
     # These special functions get installed in the builtin namespace, to
     # provide programmatic (pure python) access to magics, aliases and system
     # calls.  This is important for logging, user scripting, and more.
 
     # We are basically exposing, via normal python functions, the three
     # mechanisms in which ipython offers special call modes (magics for
     # internal control, aliases for direct system access via pre-selected
     # names, and !cmd for calling arbitrary system commands).
 
     def ipmagic(self,arg_s):
         """Call a magic function by name.
 
         Input: a string containing the name of the magic function to call and any
         additional arguments to be passed to the magic.
 
         ipmagic('name -opt foo bar') is equivalent to typing at the ipython
         prompt:
 
         In[1]: %name -opt foo bar
 
         To call a magic without arguments, simply use ipmagic('name').
 
         This provides a proper Python function to call IPython's magics in any
         valid Python code you can type at the interpreter, including loops and
         compound statements.  It is added by IPython to the Python builtin
         namespace upon initialization."""
 
         args = arg_s.split(' ',1)
         magic_name = args[0]
         magic_name = magic_name.lstrip(self.ESC_MAGIC)
 
         try:
             magic_args = args[1]
         except IndexError:
             magic_args = ''
         fn = getattr(self,'magic_'+magic_name,None)
         if fn is None:
             error("Magic function `%s` not found." % magic_name)
         else:
             magic_args = self.var_expand(magic_args,1)
             return fn(magic_args)
 
     def ipalias(self,arg_s):
         """Call an alias by name.
 
         Input: a string containing the name of the alias to call and any
         additional arguments to be passed to the magic.
 
         ipalias('name -opt foo bar') is equivalent to typing at the ipython
         prompt:
 
         In[1]: name -opt foo bar
 
         To call an alias without arguments, simply use ipalias('name').
 
         This provides a proper Python function to call IPython's aliases in any
         valid Python code you can type at the interpreter, including loops and
         compound statements.  It is added by IPython to the Python builtin
         namespace upon initialization."""
 
         args = arg_s.split(' ',1)
         alias_name = args[0]
         try:
             alias_args = args[1]
         except IndexError:
             alias_args = ''
         if alias_name in self.alias_table:
             self.call_alias(alias_name,alias_args)
         else:
             error("Alias `%s` not found." % alias_name)
 
     def ipsystem(self,arg_s):
         """Make a system call, using IPython."""
 
         self.system(arg_s)
 
     def complete(self,text):
         """Return a sorted list of all possible completions on text.
 
         Inputs:
 
           - text: a string of text to be completed on.
 
         This is a wrapper around the completion mechanism, similar to what
         readline does at the command line when the TAB key is hit.  By
         exposing it as a method, it can be used by other non-readline
         environments (such as GUIs) for text completion.
 
         Simple usage example:
 
         In [7]: x = 'hello'
 
         In [8]: x
         Out[8]: 'hello'
 
         In [9]: print x
         hello
 
         In [10]: _ip.IP.complete('x.l')
         Out[10]: ['x.ljust', 'x.lower', 'x.lstrip']
         """
         
         complete = self.Completer.complete
         state = 0
         # use a dict so we get unique keys, since ipyhton's multiple
         # completers can return duplicates.  When we make 2.4 a requirement,
         # start using sets instead, which are faster.
         comps = {}
         while True:
             newcomp = complete(text,state,line_buffer=text)
             if newcomp is None:
                 break
             comps[newcomp] = 1
             state += 1
         outcomps = comps.keys()
         outcomps.sort()
         #print "T:",text,"OC:",outcomps  # dbg
         #print "vars:",self.user_ns.keys()
         return outcomps
         
     def set_completer_frame(self, frame=None):
         if frame:
             self.Completer.namespace = frame.f_locals
             self.Completer.global_namespace = frame.f_globals
         else:
             self.Completer.namespace = self.user_ns
             self.Completer.global_namespace = self.user_global_ns
 
     def init_auto_alias(self):
         """Define some aliases automatically.
 
         These are ALL parameter-less aliases"""
 
         for alias,cmd in self.auto_alias:
             self.getapi().defalias(alias,cmd)
             
 
     def alias_table_validate(self,verbose=0):
         """Update information about the alias table.
 
         In particular, make sure no Python keywords/builtins are in it."""
 
         no_alias = self.no_alias
         for k in self.alias_table.keys():
             if k in no_alias:
                 del self.alias_table[k]
                 if verbose:
                     print ("Deleting alias <%s>, it's a Python "
                            "keyword or builtin." % k)
     
     def set_autoindent(self,value=None):
         """Set the autoindent flag, checking for readline support.
 
         If called with no arguments, it acts as a toggle."""
 
         if not self.has_readline:
             if os.name == 'posix':
                 warn("The auto-indent feature requires the readline library")
             self.autoindent = 0
             return
         if value is None:
             self.autoindent = not self.autoindent
         else:
             self.autoindent = value
 
     def rc_set_toggle(self,rc_field,value=None):
         """Set or toggle a field in IPython's rc config. structure.
 
         If called with no arguments, it acts as a toggle.
 
         If called with a non-existent field, the resulting AttributeError
         exception will propagate out."""
 
         rc_val = getattr(self.rc,rc_field)
         if value is None:
             value = not rc_val
         setattr(self.rc,rc_field,value)
 
     def user_setup(self,ipythondir,rc_suffix,mode='install'):
         """Install the user configuration directory.
 
         Note
         ----
         DEPRECATED: use the top-level user_setup() function instead.
         """
         return user_setup(ipythondir,rc_suffix,mode)
 
     def atexit_operations(self):
         """This will be executed at the time of exit.
 
         Saving of persistent data should be performed here. """
 
         #print '*** IPython exit cleanup ***' # dbg
         # input history
         self.savehist()
 
         # Cleanup all tempfiles left around
         for tfile in self.tempfiles:
             try:
                 os.unlink(tfile)
             except OSError:
                 pass
 
         # Clear all user namespaces to release all references cleanly.
         self.reset()
 
         # Run user hooks
         self.hooks.shutdown_hook()
 
     def reset(self):
         """Clear all internal namespaces.
 
         Note that this is much more aggressive than %reset, since it clears
         fully all namespaces, as well as all input/output lists.
         """
         for ns in self.ns_refs_table:
             ns.clear()
 
         # Clear input and output histories
         self.input_hist[:] = []
         self.input_hist_raw[:] = []
         self.output_hist.clear()
         # Restore the user namespaces to minimal usability
         self.init_namespaces()
         
     def savehist(self):
         """Save input history to a file (via readline library)."""
 
         if not self.has_readline:
             return
         
         try:
             self.readline.write_history_file(self.histfile)
         except:
             print 'Unable to save IPython command history to file: ' + \
                   `self.histfile`
 
     def reloadhist(self):
         """Reload the input history from disk file."""
 
         if self.has_readline:
             try:
                 self.readline.clear_history()
                 self.readline.read_history_file(self.shell.histfile)
             except AttributeError:
                 pass
             
 
     def history_saving_wrapper(self, func):
         """ Wrap func for readline history saving
         
         Convert func into callable that saves & restores
         history around the call """
         
         if not self.has_readline:
             return func
         
         def wrapper():
             self.savehist()
             try:
                 func()
             finally:
                 readline.read_history_file(self.histfile)
         return wrapper
             
     def pre_readline(self):
         """readline hook to be used at the start of each line.
 
         Currently it handles auto-indent only."""
 
         #debugx('self.indent_current_nsp','pre_readline:')
 
         if self.rl_do_indent:
             self.readline.insert_text(self.indent_current_str())
         if self.rl_next_input is not None:
             self.readline.insert_text(self.rl_next_input)
             self.rl_next_input = None
 
     def init_readline(self):
         """Command history completion/saving/reloading."""
 
 
         import IPython.rlineimpl as readline
                   
         if not readline.have_readline:
             self.has_readline = 0
             self.readline = None
             # no point in bugging windows users with this every time:
             warn('Readline services not available on this platform.')
         else:
             sys.modules['readline'] = readline
             import atexit
             from IPython.completer import IPCompleter
             self.Completer = IPCompleter(self,
                                             self.user_ns,
                                             self.user_global_ns,
                                             self.rc.readline_omit__names,
                                             self.alias_table)
             sdisp = self.strdispatchers.get('complete_command', StrDispatch())
             self.strdispatchers['complete_command'] = sdisp
             self.Completer.custom_completers = sdisp
             # Platform-specific configuration
             if os.name == 'nt':
                 self.readline_startup_hook = readline.set_pre_input_hook
             else:
                 self.readline_startup_hook = readline.set_startup_hook
 
             # Load user's initrc file (readline config)
             # Or if libedit is used, load editrc.
             inputrc_name = os.environ.get('INPUTRC')
             if inputrc_name is None:
                 home_dir = get_home_dir()
                 if home_dir is not None:
                     inputrc_name = '.inputrc'
                     if readline.uses_libedit:
                         inputrc_name = '.editrc'
                     inputrc_name = os.path.join(home_dir, inputrc_name)
             if os.path.isfile(inputrc_name):
                 try:
                     readline.read_init_file(inputrc_name)
                 except:
                     warn('Problems reading readline initialization file <%s>'
                          % inputrc_name)
             
             self.has_readline = 1
             self.readline = readline
             # save this in sys so embedded copies can restore it properly
             sys.ipcompleter = self.Completer.complete
             self.set_completer()
 
             # Configure readline according to user's prefs
             # This is only done if GNU readline is being used.  If libedit
             # is being used (as on Leopard) the readline config is
             # not run as the syntax for libedit is different.
             if not readline.uses_libedit:
                 for rlcommand in self.rc.readline_parse_and_bind:
                     #print "loading rl:",rlcommand  # dbg
                     readline.parse_and_bind(rlcommand)
 
-            # remove some chars from the delimiters list
-            delims = readline.get_completer_delims()
+            # Remove some chars from the delimiters list.  If we encounter
+            # unicode chars, discard them.
+            delims = readline.get_completer_delims().encode("ascii", "ignore")
             delims = delims.translate(string._idmap,
                                       self.rc.readline_remove_delims)
             readline.set_completer_delims(delims)
             # otherwise we end up with a monster history after a while:
             readline.set_history_length(1000)
             try:
                 #print '*** Reading readline history'  # dbg
                 readline.read_history_file(self.histfile)
             except IOError:
                 pass  # It doesn't exist yet.
 
             atexit.register(self.atexit_operations)
             del atexit
 
         # Configure auto-indent for all platforms
         self.set_autoindent(self.rc.autoindent)
 
     def ask_yes_no(self,prompt,default=True):
         if self.rc.quiet:
             return True
         return ask_yes_no(prompt,default)
 
     def new_main_mod(self,ns=None):
         """Return a new 'main' module object for user code execution.
         """
         main_mod = self._user_main_module
         init_fakemod_dict(main_mod,ns)
         return main_mod
 
     def cache_main_mod(self,ns,fname):
         """Cache a main module's namespace.
 
         When scripts are executed via %run, we must keep a reference to the
         namespace of their __main__ module (a FakeModule instance) around so
         that Python doesn't clear it, rendering objects defined therein
         useless.
 
         This method keeps said reference in a private dict, keyed by the
         absolute path of the module object (which corresponds to the script
         path).  This way, for multiple executions of the same script we only
         keep one copy of the namespace (the last one), thus preventing memory
         leaks from old references while allowing the objects from the last
         execution to be accessible.
 
         Note: we can not allow the actual FakeModule instances to be deleted,
         because of how Python tears down modules (it hard-sets all their
         references to None without regard for reference counts).  This method
         must therefore make a *copy* of the given namespace, to allow the
         original module's __dict__ to be cleared and reused.
 
         
         Parameters
         ----------
           ns : a namespace (a dict, typically)
 
           fname : str
             Filename associated with the namespace.
 
         Examples
         --------
 
         In [10]: import IPython
 
         In [11]: _ip.IP.cache_main_mod(IPython.__dict__,IPython.__file__)
 
         In [12]: IPython.__file__ in _ip.IP._main_ns_cache
         Out[12]: True
         """
         self._main_ns_cache[os.path.abspath(fname)] = ns.copy()
 
     def clear_main_mod_cache(self):
         """Clear the cache of main modules.
 
         Mainly for use by utilities like %reset.
 
         Examples
         --------
 
         In [15]: import IPython
 
         In [16]: _ip.IP.cache_main_mod(IPython.__dict__,IPython.__file__)
 
         In [17]: len(_ip.IP._main_ns_cache) > 0
         Out[17]: True
 
         In [18]: _ip.IP.clear_main_mod_cache()
 
         In [19]: len(_ip.IP._main_ns_cache) == 0
         Out[19]: True
         """
         self._main_ns_cache.clear()
 
     def _should_recompile(self,e):
         """Utility routine for edit_syntax_error"""
 
         if e.filename in ('<ipython console>','<input>','<string>',
                           '<console>','<BackgroundJob compilation>',
                           None):
                               
             return False
         try:
             if (self.rc.autoedit_syntax and 
                 not self.ask_yes_no('Return to editor to correct syntax error? '
                               '[Y/n] ','y')):
                 return False
         except EOFError:
             return False
 
         def int0(x):
             try:
                 return int(x)
             except TypeError:
                 return 0
         # always pass integer line and offset values to editor hook
         try:
             self.hooks.fix_error_editor(e.filename,
                 int0(e.lineno),int0(e.offset),e.msg)
         except IPython.ipapi.TryNext:
             warn('Could not open editor')
             return False
         return True
         
     def edit_syntax_error(self):
         """The bottom half of the syntax error handler called in the main loop.
 
         Loop until syntax error is fixed or user cancels.
         """
 
         while self.SyntaxTB.last_syntax_error:
             # copy and clear last_syntax_error
             err = self.SyntaxTB.clear_err_state()
             if not self._should_recompile(err):
                 return
             try:
                 # may set last_syntax_error again if a SyntaxError is raised
                 self.safe_execfile(err.filename,self.user_ns)
             except:
                 self.showtraceback()
             else:
                 try:
                     f = file(err.filename)
                     try:
                         sys.displayhook(f.read())
                     finally:
                         f.close()
                 except:
                     self.showtraceback()
 
     def showsyntaxerror(self, filename=None):
         """Display the syntax error that just occurred.
 
         This doesn't display a stack trace because there isn't one.
 
         If a filename is given, it is stuffed in the exception instead
         of what was there before (because Python's parser always uses
         "<string>" when reading from a string).
         """
         etype, value, last_traceback = sys.exc_info()
 
         # See note about these variables in showtraceback() below
         sys.last_type = etype
         sys.last_value = value
         sys.last_traceback = last_traceback
         
         if filename and etype is SyntaxError:
             # Work hard to stuff the correct filename in the exception
             try:
                 msg, (dummy_filename, lineno, offset, line) = value
             except:
                 # Not the format we expect; leave it alone
                 pass
             else:
                 # Stuff in the right filename
                 try:
                     # Assume SyntaxError is a class exception
                     value = SyntaxError(msg, (filename, lineno, offset, line))
                 except:
                     # If that failed, assume SyntaxError is a string
                     value = msg, (filename, lineno, offset, line)
         self.SyntaxTB(etype,value,[])
 
     def debugger(self,force=False):
         """Call the pydb/pdb debugger.
 
         Keywords:
 
           - force(False): by default, this routine checks the instance call_pdb
           flag and does not actually invoke the debugger if the flag is false.
           The 'force' option forces the debugger to activate even if the flag
           is false.
         """
 
         if not (force or self.call_pdb):
             return
 
         if not hasattr(sys,'last_traceback'):
             error('No traceback has been produced, nothing to debug.')
             return
 
         # use pydb if available
         if Debugger.has_pydb:
             from pydb import pm
         else:
             # fallback to our internal debugger
             pm = lambda : self.InteractiveTB.debugger(force=True)
         self.history_saving_wrapper(pm)()
 
     def showtraceback(self,exc_tuple = None,filename=None,tb_offset=None):
         """Display the exception that just occurred.
 
         If nothing is known about the exception, this is the method which
         should be used throughout the code for presenting user tracebacks,
         rather than directly invoking the InteractiveTB object.
 
         A specific showsyntaxerror() also exists, but this method can take
         care of calling it if needed, so unless you are explicitly catching a
         SyntaxError exception, don't try to analyze the stack manually and
         simply call this method."""
 
         
         # Though this won't be called by syntax errors in the input line,
         # there may be SyntaxError cases whith imported code.
         
         try:
             if exc_tuple is None:
                 etype, value, tb = sys.exc_info()
             else:
                 etype, value, tb = exc_tuple
     
             if etype is SyntaxError:
                 self.showsyntaxerror(filename)
             elif etype is IPython.ipapi.UsageError:
                 print "UsageError:", value
             else:
                 # WARNING: these variables are somewhat deprecated and not
                 # necessarily safe to use in a threaded environment, but tools
                 # like pdb depend on their existence, so let's set them.  If we
                 # find problems in the field, we'll need to revisit their use.
                 sys.last_type = etype
                 sys.last_value = value
                 sys.last_traceback = tb
     
                 if etype in self.custom_exceptions:
                     self.CustomTB(etype,value,tb)
                 else:
                     self.InteractiveTB(etype,value,tb,tb_offset=tb_offset)
                     if self.InteractiveTB.call_pdb and self.has_readline:
                         # pdb mucks up readline, fix it back
                         self.set_completer()
         except KeyboardInterrupt:
             self.write("\nKeyboardInterrupt\n")
 
     def mainloop(self,banner=None):
         """Creates the local namespace and starts the mainloop.
 
         If an optional banner argument is given, it will override the
         internally created default banner."""
 
         if self.rc.c:  # Emulate Python's -c option
             self.exec_init_cmd()
         if banner is None:
             if not self.rc.banner:
                 banner = ''
             # banner is string? Use it directly!
             elif isinstance(self.rc.banner,basestring):
                 banner = self.rc.banner
             else:                
                 banner = self.BANNER+self.banner2
 
         # if you run stuff with -c <cmd>, raw hist is not updated
         # ensure that it's in sync
         if len(self.input_hist) != len (self.input_hist_raw):
             self.input_hist_raw = InputList(self.input_hist)
 
         while 1:
             try:
                 self.interact(banner)
                 #self.interact_with_readline()                
 
                 # XXX for testing of a readline-decoupled repl loop, call
                 # interact_with_readline above
 
                 break
             except KeyboardInterrupt:
                 # this should not be necessary, but KeyboardInterrupt
                 # handling seems rather unpredictable...
                 self.write("\nKeyboardInterrupt in interact()\n")
 
     def exec_init_cmd(self):
         """Execute a command given at the command line.
 
         This emulates Python's -c option."""
 
         #sys.argv = ['-c']
         self.push(self.prefilter(self.rc.c, False))
         if not self.rc.interact:
             self.ask_exit()
 
     def embed_mainloop(self,header='',local_ns=None,global_ns=None,stack_depth=0):
         """Embeds IPython into a running python program.
 
         Input:
 
           - header: An optional header message can be specified.
 
           - local_ns, global_ns: working namespaces. If given as None, the
           IPython-initialized one is updated with __main__.__dict__, so that
           program variables become visible but user-specific configuration
           remains possible.
 
           - stack_depth: specifies how many levels in the stack to go to
           looking for namespaces (when local_ns and global_ns are None).  This
           allows an intermediate caller to make sure that this function gets
           the namespace from the intended level in the stack.  By default (0)
           it will get its locals and globals from the immediate caller.
 
         Warning: it's possible to use this in a program which is being run by
         IPython itself (via %run), but some funny things will happen (a few
         globals get overwritten). In the future this will be cleaned up, as
         there is no fundamental reason why it can't work perfectly."""
 
         # Get locals and globals from caller
         if local_ns is None or global_ns is None:
             call_frame = sys._getframe(stack_depth).f_back
 
             if local_ns is None:
                 local_ns = call_frame.f_locals
             if global_ns is None:
                 global_ns = call_frame.f_globals
 
         # Update namespaces and fire up interpreter
 
         # The global one is easy, we can just throw it in
         self.user_global_ns = global_ns
 
         # but the user/local one is tricky: ipython needs it to store internal
         # data, but we also need the locals.  We'll copy locals in the user
         # one, but will track what got copied so we can delete them at exit.
         # This is so that a later embedded call doesn't see locals from a
         # previous call (which most likely existed in a separate scope).
         local_varnames = local_ns.keys()
         self.user_ns.update(local_ns)
         #self.user_ns['local_ns'] = local_ns  # dbg
 
         # Patch for global embedding to make sure that things don't overwrite
         # user globals accidentally. Thanks to Richard <[e-mail]>
         # FIXME. Test this a bit more carefully (the if.. is new)
         if local_ns is None and global_ns is None:
             self.user_global_ns.update(__main__.__dict__)
 
         # make sure the tab-completer has the correct frame information, so it
         # actually completes using the frame's locals/globals
         self.set_completer_frame()
 
         # before activating the interactive mode, we need to make sure that
         # all names in the builtin namespace needed by ipython point to
         # ourselves, and not to other instances.
         self.add_builtins()
 
         self.interact(header)
         
         # now, purge out the user namespace from anything we might have added
         # from the caller's local namespace
         delvar = self.user_ns.pop
         for var in local_varnames:
             delvar(var,None)
         # and clean builtins we may have overridden
         self.clean_builtins()
 
     def interact_prompt(self):
         """ Print the prompt (in read-eval-print loop) 
 
         Provided for those who want to implement their own read-eval-print loop (e.g. GUIs), not 
         used in standard IPython flow.
         """
         if self.more:
             try:
                 prompt = self.hooks.generate_prompt(True)
             except:
                 self.showtraceback()
             if self.autoindent:
                 self.rl_do_indent = True
 
         else:
             try:
                 prompt = self.hooks.generate_prompt(False)
             except:
                 self.showtraceback()
         self.write(prompt)
 
     def interact_handle_input(self,line):
         """ Handle the input line (in read-eval-print loop)
         
         Provided for those who want to implement their own read-eval-print loop (e.g. GUIs), not 
         used in standard IPython flow.        
         """
         if line.lstrip() == line:
             self.shadowhist.add(line.strip())
         lineout = self.prefilter(line,self.more)
 
         if line.strip():
             if self.more:
                 self.input_hist_raw[-1] += '%s\n' % line
             else:
                 self.input_hist_raw.append('%s\n' % line)                
 
         
         self.more = self.push(lineout)
         if (self.SyntaxTB.last_syntax_error and
             self.rc.autoedit_syntax):
             self.edit_syntax_error()
 
     def interact_with_readline(self):
         """ Demo of using interact_handle_input, interact_prompt
         
         This is the main read-eval-print loop. If you need to implement your own (e.g. for GUI),
         it should work like this.
         """ 
         self.readline_startup_hook(self.pre_readline)
         while not self.exit_now:
             self.interact_prompt()
             if self.more:
                 self.rl_do_indent = True
             else:
                 self.rl_do_indent = False
             line = raw_input_original().decode(self.stdin_encoding)
             self.interact_handle_input(line)
 
         
     def interact(self, banner=None):
         """Closely emulate the interactive Python console.
 
         The optional banner argument specify the banner to print
         before the first interaction; by default it prints a banner
         similar to the one printed by the real Python interpreter,
         followed by the current class name in parentheses (so as not
         to confuse this with the real interpreter -- since it's so
         close!).
 
         """
         
         if self.exit_now:
             # batch run -> do not interact
             return
         cprt = 'Type "copyright", "credits" or "license" for more information.'
         if banner is None:
             self.write("Python %s on %s\n%s\n(%s)\n" %
                        (sys.version, sys.platform, cprt,
                         self.__class__.__name__))
         else:
             self.write(banner)
 
         more = 0
         
         # Mark activity in the builtins
         __builtin__.__dict__['__IPYTHON__active'] += 1
         
         if self.has_readline:
             self.readline_startup_hook(self.pre_readline)
         # exit_now is set by a call to %Exit or %Quit, through the
         # ask_exit callback.
         
         while not self.exit_now:
             self.hooks.pre_prompt_hook()
             if more:
                 try:
                     prompt = self.hooks.generate_prompt(True)
                 except:
                     self.showtraceback()
                 if self.autoindent:
                     self.rl_do_indent = True
                     
             else:
                 try:
                     prompt = self.hooks.generate_prompt(False)
                 except:
                     self.showtraceback()
             try:
                 line = self.raw_input(prompt,more)
                 if self.exit_now:
                     # quick exit on sys.std[in|out] close
                     break
                 if self.autoindent:
                     self.rl_do_indent = False
                     
             except KeyboardInterrupt:
                 #double-guard against keyboardinterrupts during kbdint handling
                 try:
                     self.write('\nKeyboardInterrupt\n')
                     self.resetbuffer()
                     # keep cache in sync with the prompt counter:
                     self.outputcache.prompt_count -= 1
     
                     if self.autoindent:
                         self.indent_current_nsp = 0
                     more = 0
                 except KeyboardInterrupt:
                     pass
             except EOFError:
                 if self.autoindent:
                     self.rl_do_indent = False
                     self.readline_startup_hook(None)
                 self.write('\n')
                 self.exit()
             except bdb.BdbQuit:
                 warn('The Python debugger has exited with a BdbQuit exception.\n'
                      'Because of how pdb handles the stack, it is impossible\n'
                      'for IPython to properly format this particular exception.\n'
                      'IPython will resume normal operation.')
             except:
                 # exceptions here are VERY RARE, but they can be triggered
                 # asynchronously by signal handlers, for example.
                 self.showtraceback()
             else:
                 more = self.push(line)
                 if (self.SyntaxTB.last_syntax_error and
                     self.rc.autoedit_syntax):
                     self.edit_syntax_error()
             
         # We are off again...
         __builtin__.__dict__['__IPYTHON__active'] -= 1
 
     def excepthook(self, etype, value, tb):
       """One more defense for GUI apps that call sys.excepthook.
 
       GUI frameworks like wxPython trap exceptions and call
       sys.excepthook themselves.  I guess this is a feature that
       enables them to keep running after exceptions that would
       otherwise kill their mainloop. This is a bother for IPython
       which excepts to catch all of the program exceptions with a try:
       except: statement.
 
       Normally, IPython sets sys.excepthook to a CrashHandler instance, so if
       any app directly invokes sys.excepthook, it will look to the user like
       IPython crashed.  In order to work around this, we can disable the
       CrashHandler and replace it with this excepthook instead, which prints a
       regular traceback using our InteractiveTB.  In this fashion, apps which
       call sys.excepthook will generate a regular-looking exception from
       IPython, and the CrashHandler will only be triggered by real IPython
       crashes.
 
       This hook should be used sparingly, only in places which are not likely
       to be true IPython errors.
       """
       self.showtraceback((etype,value,tb),tb_offset=0)
 
     def expand_aliases(self,fn,rest):
         """ Expand multiple levels of aliases:
         
         if:
         
         alias foo bar /tmp
         alias baz foo
         
         then:
         
         baz huhhahhei -> bar /tmp huhhahhei
         
         """
         line = fn + " " + rest
         
         done = set()
         while 1:
             pre,fn,rest = prefilter.splitUserInput(line,
                                                    prefilter.shell_line_split)
             if fn in self.alias_table:
                 if fn in done:
                     warn("Cyclic alias definition, repeated '%s'" % fn)
                     return ""
                 done.add(fn)
 
                 l2 = self.transform_alias(fn,rest)
                 # dir -> dir 
                 # print "alias",line, "->",l2  #dbg
                 if l2 == line:
                     break
                 # ls -> ls -F should not recurse forever
                 if l2.split(None,1)[0] == line.split(None,1)[0]:
                     line = l2
                     break
                 
                 line=l2
                 
                 
                 # print "al expand to",line #dbg
             else:
                 break
                 
         return line
 
     def transform_alias(self, alias,rest=''):
         """ Transform alias to system command string.
         """
         trg = self.alias_table[alias]
 
         nargs,cmd = trg
         # print trg #dbg
         if ' ' in cmd and os.path.isfile(cmd):
             cmd = '"%s"' % cmd
 
         # Expand the %l special to be the user's input line
         if cmd.find('%l') >= 0:
             cmd = cmd.replace('%l',rest)
             rest = ''
         if nargs==0:
             # Simple, argument-less aliases
             cmd = '%s %s' % (cmd,rest)
         else:
             # Handle aliases with positional arguments
             args = rest.split(None,nargs)
             if len(args)< nargs:
                 error('Alias <%s> requires %s arguments, %s given.' %
                       (alias,nargs,len(args)))
                 return None
             cmd = '%s %s' % (cmd % tuple(args[:nargs]),' '.join(args[nargs:]))
         # Now call the macro, evaluating in the user's namespace
         #print 'new command: <%r>' % cmd  # dbg
         return cmd
         
     def call_alias(self,alias,rest=''):
         """Call an alias given its name and the rest of the line.
 
         This is only used to provide backwards compatibility for users of
         ipalias(), use of which is not recommended for anymore."""
 
         # Now call the macro, evaluating in the user's namespace
         cmd = self.transform_alias(alias, rest)
         try:
             self.system(cmd)
         except:
             self.showtraceback()
 
     def indent_current_str(self):
         """return the current level of indentation as a string"""
         return self.indent_current_nsp * ' '
 
     def autoindent_update(self,line):
         """Keep track of the indent level."""
 
         #debugx('line')
         #debugx('self.indent_current_nsp')
         if self.autoindent:
             if line:
                 inisp = num_ini_spaces(line)
                 if inisp < self.indent_current_nsp:
                     self.indent_current_nsp = inisp
 
                 if line[-1] == ':':
                     self.indent_current_nsp += 4
                 elif dedent_re.match(line):
                     self.indent_current_nsp -= 4
             else:
                 self.indent_current_nsp = 0
 
     def runlines(self,lines):
         """Run a string of one or more lines of source.
 
         This method is capable of running a string containing multiple source
         lines, as if they had been entered at the IPython prompt.  Since it
         exposes IPython's processing machinery, the given strings can contain
         magic calls (%magic), special shell access (!cmd), etc."""
 
         # We must start with a clean buffer, in case this is run from an
         # interactive IPython session (via a magic, for example).
         self.resetbuffer()
         lines = lines.split('\n')
         more = 0
     
         for line in lines:
             # skip blank lines so we don't mess up the prompt counter, but do
             # NOT skip even a blank line if we are in a code block (more is
             # true)
             
             if line or more:
                 # push to raw history, so hist line numbers stay in sync
                 self.input_hist_raw.append("# " + line + "\n")
                 more = self.push(self.prefilter(line,more))
                 # IPython's runsource returns None if there was an error
                 # compiling the code.  This allows us to stop processing right
                 # away, so the user gets the error message at the right place.
                 if more is None:
                     break
             else:
                 self.input_hist_raw.append("\n")
         # final newline in case the input didn't have it, so that the code
         # actually does get executed
         if more:
             self.push('\n')
 
     def runsource(self, source, filename='<input>', symbol='single'):
         """Compile and run some source in the interpreter.
 
         Arguments are as for compile_command().
 
         One several things can happen:
 
         1) The input is incorrect; compile_command() raised an
         exception (SyntaxError or OverflowError).  A syntax traceback
         will be printed by calling the showsyntaxerror() method.
 
         2) The input is incomplete, and more input is required;
         compile_command() returned None.  Nothing happens.
 
         3) The input is complete; compile_command() returned a code
         object.  The code is executed by calling self.runcode() (which
         also handles run-time exceptions, except for SystemExit).
 
         The return value is:
 
           - True in case 2
 
           - False in the other cases, unless an exception is raised, where
           None is returned instead.  This can be used by external callers to
           know whether to continue feeding input or not.
 
         The return value can be used to decide whether to use sys.ps1 or
         sys.ps2 to prompt the next line."""
 
         # if the source code has leading blanks, add 'if 1:\n' to it
         # this allows execution of indented pasted code. It is tempting
         # to add '\n' at the end of source to run commands like ' a=1'
         # directly, but this fails for more complicated scenarios
         source=source.encode(self.stdin_encoding)
         if source[:1] in [' ', '\t']:
             source = 'if 1:\n%s' % source
 
         try:
             code = self.compile(source,filename,symbol)
         except (OverflowError, SyntaxError, ValueError, TypeError):
             # Case 1
             self.showsyntaxerror(filename)
             return None
 
         if code is None:
             # Case 2
             return True
 
         # Case 3
         # We store the code object so that threaded shells and
         # custom exception handlers can access all this info if needed.
         # The source corresponding to this can be obtained from the
         # buffer attribute as '\n'.join(self.buffer).
         self.code_to_run = code
         # now actually execute the code object
         if self.runcode(code) == 0:
             return False
         else:
             return None
 
     def runcode(self,code_obj):
         """Execute a code object.
 
         When an exception occurs, self.showtraceback() is called to display a
         traceback.
 
         Return value: a flag indicating whether the code to be run completed
         successfully:
 
           - 0: successful execution.
           - 1: an error occurred.
         """
 
         # Set our own excepthook in case the user code tries to call it
         # directly, so that the IPython crash handler doesn't get triggered
         old_excepthook,sys.excepthook = sys.excepthook, self.excepthook
 
         # we save the original sys.excepthook in the instance, in case config
         # code (such as magics) needs access to it.
         self.sys_excepthook = old_excepthook
         outflag = 1  # happens in more places, so it's easier as default
         try:
             try:
                 self.hooks.pre_runcode_hook()
                 exec code_obj in self.user_global_ns, self.user_ns
             finally:
                 # Reset our crash handler in place
                 sys.excepthook = old_excepthook
         except SystemExit:
             self.resetbuffer()
             self.showtraceback()
             warn("Type %exit or %quit to exit IPython "
                  "(%Exit or %Quit do so unconditionally).",level=1)
         except self.custom_exceptions:
             etype,value,tb = sys.exc_info()
             self.CustomTB(etype,value,tb)
         except:
             self.showtraceback()
         else:
             outflag = 0
             if softspace(sys.stdout, 0):
                 print
         # Flush out code object which has been run (and source)
         self.code_to_run = None
         return outflag
         
     def push(self, line):
         """Push a line to the interpreter.
 
         The line should not have a trailing newline; it may have
         internal newlines.  The line is appended to a buffer and the
         interpreter's runsource() method is called with the
         concatenated contents of the buffer as source.  If this
         indicates that the command was executed or invalid, the buffer
         is reset; otherwise, the command is incomplete, and the buffer
         is left as it was after the line was appended.  The return
         value is 1 if more input is required, 0 if the line was dealt
         with in some way (this is the same as runsource()).
         """
 
         # autoindent management should be done here, and not in the
         # interactive loop, since that one is only seen by keyboard input.  We
         # need this done correctly even for code run via runlines (which uses
         # push).
 
         #print 'push line: <%s>' % line  # dbg
         for subline in line.splitlines():
             self.autoindent_update(subline)
         self.buffer.append(line)
         more = self.runsource('\n'.join(self.buffer), self.filename)
         if not more:
             self.resetbuffer()
         return more
 
     def split_user_input(self, line):
         # This is really a hold-over to support ipapi and some extensions
         return prefilter.splitUserInput(line)
 
     def resetbuffer(self):
         """Reset the input buffer."""
         self.buffer[:] = []
         
     def raw_input(self,prompt='',continue_prompt=False):
         """Write a prompt and read a line.
 
         The returned line does not include the trailing newline.
         When the user enters the EOF key sequence, EOFError is raised.
 
         Optional inputs:
 
           - prompt(''): a string to be printed to prompt the user.
 
           - continue_prompt(False): whether this line is the first one or a
           continuation in a sequence of inputs.
         """
 
         # Code run by the user may have modified the readline completer state.
         # We must ensure that our completer is back in place.
         if self.has_readline:
             self.set_completer()
         
         try:
             line = raw_input_original(prompt).decode(self.stdin_encoding)
         except ValueError:
             warn("\n********\nYou or a %run:ed script called sys.stdin.close()"
                  " or sys.stdout.close()!\nExiting IPython!")
             self.ask_exit()
             return ""
 
         # Try to be reasonably smart about not re-indenting pasted input more
         # than necessary.  We do this by trimming out the auto-indent initial
         # spaces, if the user's actual input started itself with whitespace.
         #debugx('self.buffer[-1]')
 
         if self.autoindent:
             if num_ini_spaces(line) > self.indent_current_nsp:
                 line = line[self.indent_current_nsp:]
                 self.indent_current_nsp = 0
             
         # store the unfiltered input before the user has any chance to modify
         # it.
         if line.strip():
             if continue_prompt:
                 self.input_hist_raw[-1] += '%s\n' % line
                 if self.has_readline: # and some config option is set?
                     try:
                         histlen = self.readline.get_current_history_length()
                         if histlen > 1:
                             newhist = self.input_hist_raw[-1].rstrip()
                             self.readline.remove_history_item(histlen-1)
                             self.readline.replace_history_item(histlen-2,
                                             newhist.encode(self.stdin_encoding))
                     except AttributeError:
                         pass # re{move,place}_history_item are new in 2.4.                
             else:
                 self.input_hist_raw.append('%s\n' % line)                
             # only entries starting at first column go to shadow history
             if line.lstrip() == line:
                 self.shadowhist.add(line.strip())
         elif not continue_prompt:
             self.input_hist_raw.append('\n')
         try:
             lineout = self.prefilter(line,continue_prompt)
         except:
             # blanket except, in case a user-defined prefilter crashes, so it
             # can't take all of ipython with it.
             self.showtraceback()
             return ''
         else:
             return lineout
 
     def _prefilter(self, line, continue_prompt):
         """Calls different preprocessors, depending on the form of line."""
 
         # All handlers *must* return a value, even if it's blank ('').
 
         # Lines are NOT logged here. Handlers should process the line as
         # needed, update the cache AND log it (so that the input cache array
         # stays synced).
 
         #.....................................................................
         # Code begins
 
         #if line.startswith('%crash'): raise RuntimeError,'Crash now!'  # dbg
 
         # save the line away in case we crash, so the post-mortem handler can
         # record it
         self._last_input_line = line
 
         #print '***line: <%s>' % line # dbg
 
         if not line:
             # Return immediately on purely empty lines, so that if the user
             # previously typed some whitespace that started a continuation
             # prompt, he can break out of that loop with just an empty line.
             # This is how the default python prompt works.
 
             # Only return if the accumulated input buffer was just whitespace!
             if ''.join(self.buffer).isspace():
                 self.buffer[:] = []
             return ''
         
         line_info = prefilter.LineInfo(line, continue_prompt)
         
         # the input history needs to track even empty lines
         stripped = line.strip()
         
         if not stripped:
             if not continue_prompt:
                 self.outputcache.prompt_count -= 1
             return self.handle_normal(line_info)
 
         # print '***cont',continue_prompt  # dbg
         # special handlers are only allowed for single line statements
         if continue_prompt and not self.rc.multi_line_specials:
             return self.handle_normal(line_info)
 
 
         # See whether any pre-existing handler can take care of it  
         rewritten = self.hooks.input_prefilter(stripped)
         if rewritten != stripped: # ok, some prefilter did something
             rewritten = line_info.pre + rewritten  # add indentation
             return self.handle_normal(prefilter.LineInfo(rewritten,
                                                          continue_prompt))
             
         #print 'pre <%s> iFun <%s> rest <%s>' % (pre,iFun,theRest)  # dbg
         
         return prefilter.prefilter(line_info, self)
 
 
     def _prefilter_dumb(self, line, continue_prompt):
         """simple prefilter function, for debugging"""
         return self.handle_normal(line,continue_prompt)
 
     
     def multiline_prefilter(self, line, continue_prompt):
         """ Run _prefilter for each line of input
         
         Covers cases where there are multiple lines in the user entry,
         which is the case when the user goes back to a multiline history
         entry and presses enter.
         
         """
         out = []
         for l in line.rstrip('\n').split('\n'):
             out.append(self._prefilter(l, continue_prompt))
         return '\n'.join(out)
     
     # Set the default prefilter() function (this can be user-overridden)
     prefilter = multiline_prefilter
 
     def handle_normal(self,line_info):
         """Handle normal input lines. Use as a template for handlers."""
 
         # With autoindent on, we need some way to exit the input loop, and I
         # don't want to force the user to have to backspace all the way to
         # clear the line.  The rule will be in this case, that either two
         # lines of pure whitespace in a row, or a line of pure whitespace but
         # of a size different to the indent level, will exit the input loop.
         line = line_info.line
         continue_prompt = line_info.continue_prompt
         
         if (continue_prompt and self.autoindent and line.isspace() and
             (0 < abs(len(line) - self.indent_current_nsp) <= 2 or
              (self.buffer[-1]).isspace() )):
             line = ''
 
         self.log(line,line,continue_prompt)
         return line
 
     def handle_alias(self,line_info):
         """Handle alias input lines. """
         tgt = self.alias_table[line_info.iFun]
         # print "=>",tgt #dbg
         if callable(tgt):
             if '$' in line_info.line:
                 call_meth = '(_ip, _ip.itpl(%s))'
             else:
                 call_meth = '(_ip,%s)'
             line_out = ("%s_sh.%s" + call_meth) % (line_info.preWhitespace,
                                          line_info.iFun, 
             make_quoted_expr(line_info.line))
         else:
             transformed = self.expand_aliases(line_info.iFun,line_info.theRest)
 
             # pre is needed, because it carries the leading whitespace.  Otherwise
             # aliases won't work in indented sections.
             line_out = '%s_ip.system(%s)' % (line_info.preWhitespace,
                                              make_quoted_expr( transformed ))
         
         self.log(line_info.line,line_out,line_info.continue_prompt)
         #print 'line out:',line_out # dbg
         return line_out
 
     def handle_shell_escape(self, line_info):
         """Execute the line in a shell, empty return value"""
         #print 'line in :', `line` # dbg
         line = line_info.line
         if line.lstrip().startswith('!!'):
             # rewrite LineInfo's line, iFun and theRest to properly hold the
             # call to %sx and the actual command to be executed, so
             # handle_magic can work correctly.  Note that this works even if
             # the line is indented, so it handles multi_line_specials
             # properly.
             new_rest = line.lstrip()[2:]
             line_info.line = '%ssx %s' % (self.ESC_MAGIC,new_rest)
             line_info.iFun = 'sx'
             line_info.theRest = new_rest
             return self.handle_magic(line_info)
         else:
             cmd = line.lstrip().lstrip('!')
             line_out = '%s_ip.system(%s)' % (line_info.preWhitespace,
                                              make_quoted_expr(cmd))
         # update cache/log and return
         self.log(line,line_out,line_info.continue_prompt)
         return line_out
 
     def handle_magic(self, line_info):
         """Execute magic functions."""
         iFun    = line_info.iFun
         theRest = line_info.theRest
         cmd = '%s_ip.magic(%s)' % (line_info.preWhitespace,
                                    make_quoted_expr(iFun + " " + theRest))
         self.log(line_info.line,cmd,line_info.continue_prompt)
         #print 'in handle_magic, cmd=<%s>' % cmd  # dbg
         return cmd
 
     def handle_auto(self, line_info):
         """Hande lines which can be auto-executed, quoting if requested."""
 
         line    = line_info.line
         iFun    = line_info.iFun
         theRest = line_info.theRest
         pre     = line_info.pre
         continue_prompt = line_info.continue_prompt
         obj = line_info.ofind(self)['obj']
 
         #print 'pre <%s> iFun <%s> rest <%s>' % (pre,iFun,theRest)  # dbg
 
         # This should only be active for single-line input!
         if continue_prompt:
             self.log(line,line,continue_prompt)
             return line
 
         force_auto = isinstance(obj, IPython.ipapi.IPyAutocall)
         auto_rewrite = True
         
         if pre == self.ESC_QUOTE:
             # Auto-quote splitting on whitespace
             newcmd = '%s("%s")' % (iFun,'", "'.join(theRest.split()) )
         elif pre == self.ESC_QUOTE2:
             # Auto-quote whole string
             newcmd = '%s("%s")' % (iFun,theRest)
         elif pre == self.ESC_PAREN:
             newcmd = '%s(%s)' % (iFun,",".join(theRest.split()))
         else:
             # Auto-paren.
             # We only apply it to argument-less calls if the autocall
             # parameter is set to 2.  We only need to check that autocall is <
             # 2, since this function isn't called unless it's at least 1.
             if not theRest and (self.rc.autocall < 2) and not force_auto:
                 newcmd = '%s %s' % (iFun,theRest)
                 auto_rewrite = False
             else:
                 if not force_auto and theRest.startswith('['):
                     if hasattr(obj,'__getitem__'):
                         # Don't autocall in this case: item access for an object
                         # which is BOTH callable and implements __getitem__.
                         newcmd = '%s %s' % (iFun,theRest)
                         auto_rewrite = False
                     else:
                         # if the object doesn't support [] access, go ahead and
                         # autocall
                         newcmd = '%s(%s)' % (iFun.rstrip(),theRest)
                 elif theRest.endswith(';'):
                     newcmd = '%s(%s);' % (iFun.rstrip(),theRest[:-1])
                 else:
                     newcmd = '%s(%s)' % (iFun.rstrip(), theRest)
 
         if auto_rewrite:
             rw = self.outputcache.prompt1.auto_rewrite() + newcmd
             
             try:
                 # plain ascii works better w/ pyreadline, on some machines, so
                 # we use it and only print uncolored rewrite if we have unicode
                 rw = str(rw)
                 print >>Term.cout, rw
             except UnicodeEncodeError:
                 print "-------------->" + newcmd
             
         # log what is now valid Python, not the actual user input (without the
         # final newline)
         self.log(line,newcmd,continue_prompt)
         return newcmd
 
     def handle_help(self, line_info):
         """Try to get some help for the object.
 
         obj? or ?obj   -> basic information.
         obj?? or ??obj -> more details.
         """
         
         line = line_info.line
         # We need to make sure that we don't process lines which would be
         # otherwise valid python, such as "x=1 # what?"
         try:
             codeop.compile_command(line)
         except SyntaxError:
             # We should only handle as help stuff which is NOT valid syntax
             if line[0]==self.ESC_HELP:
                 line = line[1:]
             elif line[-1]==self.ESC_HELP:
                 line = line[:-1]
             self.log(line,'#?'+line,line_info.continue_prompt)
             if line:
                 #print 'line:<%r>' % line  # dbg
                 self.magic_pinfo(line)
             else:
                 page(self.usage,screen_lines=self.rc.screen_length)
             return '' # Empty string is needed here!
         except:
             # Pass any other exceptions through to the normal handler
             return self.handle_normal(line_info)
         else:
             # If the code compiles ok, we should handle it normally
             return self.handle_normal(line_info)
 
     def getapi(self):
         """ Get an IPApi object for this shell instance
         
         Getting an IPApi object is always preferable to accessing the shell
         directly, but this holds true especially for extensions.
         
         It should always be possible to implement an extension with IPApi
         alone. If not, contact maintainer to request an addition.
         
         """
         return self.api
 
     def handle_emacs(self, line_info):
         """Handle input lines marked by python-mode."""
 
         # Currently, nothing is done.  Later more functionality can be added
         # here if needed.
 
         # The input cache shouldn't be updated
         return line_info.line
     
 
     def mktempfile(self,data=None):
         """Make a new tempfile and return its filename.
 
         This makes a call to tempfile.mktemp, but it registers the created
         filename internally so ipython cleans it up at exit time.
 
         Optional inputs:
 
           - data(None): if data is given, it gets written out to the temp file
           immediately, and the file is closed again."""
 
         filename = tempfile.mktemp('.py','ipython_edit_')
         self.tempfiles.append(filename)
         
         if data:
             tmp_file = open(filename,'w')
             tmp_file.write(data)
             tmp_file.close()
         return filename
 
     def write(self,data):
         """Write a string to the default output"""
         Term.cout.write(data)
 
     def write_err(self,data):
         """Write a string to the default error output"""
         Term.cerr.write(data)
 
     def ask_exit(self):
         """ Call for exiting. Can be overiden and used as a callback. """
         self.exit_now = True
 
     def exit(self):
         """Handle interactive exit.
 
         This method calls the ask_exit callback."""
 
         if self.rc.confirm_exit:
             if self.ask_yes_no('Do you really want to exit ([y]/n)?','y'):
                 self.ask_exit()
         else:
             self.ask_exit()
 
     def safe_execfile(self,fname,*where,**kw):
         """A safe version of the builtin execfile().
 
         This version will never throw an exception, and knows how to handle
         ipython logs as well.
 
         :Parameters:
           fname : string
             Name of the file to be executed.
             
           where : tuple
             One or two namespaces, passed to execfile() as (globals,locals).
             If only one is given, it is passed as both.
 
         :Keywords:
           islog : boolean (False)
 
           quiet : boolean (True)
 
           exit_ignore : boolean (False)
           """
 
         def syspath_cleanup():
             """Internal cleanup routine for sys.path."""
             if add_dname:
                 try:
                     sys.path.remove(dname)
                 except ValueError:
                     # For some reason the user has already removed it, ignore.
                     pass
         
         fname = os.path.expanduser(fname)
 
         # Find things also in current directory.  This is needed to mimic the
         # behavior of running a script from the system command line, where
         # Python inserts the script's directory into sys.path
         dname = os.path.dirname(os.path.abspath(fname))
         add_dname = False
         if dname not in sys.path:
             sys.path.insert(0,dname)
             add_dname = True
 
         try:
             xfile = open(fname)
         except:
             print >> Term.cerr, \
                   'Could not open file <%s> for safe execution.' % fname
             syspath_cleanup()
             return None
 
         kw.setdefault('islog',0)
         kw.setdefault('quiet',1)
         kw.setdefault('exit_ignore',0)
         
         first = xfile.readline()
         loghead = str(self.loghead_tpl).split('\n',1)[0].strip()
         xfile.close()
         # line by line execution
         if first.startswith(loghead) or kw['islog']:
             print 'Loading log file <%s> one line at a time...' % fname
             if kw['quiet']:
                 stdout_save = sys.stdout
                 sys.stdout = StringIO.StringIO()
             try:
                 globs,locs = where[0:2]
             except:
                 try:
                     globs = locs = where[0]
                 except:
                     globs = locs = globals()
             badblocks = []
 
             # we also need to identify indented blocks of code when replaying
             # logs and put them together before passing them to an exec
             # statement. This takes a bit of regexp and look-ahead work in the
             # file. It's easiest if we swallow the whole thing in memory
             # first, and manually walk through the lines list moving the
             # counter ourselves.
             indent_re = re.compile('\s+\S')
             xfile = open(fname)
             filelines = xfile.readlines()
             xfile.close()
             nlines = len(filelines)
             lnum = 0
             while lnum < nlines:
                 line = filelines[lnum]
                 lnum += 1
                 # don't re-insert logger status info into cache
                 if line.startswith('#log#'):
                     continue
                 else:
                     # build a block of code (maybe a single line) for execution
                     block = line
                     try:
                         next = filelines[lnum] # lnum has already incremented
                     except:
                         next = None
                     while next and indent_re.match(next):
                         block += next
                         lnum += 1
                         try:
                             next = filelines[lnum]
                         except:
                             next = None
                     # now execute the block of one or more lines
                     try:
                         exec block in globs,locs
                     except SystemExit:
                         pass
                     except:
                         badblocks.append(block.rstrip())
             if kw['quiet']:  # restore stdout
                 sys.stdout.close()
                 sys.stdout = stdout_save
             print 'Finished replaying log file <%s>' % fname
             if badblocks:
                 print >> sys.stderr, ('\nThe following lines/blocks in file '
                                       '<%s> reported errors:' % fname)
                     
                 for badline in badblocks:
                     print >> sys.stderr, badline
         else:  # regular file execution
             try:
                 if sys.platform == 'win32' and sys.version_info < (2,5,1):
                     # Work around a bug in Python for Windows.  The bug was
                     # fixed in in Python 2.5 r54159 and 54158, but that's still
                     # SVN Python as of March/07.  For details, see:
                     # http://projects.scipy.org/ipython/ipython/ticket/123
                     try:
                         globs,locs = where[0:2]
                     except:
                         try:
                             globs = locs = where[0]
                         except:
                             globs = locs = globals()
                     exec file(fname) in globs,locs
                 else:
                     execfile(fname,*where)
             except SyntaxError:
                 self.showsyntaxerror()
                 warn('Failure executing file: <%s>' % fname)
             except SystemExit,status:
                 # Code that correctly sets the exit status flag to success (0)
                 # shouldn't be bothered with a traceback.  Note that a plain
                 # sys.exit() does NOT set the message to 0 (it's empty) so that
                 # will still get a traceback.  Note that the structure of the
                 # SystemExit exception changed between Python 2.4 and 2.5, so
                 # the checks must be done in a version-dependent way.
                 show = False
 
                 if sys.version_info[:2] > (2,5):
                     if status.message!=0 and not kw['exit_ignore']:
                         show = True
                 else:
                     if status.code and not kw['exit_ignore']:
                         show = True
                 if show:
                     self.showtraceback()
                     warn('Failure executing file: <%s>' % fname)
             except:
                 self.showtraceback()
                 warn('Failure executing file: <%s>' % fname)
 
         syspath_cleanup()
 
 #************************* end of file <iplib.py> *****************************

IPython/ipmaker.py

@@ -1,773 +1,773 b''
 # -*- coding: utf-8 -*-
 """
 IPython -- An enhanced Interactive Python
 
 Requires Python 2.1 or better.
 
 This file contains the main make_IPython() starter function.
 """
 
 #*****************************************************************************
 #       Copyright (C) 2008-2009 The IPython Development Team
 #       Copyright (C) 2001-2007 Fernando Perez. <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 try:
     credits._Printer__data = """
     Python: %s
 
     IPython: The IPython Development Team.
     See http://ipython.scipy.org for more information.""" \
     % credits._Printer__data
 
     copyright._Printer__data += """
 
     Copyright (c) 2008-2009 The IPython Development Team.
     Copyright (c) 2001-2007 Fernando Perez, Janko Hauser, Nathan Gray.
     All Rights Reserved."""
 except NameError:
     # Can happen if ipython was started with 'python -S', so that site.py is
     # not loaded
     pass
 
 #****************************************************************************
 # Required modules
 
 # From the standard library
 import __main__
 import __builtin__
 import os
 import re
 import sys
 import types
 from pprint import pprint,pformat
 
 # Our own
 from IPython import DPyGetOpt
 from IPython import Release
 from IPython.ipstruct import Struct
 from IPython.OutputTrap import OutputTrap
 from IPython.ConfigLoader import ConfigLoader
 from IPython.iplib import InteractiveShell
 from IPython.usage import cmd_line_usage,interactive_usage
 from IPython.genutils import *
 
-def force_import(modname):
-    if modname in sys.modules:
-        print "reload",modname
+def force_import(modname,force_reload=False):
+    if modname in sys.modules and force_reload:
+        info("reloading: %s" % modname)
         reload(sys.modules[modname])
     else:
         __import__(modname)
         
 
 #-----------------------------------------------------------------------------
 def make_IPython(argv=None,user_ns=None,user_global_ns=None,debug=1,
                  rc_override=None,shell_class=InteractiveShell,
                  embedded=False,**kw):
     """This is a dump of IPython into a single function.
 
     Later it will have to be broken up in a sensible manner.
 
     Arguments:
 
     - argv: a list similar to sys.argv[1:].  It should NOT contain the desired
     script name, b/c DPyGetOpt strips the first argument only for the real
     sys.argv.
 
     - user_ns: a dict to be used as the user's namespace."""
 
     #----------------------------------------------------------------------
     # Defaults and initialization
     
     # For developer debugging, deactivates crash handler and uses pdb.
     DEVDEBUG = False
 
     if argv is None:
         argv = sys.argv
 
     # __IP is the main global that lives throughout and represents the whole
     # application. If the user redefines it, all bets are off as to what
     # happens.
 
     # __IP is the name of he global which the caller will have accessible as
     # __IP.name. We set its name via the first parameter passed to
     # InteractiveShell:
 
     IP = shell_class('__IP',user_ns=user_ns,user_global_ns=user_global_ns,
                      embedded=embedded,**kw)
 
     # Put 'help' in the user namespace
     try:
         from site import _Helper
         IP.user_ns['help'] = _Helper()
     except ImportError:
         warn('help() not available - check site.py')
 
     if DEVDEBUG:
         # For developer debugging only (global flag)
         from IPython import ultraTB
         sys.excepthook = ultraTB.VerboseTB(call_pdb=1)
 
     IP.BANNER_PARTS = ['Python %s\n'
                          'Type "copyright", "credits" or "license" '
                          'for more information.\n'
                          % (sys.version.split('\n')[0],),
                          "IPython %s -- An enhanced Interactive Python."
                          % (Release.version,),
 """\
 ?         -> Introduction and overview of IPython's features.
 %quickref -> Quick reference.
 help      -> Python's own help system.
 object?   -> Details about 'object'. ?object also works, ?? prints more.
 """ ]
 
     IP.usage = interactive_usage
 
     # Platform-dependent suffix.
     if os.name == 'posix':
         rc_suffix = ''
     else:
         rc_suffix = '.ini'
 
     # default directory for configuration
     ipythondir_def = get_ipython_dir() 
     
     sys.path.insert(0, '') # add . to sys.path. Fix from Prabhu Ramachandran
 
     # we need the directory where IPython itself is installed
     import IPython
     IPython_dir = os.path.dirname(IPython.__file__)
     del IPython
     
     #-------------------------------------------------------------------------
     # Command line handling
 
     # Valid command line options (uses DPyGetOpt syntax, like Perl's
     # GetOpt::Long)
 
     # Any key not listed here gets deleted even if in the file (like session
     # or profile). That's deliberate, to maintain the rc namespace clean.
 
     # Each set of options appears twice: under _conv only the names are
     # listed, indicating which type they must be converted to when reading the
     # ipythonrc file. And under DPyGetOpt they are listed with the regular
     # DPyGetOpt syntax (=s,=i,:f,etc).
 
     # Make sure there's a space before each end of line (they get auto-joined!)
     cmdline_opts = ('autocall=i autoindent! automagic! banner! cache_size|cs=i '
                     'c=s classic|cl color_info! colors=s confirm_exit! '
                     'debug! deep_reload! editor=s log|l messages! nosep '
                     'object_info_string_level=i pdb! '
                     'pprint! prompt_in1|pi1=s prompt_in2|pi2=s prompt_out|po=s '
                     'pydb! '
                     'pylab_import_all! '
                     'quick screen_length|sl=i prompts_pad_left=i '
                     'logfile|lf=s logplay|lp=s profile|p=s '
                     'readline! readline_merge_completions! '
                     'readline_omit__names! '
                     'rcfile=s separate_in|si=s separate_out|so=s '
                     'separate_out2|so2=s xmode=s wildcards_case_sensitive! '
                     'magic_docstrings system_verbose! '
                     'multi_line_specials! '
                     'term_title! wxversion=s '
                     'autoedit_syntax!')
 
     # Options that can *only* appear at the cmd line (not in rcfiles).
     
     cmdline_only = ('help interact|i ipythondir=s Version upgrade '
                     'gthread! qthread! q4thread! wthread! tkthread! pylab! tk! '
                     # 'twisted!'  # disabled for now.
                     )
 
     # Build the actual name list to be used by DPyGetOpt
     opts_names = qw(cmdline_opts) + qw(cmdline_only)
 
     # Set sensible command line defaults.
     # This should have everything from  cmdline_opts and cmdline_only
     opts_def = Struct(autocall = 1,
                       autoedit_syntax = 0,
                       autoindent = 0,
                       automagic = 1,
                       autoexec = [],
                       banner = 1,
                       c = '',
                       cache_size = 1000,
                       classic = 0,
                       color_info = 0,
                       colors = 'NoColor',
                       confirm_exit = 1,
                       debug = 0,
                       deep_reload = 0,
                       editor = '0',
                       gthread = 0,
                       help = 0,
                       interact = 0,
                       ipythondir = ipythondir_def,
                       log = 0,
                       logfile = '',
                       logplay = '',
                       messages = 1,
                       multi_line_specials = 1,
                       nosep = 0,
                       object_info_string_level = 0,
                       pdb = 0,
                       pprint = 0,
                       profile = '',
                       prompt_in1 = 'In [\\#]: ',
                       prompt_in2 = '   .\\D.: ',
                       prompt_out = 'Out[\\#]: ',
                       prompts_pad_left = 1,
                       pydb = 0,
                       pylab = 0,
                       pylab_import_all = 1,
                       q4thread = 0,
                       qthread = 0,
                       quick = 0,
                       quiet = 0,
                       rcfile = 'ipythonrc' + rc_suffix,
                       readline = 1,
                       readline_merge_completions = 1,
                       readline_omit__names = 0,
                       screen_length = 0,
                       separate_in = '\n',
                       separate_out = '\n',
                       separate_out2 = '',
                       system_header = 'IPython system call: ',
                       system_verbose = 0,
                       term_title = 1,
                       tk = 0,
                       #twisted= 0,  # disabled for now
                       upgrade = 0,
                       Version = 0,
                       wildcards_case_sensitive = 1,
                       wthread = 0,
                       wxversion = '0',
                       xmode = 'Context',
                       magic_docstrings = 0,  # undocumented, for doc generation
                       )
     
     # Things that will *only* appear in rcfiles (not at the command line).
     # Make sure there's a space before each end of line (they get auto-joined!)
     rcfile_opts = { qwflat: 'include import_mod import_all execfile ',
                     qw_lol: 'import_some ',
                     # for things with embedded whitespace:
                     list_strings:'execute alias readline_parse_and_bind ',
                     # Regular strings need no conversion:
                     None:'readline_remove_delims ',
                     }
     # Default values for these
     rc_def = Struct(include = [],
                     import_mod = [], 
                     import_all = [],
                     import_some = [[]],
                     execute = [],
                     execfile = [],
                     alias = [],
                     readline_parse_and_bind = [],
                     readline_remove_delims = '',
                     )
 
     # Build the type conversion dictionary from the above tables:
     typeconv = rcfile_opts.copy()
     typeconv.update(optstr2types(cmdline_opts))
 
     # FIXME: the None key appears in both, put that back together by hand. Ugly!
     typeconv[None] += ' ' + rcfile_opts[None]
 
     # Remove quotes at ends of all strings (used to protect spaces)
     typeconv[unquote_ends] = typeconv[None]
     del typeconv[None]
 
     # Build the list we'll use to make all config decisions with defaults:
     opts_all = opts_def.copy()
     opts_all.update(rc_def)
 
     # Build conflict resolver for recursive loading of config files:
     # - preserve means the outermost file maintains the value, it is not
     # overwritten if an included file has the same key.
     # - add_flip applies + to the two values, so it better make sense to add
     # those types of keys. But it flips them first so that things loaded
     # deeper in the inclusion chain have lower precedence.
     conflict = {'preserve': ' '.join([ typeconv[int],
                                        typeconv[unquote_ends] ]),
                 'add_flip': ' '.join([ typeconv[qwflat],
                                        typeconv[qw_lol],
                                        typeconv[list_strings] ])
                 }
 
     # Now actually process the command line
     getopt = DPyGetOpt.DPyGetOpt()
     getopt.setIgnoreCase(0)
 
     getopt.parseConfiguration(opts_names)
 
     try:
         getopt.processArguments(argv)
     except DPyGetOpt.ArgumentError, exc:
         print cmd_line_usage
         warn('\nError in Arguments: "%s"' % exc)
         sys.exit(1)
 
     # convert the options dict to a struct for much lighter syntax later
     opts = Struct(getopt.optionValues)
     args = getopt.freeValues
 
     # this is the struct (which has default values at this point) with which
     # we make all decisions:
     opts_all.update(opts)
 
     # Options that force an immediate exit
     if opts_all.help:
         page(cmd_line_usage)
         sys.exit()
 
     if opts_all.Version:
         print Release.version
         sys.exit()
 
     if opts_all.magic_docstrings:
         IP.magic_magic('-latex')
         sys.exit()
 
     # add personal ipythondir to sys.path so that users can put things in
     # there for customization
     sys.path.append(os.path.abspath(opts_all.ipythondir))
 
     # Create user config directory if it doesn't exist. This must be done
     # *after* getting the cmd line options.
     if not os.path.isdir(opts_all.ipythondir):
         IP.user_setup(opts_all.ipythondir,rc_suffix,'install')
 
     # upgrade user config files while preserving a copy of the originals
     if opts_all.upgrade:
         IP.user_setup(opts_all.ipythondir,rc_suffix,'upgrade')
 
     # check mutually exclusive options in the *original* command line
     mutex_opts(opts,[qw('log logfile'),qw('rcfile profile'),
                      qw('classic profile'),qw('classic rcfile')])
 
     #---------------------------------------------------------------------------
     # Log replay
     
     # if -logplay, we need to 'become' the other session. That basically means
     # replacing the current command line environment with that of the old
     # session and moving on.
 
     # this is needed so that later we know we're in session reload mode, as
     # opts_all will get overwritten:
     load_logplay = 0
 
     if opts_all.logplay:
         load_logplay = opts_all.logplay 
         opts_debug_save = opts_all.debug
         try:
             logplay = open(opts_all.logplay)
         except IOError:
             if opts_all.debug: IP.InteractiveTB()
             warn('Could not open logplay file '+`opts_all.logplay`)
             # restore state as if nothing had happened and move on, but make
             # sure that later we don't try to actually load the session file
             logplay = None
             load_logplay = 0
             del opts_all.logplay
         else:
             try: 
                 logplay.readline()
                 logplay.readline();
                 # this reloads that session's command line
                 cmd = logplay.readline()[6:] 
                 exec cmd
                 # restore the true debug flag given so that the process of
                 # session loading itself can be monitored.
                 opts.debug = opts_debug_save
                 # save the logplay flag so later we don't overwrite the log
                 opts.logplay = load_logplay
                 # now we must update our own structure with defaults
                 opts_all.update(opts)
                 # now load args
                 cmd = logplay.readline()[6:] 
                 exec cmd
                 logplay.close()
             except:
                 logplay.close()
                 if opts_all.debug: IP.InteractiveTB()
                 warn("Logplay file lacking full configuration information.\n"
                      "I'll try to read it, but some things may not work.")
 
     #-------------------------------------------------------------------------
     # set up output traps: catch all output from files, being run, modules
     # loaded, etc. Then give it to the user in a clean form at the end.
     
     msg_out = 'Output messages. '
     msg_err = 'Error messages. '
     msg_sep = '\n'
     msg = Struct(config    = OutputTrap('Configuration Loader',msg_out,
                                         msg_err,msg_sep,debug,
                                         quiet_out=1),
                  user_exec = OutputTrap('User File Execution',msg_out,
                                         msg_err,msg_sep,debug),
                  logplay   = OutputTrap('Log Loader',msg_out,
                                         msg_err,msg_sep,debug),
                  summary = ''
                  )
 
     #-------------------------------------------------------------------------
     # Process user ipythonrc-type configuration files
 
     # turn on output trapping and log to msg.config
     # remember that with debug on, trapping is actually disabled
     msg.config.trap_all()
 
     # look for rcfile in current or default directory
     try:
         opts_all.rcfile = filefind(opts_all.rcfile,opts_all.ipythondir)
     except IOError:
         if opts_all.debug:  IP.InteractiveTB()
         warn('Configuration file %s not found. Ignoring request.'
              % (opts_all.rcfile) )
 
     # 'profiles' are a shorthand notation for config filenames
     profile_handled_by_legacy = False
     if opts_all.profile:
             
         try:
             opts_all.rcfile = filefind('ipythonrc-' + opts_all.profile
                                        + rc_suffix,
                                        opts_all.ipythondir)
             profile_handled_by_legacy = True
         except IOError:
            if opts_all.debug:  IP.InteractiveTB()
            opts.profile = ''  # remove profile from options if invalid
            # We won't warn anymore, primary method is ipy_profile_PROFNAME
            # which does trigger a warning.
 
     # load the config file
     rcfiledata = None
     if opts_all.quick:
         print 'Launching IPython in quick mode. No config file read.'
     elif opts_all.rcfile:
         try:
             cfg_loader = ConfigLoader(conflict)
             rcfiledata = cfg_loader.load(opts_all.rcfile,typeconv,
                                          'include',opts_all.ipythondir,
                                          purge = 1,
                                          unique = conflict['preserve'])
         except:
             IP.InteractiveTB()
             warn('Problems loading configuration file '+
                  `opts_all.rcfile`+
                  '\nStarting with default -bare bones- configuration.')
     else:
         warn('No valid configuration file found in either currrent directory\n'+
              'or in the IPython config. directory: '+`opts_all.ipythondir`+
              '\nProceeding with internal defaults.')
 
     #------------------------------------------------------------------------
     # Set exception handlers in mode requested by user.
     otrap = OutputTrap(trap_out=1)  # trap messages from magic_xmode
     IP.magic_xmode(opts_all.xmode)
     otrap.release_out()
 
     #------------------------------------------------------------------------
     # Execute user config
 
     # Create a valid config structure with the right precedence order:
     # defaults < rcfile < command line.  This needs to be in the instance, so
     # that method calls below that rely on it find it.
     IP.rc = rc_def.copy()
 
     # Work with a local alias inside this routine to avoid unnecessary
     # attribute lookups.
     IP_rc = IP.rc
     
     IP_rc.update(opts_def)
     if rcfiledata:
         # now we can update 
         IP_rc.update(rcfiledata)
     IP_rc.update(opts)
     IP_rc.update(rc_override)
 
     # Store the original cmd line for reference:
     IP_rc.opts = opts
     IP_rc.args = args
 
     # create a *runtime* Struct like rc for holding parameters which may be
     # created and/or modified by runtime user extensions.
     IP.runtime_rc = Struct()
 
     # from this point on, all config should be handled through IP_rc,
     # opts* shouldn't be used anymore.
 
     
     # update IP_rc with some special things that need manual
     # tweaks. Basically options which affect other options. I guess this
     # should just be written so that options are fully orthogonal and we
     # wouldn't worry about this stuff!
 
     if IP_rc.classic:
         IP_rc.quick = 1
         IP_rc.cache_size = 0
         IP_rc.pprint = 0
         IP_rc.prompt_in1 = '>>> '
         IP_rc.prompt_in2 = '... '
         IP_rc.prompt_out = ''
         IP_rc.separate_in = IP_rc.separate_out = IP_rc.separate_out2 = '0'
         IP_rc.colors = 'NoColor'
         IP_rc.xmode = 'Plain'
 
     IP.pre_config_initialization()
     # configure readline
 
     # update exception handlers with rc file status
     otrap.trap_out()  # I don't want these messages ever.
     IP.magic_xmode(IP_rc.xmode)
     otrap.release_out()
 
     # activate logging if requested and not reloading a log
     if IP_rc.logplay:
         IP.magic_logstart(IP_rc.logplay + ' append')
     elif  IP_rc.logfile:
         IP.magic_logstart(IP_rc.logfile)
     elif IP_rc.log:
         IP.magic_logstart()
 
     # find user editor so that it we don't have to look it up constantly
     if IP_rc.editor.strip()=='0':
         try:
             ed = os.environ['EDITOR']
         except KeyError:
             if os.name == 'posix':
                 ed = 'vi'  # the only one guaranteed to be there!
             else:
                 ed = 'notepad' # same in Windows!
         IP_rc.editor = ed
 
     # Keep track of whether this is an embedded instance or not (useful for
     # post-mortems).
     IP_rc.embedded = IP.embedded
 
     # Recursive reload
     try:
         from IPython import deep_reload
         if IP_rc.deep_reload:
             __builtin__.reload = deep_reload.reload
         else:
             __builtin__.dreload = deep_reload.reload
         del deep_reload
     except ImportError:
         pass
 
     # Save the current state of our namespace so that the interactive shell
     # can later know which variables have been created by us from config files
     # and loading. This way, loading a file (in any way) is treated just like
     # defining things on the command line, and %who works as expected.
 
     # DON'T do anything that affects the namespace beyond this point!
     IP.internal_ns.update(__main__.__dict__)
 
     #IP.internal_ns.update(locals()) # so our stuff doesn't show up in %who
 
     # Now run through the different sections of the users's config
     if IP_rc.debug:    
         print 'Trying to execute the following configuration structure:'
         print '(Things listed first are deeper in the inclusion tree and get'
         print 'loaded first).\n'
         pprint(IP_rc.__dict__)
         
     for mod in IP_rc.import_mod:
         try:
             exec 'import '+mod in IP.user_ns
         except :
             IP.InteractiveTB()
             import_fail_info(mod)
 
     for mod_fn in IP_rc.import_some:
         if not mod_fn == []: 
             mod,fn = mod_fn[0],','.join(mod_fn[1:])
             try:
                 exec 'from '+mod+' import '+fn in IP.user_ns
             except :
                 IP.InteractiveTB()
                 import_fail_info(mod,fn)
 
     for mod in IP_rc.import_all:
         try:
             exec 'from '+mod+' import *' in IP.user_ns
         except :
             IP.InteractiveTB()
             import_fail_info(mod)
 
     for code in IP_rc.execute:
         try:
             exec code in IP.user_ns
         except:
             IP.InteractiveTB()
             warn('Failure executing code: ' + `code`)
 
     # Execute the files the user wants in ipythonrc
     for file in IP_rc.execfile:
         try:
             file = filefind(file,sys.path+[IPython_dir])
         except IOError:
             warn(itpl('File $file not found. Skipping it.'))
         else:
             IP.safe_execfile(os.path.expanduser(file),IP.user_ns)
 
     # finally, try importing ipy_*_conf for final configuration
     try:
         import ipy_system_conf
     except ImportError:
         if opts_all.debug:  IP.InteractiveTB()
         warn("Could not import 'ipy_system_conf'")        
     except:
         IP.InteractiveTB()
         import_fail_info('ipy_system_conf')
-        
+
     # only import prof module if ipythonrc-PROF was not found
     if opts_all.profile and not profile_handled_by_legacy:
         profmodname = 'ipy_profile_' + opts_all.profile
         try:
-            
             force_import(profmodname)
         except:
             IP.InteractiveTB()
-            print "Error importing",profmodname,"- perhaps you should run %upgrade?"
+            print "Error importing",profmodname,\
+                  "- perhaps you should run %upgrade?"
             import_fail_info(profmodname)
         else:
             opts.profile = opts_all.profile
     else:
         force_import('ipy_profile_none')
+    # XXX - this is wrong: ipy_user_conf should not be loaded unconditionally,
+    # since the user could have specified a config file path by hand.
     try:
-            
         force_import('ipy_user_conf')
-        
     except:
         conf = opts_all.ipythondir + "/ipy_user_conf.py"
         IP.InteractiveTB()
         if not os.path.isfile(conf):
             warn(conf + ' does not exist, please run %upgrade!')
 
         import_fail_info("ipy_user_conf")
 
     # Define the history file for saving commands in between sessions
     try:
         histfname = 'history-%s' % opts.profile
     except AttributeError:
         histfname = 'history'
     IP.histfile = os.path.join(opts_all.ipythondir,histfname)
 
     # finally, push the argv to options again to ensure highest priority
     IP_rc.update(opts)
     
     # release stdout and stderr and save config log into a global summary
     msg.config.release_all()
     if IP_rc.messages:
         msg.summary += msg.config.summary_all()
 
     #------------------------------------------------------------------------
     # Setup interactive session
 
     # Now we should be fully configured. We can then execute files or load
     # things only needed for interactive use. Then we'll open the shell.
 
     # Take a snapshot of the user namespace before opening the shell. That way
     # we'll be able to identify which things were interactively defined and
     # which were defined through config files.
     IP.user_config_ns.update(IP.user_ns)
 
     # Force reading a file as if it were a session log. Slower but safer.
     if load_logplay:
         print 'Replaying log...'
         try:
             if IP_rc.debug:
                 logplay_quiet = 0
             else:
                  logplay_quiet = 1
 
             msg.logplay.trap_all()
             IP.safe_execfile(load_logplay,IP.user_ns,
                              islog = 1, quiet = logplay_quiet)
             msg.logplay.release_all()
             if IP_rc.messages:
                 msg.summary += msg.logplay.summary_all()
         except:
             warn('Problems replaying logfile %s.' % load_logplay)
             IP.InteractiveTB()
 
     # Load remaining files in command line
     msg.user_exec.trap_all()
 
     # Do NOT execute files named in the command line as scripts to be loaded
     # by embedded instances.  Doing so has the potential for an infinite
     # recursion if there are exceptions thrown in the process.
 
     # XXX FIXME: the execution of user files should be moved out to after
     # ipython is fully initialized, just as if they were run via %run at the
     # ipython prompt.  This would also give them the benefit of ipython's
     # nice tracebacks.
 
     if (not embedded and IP_rc.args and 
         not IP_rc.args[0].lower().endswith('.ipy')):
         name_save = IP.user_ns['__name__']
         IP.user_ns['__name__'] = '__main__'
         # Set our own excepthook in case the user code tries to call it
         # directly. This prevents triggering the IPython crash handler.
         old_excepthook,sys.excepthook = sys.excepthook, IP.excepthook
 
         save_argv = sys.argv[1:] # save it for later restoring
         
         sys.argv = args
         
         try:
             IP.safe_execfile(args[0], IP.user_ns)
         finally:
             # Reset our crash handler in place
             sys.excepthook = old_excepthook
             sys.argv[:] = save_argv
             IP.user_ns['__name__'] = name_save
         
     msg.user_exec.release_all()
 
     if IP_rc.messages:
         msg.summary += msg.user_exec.summary_all()
 
     # since we can't specify a null string on the cmd line, 0 is the equivalent:
     if IP_rc.nosep:
         IP_rc.separate_in = IP_rc.separate_out = IP_rc.separate_out2 = '0'
     if IP_rc.separate_in == '0': IP_rc.separate_in = ''
     if IP_rc.separate_out == '0': IP_rc.separate_out = ''
     if IP_rc.separate_out2 == '0': IP_rc.separate_out2 = ''
     IP_rc.separate_in = IP_rc.separate_in.replace('\\n','\n')
     IP_rc.separate_out = IP_rc.separate_out.replace('\\n','\n')
     IP_rc.separate_out2 = IP_rc.separate_out2.replace('\\n','\n')
 
     # Determine how many lines at the bottom of the screen are needed for
     # showing prompts, so we can know wheter long strings are to be printed or
     # paged:
     num_lines_bot = IP_rc.separate_in.count('\n')+1
     IP_rc.screen_length = IP_rc.screen_length - num_lines_bot
 
     # configure startup banner
     if IP_rc.c:  # regular python doesn't print the banner with -c
         IP_rc.banner = 0
     if IP_rc.banner:
         BANN_P = IP.BANNER_PARTS
     else:
         BANN_P = []
 
     if IP_rc.profile: BANN_P.append('IPython profile: %s\n' % IP_rc.profile)
 
     # add message log (possibly empty)
     if msg.summary: BANN_P.append(msg.summary)
     # Final banner is a string
     IP.BANNER = '\n'.join(BANN_P)
 
     # Finalize the IPython instance.  This assumes the rc structure is fully
     # in place.
     IP.post_config_initialization()
 
     return IP
 #************************ end of file <ipmaker.py> **************************

IPython/kernel/__init__.py

@@ -1,24 +1,25 b''
 # encoding: utf-8
 """The IPython1 kernel.
 
 The IPython kernel actually refers to three things:
 
  * The IPython Engine
  * The IPython Controller
  * Clients to the IPython Controller
 
 The kernel module implements the engine, controller and client and all the 
 network protocols needed for the various entities to talk to each other.
 
 An end user should probably begin by looking at the `client.py` module
 if they need blocking clients or in `asyncclient.py` if they want asynchronous,
 deferred/Twisted using clients.
 """
 __docformat__ = "restructuredtext en"
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
-    
  No newline at end of file
+#-----------------------------------------------------------------------------
+
+from IPython.kernel.error import TaskRejectError
  No newline at end of file

IPython/kernel/config/__init__.py

@@ -1,124 +1,126 b''
 # encoding: utf-8
 
 """Default kernel configuration."""
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
+import os, sys
 from os.path import join as pjoin
 
 from IPython.external.configobj import ConfigObj
 from IPython.config.api import ConfigObjManager
 from IPython.genutils import get_ipython_dir, get_security_dir
 
 default_kernel_config = ConfigObj()
 
+# This will raise OSError if ipythondir doesn't exist.
 security_dir = get_security_dir()
 
 #-------------------------------------------------------------------------------
 # Engine Configuration
 #-------------------------------------------------------------------------------
 
 engine_config = dict(
     logfile = '',    # Empty means log to stdout
     furl_file = pjoin(security_dir, 'ipcontroller-engine.furl')
 )
 
 #-------------------------------------------------------------------------------
 # MPI Configuration
 #-------------------------------------------------------------------------------
 
 mpi_config = dict(
     mpi4py = """from mpi4py import MPI as mpi
 mpi.size = mpi.COMM_WORLD.Get_size()
 mpi.rank = mpi.COMM_WORLD.Get_rank()
 """,
     pytrilinos = """from PyTrilinos import Epetra
 class SimpleStruct:
     pass
 mpi = SimpleStruct()
 mpi.rank = 0
 mpi.size = 0
 """,
     default = ''
 )
 
 #-------------------------------------------------------------------------------
 # Controller Configuration
 #-------------------------------------------------------------------------------
 
 controller_config = dict(
 
     logfile = '',    # Empty means log to stdout
     import_statement = '',
     reuse_furls = False, # If False, old furl files are deleted
 
     engine_tub = dict(
         ip = '',         # Empty string means all interfaces
         port = 0,        # 0 means pick a port for me
         location = '',    # Empty string means try to set automatically
         secure = True,
         cert_file = pjoin(security_dir, 'ipcontroller-engine.pem'),
     ),
     engine_fc_interface = 'IPython.kernel.enginefc.IFCControllerBase',
     engine_furl_file = pjoin(security_dir, 'ipcontroller-engine.furl'),
     
     controller_interfaces = dict(
         # multiengine = dict(
         #     controller_interface = 'IPython.kernel.multiengine.IMultiEngine',
         #     fc_interface = 'IPython.kernel.multienginefc.IFCMultiEngine',
         #     furl_file = 'ipcontroller-mec.furl'
         # ),
         task = dict(
             controller_interface = 'IPython.kernel.task.ITaskController',
             fc_interface = 'IPython.kernel.taskfc.IFCTaskController',
             furl_file = pjoin(security_dir, 'ipcontroller-tc.furl')
         ),
         multiengine = dict(
             controller_interface = 'IPython.kernel.multiengine.IMultiEngine',
             fc_interface = 'IPython.kernel.multienginefc.IFCSynchronousMultiEngine',
             furl_file = pjoin(security_dir, 'ipcontroller-mec.furl')
         )
     ),
 
     client_tub = dict(
         ip = '',         # Empty string means all interfaces
         port = 0,        # 0 means pick a port for me
         location = '',    # Empty string means try to set automatically
         secure = True,
         cert_file = pjoin(security_dir, 'ipcontroller-client.pem')
     )
 )
 
 #-------------------------------------------------------------------------------
 # Client Configuration
 #-------------------------------------------------------------------------------
 
 client_config = dict(
     client_interfaces = dict(
         task = dict(
             furl_file = pjoin(security_dir, 'ipcontroller-tc.furl')
         ),
         multiengine = dict(
             furl_file = pjoin(security_dir, 'ipcontroller-mec.furl')
         )
     )
 )
 
 default_kernel_config['engine'] = engine_config
 default_kernel_config['mpi'] = mpi_config
 default_kernel_config['controller'] = controller_config
 default_kernel_config['client'] = client_config
 
 
 config_manager = ConfigObjManager(default_kernel_config, 'IPython.kernel.ini')
  No newline at end of file

IPython/kernel/core/interpreter.py

@@ -1,754 +1,761 b''
 # encoding: utf-8
 
 """Central interpreter object for an IPython engine.
 
 The interpreter is the object whose job is to process lines of user input and
 actually execute them in the user's namespace.
 """
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 # Standard library imports.
 from types import FunctionType
 
 import __builtin__
 import codeop
 import compiler
 import sys
 import traceback
 
 # Local imports.
 from IPython.kernel.core import ultraTB
 from IPython.kernel.core.display_trap import DisplayTrap
 from IPython.kernel.core.macro import Macro
 from IPython.kernel.core.prompts import CachedOutput
 from IPython.kernel.core.traceback_trap import TracebackTrap
 from IPython.kernel.core.util import Bunch, system_shell
 from IPython.external.Itpl import ItplNS
 
 # Global constants
 COMPILER_ERROR = 'error'
 INCOMPLETE_INPUT = 'incomplete'
 COMPLETE_INPUT = 'complete'
 
 ##############################################################################
 # TEMPORARY!!! fake configuration, while we decide whether to use tconfig or
 # not
 
 rc = Bunch()
 rc.cache_size = 100
 rc.pprint = True
 rc.separate_in = '\n'
 rc.separate_out = '\n'
 rc.separate_out2 = ''
 rc.prompt_in1 = r'In [\#]: '
 rc.prompt_in2 = r'   .\\D.: '
 rc.prompt_out = ''
 rc.prompts_pad_left = False
 
 ##############################################################################
 
 # Top-level utilities
 def default_display_formatters():
     """ Return a list of default display formatters.
     """
 
     from display_formatter import PPrintDisplayFormatter, ReprDisplayFormatter
     return [PPrintDisplayFormatter(), ReprDisplayFormatter()]
 
 def default_traceback_formatters():
     """ Return a list of default traceback formatters.
     """
 
     from traceback_formatter import PlainTracebackFormatter
     return [PlainTracebackFormatter()]
 
 # Top-level classes
 class NotDefined(object):    pass
 
 class Interpreter(object):
     """ An interpreter object.
 
     fixme: needs to negotiate available formatters with frontends.
 
     Important: the interpeter should be built so that it exposes a method
     for each attribute/method of its sub-object. This way it can be
     replaced by a network adapter.
     """
 
     def __init__(self, user_ns=None, global_ns=None,translator=None,
                  magic=None, display_formatters=None,
                  traceback_formatters=None, output_trap=None, history=None,
                  message_cache=None, filename='<string>', config=None):
 
         # The local/global namespaces for code execution
         local_ns = user_ns  # compatibility name
         if local_ns is None:
             local_ns = {}
         self.user_ns = local_ns
         # The local namespace
         if global_ns is None:
             global_ns = {}
         self.user_global_ns = global_ns
 
         # An object that will translate commands into executable Python.
         # The current translator does not work properly so for now we are going
         # without!
         # if translator is None:
         #             from IPython.kernel.core.translator import IPythonTranslator
         #             translator = IPythonTranslator()
         self.translator = translator
 
         # An object that maintains magic commands.
         if magic is None:
             from IPython.kernel.core.magic import Magic
             magic = Magic(self)
         self.magic = magic
 
         # A list of formatters for the displayhook.
         if display_formatters is None:
             display_formatters = default_display_formatters()
         self.display_formatters = display_formatters
 
         # A list of formatters for tracebacks.
         if traceback_formatters is None:
             traceback_formatters = default_traceback_formatters()
         self.traceback_formatters = traceback_formatters
 
         # The object trapping stdout/stderr.
         if output_trap is None:
             from IPython.kernel.core.output_trap import OutputTrap
             output_trap = OutputTrap()
         self.output_trap = output_trap
 
         # An object that manages the history.
         if history is None:
             from IPython.kernel.core.history import InterpreterHistory
             history = InterpreterHistory()
         self.history = history
         self.get_history_item = history.get_history_item
         self.get_history_input_cache = history.get_input_cache
         self.get_history_input_after = history.get_input_after
 
         # An object that caches all of the return messages.
         if message_cache is None:
             from IPython.kernel.core.message_cache import SimpleMessageCache
             message_cache = SimpleMessageCache()
         self.message_cache = message_cache
 
         # The "filename" of the code that is executed in this interpreter.
         self.filename = filename
 
         # An object that contains much configuration information.
         if config is None:
             # fixme: Move this constant elsewhere!
             config = Bunch(ESC_MAGIC='%')
         self.config = config
 
         # Hook managers.
         # fixme: make the display callbacks configurable. In the meantime,
         # enable macros.
         self.display_trap = DisplayTrap(
             formatters=self.display_formatters,
             callbacks=[self._possible_macro],
         )
         self.traceback_trap = TracebackTrap(
             formatters=self.traceback_formatters)
 
         # This is used temporarily for reformating exceptions in certain
         # cases.  It will go away once the ultraTB stuff is ported
         # to ipython1
         self.tbHandler = ultraTB.FormattedTB(color_scheme='NoColor',
                                                  mode='Context',
                                                  tb_offset=2)
 
         # An object that can compile commands and remember __future__
         # statements.
         self.command_compiler = codeop.CommandCompiler()
 
         # A replacement for the raw_input() and input() builtins. Change these
         # attributes later to configure them.
         self.raw_input_builtin = raw_input
         self.input_builtin = input
 
         # The number of the current cell.
         self.current_cell_number = 1
 
         # Initialize cache, set in/out prompts and printing system
         self.outputcache = CachedOutput(self,
                                         rc.cache_size,
                                         rc.pprint,
                                         input_sep = rc.separate_in,
                                         output_sep = rc.separate_out,
                                         output_sep2 = rc.separate_out2,
                                         ps1 = rc.prompt_in1,
                                         ps2 = rc.prompt_in2,
                                         ps_out = rc.prompt_out,
                                         pad_left = rc.prompts_pad_left)
 
         # Need to decide later if this is the right approach, but clients
         # commonly use sys.ps1/2, so it may be best to just set them here
         sys.ps1 = self.outputcache.prompt1.p_str
         sys.ps2 = self.outputcache.prompt2.p_str
 
         # This is the message dictionary assigned temporarily when running the
         # code.
         self.message = None
 
         self.setup_namespace()
 
 
     #### Public 'Interpreter' interface ########################################
 
     def formatTraceback(self, et, ev, tb, message=''):
         """Put a formatted version of the traceback into value and reraise.
         
         When exceptions have to be sent over the network, the traceback 
         needs to be put into the value of the exception in a nicely
         formatted way.  The method takes the type, value and tb of an 
         exception and puts a string representation of the tb into the 
         value of the exception and reraises it.
         
         Currently this method uses the ultraTb formatter from IPython trunk.
         Eventually it should simply use the traceback formatters in core
         that are loaded into self.tracback_trap.formatters.
         """
         tbinfo = self.tbHandler.text(et,ev,tb)
         ev._ipython_traceback_text = tbinfo
         return et, ev, tb
         
     def execute(self, commands, raiseException=True):
         """ Execute some IPython commands.
 
         1. Translate them into Python.
         2. Run them.
         3. Trap stdout/stderr.
         4. Trap sys.displayhook().
         5. Trap exceptions.
         6. Return a message object.
 
         Parameters
         ----------
         commands : str
             The raw commands that the user typed into the prompt.
         
         Returns
         -------
         message : dict
             The dictionary of responses. See the README.txt in this directory
             for an explanation of the format.
         """
 
         # Create a message dictionary with all of the information we will be
         # returning to the frontend and other listeners.
         message = self.setup_message()
 
         # Massage the input and store the raw and translated commands into
         # a dict.
         user_input = dict(raw=commands)
         if self.translator is not None:
             python = self.translator(commands, message)
             if python is None:
                 # Something went wrong with the translation. The translator
                 # should have added an appropriate entry to the message object.
                 return message
         else:
             python = commands
         user_input['translated'] = python
         message['input'] = user_input
 
         # Set the message object so that any magics executed in the code have
         # access.
         self.message = message
 
         # Set all of the output/exception traps.
         self.set_traps()
         
         # Actually execute the Python code.
         status = self.execute_python(python)
 
         # Unset all of the traps.
         self.unset_traps()
 
         # Unset the message object.
         self.message = None
 
         # Update the history variables in the namespace.
         # E.g. In, Out, _, __, ___
         if self.history is not None:
             self.history.update_history(self, python)
 
         # Let all of the traps contribute to the message and then clear their
         # stored information.
         self.output_trap.add_to_message(message)
         self.output_trap.clear()
         self.display_trap.add_to_message(message)
         self.display_trap.clear()
         self.traceback_trap.add_to_message(message)
         # Pull out the type, value and tb of the current exception
         # before clearing it.
         einfo = self.traceback_trap.args
         self.traceback_trap.clear()
         
         # Cache the message.
         self.message_cache.add_message(self.current_cell_number, message)
 
         # Bump the number.
         self.current_cell_number += 1
         
         # This conditional lets the execute method either raise any
         # exception that has occured in user code OR return the message
         # dict containing the traceback and other useful info.
         if raiseException and einfo:
             raise einfo[0],einfo[1],einfo[2]
         else:
             return message
 
     def generate_prompt(self, is_continuation):
         """Calculate and return a string with the prompt to display.
 
         :Parameters:
           is_continuation : bool
             Whether the input line is continuing multiline input or not, so
           that a proper continuation prompt can be computed."""
         
         if is_continuation:
             return str(self.outputcache.prompt2)
         else:
             return str(self.outputcache.prompt1)
 
     def execute_python(self, python):
         """ Actually run the Python code in the namespace.
 
         :Parameters:
 
         python : str
             Pure, exec'able Python code. Special IPython commands should have
             already been translated into pure Python.
         """
 
         # We use a CommandCompiler instance to compile the code so as to keep
         # track of __future__ imports.
         try:
             commands = self.split_commands(python)
         except (SyntaxError, IndentationError), e:
             # Save the exc_info so compilation related exceptions can be
             # reraised
             self.traceback_trap.args = sys.exc_info()
             self.pack_exception(self.message,e)
             return None
 
         for cmd in commands:
             try:
                 code = self.command_compiler(cmd, self.filename, 'single')
             except (SyntaxError, OverflowError, ValueError), e:
                 self.traceback_trap.args = sys.exc_info()
                 self.pack_exception(self.message,e)
                 # No point in continuing if one block raised
                 return None
             else:
                 self.execute_block(code)
 
     def execute_block(self,code):
         """Execute a single block of code in the user namespace.
 
         Return value: a flag indicating whether the code to be run completed
         successfully:
 
           - 0: successful execution.
           - 1: an error occurred.
           """
         
         outflag = 1 # start by assuming error, success will reset it
         try:
             exec code in self.user_ns
             outflag = 0
         except SystemExit:
             self.resetbuffer()
             self.traceback_trap.args = sys.exc_info()
         except:
             self.traceback_trap.args = sys.exc_info()
 
         return outflag
 
     def execute_macro(self, macro):
         """ Execute the value of a macro.
 
         Parameters
         ----------
         macro : Macro
         """
 
         python = macro.value
         if self.translator is not None:
             python = self.translator(python)
         self.execute_python(python)
 
     def getCommand(self, i=None):
         """Gets the ith message in the message_cache.
         
         This is implemented here for compatibility with the old ipython1 shell
         I am not sure we need this though.  I even seem to remember that we
         were going to get rid of it.
         """
         return self.message_cache.get_message(i)
 
     def reset(self):
         """Reset the interpreter.
         
         Currently this only resets the users variables in the namespace.
         In the future we might want to also reset the other stateful
         things like that the Interpreter has, like In, Out, etc.
         """
         self.user_ns.clear()
         self.setup_namespace()
 
     def complete(self,line,text=None, pos=None):
         """Complete the given text.
 
         :Parameters:
 
           text : str
             Text fragment to be completed on.  Typically this is
         """
         # fixme: implement
         raise NotImplementedError
 
     def push(self, ns):
         """ Put value into the namespace with name key.
         
         Parameters
         ----------
         **kwds
         """
 
         self.user_ns.update(ns)
 
     def push_function(self, ns):
         # First set the func_globals for all functions to self.user_ns
         new_kwds = {}
         for k, v in ns.iteritems():
             if not isinstance(v, FunctionType):
                 raise TypeError("function object expected")
             new_kwds[k] = FunctionType(v.func_code, self.user_ns)
         self.user_ns.update(new_kwds)        
 
     def pack_exception(self,message,exc):
         message['exception'] = exc.__class__
         message['exception_value'] = \
             traceback.format_exception_only(exc.__class__, exc)
 
     def feed_block(self, source, filename='<input>', symbol='single'):
         """Compile some source in the interpreter.
 
         One several things can happen:
 
         1) The input is incorrect; compile_command() raised an
         exception (SyntaxError or OverflowError).
 
         2) The input is incomplete, and more input is required;
         compile_command() returned None.  Nothing happens.
 
         3) The input is complete; compile_command() returned a code
         object.  The code is executed by calling self.runcode() (which
         also handles run-time exceptions, except for SystemExit).
 
         The return value is:
 
           - True in case 2
 
           - False in the other cases, unless an exception is raised, where
           None is returned instead.  This can be used by external callers to
           know whether to continue feeding input or not.
 
         The return value can be used to decide whether to use sys.ps1 or
         sys.ps2 to prompt the next line."""
 
         self.message = self.setup_message()
 
         try:
             code = self.command_compiler(source,filename,symbol)
         except (OverflowError, SyntaxError, IndentationError, ValueError ), e:
             # Case 1
             self.traceback_trap.args = sys.exc_info()
             self.pack_exception(self.message,e)
             return COMPILER_ERROR,False
 
         if code is None:
             # Case 2: incomplete input.  This means that the input can span
             # multiple lines.  But we still need to decide when to actually
             # stop taking user input.  Later we'll add auto-indentation support
             # somehow.  In the meantime, we'll just stop if there are two lines
             # of pure whitespace at the end.
             last_two = source.rsplit('\n',2)[-2:]
             print 'last two:',last_two  # dbg
             if len(last_two)==2 and all(s.isspace() for s in last_two):
                 return COMPLETE_INPUT,False
             else:
                 return INCOMPLETE_INPUT, True
         else:
             # Case 3
             return COMPLETE_INPUT, False
         
     def pull(self, keys):
         """ Get an item out of the namespace by key.
         
         Parameters
         ----------
         key : str
 
         Returns
         -------
         value : object
 
         Raises
         ------
         TypeError if the key is not a string.
         NameError if the object doesn't exist.
         """
 
         if isinstance(keys, str):
             result = self.user_ns.get(keys, NotDefined())
             if isinstance(result, NotDefined):
                 raise NameError('name %s is not defined' % keys)        
         elif isinstance(keys, (list, tuple)):
             result = []
             for key in keys:
                 if not isinstance(key, str):
                     raise TypeError("objects must be keyed by strings.")
                 else:
                     r = self.user_ns.get(key, NotDefined())
                     if isinstance(r, NotDefined):
                         raise NameError('name %s is not defined' % key)
                     else:
                         result.append(r)
                     if len(keys)==1:
                         result = result[0]
         else:
             raise TypeError("keys must be a strong or a list/tuple of strings")
         return result
 
     def pull_function(self, keys):
         return self.pull(keys)
 
     #### Interactive user API ##################################################
 
     def ipsystem(self, command):
         """ Execute a command in a system shell while expanding variables in the
         current namespace.
 
         Parameters
         ----------
         command : str
         """
 
         # Expand $variables.
         command = self.var_expand(command)
 
         system_shell(command,
             header='IPython system call: ',
             verbose=self.rc.system_verbose,
         )
 
     def ipmagic(self, arg_string):
         """ Call a magic function by name.
 
         ipmagic('name -opt foo bar') is equivalent to typing at the ipython
         prompt:
 
         In[1]: %name -opt foo bar
 
         To call a magic without arguments, simply use ipmagic('name').
 
         This provides a proper Python function to call IPython's magics in any
         valid Python code you can type at the interpreter, including loops and
         compound statements.  It is added by IPython to the Python builtin
         namespace upon initialization.
 
         Parameters
         ----------
         arg_string : str
             A string containing the name of the magic function to call and any
             additional arguments to be passed to the magic.
 
         Returns
         -------
         something : object
             The return value of the actual object.
         """
 
         # Taken from IPython.
         raise NotImplementedError('Not ported yet')
 
         args = arg_string.split(' ', 1)
         magic_name = args[0]
         magic_name = magic_name.lstrip(self.config.ESC_MAGIC)
 
         try:
             magic_args = args[1]
         except IndexError:
             magic_args = ''
         fn = getattr(self.magic, 'magic_'+magic_name, None)
         if fn is None:
             self.error("Magic function `%s` not found." % magic_name)
         else:
             magic_args = self.var_expand(magic_args)
             return fn(magic_args)
 
 
     #### Private 'Interpreter' interface #######################################
 
     def setup_message(self):
         """Return a message object.
 
         This method prepares and returns a message dictionary.  This dict
         contains the various fields that are used to transfer information about
         execution, results, tracebacks, etc, to clients (either in or out of
         process ones).  Because of the need to work with possibly out of
         process clients, this dict MUST contain strictly pickle-safe values.
         """
 
         return dict(number=self.current_cell_number)
     
     def setup_namespace(self):
         """ Add things to the namespace.
         """
 
         self.user_ns.setdefault('__name__', '__main__')
         self.user_ns.setdefault('__builtins__', __builtin__)
         self.user_ns['__IP'] = self
         if self.raw_input_builtin is not None:
             self.user_ns['raw_input'] = self.raw_input_builtin
         if self.input_builtin is not None:
             self.user_ns['input'] = self.input_builtin
 
         builtin_additions = dict(
             ipmagic=self.ipmagic,
         )
         __builtin__.__dict__.update(builtin_additions)
 
         if self.history is not None:
             self.history.setup_namespace(self.user_ns)
 
     def set_traps(self):
         """ Set all of the output, display, and traceback traps.
         """
 
         self.output_trap.set()
         self.display_trap.set()
         self.traceback_trap.set()
 
     def unset_traps(self):
         """ Unset all of the output, display, and traceback traps.
         """
 
         self.output_trap.unset()
         self.display_trap.unset()
         self.traceback_trap.unset()
 
     def split_commands(self, python):
         """ Split multiple lines of code into discrete commands that can be
         executed singly.
 
         Parameters
         ----------
         python : str
             Pure, exec'able Python code.
 
         Returns
         -------
         commands : list of str
             Separate commands that can be exec'ed independently.
         """
 
         # compiler.parse treats trailing spaces after a newline as a
         # SyntaxError.  This is different than codeop.CommandCompiler, which
         # will compile the trailng spaces just fine.  We simply strip any
         # trailing whitespace off.  Passing a string with trailing whitespace
         # to exec will fail however.  There seems to be some inconsistency in
         # how trailing whitespace is handled, but this seems to work.
         python = python.strip()
-
+        
         # The compiler module does not like unicode. We need to convert
         # it encode it:
         if isinstance(python, unicode):
             # Use the utf-8-sig BOM so the compiler detects this a UTF-8
             # encode string.
             python = '\xef\xbb\xbf' + python.encode('utf-8')
-
+        
         # The compiler module will parse the code into an abstract syntax tree.
+        # This has a bug with str("a\nb"), but not str("""a\nb""")!!!
         ast = compiler.parse(python)
-
+        
         # Uncomment to help debug the ast tree
         # for n in ast.node:
         #     print n.lineno,'->',n
-                
+        
         # Each separate command is available by iterating over ast.node. The
         # lineno attribute is the line number (1-indexed) beginning the commands
         # suite.
         # lines ending with ";" yield a Discard Node that doesn't have a lineno
         # attribute.  These nodes can and should be discarded.  But there are
         # other situations that cause Discard nodes that shouldn't be discarded.
         # We might eventually discover other cases where lineno is None and have
         # to put in a more sophisticated test.
         linenos = [x.lineno-1 for x in ast.node if x.lineno is not None]
-
+        
         # When we finally get the slices, we will need to slice all the way to
         # the end even though we don't have a line number for it. Fortunately,
         # None does the job nicely.
         linenos.append(None)
+        
+        # Same problem at the other end: sometimes the ast tree has its
+        # first complete statement not starting on line 0. In this case
+        # we might miss part of it.  This fixes ticket 266993.  Thanks Gael!
+        linenos[0] = 0
+        
         lines = python.splitlines()
-
+        
         # Create a list of atomic commands.
         cmds = []
         for i, j in zip(linenos[:-1], linenos[1:]):
             cmd = lines[i:j]
             if cmd:
                 cmds.append('\n'.join(cmd)+'\n')
-
+        
         return cmds
 
     def error(self, text):
         """ Pass an error message back to the shell.
 
         Preconditions
         -------------
         This should only be called when self.message is set. In other words,
         when code is being executed.
 
         Parameters
         ----------
         text : str
         """
 
         errors = self.message.get('IPYTHON_ERROR', [])
         errors.append(text)
 
     def var_expand(self, template):
         """ Expand $variables in the current namespace using Itpl.
 
         Parameters
         ----------
         template : str
         """
 
         return str(ItplNS(template, self.user_ns))
 
     def _possible_macro(self, obj):
         """ If the object is a macro, execute it.
         """
 
         if isinstance(obj, Macro):
             self.execute_macro(obj)
 

IPython/kernel/core/notification.py

@@ -1,123 +1,125 b''
 # encoding: utf-8
 
 """The IPython Core Notification Center.
 
 See docs/source/development/notification_blueprint.txt for an overview of the
 notification module.
 """
 
 __docformat__ = "restructuredtext en"
                                                                               
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team                           
 #                                                                             
 #  Distributed under the terms of the BSD License.  The full license is in    
 #  the file COPYING, distributed as part of this software.                    
 #-----------------------------------------------------------------------------
 
+# Tell nose to skip the testing of this module
+__test__ = {}
 
 class NotificationCenter(object):
     """Synchronous notification center
     
     Example
     -------
     >>> import IPython.kernel.core.notification as notification
     >>> def callback(theType, theSender, args={}):
     ...     print theType,theSender,args
     ...     
     >>> notification.sharedCenter.add_observer(callback, 'NOTIFICATION_TYPE', None)
     >>> notification.sharedCenter.post_notification('NOTIFICATION_TYPE', object()) # doctest:+ELLIPSIS
     NOTIFICATION_TYPE ...
         
     """
     def __init__(self):
         super(NotificationCenter, self).__init__()
         self._init_observers()
     
     
     def _init_observers(self):
         """Initialize observer storage"""
         
         self.registered_types = set() #set of types that are observed
         self.registered_senders = set() #set of senders that are observed
         self.observers = {} #map (type,sender) => callback (callable)
     
     
     def post_notification(self, theType, sender, **kwargs):
         """Post notification (type,sender,**kwargs) to all registered
         observers. 
         
         Implementation
         --------------
         * If no registered observers, performance is O(1).
         * Notificaiton order is undefined.
         * Notifications are posted synchronously.
         """
         
         if(theType==None or sender==None):
             raise Exception("NotificationCenter.post_notification requires \
                 type and sender.")
         
         # If there are no registered observers for the type/sender pair
         if((theType not in self.registered_types and 
                 None not in self.registered_types) or
             (sender not in self.registered_senders and 
                 None not in self.registered_senders)):
             return
         
         for o in self._observers_for_notification(theType, sender):
             o(theType, sender, args=kwargs)
     
         
     def _observers_for_notification(self, theType, sender):
         """Find all registered observers that should recieve notification"""
         
         keys = (
                     (theType,sender),
                     (theType, None),
                     (None, sender),  
                     (None,None)
                 )
         
         
         obs = set()
         for k in keys:
             obs.update(self.observers.get(k, set()))
         
         return obs
     
     
     def add_observer(self, callback, theType, sender):
         """Add an observer callback to this notification center.
         
         The given callback will be called upon posting of notifications of
         the given type/sender and will receive any additional kwargs passed
         to post_notification.
         
         Parameters
         ----------
         observerCallback : callable
             Callable. Must take at least two arguments::
                 observerCallback(type, sender, args={})
         
         theType : hashable
             The notification type. If None, all notifications from sender
             will be posted.
         
         sender : hashable
             The notification sender. If None, all notifications of theType
             will be posted.
         """
         assert(callback != None)
         self.registered_types.add(theType)
         self.registered_senders.add(sender)
         self.observers.setdefault((theType,sender), set()).add(callback)
     
     def remove_all_observers(self):
         """Removes all observers from this notification center"""
         
         self._init_observers()
     
 
 
 sharedCenter = NotificationCenter()
  No newline at end of file

IPython/kernel/core/tests/test_interpreter.py

@@ -1,26 +1,62 b''
 # encoding: utf-8
 
 """This file contains unittests for the interpreter.py module."""
 
-__docformat__ = "restructuredtext en"
-
 #-----------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team                           
-#                                                                             
-#  Distributed under the terms of the BSD License.  The full license is in    
-#  the file COPYING, distributed as part of this software.                    
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is
+#  in the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
-                                                                              
+
 #-----------------------------------------------------------------------------
-# Imports                                                                     
+# Imports
 #-----------------------------------------------------------------------------
 
+# Tell nose to skip this module
+__test__ = {}
+
+from twisted.trial import unittest
 from IPython.kernel.core.interpreter import Interpreter
 
-def test_unicode():
-    """ Test unicode handling with the interpreter.
-    """
-    i = Interpreter()
-    i.execute_python(u'print "ù"')
-    i.execute_python('print "ù"')
+#-----------------------------------------------------------------------------
+# Tests
+#-----------------------------------------------------------------------------
+
+class TestInterpreter(unittest.TestCase):
+
+    def test_unicode(self):
+        """ Test unicode handling with the interpreter."""
+        i = Interpreter()
+        i.execute_python(u'print "ù"')
+        i.execute_python('print "ù"')
+
+    def test_ticket266993(self):
+        """ Test for ticket 266993."""
+        i = Interpreter()
+        i.execute('str("""a\nb""")')
+
+    def test_ticket364347(self):
+        """Test for ticket 364347."""
+        i = Interpreter()
+        i.split_commands('str("a\\nb")')
+
+    def test_split_commands(self):
+        """ Test that commands are indeed individually split."""
+        i = Interpreter()
+        test_atoms = [('(1\n + 1)', ),
+                      ('1', '1', ),
+                      ]
+        for atoms in test_atoms:
+            atoms = [atom.rstrip() + '\n' for atom in atoms]
+            self.assertEquals(i.split_commands(''.join(atoms)),atoms)
+
+    def test_long_lines(self):
+        """ Test for spurious syntax error created by the interpreter."""
+        test_strings = [u'( 1 +\n 1\n )\n\n',
+                        u'(1 \n + 1\n )\n\n',
+                       ]
+        i = Interpreter()
+        for s in test_strings:
+            i.execute(s)
 

IPython/kernel/core/tests/test_notification.py

@@ -1,171 +1,161 b''
 # encoding: utf-8
 
 """This file contains unittests for the notification.py module."""
 
-__docformat__ = "restructuredtext en"
-
 #-----------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team                           
-#                                                                             
-#  Distributed under the terms of the BSD License.  The full license is in    
-#  the file COPYING, distributed as part of this software.                    
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is
+#  in the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
-                                                                              
+
 #-----------------------------------------------------------------------------
-# Imports                                                                     
+# Imports
 #-----------------------------------------------------------------------------
 
-import unittest
+# Tell nose to skip this module
+__test__ = {}
+
+from twisted.trial import unittest
 import IPython.kernel.core.notification as notification
-from nose.tools import timed
 
-#
-# Supporting test classes
-#
+#-----------------------------------------------------------------------------
+# Support Classes
+#-----------------------------------------------------------------------------
 
 class Observer(object):
     """docstring for Observer"""
     def __init__(self, expectedType, expectedSender, 
                     center=notification.sharedCenter, **kwargs):
         super(Observer, self).__init__()
         self.expectedType = expectedType
         self.expectedSender = expectedSender
         self.expectedKwArgs = kwargs
         self.recieved = False
         center.add_observer(self.callback, 
                             self.expectedType, 
                             self.expectedSender)
     
-    
     def callback(self, theType, sender, args={}):
         """callback"""
         
         assert(theType == self.expectedType or
                 self.expectedType == None)
         assert(sender == self.expectedSender or
                 self.expectedSender == None)
         assert(args == self.expectedKwArgs)
         self.recieved = True
     
-    
     def verify(self):
         """verify"""
         
         assert(self.recieved)
     
     def reset(self):
         """reset"""
         
         self.recieved = False
-    
 
 
 class Notifier(object):
     """docstring for Notifier"""
     def __init__(self, theType, **kwargs):
         super(Notifier, self).__init__()
         self.theType = theType
         self.kwargs = kwargs
     
     def post(self, center=notification.sharedCenter):
         """fire"""
         
         center.post_notification(self.theType, self,
             **self.kwargs)
-    
 
-#
-# Test Cases
-#
+#-----------------------------------------------------------------------------
+# Tests
+#-----------------------------------------------------------------------------
 
 class NotificationTests(unittest.TestCase):
     """docstring for NotificationTests"""
     
     def tearDown(self):
         notification.sharedCenter.remove_all_observers()
     
     def test_notification_delivered(self):
         """Test that notifications are delivered"""
         expectedType = 'EXPECTED_TYPE'
         sender = Notifier(expectedType)
         observer = Observer(expectedType, sender)
         
         sender.post()
         
         observer.verify()
     
-    
     def test_type_specificity(self):
         """Test that observers are registered by type"""
         
         expectedType = 1
         unexpectedType = "UNEXPECTED_TYPE"
         sender = Notifier(expectedType)
         unexpectedSender = Notifier(unexpectedType)
         observer = Observer(expectedType, sender)
     
         sender.post()
         unexpectedSender.post()
     
         observer.verify()
     
-    
     def test_sender_specificity(self):
         """Test that observers are registered by sender"""
         
         expectedType = "EXPECTED_TYPE"
         sender1 = Notifier(expectedType)
         sender2 = Notifier(expectedType)
         observer = Observer(expectedType, sender1)
         
         sender1.post()
         sender2.post()
         
         observer.verify()
     
-    
     def test_remove_all_observers(self):
         """White-box test for remove_all_observers"""
         
         for i in xrange(10):
             Observer('TYPE', None, center=notification.sharedCenter)
         
         self.assert_(len(notification.sharedCenter.observers[('TYPE',None)]) >= 10, 
             "observers registered")
         
         notification.sharedCenter.remove_all_observers()
         
         self.assert_(len(notification.sharedCenter.observers) == 0, "observers removed")
-    
-    
+
     def test_any_sender(self):
         """test_any_sender"""
         
         expectedType = "EXPECTED_TYPE"
         sender1 = Notifier(expectedType)
         sender2 = Notifier(expectedType)
         observer = Observer(expectedType, None)
         
         
         sender1.post()
         observer.verify()
         
         observer.reset()
         sender2.post()
         observer.verify()
-        
-    
-    @timed(.01)
+
     def test_post_performance(self):
         """Test that post_notification, even with many registered irrelevant
         observers is fast"""
         
         for i in xrange(10):
             Observer("UNRELATED_TYPE", None)
         
         o = Observer('EXPECTED_TYPE', None)
         
         notification.sharedCenter.post_notification('EXPECTED_TYPE', self)
         
         o.verify()
-    
+

IPython/kernel/core/tests/test_redirectors.py

@@ -1,70 +1,78 b''
 # encoding: utf-8
 """
 Test the output capture at the OS level, using file descriptors.
 """
-
-__docformat__ = "restructuredtext en"
-
-#-------------------------------------------------------------------------------
-#  Copyright (C) 2008  The IPython Development Team
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is
 #  in the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
+#-----------------------------------------------------------------------------
 
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+# Tell nose to skip this module
+__test__ = {}
 
-# Stdlib imports
-import os
 from cStringIO import StringIO
+import os
+
+from twisted.trial import unittest
 
-# Our own imports
-from IPython.testing import decorators as dec
+from IPython.testing import decorators_trial as dec
 
 #-----------------------------------------------------------------------------
-# Test functions
+# Tests
+#-----------------------------------------------------------------------------
 
[e-mail]_win32
-def test_redirector():
-    """ Checks that the redirector can be used to do synchronous capture.
-    """
-    from IPython.kernel.core.fd_redirector import FDRedirector
-    r = FDRedirector()
-    out = StringIO()
-    try:
-        r.start()
-        for i in range(10):
-            os.system('echo %ic' % i)
-            print >>out, r.getvalue(),
-            print >>out, i
-    except:
-        r.stop()
-        raise
-    r.stop()
-    result1 = out.getvalue()
-    result2 = "".join("%ic\n%i\n" %(i, i) for i in range(10))
-    assert result1 == result2
 
+class TestRedirector(unittest.TestCase):
 
[e-mail]_win32
-def test_redirector_output_trap():
-    """ This test check not only that the redirector_output_trap does
+    @dec.skip_win32
+    def test_redirector(self):
+        """Checks that the redirector can be used to do synchronous capture.
+        """
+        from IPython.kernel.core.fd_redirector import FDRedirector
+        r = FDRedirector()
+        out = StringIO()
+        try:
+            r.start()
+            for i in range(10):
+                os.system('echo %ic' % i)
+                print >>out, r.getvalue(),
+                print >>out, i
+        except:
+            r.stop()
+            raise
+        r.stop()
+        result1 = out.getvalue()
+        result2 = "".join("%ic\n%i\n" %(i, i) for i in range(10))
+        self.assertEquals(result1, result2)
+
+    @dec.skip_win32
+    def test_redirector_output_trap(self):
+        """Check the greedy trapping behavior of the traps.
+        
+        This test check not only that the redirector_output_trap does
         trap the output, but also that it does it in a gready way, that
         is by calling the callback ASAP.
-    """
-    from IPython.kernel.core.redirector_output_trap import RedirectorOutputTrap
-    out = StringIO()
-    trap = RedirectorOutputTrap(out.write, out.write)
-    try:
-        trap.set()
-        for i in range(10):
-            os.system('echo %ic' % i)
-            print "%ip" % i
-            print >>out, i
-    except:
+        """
+        from IPython.kernel.core.redirector_output_trap import RedirectorOutputTrap
+        out = StringIO()
+        trap = RedirectorOutputTrap(out.write, out.write)
+        try:
+            trap.set()
+            for i in range(10):
+                os.system('echo %ic' % i)
+                print "%ip" % i
+                print >>out, i
+        except:
+            trap.unset()
+            raise
         trap.unset()
-        raise
-    trap.unset()
-    result1 = out.getvalue()
-    result2 = "".join("%ic\n%ip\n%i\n" %(i, i, i) for i in range(10))
-    assert result1 == result2
-    
+        result1 = out.getvalue()
+        result2 = "".join("%ic\n%ip\n%i\n" %(i, i, i) for i in range(10))
+        self.assertEquals(result1, result2)
+

IPython/kernel/core/ultraTB.py

@@ -1,1056 +1,1066 b''
 # -*- coding: utf-8 -*-
 """
 ultraTB.py -- Spice up your tracebacks!
 
 * ColorTB
 I've always found it a bit hard to visually parse tracebacks in Python.  The
 ColorTB class is a solution to that problem.  It colors the different parts of a
 traceback in a manner similar to what you would expect from a syntax-highlighting
 text editor.
 
 Installation instructions for ColorTB:
     import sys,ultraTB
     sys.excepthook = ultraTB.ColorTB()
 
 * VerboseTB  
 I've also included a port of Ka-Ping Yee's "cgitb.py" that produces all kinds
 of useful info when a traceback occurs.  Ping originally had it spit out HTML
 and intended it for CGI programmers, but why should they have all the fun?  I
 altered it to spit out colored text to the terminal.  It's a bit overwhelming,
 but kind of neat, and maybe useful for long-running programs that you believe
 are bug-free.  If a crash *does* occur in that type of program you want details.
 Give it a shot--you'll love it or you'll hate it.
 
 Note:
 
   The Verbose mode prints the variables currently visible where the exception
   happened (shortening their strings if too long). This can potentially be
   very slow, if you happen to have a huge data structure whose string
   representation is complex to compute. Your computer may appear to freeze for
   a while with cpu usage at 100%. If this occurs, you can cancel the traceback
   with Ctrl-C (maybe hitting it more than once).
 
   If you encounter this kind of situation often, you may want to use the
   Verbose_novars mode instead of the regular Verbose, which avoids formatting
   variables (but otherwise includes the information and context given by
   Verbose).
   
 
 Installation instructions for ColorTB:
     import sys,ultraTB
     sys.excepthook = ultraTB.VerboseTB()
 
 Note:  Much of the code in this module was lifted verbatim from the standard
 library module 'traceback.py' and Ka-Ping Yee's 'cgitb.py'.
 
 * Color schemes
 The colors are defined in the class TBTools through the use of the
 ColorSchemeTable class. Currently the following exist:
 
   - NoColor: allows all of this module to be used in any terminal (the color
   escapes are just dummy blank strings).
 
   - Linux: is meant to look good in a terminal like the Linux console (black
   or very dark background).
 
   - LightBG: similar to Linux but swaps dark/light colors to be more readable
   in light background terminals.
 
 You can implement other color schemes easily, the syntax is fairly
 self-explanatory. Please send back new schemes you develop to the author for
 possible inclusion in future releases.
 """
 
 #*****************************************************************************
 #       Copyright (C) 2001 Nathaniel Gray <[e-mail]>
 #       Copyright (C) 2001-2004 Fernando Perez <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 # Required modules
 import inspect
 import keyword
 import linecache
 import os
 import pydoc
 import re
 import string
 import sys
 import time
 import tokenize
 import traceback
 import types
 
 # For purposes of monkeypatching inspect to fix a bug in it.
 from inspect import getsourcefile, getfile, getmodule,\
      ismodule,  isclass, ismethod, isfunction, istraceback, isframe, iscode
 
 
 # IPython's own modules
 # Modified pdb which doesn't damage IPython's readline handling
 from IPython import Debugger, PyColorize
 from IPython.ipstruct import Struct
 from IPython.excolors import exception_colors
 from IPython.genutils import Term,uniq_stable,error,info
 
 # Globals
 # amount of space to put line numbers before verbose tracebacks
 INDENT_SIZE = 8
 
 # Default color scheme.  This is used, for example, by the traceback
 # formatter.  When running in an actual IPython instance, the user's rc.colors
 # value is used, but havinga module global makes this functionality available
 # to users of ultraTB who are NOT running inside ipython.
 DEFAULT_SCHEME = 'NoColor'
 
 #---------------------------------------------------------------------------
 # Code begins
 
 # Utility functions
 def inspect_error():
     """Print a message about internal inspect errors.
 
     These are unfortunately quite common."""
     
     error('Internal Python error in the inspect module.\n'
           'Below is the traceback from this internal error.\n')
 
 
 def findsource(object):
     """Return the entire source file and starting line number for an object.
 
     The argument may be a module, class, method, function, traceback, frame,
     or code object.  The source code is returned as a list of all the lines
     in the file and the line number indexes a line in that list.  An IOError
     is raised if the source code cannot be retrieved.
 
     FIXED version with which we monkeypatch the stdlib to work around a bug."""
 
     file = getsourcefile(object) or getfile(object)
     # If the object is a frame, then trying to get the globals dict from its
     # module won't work. Instead, the frame object itself has the globals
     # dictionary.
     globals_dict = None
     if inspect.isframe(object):
         # XXX: can this ever be false?
         globals_dict = object.f_globals
     else:
         module = getmodule(object, file)
         if module:
             globals_dict = module.__dict__
     lines = linecache.getlines(file, globals_dict)
     if not lines:
         raise IOError('could not get source code')
 
     if ismodule(object):
         return lines, 0
 
     if isclass(object):
         name = object.__name__
         pat = re.compile(r'^(\s*)class\s*' + name + r'\b')
         # make some effort to find the best matching class definition:
         # use the one with the least indentation, which is the one
         # that's most probably not inside a function definition.
         candidates = []
         for i in range(len(lines)):
             match = pat.match(lines[i])
             if match:
                 # if it's at toplevel, it's already the best one
                 if lines[i][0] == 'c':
                     return lines, i
                 # else add whitespace to candidate list
                 candidates.append((match.group(1), i))
         if candidates:
             # this will sort by whitespace, and by line number,
             # less whitespace first
             candidates.sort()
             return lines, candidates[0][1]
         else:
             raise IOError('could not find class definition')
 
     if ismethod(object):
         object = object.im_func
     if isfunction(object):
         object = object.func_code
     if istraceback(object):
         object = object.tb_frame
     if isframe(object):
         object = object.f_code
     if iscode(object):
         if not hasattr(object, 'co_firstlineno'):
             raise IOError('could not find function definition')
         pat = re.compile(r'^(\s*def\s)|(.*(?<!\w)lambda(:|\s))|^(\s*@)')
         pmatch = pat.match
         # fperez - fix: sometimes, co_firstlineno can give a number larger than
         # the length of lines, which causes an error.  Safeguard against that.
         lnum = min(object.co_firstlineno,len(lines))-1
         while lnum > 0:
             if pmatch(lines[lnum]): break
             lnum -= 1
  
         return lines, lnum
     raise IOError('could not find code object')
 
 # Monkeypatch inspect to apply our bugfix.  This code only works with py25
 if sys.version_info[:2] >= (2,5):
     inspect.findsource = findsource
 
 def fix_frame_records_filenames(records):
     """Try to fix the filenames in each record from inspect.getinnerframes().
 
     Particularly, modules loaded from within zip files have useless filenames
     attached to their code object, and inspect.getinnerframes() just uses it.
     """
     fixed_records = []
     for frame, filename, line_no, func_name, lines, index in records:
         # Look inside the frame's globals dictionary for __file__, which should
         # be better.
         better_fn = frame.f_globals.get('__file__', None)
         if isinstance(better_fn, str):
             # Check the type just in case someone did something weird with
             # __file__. It might also be None if the error occurred during
             # import.
             filename = better_fn
         fixed_records.append((frame, filename, line_no, func_name, lines, index))
     return fixed_records
 
 
 def _fixed_getinnerframes(etb, context=1,tb_offset=0):
     import linecache
     LNUM_POS, LINES_POS, INDEX_POS =  2, 4, 5
 
     records  = fix_frame_records_filenames(inspect.getinnerframes(etb, context))
 
     # If the error is at the console, don't build any context, since it would
     # otherwise produce 5 blank lines printed out (there is no file at the
     # console)
     rec_check = records[tb_offset:]
     try:
         rname = rec_check[0][1]
         if rname == '<ipython console>' or rname.endswith('<string>'):
             return rec_check
     except IndexError:
         pass
 
     aux = traceback.extract_tb(etb)
     assert len(records) == len(aux)
     for i, (file, lnum, _, _) in zip(range(len(records)), aux):
         maybeStart = lnum-1 - context//2
         start =  max(maybeStart, 0)
         end   = start + context
         lines = linecache.getlines(file)[start:end]
         # pad with empty lines if necessary
         if maybeStart < 0:
             lines = (['\n'] * -maybeStart) + lines
         if len(lines) < context:
             lines += ['\n'] * (context - len(lines))
         buf = list(records[i])
         buf[LNUM_POS] = lnum
         buf[INDEX_POS] = lnum - 1 - start
         buf[LINES_POS] = lines
         records[i] = tuple(buf)
     return records[tb_offset:]
 
 # Helper function -- largely belongs to VerboseTB, but we need the same
 # functionality to produce a pseudo verbose TB for SyntaxErrors, so that they
 # can be recognized properly by ipython.el's py-traceback-line-re
 # (SyntaxErrors have to be treated specially because they have no traceback)
 
 _parser = PyColorize.Parser()
     
 def _formatTracebackLines(lnum, index, lines, Colors, lvals=None,scheme=None):
     numbers_width = INDENT_SIZE - 1
     res = []
     i = lnum - index
 
     # This lets us get fully syntax-highlighted tracebacks.
     if scheme is None:
         try:
+            # Again, reference to a global __IPYTHON__ that doesn't exist.
+            # XXX
             scheme = __IPYTHON__.rc.colors
         except:
             scheme = DEFAULT_SCHEME
     _line_format = _parser.format2
 
     for line in lines:
         new_line, err = _line_format(line,'str',scheme)
         if not err: line = new_line
         
         if i == lnum:
             # This is the line with the error
             pad = numbers_width - len(str(i))
             if pad >= 3:
                 marker = '-'*(pad-3) + '-> '
             elif pad == 2:
                 marker = '> '
             elif pad == 1:
                 marker = '>'
             else:
                 marker = ''
             num = marker + str(i)
             line = '%s%s%s %s%s' %(Colors.linenoEm, num, 
                                    Colors.line, line, Colors.Normal)
         else:
             num = '%*s' % (numbers_width,i)
             line = '%s%s%s %s' %(Colors.lineno, num, 
                                  Colors.Normal, line)
 
         res.append(line)
         if lvals and i == lnum:
             res.append(lvals + '\n')
         i = i + 1
     return res
 
 
 #---------------------------------------------------------------------------
 # Module classes
 class TBTools:
     """Basic tools used by all traceback printer classes."""
 
     def __init__(self,color_scheme = 'NoColor',call_pdb=False):
         # Whether to call the interactive pdb debugger after printing
         # tracebacks or not
         self.call_pdb = call_pdb
 
         # Create color table
         self.color_scheme_table = exception_colors()
 
         self.set_colors(color_scheme)
         self.old_scheme = color_scheme  # save initial value for toggles
 
         if call_pdb:
             self.pdb = Debugger.Pdb(self.color_scheme_table.active_scheme_name)
         else:
             self.pdb = None
 
     def set_colors(self,*args,**kw):
         """Shorthand access to the color table scheme selector method."""
 
         # Set own color table
         self.color_scheme_table.set_active_scheme(*args,**kw)
         # for convenience, set Colors to the active scheme
         self.Colors = self.color_scheme_table.active_colors
         # Also set colors of debugger
         if hasattr(self,'pdb') and self.pdb is not None:
             self.pdb.set_colors(*args,**kw)
 
     def color_toggle(self):
         """Toggle between the currently active color scheme and NoColor."""
         
         if self.color_scheme_table.active_scheme_name == 'NoColor':
             self.color_scheme_table.set_active_scheme(self.old_scheme)
             self.Colors = self.color_scheme_table.active_colors
         else:
             self.old_scheme = self.color_scheme_table.active_scheme_name
             self.color_scheme_table.set_active_scheme('NoColor')
             self.Colors = self.color_scheme_table.active_colors
 
 #---------------------------------------------------------------------------
 class ListTB(TBTools):
     """Print traceback information from a traceback list, with optional color.
         
     Calling: requires 3 arguments:
       (etype, evalue, elist)
     as would be obtained by:
       etype, evalue, tb = sys.exc_info()
       if tb:
         elist = traceback.extract_tb(tb)
       else:
         elist = None
 
     It can thus be used by programs which need to process the traceback before
     printing (such as console replacements based on the code module from the
     standard library).
 
     Because they are meant to be called without a full traceback (only a
     list), instances of this class can't call the interactive pdb debugger."""
 
     def __init__(self,color_scheme = 'NoColor'):
         TBTools.__init__(self,color_scheme = color_scheme,call_pdb=0)
         
     def __call__(self, etype, value, elist):
         Term.cout.flush()
         print >> Term.cerr, self.text(etype,value,elist)
         Term.cerr.flush()
 
     def text(self,etype, value, elist,context=5):
         """Return a color formatted string with the traceback info."""
 
         Colors = self.Colors
         out_string = ['%s%s%s\n' % (Colors.topline,'-'*60,Colors.Normal)]
         if elist:
             out_string.append('Traceback %s(most recent call last)%s:' % \
                                 (Colors.normalEm, Colors.Normal) + '\n')
             out_string.extend(self._format_list(elist))
         lines = self._format_exception_only(etype, value)
         for line in lines[:-1]:
             out_string.append(" "+line)
         out_string.append(lines[-1])
         return ''.join(out_string)
 
     def _format_list(self, extracted_list):
         """Format a list of traceback entry tuples for printing.
 
         Given a list of tuples as returned by extract_tb() or
         extract_stack(), return a list of strings ready for printing.
         Each string in the resulting list corresponds to the item with the
         same index in the argument list.  Each string ends in a newline;
         the strings may contain internal newlines as well, for those items
         whose source text line is not None.
         
         Lifted almost verbatim from traceback.py
         """
 
         Colors = self.Colors
         list = []
         for filename, lineno, name, line in extracted_list[:-1]:
             item = '  File %s"%s"%s, line %s%d%s, in %s%s%s\n' % \
                     (Colors.filename, filename, Colors.Normal, 
                      Colors.lineno, lineno, Colors.Normal,
                      Colors.name, name, Colors.Normal)
             if line:
                 item = item + '    %s\n' % line.strip()
             list.append(item)
         # Emphasize the last entry
         filename, lineno, name, line = extracted_list[-1]
         item = '%s  File %s"%s"%s, line %s%d%s, in %s%s%s%s\n' % \
                 (Colors.normalEm,
                  Colors.filenameEm, filename, Colors.normalEm,
                  Colors.linenoEm, lineno, Colors.normalEm,
                  Colors.nameEm, name, Colors.normalEm,
                  Colors.Normal)
         if line:
             item = item + '%s    %s%s\n' % (Colors.line, line.strip(),
                                             Colors.Normal)
         list.append(item)
         return list
         
     def _format_exception_only(self, etype, value):
         """Format the exception part of a traceback.
 
         The arguments are the exception type and value such as given by
         sys.exc_info()[:2]. The return value is a list of strings, each ending
         in a newline.  Normally, the list contains a single string; however,
         for SyntaxError exceptions, it contains several lines that (when
         printed) display detailed information about where the syntax error
         occurred.  The message indicating which exception occurred is the
         always last string in the list.
         
         Also lifted nearly verbatim from traceback.py
         """
 
         have_filedata = False
         Colors = self.Colors
         list = []
         try:
             stype = Colors.excName + etype.__name__ + Colors.Normal
         except AttributeError:
             stype = etype  # String exceptions don't get special coloring
         if value is None:
             list.append( str(stype) + '\n')
         else:
             if etype is SyntaxError:
                 try:
                     msg, (filename, lineno, offset, line) = value
                 except:
                     have_filedata = False
                 else:
                     have_filedata = True
                     #print 'filename is',filename  # dbg
                     if not filename: filename = "<string>"
                     list.append('%s  File %s"%s"%s, line %s%d%s\n' % \
                             (Colors.normalEm,
                              Colors.filenameEm, filename, Colors.normalEm,
                              Colors.linenoEm, lineno, Colors.Normal  ))
                     if line is not None:
                         i = 0
                         while i < len(line) and line[i].isspace():
                             i = i+1
                         list.append('%s    %s%s\n' % (Colors.line,
                                                       line.strip(), 
                                                       Colors.Normal))
                         if offset is not None:
                             s = '    '
                             for c in line[i:offset-1]:
                                 if c.isspace():
                                     s = s + c
                                 else:
                                     s = s + ' '
                             list.append('%s%s^%s\n' % (Colors.caret, s,
                                                        Colors.Normal) )
                         value = msg
             s = self._some_str(value)
             if s:
                 list.append('%s%s:%s %s\n' % (str(stype), Colors.excName,
                                               Colors.Normal, s))
             else:
                 list.append('%s\n' % str(stype))
 
-        # vds:>>
-        if have_filedata:
-            __IPYTHON__.hooks.synchronize_with_editor(filename, lineno, 0)
-        # vds:<<
+        # This is being commented out for now as the __IPYTHON__ variable
+        # referenced here is not resolved and causes massive test failures
+        # and errors. B. Granger, 04/2009. XXX
+        # See https://bugs.launchpad.net/bugs/362137
+        # # vds:>>
+        # if have_filedata:
+        #     __IPYTHON__.hooks.synchronize_with_editor(filename, lineno, 0)
+        # # vds:<<
 
         return list
 
     def _some_str(self, value):
         # Lifted from traceback.py
         try:
             return str(value)
         except:
             return '<unprintable %s object>' % type(value).__name__
 
 #----------------------------------------------------------------------------
 class VerboseTB(TBTools):
     """A port of Ka-Ping Yee's cgitb.py module that outputs color text instead
     of HTML.  Requires inspect and pydoc.  Crazy, man.
 
     Modified version which optionally strips the topmost entries from the
     traceback, to be used with alternate interpreters (because their own code
     would appear in the traceback)."""
 
     def __init__(self,color_scheme = 'Linux',tb_offset=0,long_header=0,
                  call_pdb = 0, include_vars=1):
         """Specify traceback offset, headers and color scheme.
 
         Define how many frames to drop from the tracebacks. Calling it with
         tb_offset=1 allows use of this handler in interpreters which will have
         their own code at the top of the traceback (VerboseTB will first
         remove that frame before printing the traceback info)."""
         TBTools.__init__(self,color_scheme=color_scheme,call_pdb=call_pdb)
         self.tb_offset = tb_offset
         self.long_header = long_header
         self.include_vars = include_vars
 
     def text(self, etype, evalue, etb, context=5):
         """Return a nice text document describing the traceback."""
 
         # some locals
         try:
             etype = etype.__name__
         except AttributeError:
             pass
         Colors        = self.Colors   # just a shorthand + quicker name lookup
         ColorsNormal  = Colors.Normal  # used a lot
         col_scheme    = self.color_scheme_table.active_scheme_name
         indent        = ' '*INDENT_SIZE
         em_normal     = '%s\n%s%s' % (Colors.valEm, indent,ColorsNormal)
         undefined     = '%sundefined%s' % (Colors.em, ColorsNormal)
         exc = '%s%s%s' % (Colors.excName,etype,ColorsNormal)
 
         # some internal-use functions
         def text_repr(value):
             """Hopefully pretty robust repr equivalent."""
             # this is pretty horrible but should always return *something*
             try:
                 return pydoc.text.repr(value)
             except KeyboardInterrupt:
                 raise
             except:
                 try:
                     return repr(value)
                 except KeyboardInterrupt:
                     raise
                 except:
                     try:
                         # all still in an except block so we catch
                         # getattr raising
                         name = getattr(value, '__name__', None)
                         if name:
                             # ick, recursion
                             return text_repr(name)
                         klass = getattr(value, '__class__', None)
                         if klass:
                             return '%s instance' % text_repr(klass)
                     except KeyboardInterrupt:
                         raise
                     except:
                         return 'UNRECOVERABLE REPR FAILURE'
         def eqrepr(value, repr=text_repr): return '=%s' % repr(value)
         def nullrepr(value, repr=text_repr): return ''
 
         # meat of the code begins
         try:
             etype = etype.__name__
         except AttributeError:
             pass
 
         if self.long_header:
             # Header with the exception type, python version, and date
             pyver = 'Python ' + string.split(sys.version)[0] + ': ' + sys.executable
             date = time.ctime(time.time())
             
             head = '%s%s%s\n%s%s%s\n%s' % (Colors.topline, '-'*75, ColorsNormal,
                                            exc, ' '*(75-len(str(etype))-len(pyver)),
                                            pyver, string.rjust(date, 75) )
             head += "\nA problem occured executing Python code.  Here is the sequence of function"\
                     "\ncalls leading up to the error, with the most recent (innermost) call last."
         else:
             # Simplified header
             head = '%s%s%s\n%s%s' % (Colors.topline, '-'*75, ColorsNormal,exc,
                                      string.rjust('Traceback (most recent call last)',
                                                   75 - len(str(etype)) ) )
         frames = []
         # Flush cache before calling inspect.  This helps alleviate some of the
         # problems with python 2.3's inspect.py.
         linecache.checkcache()
         # Drop topmost frames if requested
         try:
             # Try the default getinnerframes and Alex's: Alex's fixes some
             # problems, but it generates empty tracebacks for console errors
             # (5 blanks lines) where none should be returned.
             #records = inspect.getinnerframes(etb, context)[self.tb_offset:]
             #print 'python records:', records # dbg
             records = _fixed_getinnerframes(etb, context,self.tb_offset)
             #print 'alex   records:', records # dbg
         except:
 
             # FIXME: I've been getting many crash reports from python 2.3
             # users, traceable to inspect.py.  If I can find a small test-case
             # to reproduce this, I should either write a better workaround or
             # file a bug report against inspect (if that's the real problem).
             # So far, I haven't been able to find an isolated example to
             # reproduce the problem.
             inspect_error()
             traceback.print_exc(file=Term.cerr)
             info('\nUnfortunately, your original traceback can not be constructed.\n')
             return ''
 
         # build some color string templates outside these nested loops
         tpl_link       = '%s%%s%s' % (Colors.filenameEm,ColorsNormal)
         tpl_call       = 'in %s%%s%s%%s%s' % (Colors.vName, Colors.valEm,
                                               ColorsNormal)
         tpl_call_fail  = 'in %s%%s%s(***failed resolving arguments***)%s' % \
                          (Colors.vName, Colors.valEm, ColorsNormal)
         tpl_local_var  = '%s%%s%s' % (Colors.vName, ColorsNormal)
         tpl_global_var = '%sglobal%s %s%%s%s' % (Colors.em, ColorsNormal,
                                                  Colors.vName, ColorsNormal)
         tpl_name_val   = '%%s %s= %%s%s' % (Colors.valEm, ColorsNormal)
         tpl_line       = '%s%%s%s %%s' % (Colors.lineno, ColorsNormal)
         tpl_line_em    = '%s%%s%s %%s%s' % (Colors.linenoEm,Colors.line,
                                             ColorsNormal)
 
         # now, loop over all records printing context and info
         abspath = os.path.abspath
         for frame, file, lnum, func, lines, index in records:
             #print '*** record:',file,lnum,func,lines,index  # dbg
             try:
                 file = file and abspath(file) or '?'
             except OSError:
                 # if file is '<console>' or something not in the filesystem,
                 # the abspath call will throw an OSError.  Just ignore it and
                 # keep the original file string.
                 pass
             link = tpl_link % file
             try:
                 args, varargs, varkw, locals = inspect.getargvalues(frame)
             except:
                 # This can happen due to a bug in python2.3.  We should be
                 # able to remove this try/except when 2.4 becomes a
                 # requirement.  Bug details at http://python.org/sf/1005466
                 inspect_error()
                 traceback.print_exc(file=Term.cerr)
                 info("\nIPython's exception reporting continues...\n")
                 
             if func == '?':
                 call = ''
             else:
                 # Decide whether to include variable details or not
                 var_repr = self.include_vars and eqrepr or nullrepr
                 try:
                     call = tpl_call % (func,inspect.formatargvalues(args,
                                                 varargs, varkw,
                                                 locals,formatvalue=var_repr))
                 except KeyError:
                     # Very odd crash from inspect.formatargvalues().  The
                     # scenario under which it appeared was a call to
                     # view(array,scale) in NumTut.view.view(), where scale had
                     # been defined as a scalar (it should be a tuple). Somehow
                     # inspect messes up resolving the argument list of view()
                     # and barfs out. At some point I should dig into this one
                     # and file a bug report about it.
                     inspect_error()
                     traceback.print_exc(file=Term.cerr)
                     info("\nIPython's exception reporting continues...\n")
                     call = tpl_call_fail % func
 
             # Initialize a list of names on the current line, which the
             # tokenizer below will populate.
             names = []
 
             def tokeneater(token_type, token, start, end, line):
                 """Stateful tokeneater which builds dotted names.
 
                 The list of names it appends to (from the enclosing scope) can
                 contain repeated composite names.  This is unavoidable, since
                 there is no way to disambguate partial dotted structures until
                 the full list is known.  The caller is responsible for pruning
                 the final list of duplicates before using it."""
                 
                 # build composite names
                 if token == '.':
                     try:
                         names[-1] += '.'
                         # store state so the next token is added for x.y.z names
                         tokeneater.name_cont = True
                         return
                     except IndexError:
                         pass
                 if token_type == tokenize.NAME and token not in keyword.kwlist:
                     if tokeneater.name_cont:
                         # Dotted names
                         names[-1] += token
                         tokeneater.name_cont = False
                     else:
                         # Regular new names.  We append everything, the caller
                         # will be responsible for pruning the list later.  It's
                         # very tricky to try to prune as we go, b/c composite
                         # names can fool us.  The pruning at the end is easy
                         # to do (or the caller can print a list with repeated
                         # names if so desired.
                         names.append(token)
                 elif token_type == tokenize.NEWLINE:
                     raise IndexError
             # we need to store a bit of state in the tokenizer to build
             # dotted names
             tokeneater.name_cont = False
 
             def linereader(file=file, lnum=[lnum], getline=linecache.getline):
                 line = getline(file, lnum[0])
                 lnum[0] += 1
                 return line
 
             # Build the list of names on this line of code where the exception
             # occurred.
             try:
                 # This builds the names list in-place by capturing it from the
                 # enclosing scope.
                 tokenize.tokenize(linereader, tokeneater)
             except IndexError:
                 # signals exit of tokenizer
                 pass
             except tokenize.TokenError,msg:
                 _m = ("An unexpected error occurred while tokenizing input\n"
                       "The following traceback may be corrupted or invalid\n"
                       "The error message is: %s\n" % msg)
                 error(_m)
             
             # prune names list of duplicates, but keep the right order
             unique_names = uniq_stable(names)
 
             # Start loop over vars
             lvals = []
             if self.include_vars:
                 for name_full in unique_names:
                     name_base = name_full.split('.',1)[0]
                     if name_base in frame.f_code.co_varnames:
                         if locals.has_key(name_base):
                             try:
                                 value = repr(eval(name_full,locals))
                             except:
                                 value = undefined
                         else:
                             value = undefined
                         name = tpl_local_var % name_full
                     else:
                         if frame.f_globals.has_key(name_base):
                             try:
                                 value = repr(eval(name_full,frame.f_globals))
                             except:
                                 value = undefined
                         else:
                             value = undefined
                         name = tpl_global_var % name_full
                     lvals.append(tpl_name_val % (name,value))
             if lvals:
                 lvals = '%s%s' % (indent,em_normal.join(lvals))
             else:
                 lvals = ''
 
             level = '%s %s\n' % (link,call)
 
             if index is None:
                 frames.append(level)
             else:
                 frames.append('%s%s' % (level,''.join(
                     _formatTracebackLines(lnum,index,lines,Colors,lvals,
                                           col_scheme))))
 
         # Get (safely) a string form of the exception info
         try:
             etype_str,evalue_str = map(str,(etype,evalue))
         except:
             # User exception is improperly defined.
             etype,evalue = str,sys.exc_info()[:2]
             etype_str,evalue_str = map(str,(etype,evalue))
         # ... and format it
         exception = ['%s%s%s: %s' % (Colors.excName, etype_str,
                                      ColorsNormal, evalue_str)]
         if type(evalue) is types.InstanceType:
             try:
                 names = [w for w in dir(evalue) if isinstance(w, basestring)]
             except:
                 # Every now and then, an object with funny inernals blows up
                 # when dir() is called on it.  We do the best we can to report
                 # the problem and continue
                 _m = '%sException reporting error (object with broken dir())%s:'
                 exception.append(_m % (Colors.excName,ColorsNormal))
                 etype_str,evalue_str = map(str,sys.exc_info()[:2])
                 exception.append('%s%s%s: %s' % (Colors.excName,etype_str,
                                      ColorsNormal, evalue_str))
                 names = []
             for name in names:
                 value = text_repr(getattr(evalue, name))
                 exception.append('\n%s%s = %s' % (indent, name, value))
 
-        # vds: >>
-        if records:
-             filepath, lnum = records[-1][1:3]
-             #print "file:", str(file), "linenb", str(lnum) # dbg
-             filepath = os.path.abspath(filepath)
-             __IPYTHON__.hooks.synchronize_with_editor(filepath, lnum, 0)
-        # vds: <<
+        # This is being commented out for now as the __IPYTHON__ variable
+        # referenced here is not resolved and causes massive test failures
+        # and errors. B. Granger, 04/2009. XXX
+        # See https://bugs.launchpad.net/bugs/362137
+        # # vds: >>
+        # if records:
+        #      filepath, lnum = records[-1][1:3]
+        #      #print "file:", str(file), "linenb", str(lnum) # dbg
+        #      filepath = os.path.abspath(filepath)
+        #      __IPYTHON__.hooks.synchronize_with_editor(filepath, lnum, 0)
+        # # vds: <<
                 
         # return all our info assembled as a single string
         return '%s\n\n%s\n%s' % (head,'\n'.join(frames),''.join(exception[0]) )
 
     def debugger(self,force=False):
         """Call up the pdb debugger if desired, always clean up the tb
         reference.
 
         Keywords:
 
           - force(False): by default, this routine checks the instance call_pdb
           flag and does not actually invoke the debugger if the flag is false.
           The 'force' option forces the debugger to activate even if the flag
           is false.
 
         If the call_pdb flag is set, the pdb interactive debugger is
         invoked. In all cases, the self.tb reference to the current traceback
         is deleted to prevent lingering references which hamper memory
         management.
 
         Note that each call to pdb() does an 'import readline', so if your app
         requires a special setup for the readline completers, you'll have to
         fix that by hand after invoking the exception handler."""
 
         if force or self.call_pdb:
             if self.pdb is None:
                 self.pdb = Debugger.Pdb(
                     self.color_scheme_table.active_scheme_name)
             # the system displayhook may have changed, restore the original
             # for pdb
             dhook = sys.displayhook
             sys.displayhook = sys.__displayhook__
             self.pdb.reset()
             # Find the right frame so we don't pop up inside ipython itself
             if hasattr(self,'tb'):
                 etb = self.tb
             else:
                 etb = self.tb = sys.last_traceback
             while self.tb.tb_next is not None:
                 self.tb = self.tb.tb_next
             try:
                 if etb and etb.tb_next:
                     etb = etb.tb_next
                 self.pdb.botframe = etb.tb_frame
                 self.pdb.interaction(self.tb.tb_frame, self.tb)
             finally:
                 sys.displayhook = dhook
             
         if hasattr(self,'tb'):
             del self.tb
 
     def handler(self, info=None):
         (etype, evalue, etb) = info or sys.exc_info()
         self.tb = etb
         Term.cout.flush()
         print >> Term.cerr, self.text(etype, evalue, etb)
         Term.cerr.flush()
 
     # Changed so an instance can just be called as VerboseTB_inst() and print
     # out the right info on its own.
     def __call__(self, etype=None, evalue=None, etb=None):
         """This hook can replace sys.excepthook (for Python 2.1 or higher)."""
         if etb is None:
             self.handler()
         else:
             self.handler((etype, evalue, etb))
         try:
             self.debugger()
         except KeyboardInterrupt:
             print "\nKeyboardInterrupt"
 
 #----------------------------------------------------------------------------
 class FormattedTB(VerboseTB,ListTB):
     """Subclass ListTB but allow calling with a traceback.
 
     It can thus be used as a sys.excepthook for Python > 2.1.
 
     Also adds 'Context' and 'Verbose' modes, not available in ListTB.
 
     Allows a tb_offset to be specified. This is useful for situations where
     one needs to remove a number of topmost frames from the traceback (such as
     occurs with python programs that themselves execute other python code,
     like Python shells).  """
     
     def __init__(self, mode = 'Plain', color_scheme='Linux',
                  tb_offset = 0,long_header=0,call_pdb=0,include_vars=0):
 
         # NEVER change the order of this list. Put new modes at the end:
         self.valid_modes = ['Plain','Context','Verbose']
         self.verbose_modes = self.valid_modes[1:3]
 
         VerboseTB.__init__(self,color_scheme,tb_offset,long_header,
                            call_pdb=call_pdb,include_vars=include_vars)
         self.set_mode(mode)
         
     def _extract_tb(self,tb):
         if tb:
             return traceback.extract_tb(tb)
         else:
             return None
 
     def text(self, etype, value, tb,context=5,mode=None):
         """Return formatted traceback.
 
         If the optional mode parameter is given, it overrides the current
         mode."""
 
         if mode is None:
             mode = self.mode
         if mode in self.verbose_modes:
             # verbose modes need a full traceback
             return VerboseTB.text(self,etype, value, tb,context=5)
         else:
             # We must check the source cache because otherwise we can print
             # out-of-date source code.
             linecache.checkcache()
             # Now we can extract and format the exception
             elist = self._extract_tb(tb)
             if len(elist) > self.tb_offset:
                 del elist[:self.tb_offset]
             return ListTB.text(self,etype,value,elist)
 
     def set_mode(self,mode=None):
         """Switch to the desired mode.
 
         If mode is not specified, cycles through the available modes."""
 
         if not mode:
             new_idx = ( self.valid_modes.index(self.mode) + 1 ) % \
                       len(self.valid_modes)
             self.mode = self.valid_modes[new_idx]
         elif mode not in self.valid_modes:
             raise ValueError, 'Unrecognized mode in FormattedTB: <'+mode+'>\n'\
                   'Valid modes: '+str(self.valid_modes)
         else:
             self.mode = mode
         # include variable details only in 'Verbose' mode
         self.include_vars = (self.mode == self.valid_modes[2])
 
     # some convenient shorcuts
     def plain(self):
         self.set_mode(self.valid_modes[0])
 
     def context(self):
         self.set_mode(self.valid_modes[1])
 
     def verbose(self):
         self.set_mode(self.valid_modes[2])
 
 #----------------------------------------------------------------------------
 class AutoFormattedTB(FormattedTB):
     """A traceback printer which can be called on the fly.
 
     It will find out about exceptions by itself.
 
     A brief example:
     
     AutoTB = AutoFormattedTB(mode = 'Verbose',color_scheme='Linux')
     try:
       ...
     except:
       AutoTB()  # or AutoTB(out=logfile) where logfile is an open file object
     """
     def __call__(self,etype=None,evalue=None,etb=None,
                  out=None,tb_offset=None):
         """Print out a formatted exception traceback.
 
         Optional arguments:
           - out: an open file-like object to direct output to.
 
           - tb_offset: the number of frames to skip over in the stack, on a
           per-call basis (this overrides temporarily the instance's tb_offset
           given at initialization time.  """
         
         if out is None:
             out = Term.cerr
         Term.cout.flush()
         if tb_offset is not None:
             tb_offset, self.tb_offset = self.tb_offset, tb_offset
             print >> out, self.text(etype, evalue, etb)
             self.tb_offset = tb_offset
         else:
             print >> out, self.text(etype, evalue, etb)
         out.flush()
         try:
             self.debugger()
         except KeyboardInterrupt:
             print "\nKeyboardInterrupt"
 
     def text(self,etype=None,value=None,tb=None,context=5,mode=None):
         if etype is None:
             etype,value,tb = sys.exc_info()
         self.tb = tb
         return FormattedTB.text(self,etype,value,tb,context=5,mode=mode)
 
 #---------------------------------------------------------------------------
 # A simple class to preserve Nathan's original functionality.
 class ColorTB(FormattedTB):
     """Shorthand to initialize a FormattedTB in Linux colors mode."""
     def __init__(self,color_scheme='Linux',call_pdb=0):
         FormattedTB.__init__(self,color_scheme=color_scheme,
                              call_pdb=call_pdb)
 
 #----------------------------------------------------------------------------
 # module testing (minimal)
 if __name__ == "__main__":
     def spam(c, (d, e)):
         x = c + d
         y = c * d
         foo(x, y)
 
     def foo(a, b, bar=1):
         eggs(a, b + bar)
 
     def eggs(f, g, z=globals()):
         h = f + g
         i = f - g
         return h / i
 
     print ''
     print '*** Before ***'
     try:
         print spam(1, (2, 3))
     except:
         traceback.print_exc()
     print ''
     
     handler = ColorTB()
     print '*** ColorTB ***'
     try:
         print spam(1, (2, 3))
     except:
         apply(handler, sys.exc_info() )
     print ''
     
     handler = VerboseTB()
     print '*** VerboseTB ***'
     try:
         print spam(1, (2, 3))
     except:
         apply(handler, sys.exc_info() )
     print ''
     

IPython/kernel/engineconnector.py

@@ -1,92 +1,92 b''
 # encoding: utf-8
 
 """A class that manages the engines connection to the controller."""
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 import os
 import cPickle as pickle
 
 from twisted.python import log, failure
 from twisted.internet import defer
 
 from IPython.kernel.fcutil import find_furl
 from IPython.kernel.enginefc import IFCEngine
 
 #-------------------------------------------------------------------------------
 # The ClientConnector class
 #-------------------------------------------------------------------------------
 
 class EngineConnector(object):
     """Manage an engines connection to a controller.
     
     This class takes a foolscap `Tub` and provides a `connect_to_controller` 
     method that will use the `Tub` to connect to a controller and register
     the engine with the controller.
     """
     
     def __init__(self, tub):
         self.tub = tub
         
     def connect_to_controller(self, engine_service, furl_or_file):
         """
         Make a connection to a controller specified by a furl.
         
         This method takes an `IEngineBase` instance and a foolcap URL and uses
         the `tub` attribute to make a connection to the controller.  The 
         foolscap URL contains all the information needed to connect to the 
         controller, including the ip and port as well as any encryption and 
         authentication information needed for the connection.
         
         After getting a reference to the controller, this method calls the 
         `register_engine` method of the controller to actually register the 
         engine.
         
         :Parameters:
             engine_service : IEngineBase
                 An instance of an `IEngineBase` implementer
             furl_or_file : str
                 A furl or a filename containing a furl
         """
         if not self.tub.running:
             self.tub.startService()
         self.engine_service = engine_service
         self.engine_reference = IFCEngine(self.engine_service)
         try:
             self.furl = find_furl(furl_or_file)
         except ValueError:
             return defer.fail(failure.Failure())
-            # return defer.fail(failure.Failure(ValueError('not a valid furl or furl file: %r' % furl_or_file)))
-        d = self.tub.getReference(self.furl)
-        d.addCallbacks(self._register, self._log_failure)
-        return d
+        else:
+            d = self.tub.getReference(self.furl)
+            d.addCallbacks(self._register, self._log_failure)
+            return d
     
     def _log_failure(self, reason):
         log.err('EngineConnector: engine registration failed:')
         log.err(reason)
         return reason
 
     def _register(self, rr):
         self.remote_ref = rr
         # Now register myself with the controller
         desired_id = self.engine_service.id
         d = self.remote_ref.callRemote('register_engine', self.engine_reference, 
             desired_id, os.getpid(), pickle.dumps(self.engine_service.properties,2))
         return d.addCallbacks(self._reference_sent, self._log_failure)
 
     def _reference_sent(self, registration_dict):
         self.engine_service.id = registration_dict['id']
         log.msg("engine registration succeeded, got id: %r" % self.engine_service.id)
         return self.engine_service.id
 

IPython/kernel/engineservice.py

@@ -1,903 +1,906 b''
 # encoding: utf-8
 # -*- test-case-name: IPython.kernel.tests.test_engineservice -*-
 
 """A Twisted Service Representation of the IPython core.
 
 The IPython Core exposed to the network is called the Engine.  Its
 representation in Twisted in the EngineService.  Interfaces and adapters
 are used to abstract out the details of the actual network protocol used.
 The EngineService is an Engine that knows nothing about the actual protocol
 used.
 
 The EngineService is exposed with various network protocols in modules like:
 
 enginepb.py
 enginevanilla.py
 
 As of 12/12/06 the classes in this module have been simplified greatly.  It was 
 felt that we had over-engineered things.  To improve the maintainability of the
 code we have taken out the ICompleteEngine interface and the completeEngine
 method that automatically added methods to engines.
 
 """
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
+# Tell nose to skip the testing of this module
+__test__ = {}
+
 import os, sys, copy
 import cPickle as pickle
 from new import instancemethod
 
 from twisted.application import service
 from twisted.internet import defer, reactor
 from twisted.python import log, failure, components
 import zope.interface as zi
 
 from IPython.kernel.core.interpreter import Interpreter
 from IPython.kernel import newserialized, error, util
 from IPython.kernel.util import printer
 from IPython.kernel.twistedutil import gatherBoth, DeferredList
 from IPython.kernel import codeutil
 
 
 #-------------------------------------------------------------------------------
 # Interface specification for the Engine
 #-------------------------------------------------------------------------------
 
 class IEngineCore(zi.Interface):
     """The minimal required interface for the IPython Engine.
     
     This interface provides a formal specification of the IPython core.
     All these methods should return deferreds regardless of what side of a
     network connection they are on.
     
     In general, this class simply wraps a shell class and wraps its return
     values as Deferred objects.  If the underlying shell class method raises
     an exception, this class should convert it to a twisted.failure.Failure
     that will be propagated along the Deferred's errback chain.
     
     In addition, Failures are aggressive.  By this, we mean that if a method
     is performing multiple actions (like pulling multiple object) if any
     single one fails, the entire method will fail with that Failure.  It is
     all or nothing. 
     """
     
     id = zi.interface.Attribute("the id of the Engine object")
     properties = zi.interface.Attribute("A dict of properties of the Engine")
     
     def execute(lines):
         """Execute lines of Python code.
         
         Returns a dictionary with keys (id, number, stdin, stdout, stderr)
         upon success.
         
         Returns a failure object if the execution of lines raises an exception.
         """
     
     def push(namespace):
         """Push dict namespace into the user's namespace.
         
         Returns a deferred to None or a failure.
         """
     
     def pull(keys):
         """Pulls values out of the user's namespace by keys.
         
         Returns a deferred to a tuple objects or a single object.
         
         Raises NameError if any one of objects doess not exist.
         """
     
     def push_function(namespace):
         """Push a dict of key, function pairs into the user's namespace.
         
         Returns a deferred to None or a failure."""
     
     def pull_function(keys):
         """Pulls functions out of the user's namespace by keys.
         
         Returns a deferred to a tuple of functions or a single function.
         
         Raises NameError if any one of the functions does not exist.
         """
     
     def get_result(i=None):
         """Get the stdin/stdout/stderr of command i.
         
         Returns a deferred to a dict with keys
         (id, number, stdin, stdout, stderr).
         
         Raises IndexError if command i does not exist.
         Raises TypeError if i in not an int.
         """
     
     def reset():
         """Reset the shell.
         
         This clears the users namespace.  Won't cause modules to be
         reloaded.  Should also re-initialize certain variables like id.
         """
     
     def kill():
         """Kill the engine by stopping the reactor."""
     
     def keys():
         """Return the top level variables in the users namspace.
         
         Returns a deferred to a dict."""
     
 
 class IEngineSerialized(zi.Interface):
     """Push/Pull methods that take Serialized objects.  
     
     All methods should return deferreds.
     """
     
     def push_serialized(namespace):
         """Push a dict of keys and Serialized objects into the user's namespace."""
     
     def pull_serialized(keys):
         """Pull objects by key from the user's namespace as Serialized.
         
         Returns a list of or one Serialized.
         
         Raises NameError is any one of the objects does not exist.
         """
     
 
 class IEngineProperties(zi.Interface):
     """Methods for access to the properties object of an Engine"""
     
     properties = zi.Attribute("A StrictDict object, containing the properties")
     
     def set_properties(properties):
         """set properties by key and value"""
     
     def get_properties(keys=None):
         """get a list of properties by `keys`, if no keys specified, get all"""
     
     def del_properties(keys):
         """delete properties by `keys`"""
     
     def has_properties(keys):
         """get a list of bool values for whether `properties` has `keys`"""
 
     def clear_properties():
         """clear the properties dict"""
     
 class IEngineBase(IEngineCore, IEngineSerialized, IEngineProperties):
     """The basic engine interface that EngineService will implement.
     
     This exists so it is easy to specify adapters that adapt to and from the
     API that the basic EngineService implements.
     """
     pass
 
 class IEngineQueued(IEngineBase):
     """Interface for adding a queue to an IEngineBase.  
     
     This interface extends the IEngineBase interface to add methods for managing
     the engine's queue.  The implicit details of this interface are that the 
     execution of all methods declared in IEngineBase should appropriately be
     put through a queue before execution. 
     
     All methods should return deferreds.
     """
     
     def clear_queue():
         """Clear the queue."""
     
     def queue_status():
         """Get the queued and pending commands in the queue."""
     
     def register_failure_observer(obs):
         """Register an observer of pending Failures.
         
         The observer must implement IFailureObserver.
         """
     
     def unregister_failure_observer(obs):
         """Unregister an observer of pending Failures."""
     
 
 class IEngineThreaded(zi.Interface):
     """A place holder for threaded commands.  
     
     All methods should return deferreds.
     """
     pass
 
 
 #-------------------------------------------------------------------------------
 # Functions and classes to implement the EngineService
 #-------------------------------------------------------------------------------
 
 
 class StrictDict(dict):
     """This is a strict copying dictionary for use as the interface to the 
     properties of an Engine.
 
     :IMPORTANT:
         This object copies the values you set to it, and returns copies to you
         when you request them.  The only way to change properties os explicitly
         through the setitem and getitem of the dictionary interface.
 
     Example:
         >>> e = get_engine(id)
         >>> L = [1,2,3]
         >>> e.properties['L'] = L
         >>> L == e.properties['L']
         True
         >>> L.append(99)
         >>> L == e.properties['L']
         False
         
         Note that getitem copies, so calls to methods of objects do not affect
         the properties, as seen here:
         
         >>> e.properties[1] = range(2)
         >>> print e.properties[1]
         [0, 1]
         >>> e.properties[1].append(2)
         >>> print e.properties[1]
         [0, 1]
     """
     def __init__(self, *args, **kwargs):
         dict.__init__(self, *args, **kwargs)
         self.modified = True
     
     def __getitem__(self, key):
         return copy.deepcopy(dict.__getitem__(self, key))
     
     def __setitem__(self, key, value):
         # check if this entry is valid for transport around the network
         # and copying
         try:
             pickle.dumps(key, 2)
             pickle.dumps(value, 2)
             newvalue = copy.deepcopy(value)
-        except:
-            raise error.InvalidProperty(value)
+        except Exception, e:
+            raise error.InvalidProperty("can't be a value: %r" % value)
         dict.__setitem__(self, key, newvalue)
         self.modified = True
     
     def __delitem__(self, key):
         dict.__delitem__(self, key)
         self.modified = True
     
     def update(self, dikt):
         for k,v in dikt.iteritems():
             self[k] = v
     
     def pop(self, key):
         self.modified = True
         return dict.pop(self, key)
     
     def popitem(self):
         self.modified = True
         return dict.popitem(self)
     
     def clear(self):
         self.modified = True
         dict.clear(self)
     
     def subDict(self, *keys):
         d = {}
         for key in keys:
             d[key] = self[key]
         return d
     
 
 
 class EngineAPI(object):
     """This is the object through which the user can edit the `properties`
     attribute of an Engine.
     The Engine Properties object copies all object in and out of itself.
     See the EngineProperties object for details.
     """
     _fix=False
     def __init__(self, id):
         self.id = id
         self.properties = StrictDict()
         self._fix=True
     
     def __setattr__(self, k,v):
         if self._fix:
             raise error.KernelError("I am protected!")
         else:
             object.__setattr__(self, k, v)
     
     def __delattr__(self, key):
         raise error.KernelError("I am protected!")
     
 
 _apiDict = {}
 
 def get_engine(id):
     """Get the Engine API object, whcih currently just provides the properties 
     object, by ID"""
     global _apiDict
     if not _apiDict.get(id):
         _apiDict[id] = EngineAPI(id)
     return _apiDict[id]
 
 def drop_engine(id):
     """remove an engine"""
     global _apiDict
     if _apiDict.has_key(id):
         del _apiDict[id]
 
 class EngineService(object, service.Service):
     """Adapt a IPython shell into a IEngine implementing Twisted Service."""
     
     zi.implements(IEngineBase)
     name = 'EngineService'
     
     def __init__(self, shellClass=Interpreter, mpi=None):
         """Create an EngineService.
         
         shellClass: something that implements IInterpreter or core1
         mpi:        an mpi module that has rank and size attributes
         """
         self.shellClass = shellClass
         self.shell = self.shellClass()
         self.mpi = mpi
         self.id = None
         self.properties = get_engine(self.id).properties
         if self.mpi is not None:
             log.msg("MPI started with rank = %i and size = %i" % 
                 (self.mpi.rank, self.mpi.size))
             self.id = self.mpi.rank
         self._seedNamespace()
     
     # Make id a property so that the shell can get the updated id
         
     def _setID(self, id):
         self._id = id
         self.properties = get_engine(id).properties
         self.shell.push({'id': id})
     
     def _getID(self):
         return self._id
     
     id = property(_getID, _setID)
     
     def _seedNamespace(self):
         self.shell.push({'mpi': self.mpi, 'id' : self.id})
     
     def executeAndRaise(self, msg, callable, *args, **kwargs):
         """Call a method of self.shell and wrap any exception."""
         d = defer.Deferred()
         try:
             result = callable(*args, **kwargs)
         except:
             # This gives the following:
             # et=exception class
             # ev=exception class instance
             # tb=traceback object
             et,ev,tb = sys.exc_info()
             # This call adds attributes to the exception value
             et,ev,tb = self.shell.formatTraceback(et,ev,tb,msg)
             # Add another attribute
             ev._ipython_engine_info = msg
             f = failure.Failure(ev,et,None)
             d.errback(f)
         else:
             d.callback(result)
             
         return d
     
     
     # The IEngine methods.  See the interface for documentation.
     
     def execute(self, lines):
         msg = {'engineid':self.id,
                'method':'execute',
                'args':[lines]}
         d = self.executeAndRaise(msg, self.shell.execute, lines)
         d.addCallback(self.addIDToResult)
         return d
     
     def addIDToResult(self, result):
         result['id'] = self.id
         return result
     
     def push(self, namespace):
         msg = {'engineid':self.id,
                'method':'push',
                'args':[repr(namespace.keys())]}
         d = self.executeAndRaise(msg, self.shell.push, namespace)
         return d
     
     def pull(self, keys):
         msg = {'engineid':self.id,
                'method':'pull',
                'args':[repr(keys)]}
         d = self.executeAndRaise(msg, self.shell.pull, keys)
         return d
     
     def push_function(self, namespace):
         msg = {'engineid':self.id,
                'method':'push_function',
                'args':[repr(namespace.keys())]}
         d = self.executeAndRaise(msg, self.shell.push_function, namespace)
         return d
     
     def pull_function(self, keys):
         msg = {'engineid':self.id,
                'method':'pull_function',
                'args':[repr(keys)]}
         d = self.executeAndRaise(msg, self.shell.pull_function, keys)
         return d
     
     def get_result(self, i=None):
         msg = {'engineid':self.id,
                'method':'get_result',
                'args':[repr(i)]}
         d = self.executeAndRaise(msg, self.shell.getCommand, i)
         d.addCallback(self.addIDToResult)
         return d
     
     def reset(self):
         msg = {'engineid':self.id,
                'method':'reset',
                'args':[]}
         del self.shell
         self.shell = self.shellClass()
         self.properties.clear()
         d = self.executeAndRaise(msg, self._seedNamespace)
         return d
     
     def kill(self):
         drop_engine(self.id)
         try:
             reactor.stop()
         except RuntimeError:
             log.msg('The reactor was not running apparently.')
             return defer.fail()
         else:
             return defer.succeed(None)
     
     def keys(self):
         """Return a list of variables names in the users top level namespace.
         
         This used to return a dict of all the keys/repr(values) in the 
         user's namespace.  This was too much info for the ControllerService
         to handle so it is now just a list of keys.
         """
         
         remotes = []
         for k in self.shell.user_ns.iterkeys():
             if k not in ['__name__', '_ih', '_oh', '__builtins__',
                          'In', 'Out', '_', '__', '___', '__IP', 'input', 'raw_input']:
                 remotes.append(k)
         return defer.succeed(remotes)
     
     def set_properties(self, properties):
         msg = {'engineid':self.id,
                'method':'set_properties',
                'args':[repr(properties.keys())]}
         return self.executeAndRaise(msg, self.properties.update, properties)
     
     def get_properties(self, keys=None):
         msg = {'engineid':self.id,
                'method':'get_properties',
                'args':[repr(keys)]}
         if keys is None:
             keys = self.properties.keys()
         return self.executeAndRaise(msg, self.properties.subDict, *keys)
     
     def _doDel(self, keys):
         for key in keys:
             del self.properties[key]
     
     def del_properties(self, keys):
         msg = {'engineid':self.id,
                'method':'del_properties',
                'args':[repr(keys)]}
         return self.executeAndRaise(msg, self._doDel, keys)
     
     def _doHas(self, keys):
         return [self.properties.has_key(key) for key in keys]
     
     def has_properties(self, keys):
         msg = {'engineid':self.id,
                'method':'has_properties',
                'args':[repr(keys)]}
         return self.executeAndRaise(msg, self._doHas, keys)
     
     def clear_properties(self):
         msg = {'engineid':self.id,
                'method':'clear_properties',
                'args':[]}
         return self.executeAndRaise(msg, self.properties.clear)
     
     def push_serialized(self, sNamespace):
         msg = {'engineid':self.id,
                'method':'push_serialized',
                'args':[repr(sNamespace.keys())]}       
         ns = {}
         for k,v in sNamespace.iteritems():
             try:
                 unserialized = newserialized.IUnSerialized(v)
                 ns[k] = unserialized.getObject()
             except:
                 return defer.fail()
         return self.executeAndRaise(msg, self.shell.push, ns)
     
     def pull_serialized(self, keys):
         msg = {'engineid':self.id,
                'method':'pull_serialized',
                'args':[repr(keys)]}
         if isinstance(keys, str):
             keys = [keys]
         if len(keys)==1:
             d = self.executeAndRaise(msg, self.shell.pull, keys)
             d.addCallback(newserialized.serialize)
             return d
         elif len(keys)>1:
             d = self.executeAndRaise(msg, self.shell.pull, keys)         
             @d.addCallback
             def packThemUp(values):
                 serials = []
                 for v in values:
                     try:
                         serials.append(newserialized.serialize(v))
                     except:
                         return defer.fail(failure.Failure())
                 return serials
             return packThemUp
 
 
 def queue(methodToQueue):
     def queuedMethod(this, *args, **kwargs):
         name = methodToQueue.__name__
         return this.submitCommand(Command(name, *args, **kwargs))
     return queuedMethod
     
 class QueuedEngine(object):
     """Adapt an IEngineBase to an IEngineQueued by wrapping it.
     
     The resulting object will implement IEngineQueued which extends
     IEngineCore which extends (IEngineBase, IEngineSerialized). 
     
     This seems like the best way of handling it, but I am not sure.  The
     other option is to have the various base interfaces be used like
     mix-in intefaces.  The problem I have with this is adpatation is
     more difficult and complicated because there can be can multiple
     original and final Interfaces. 
     """
     
     zi.implements(IEngineQueued)
     
     def __init__(self, engine):
         """Create a QueuedEngine object from an engine
         
         engine:       An implementor of IEngineCore and IEngineSerialized
         keepUpToDate: whether to update the remote status when the 
                       queue is empty.  Defaults to False.
         """
         
         # This is the right way to do these tests rather than 
         # IEngineCore in list(zi.providedBy(engine)) which will only 
         # picks of the interfaces that are directly declared by engine.
         assert IEngineBase.providedBy(engine), \
             "engine passed to QueuedEngine doesn't provide IEngineBase"
             
         self.engine = engine
         self.id = engine.id
         self.queued = []
         self.history = {}
         self.engineStatus = {}
         self.currentCommand = None
         self.failureObservers = []
     
     def _get_properties(self):
         return self.engine.properties
     
     properties = property(_get_properties, lambda self, _: None)
     # Queue management methods.  You should not call these directly
     
     def submitCommand(self, cmd):
         """Submit command to queue."""
         
         d = defer.Deferred()
         cmd.setDeferred(d)
         if self.currentCommand is not None:
             if self.currentCommand.finished:
                 # log.msg("Running command immediately: %r" % cmd)
                 self.currentCommand = cmd
                 self.runCurrentCommand()
             else:  # command is still running  
                 # log.msg("Command is running: %r" % self.currentCommand)
                 # log.msg("Queueing: %r" % cmd)
                 self.queued.append(cmd)
         else:
             # log.msg("No current commands, running: %r" % cmd)
             self.currentCommand = cmd
             self.runCurrentCommand()
         return d
     
     def runCurrentCommand(self):
         """Run current command."""
         
         cmd = self.currentCommand
         f = getattr(self.engine, cmd.remoteMethod, None)
         if f:
             d = f(*cmd.args, **cmd.kwargs)
             if cmd.remoteMethod is 'execute':
                 d.addCallback(self.saveResult)
             d.addCallback(self.finishCommand)
             d.addErrback(self.abortCommand)
         else:
             return defer.fail(AttributeError(cmd.remoteMethod))
     
     def _flushQueue(self):
         """Pop next command in queue and run it."""
         
         if len(self.queued) > 0:
             self.currentCommand = self.queued.pop(0)
             self.runCurrentCommand()
     
     def saveResult(self, result):
         """Put the result in the history."""
         self.history[result['number']] = result
         return result
     
     def finishCommand(self, result):
         """Finish currrent command."""
         
         # The order of these commands is absolutely critical.
         self.currentCommand.handleResult(result)
         self.currentCommand.finished = True
         self._flushQueue()
         return result
     
     def abortCommand(self, reason):
         """Abort current command.
         
         This eats the Failure but first passes it onto the Deferred that the 
         user has.
         
         It also clear out the queue so subsequence commands don't run.
         """
 
         # The order of these 3 commands is absolutely critical.  The currentCommand
         # must first be marked as finished BEFORE the queue is cleared and before
         # the current command is sent the failure.
         # Also, the queue must be cleared BEFORE the current command is sent the Failure
         # otherwise the errback chain could trigger new commands to be added to the 
         # queue before we clear it.  We should clear ONLY the commands that were in
         # the queue when the error occured. 
         self.currentCommand.finished = True
         s = "%r %r %r" % (self.currentCommand.remoteMethod, self.currentCommand.args, self.currentCommand.kwargs)
         self.clear_queue(msg=s)
         self.currentCommand.handleError(reason)
         
         return None
     
     #---------------------------------------------------------------------------
     # IEngineCore methods
     #---------------------------------------------------------------------------
     
     @queue
     def execute(self, lines):
         pass
     
     @queue
     def push(self, namespace):
         pass      
     
     @queue
     def pull(self, keys):
         pass
         
     @queue
     def push_function(self, namespace):
         pass      
     
     @queue
     def pull_function(self, keys):
         pass        
 
     def get_result(self, i=None):
         if i is None:
             i = max(self.history.keys()+[None])
 
         cmd = self.history.get(i, None)
         # Uncomment this line to disable chaching of results
         #cmd = None
         if cmd is None:
             return self.submitCommand(Command('get_result', i))
         else:
             return defer.succeed(cmd)
         
     def reset(self):
         self.clear_queue()
         self.history = {}  # reset the cache - I am not sure we should do this
         return self.submitCommand(Command('reset'))
     
     def kill(self):
         self.clear_queue()
         return self.submitCommand(Command('kill'))
     
     @queue
     def keys(self):
         pass
     
     #---------------------------------------------------------------------------
     # IEngineSerialized methods
     #---------------------------------------------------------------------------
 
     @queue
     def push_serialized(self, namespace):
         pass
         
     @queue
     def pull_serialized(self, keys):
         pass
     
     #---------------------------------------------------------------------------
     # IEngineProperties methods
     #---------------------------------------------------------------------------
 
     @queue
     def set_properties(self, namespace):
         pass
         
     @queue
     def get_properties(self, keys=None):
         pass
     
     @queue
     def del_properties(self, keys):
         pass
     
     @queue
     def has_properties(self, keys):
         pass
     
     @queue
     def clear_properties(self):
         pass
     
     #---------------------------------------------------------------------------
     # IQueuedEngine methods
     #---------------------------------------------------------------------------
     
     def clear_queue(self, msg=''):
         """Clear the queue, but doesn't cancel the currently running commmand."""
         
         for cmd in self.queued:
             cmd.deferred.errback(failure.Failure(error.QueueCleared(msg)))
         self.queued = []
         return defer.succeed(None)
     
     def queue_status(self):
         if self.currentCommand is not None:
             if self.currentCommand.finished:
                 pending = repr(None)
             else:
                 pending = repr(self.currentCommand)
         else:
             pending = repr(None)
         dikt = {'queue':map(repr,self.queued), 'pending':pending}
         return defer.succeed(dikt)
         
     def register_failure_observer(self, obs):
         self.failureObservers.append(obs)
     
     def unregister_failure_observer(self, obs):
         self.failureObservers.remove(obs)
     
 
 # Now register QueuedEngine as an adpater class that makes an IEngineBase into a
 # IEngineQueued.  
 components.registerAdapter(QueuedEngine, IEngineBase, IEngineQueued)
     
 
 class Command(object):
     """A command object that encapslates queued commands.
     
     This class basically keeps track of a command that has been queued
     in a QueuedEngine.  It manages the deferreds and hold the method to be called
     and the arguments to that method.
     """
     
     
     def __init__(self, remoteMethod, *args, **kwargs):
         """Build a new Command object."""
         
         self.remoteMethod = remoteMethod
         self.args = args
         self.kwargs = kwargs
         self.finished = False
     
     def setDeferred(self, d):
         """Sets the deferred attribute of the Command."""  
           
         self.deferred = d
     
     def __repr__(self):
         if not self.args:
             args = ''
         else:
             args = str(self.args)[1:-2]  #cut off (...,)
         for k,v in self.kwargs.iteritems():
             if args:
                 args += ', '
             args += '%s=%r' %(k,v)
         return "%s(%s)" %(self.remoteMethod, args)
     
     def handleResult(self, result):
         """When the result is ready, relay it to self.deferred."""
         
         self.deferred.callback(result)
     
     def handleError(self, reason):
         """When an error has occured, relay it to self.deferred."""
         
         self.deferred.errback(reason)
 
 class ThreadedEngineService(EngineService):
     """An EngineService subclass that defers execute commands to a separate 
     thread.
     
     ThreadedEngineService uses twisted.internet.threads.deferToThread to 
     defer execute requests to a separate thread. GUI frontends may want to 
     use ThreadedEngineService as the engine in an 
     IPython.frontend.frontendbase.FrontEndBase subclass to prevent
     block execution from blocking the GUI thread.
     """
     
     zi.implements(IEngineBase)
 
     def __init__(self, shellClass=Interpreter, mpi=None):
         EngineService.__init__(self, shellClass, mpi)
     
     def wrapped_execute(self, msg, lines):
         """Wrap self.shell.execute to add extra information to tracebacks"""
         
         try:
             result = self.shell.execute(lines)
         except Exception,e:
             # This gives the following:
             # et=exception class
             # ev=exception class instance
             # tb=traceback object
             et,ev,tb = sys.exc_info()
             # This call adds attributes to the exception value
             et,ev,tb = self.shell.formatTraceback(et,ev,tb,msg)
             # Add another attribute
             
             # Create a new exception with the new attributes
             e = et(ev._ipython_traceback_text)
             e._ipython_engine_info = msg
             
             # Re-raise
             raise e
         
         return result
     
     
     def execute(self, lines):
         # Only import this if we are going to use this class
         from twisted.internet import threads
         
         msg = {'engineid':self.id,
                'method':'execute',
                'args':[lines]}
         
         d = threads.deferToThread(self.wrapped_execute, msg, lines)
         d.addCallback(self.addIDToResult)
         return d

IPython/kernel/error.py

@@ -1,185 +1,202 b''
 # encoding: utf-8
 
 """Classes and functions for kernel related errors and exceptions."""
 
 __docformat__ = "restructuredtext en"
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 from IPython.kernel.core import error
 from twisted.python import failure
 
 #-------------------------------------------------------------------------------
 # Error classes
 #-------------------------------------------------------------------------------
 
 class KernelError(error.IPythonError):
     pass
 
 class NotDefined(KernelError):
     def __init__(self, name):
         self.name = name
         self.args = (name,)
 
     def __repr__(self):
         return '<NotDefined: %s>' % self.name
     
     __str__ = __repr__
 
 class QueueCleared(KernelError):
     pass
 
 class IdInUse(KernelError):
     pass
 
 class ProtocolError(KernelError):
     pass
 
 class ConnectionError(KernelError):
     pass
 
 class InvalidEngineID(KernelError):
     pass
     
 class NoEnginesRegistered(KernelError):
     pass
     
 class InvalidClientID(KernelError):
     pass
     
 class InvalidDeferredID(KernelError):
     pass
     
 class SerializationError(KernelError):
     pass
     
 class MessageSizeError(KernelError):
     pass
     
 class PBMessageSizeError(MessageSizeError):
     pass
     
 class ResultNotCompleted(KernelError):
     pass
     
 class ResultAlreadyRetrieved(KernelError):
     pass
     
 class ClientError(KernelError):
     pass
 
 class TaskAborted(KernelError):
     pass
 
 class TaskTimeout(KernelError):
     pass
 
 class NotAPendingResult(KernelError):
     pass
 
 class UnpickleableException(KernelError):
     pass
 
 class AbortedPendingDeferredError(KernelError):
     pass
 
 class InvalidProperty(KernelError):
     pass
 
 class MissingBlockArgument(KernelError):
     pass
 
 class StopLocalExecution(KernelError):
     pass
 
 class SecurityError(KernelError):
     pass
 
+class FileTimeoutError(KernelError):
+    pass
+
+class TaskRejectError(KernelError):
+    """Exception to raise when a task should be rejected by an engine.
+    
+    This exception can be used to allow a task running on an engine to test
+    if the engine (or the user's namespace on the engine) has the needed
+    task dependencies.  If not, the task should raise this exception.  For
+    the task to be retried on another engine, the task should be created
+    with the `retries` argument > 1.
+    
+    The advantage of this approach over our older properties system is that
+    tasks have full access to the user's namespace on the engines and the
+    properties don't have to be managed or tested by the controller.
+    """
+
 class CompositeError(KernelError):
     def __init__(self, message, elist):
         Exception.__init__(self, *(message, elist))
         self.message = message
         self.elist = elist
   
     def _get_engine_str(self, ev):
         try:
             ei = ev._ipython_engine_info
         except AttributeError:
             return '[Engine Exception]'
         else:
             return '[%i:%s]: ' % (ei['engineid'], ei['method'])
     
     def _get_traceback(self, ev):
         try:
             tb = ev._ipython_traceback_text
         except AttributeError:
             return 'No traceback available'
         else:
             return tb
   
     def __str__(self):
         s = str(self.message)
         for et, ev, etb in self.elist:
             engine_str = self._get_engine_str(ev)
             s = s + '\n' + engine_str + str(et.__name__) + ': ' + str(ev)
         return s
     
     def print_tracebacks(self, excid=None):
         if excid is None:
             for (et,ev,etb) in self.elist:
                 print self._get_engine_str(ev)
                 print self._get_traceback(ev)
                 print
         else:
             try:
                 et,ev,etb = self.elist[excid]
             except:
                 raise IndexError("an exception with index %i does not exist"%excid)
             else:
                 print self._get_engine_str(ev)
                 print self._get_traceback(ev)
     
     def raise_exception(self, excid=0):
         try:
             et,ev,etb = self.elist[excid]
         except:
             raise IndexError("an exception with index %i does not exist"%excid)
         else:
             raise et, ev, etb
 
 def collect_exceptions(rlist, method):
     elist = []
     for r in rlist:
         if isinstance(r, failure.Failure):
             r.cleanFailure()
             et, ev, etb = r.type, r.value, r.tb
             # Sometimes we could have CompositeError in our list.  Just take
             # the errors out of them and put them in our new list.  This 
             # has the effect of flattening lists of CompositeErrors into one
             # CompositeError
             if et==CompositeError:
                 for e in ev.elist:
                     elist.append(e)
             else:
                 elist.append((et, ev, etb))
     if len(elist)==0:
         return rlist
     else:
         msg = "one or more exceptions from call to method: %s" % (method)
         # This silliness is needed so the debugger has access to the exception
         # instance (e in this case)
         try:
             raise CompositeError(msg, elist)
         except CompositeError, e:
             raise e