More work on refactoring things into components.

* Prefilters have been componentized (prefilter.py).
* We are not including auto_attr from NiPy in IPython.utils.
* All components are now using auto_attr when resolving refs to other
  components.
* The AliasManager component has been updated to reflect the prefilter
  refactor.
* prefilter.splitUserInput has been moved to
  IPython.core.splitinput.split_user_input as we are using this in other
  places in the IPython code base.

IPython/core/splitinput.py

@@ -0,0 +1,77 b''
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+Simple utility for splitting user input.
+
+Authors:
+
+* Brian Granger
+* Fernando Perez
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import re
+
+#-----------------------------------------------------------------------------
+# Main function
+#-----------------------------------------------------------------------------
+
+
+# RegExp for splitting line contents into pre-char//first word-method//rest.
+# For clarity, each group in on one line.
+
+# WARNING: update the regexp if the escapes in iplib are changed, as they
+# are hardwired in.
+
+# Although it's not solely driven by the regex, note that:
+# ,;/% only trigger if they are the first character on the line
+# ! and !! trigger if they are first char(s) *or* follow an indent 
+# ? triggers as first or last char.
+
+# The three parts of the regex are:
+#  1) pre:     pre_char *or* initial whitespace 
+#  2) ifun:    first word/method (mix of \w and '.')
+#  3) the_rest: rest of line (separated from ifun by space if non-empty)
+line_split = re.compile(r'^([,;/%?]|!!?|\s*)'
+                        r'\s*([\w\.]+)'
+                        r'(\s+.*$|$)')
+
+
+def split_user_input(line, pattern=None):
+    """Split user input into pre-char/whitespace, function part and rest."""
+
+    if pattern is None:
+        pattern = line_split
+    match = pattern.match(line)
+    if not match:
+        #print "match failed for line '%s'" % line
+        try:
+            ifun, the_rest = line.split(None,1)
+        except ValueError:
+            #print "split failed for line '%s'" % line
+            ifun, the_rest = line,''
+        pre = re.match('^(\s*)(.*)',line).groups()[0]
+    else:
+        pre,ifun,the_rest = match.groups()
+
+    # ifun has to be a valid python identifier, so it better be only pure
+    # ascii, no unicode:
+    try:
+        ifun = ifun.encode('ascii')
+    except UnicodeEncodeError:
+        the_rest = ifun + u' ' + the_rest
+        ifun = u''
+
+    #print 'line:<%s>' % line # dbg
+    #print 'pre <%s> ifun <%s> rest <%s>' % (pre,ifun.strip(),the_rest) # dbg
+    return pre, ifun.strip(), the_rest.lstrip()

IPython/utils/autoattr.py

@@ -0,0 +1,162 b''
+#!/usr/bin/env python
+# encoding: utf-8
+"""Descriptor support for NIPY.
+
+Utilities to support special Python descriptors [1,2], in particular the use of
+a useful pattern for properties we call 'one time properties'.  These are
+object attributes which are declared as properties, but become regular
+attributes once they've been read the first time.  They can thus be evaluated
+later in the object's life cycle, but once evaluated they become normal, static
+attributes with no function call overhead on access or any other constraints.
+
+A special ResetMixin class is provided to add a .reset() method to users who
+may want to have their objects capable of resetting these computed properties
+to their 'untriggered' state.
+
+References
+----------
+[1] How-To Guide for Descriptors, Raymond
+Hettinger. http://users.rcn.com/python/download/Descriptor.htm
+
+[2] Python data model, http://docs.python.org/reference/datamodel.html
+
+Notes
+-----
+This module is taken from the NiPy project 
+(http://neuroimaging.scipy.org/site/index.html), and is BSD licensed.
+"""
+
+#-----------------------------------------------------------------------------
+# Classes and Functions
+#-----------------------------------------------------------------------------
+
+class ResetMixin(object):
+   """A Mixin class to add a .reset() method to users of OneTimeProperty.
+
+   By default, auto attributes once computed, become static.  If they happen to
+   depend on other parts of an object and those parts change, their values may
+   now be invalid.
+
+   This class offers a .reset() method that users can call *explicitly* when
+   they know the state of their objects may have changed and they want to
+   ensure that *all* their special attributes should be invalidated.  Once
+   reset() is called, all their auto attributes are reset to their
+   OneTimeProperty descriptors, and their accessor functions will be triggered
+   again.
+
+   Example
+   -------
+
+   >>> class A(ResetMixin):
+   ...     def __init__(self,x=1.0):
+   ...         self.x = x
+   ...
+   ...     @auto_attr
+   ...     def y(self):
+   ...         print '*** y computation executed ***'
+   ...         return self.x / 2.0
+   ... 
+
+   >>> a = A(10)
+
+   About to access y twice, the second time no computation is done:
+   >>> a.y
+   *** y computation executed ***
+   5.0
+   >>> a.y
+   5.0
+
+   Changing x
+   >>> a.x = 20
+   
+   a.y doesn't change to 10, since it is a static attribute:
+   >>> a.y
+   5.0
+
+   We now reset a, and this will then force all auto attributes to recompute
+   the next time we access them:
+   >>> a.reset()
+
+   About to access y twice again after reset():
+   >>> a.y
+   *** y computation executed ***
+   10.0
+   >>> a.y
+   10.0
+   """
+
+   def reset(self):
+      """Reset all OneTimeProperty attributes that may have fired already."""
+      instdict = self.__dict__
+      classdict = self.__class__.__dict__
+      # To reset them, we simply remove them from the instance dict.  At that
+      # point, it's as if they had never been computed.  On the next access,
+      # the accessor function from the parent class will be called, simply
+      # because that's how the python descriptor protocol works.
+      for mname, mval in classdict.items():
+         if mname in instdict and isinstance(mval, OneTimeProperty):
+            delattr(self, mname)
+
+
+class OneTimeProperty(object):
+   """A descriptor to make special properties that become normal attributes.
+
+   This is meant to be used mostly by the auto_attr decorator in this module.
+   """
+   def __init__(self,func):
+       """Create a OneTimeProperty instance.
+
+        Parameters
+        ----------
+          func : method
+          
+            The method that will be called the first time to compute a value.
+            Afterwards, the method's name will be a standard attribute holding
+            the value of this computation.
+            """
+       self.getter = func
+       self.name = func.func_name
+
+   def __get__(self,obj,type=None):
+       """This will be called on attribute access on the class or instance. """
+
+       if obj is None:
+           # Being called on the class, return the original function. This way,
+           # introspection works on the class.
+           #return func
+           return self.getter
+
+       val = self.getter(obj)
+       #print "** auto_attr - loading '%s'" % self.name  # dbg
+       setattr(obj, self.name, val)
+       return val
+
+
+def auto_attr(func):
+    """Decorator to create OneTimeProperty attributes.
+
+    Parameters
+    ----------
+      func : method
+        The method that will be called the first time to compute a value.
+        Afterwards, the method's name will be a standard attribute holding the
+        value of this computation.
+
+    Examples
+    --------
+    >>> class MagicProp(object):
+    ...     @auto_attr
+    ...     def a(self):
+    ...         return 99
+    ...     
+    >>> x = MagicProp()
+    >>> 'a' in x.__dict__
+    False
+    >>> x.a
+    99
+    >>> 'a' in x.__dict__
+    True
+    """
+    return OneTimeProperty(func)
+
+

IPython/core/alias.py

@@ -22,17 +22,23 b' Authors:'
 import __builtin__
 import keyword
 import os
+import re
 import sys
 
 from IPython.core.component import Component
+from IPython.core.splitinput import split_user_input
 
 from IPython.utils.traitlets import CBool, List, Instance
 from IPython.utils.genutils import error
+from IPython.utils.autoattr import auto_attr
 
 #-----------------------------------------------------------------------------
-# Functions and classes
+# Utilities
 #-----------------------------------------------------------------------------
 
+# This is used as the pattern for calls to split_user_input.
+shell_line_split = re.compile(r'^(\s*)(\S*\s*)(.*$)')
+
 def default_aliases():
     # Make some aliases automatically
     # Prepare list of shell aliases to auto-define
@@ -88,6 +94,11 b' class InvalidAliasError(AliasError):'
     pass
 
 
+#-----------------------------------------------------------------------------
+# Main AliasManager class
+#-----------------------------------------------------------------------------
+
+
 class AliasManager(Component):
 
     auto_alias = List(default_aliases())
@@ -95,14 +106,18 b' class AliasManager(Component):'
 
     def __init__(self, parent, config=None):
         super(AliasManager, self).__init__(parent, config=config)
-        self.shell = Component.get_instances(
-            root=self.root,
-            klass='IPython.core.iplib.InteractiveShell'
-        )[0]
         self.alias_table = {}
         self.exclude_aliases()
         self.init_aliases()
 
+    @auto_attr
+    def shell(self):
+        shell = Component.get_instances(
+            root=self.root,
+            klass='IPython.core.iplib.InteractiveShell'
+        )[0]
+        return shell
+
     def __contains__(self, name):
         if name in self.alias_table:
             return True
@@ -189,3 +204,54 b' class AliasManager(Component):'
                       (alias, nargs, len(args)))
             cmd = '%s %s' % (cmd % tuple(args[:nargs]),' '.join(args[nargs:]))
         return cmd
+
+    def expand_alias(self, line):
+        """ Expand an alias in the command line 
+        
+        Returns the provided command line, possibly with the first word 
+        (command) translated according to alias expansion rules.
+        
+        [ipython]|16> _ip.expand_aliases("np myfile.txt")
+                 <16> 'q:/opt/np/notepad++.exe myfile.txt'
+        """
+        
+        pre,fn,rest = split_user_input(line)
+        res = pre + self.expand_aliases(fn, rest)
+        return res
+
+    def expand_aliases(self, fn, rest):
+        """Expand multiple levels of aliases:
+        
+        if:
+        
+        alias foo bar /tmp
+        alias baz foo
+        
+        then:
+        
+        baz huhhahhei -> bar /tmp huhhahhei
+        
+        """
+        line = fn + " " + rest
+        
+        done = set()
+        while 1:
+            pre,fn,rest = split_user_input(line, shell_line_split)
+            if fn in self.alias_table:
+                if fn in done:
+                    warn("Cyclic alias definition, repeated '%s'" % fn)
+                    return ""
+                done.add(fn)
+
+                l2 = self.transform_alias(fn, rest)
+                if l2 == line:
+                    break
+                # ls -> ls -F should not recurse forever
+                if l2.split(None,1)[0] == line.split(None,1)[0]:
+                    line = l2
+                    break
+                line=l2
+            else:
+                break
+                
+        return line

IPython/core/builtin_trap.py

@@ -24,7 +24,7 b' import __builtin__'
 from IPython.core.component import Component
 from IPython.core.quitter import Quitter
 
-from IPython.utils.traitlets import Instance
+from IPython.utils.autoattr import auto_attr
 
 #-----------------------------------------------------------------------------
 # Classes and functions
@@ -39,12 +39,15 b' class BuiltinTrap(Component):'
 
     def __init__(self, parent):
         super(BuiltinTrap, self).__init__(parent, None, None)
-        # Don't just grab parent!!!
-        self.shell = Component.get_instances(
+        self._orig_builtins = {}
+
+    @auto_attr
+    def shell(self):
+        shell = Component.get_instances(
             root=self.root,
             klass='IPython.core.iplib.InteractiveShell'
         )[0]
-        self._orig_builtins = {}
+        return shell
 
     def __enter__(self):
         self.set()

IPython/core/completer.py

@@ -74,6 +74,8 b' import itertools'
 import types
 
 from IPython.core.error import TryNext
+from IPython.core.prefilter import ESC_MAGIC
+
 import IPython.utils.rlineimpl as readline    
 from IPython.utils.ipstruct import Struct
 from IPython.utils import generics
@@ -234,7 +236,7 b' class IPCompleter(Completer):'
 
         Completer.__init__(self,namespace,global_namespace)
         self.magic_prefix = shell.name+'.magic_'
-        self.magic_escape = shell.ESC_MAGIC
+        self.magic_escape = ESC_MAGIC
         self.readline = readline
         delims = self.readline.get_completer_delims()
         delims = delims.replace(self.magic_escape,'')

IPython/core/component.py

@@ -301,5 +301,4 b' class Component(HasTraitlets):'
             self._children.append(child)
 
     def __repr__(self):
-        return "<%s('%s')>" % (self.__class__.__name__, "DummyName")
-        # return "<Component('%s')>" % self.name
+        return "<%s('%s')>" % (self.__class__.__name__, self.name)

IPython/core/display_trap.py

@@ -24,6 +24,8 b' import sys'
 
 from IPython.core.component import Component
 
+from IPython.utils.autoattr import auto_attr
+
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
@@ -42,6 +44,14 b' class DisplayTrap(Component):'
         self.hook = hook
         self.old_hook = None
 
+    @auto_attr
+    def shell(self):
+        shell = Component.get_instances(
+            root=self.root,
+            klass='IPython.core.iplib.InteractiveShell'
+        )[0]
+        return shell
+
     def __enter__(self):
         self.set()
         return self

IPython/core/iplib.py

@@ -49,10 +49,12 b' from IPython.core.logger import Logger'
 from IPython.core.magic import Magic
 from IPython.core.prompts import CachedOutput
 from IPython.core.page import page
+from IPython.core.prefilter import PrefilterManager
 from IPython.core.component import Component
 from IPython.core.oldusersetup import user_setup
 from IPython.core.usage import interactive_usage, default_banner
 from IPython.core.error import TryNext, UsageError
+from IPython.core.splitinput import split_user_input
 
 from IPython.extensions import pickleshare
 from IPython.external.Itpl import ItplNS
@@ -63,8 +65,8 b' from IPython.utils.genutils import *'
 from IPython.utils.strdispatch import StrDispatch
 from IPython.utils.platutils import toggle_set_term_title, set_term_title
 
-from IPython.utils import growl
-growl.start("IPython")
+# from IPython.utils import growl
+# growl.start("IPython")
 
 from IPython.utils.traitlets import (
     Int, Float, Str, CBool, CaselessStrEnum, Enum, List, Unicode
@@ -212,7 +214,6 b' class InteractiveShell(Component, Magic):'
     logstart = CBool(False, config_key='LOGSTART')
     logfile = Str('', config_key='LOGFILE')
     logplay = Str('', config_key='LOGPLAY')
-    multi_line_specials = CBool(True, config_key='MULTI_LINE_SPECIALS')
     object_info_string_level = Enum((0,1,2), default_value=0,
                                     config_keys='OBJECT_INFO_STRING_LEVEL')
     pager = Str('less', config_key='PAGER')
@@ -297,7 +298,7 b' class InteractiveShell(Component, Magic):'
 
         self.init_history()
         self.init_encoding()
-        self.init_handlers()
+        self.init_prefilter()
 
         Magic.__init__(self, self)
 
@@ -1595,7 +1596,7 b' class InteractiveShell(Component, Magic):'
 
         args = arg_s.split(' ',1)
         magic_name = args[0]
-        magic_name = magic_name.lstrip(self.ESC_MAGIC)
+        magic_name = magic_name.lstrip(prefilter.ESC_MAGIC)
 
         try:
             magic_args = args[1]
@@ -1608,7 +1609,6 b' class InteractiveShell(Component, Magic):'
             magic_args = self.var_expand(magic_args,1)
             with nested(self.builtin_trap, self.display_trap):
                 return fn(magic_args)
-            # return result
 
     def define_magic(self, magicname, func):
         """Expose own function as magic function for ipython 
@@ -1667,58 +1667,6 b' class InteractiveShell(Component, Magic):'
     def init_alias(self):
         self.alias_manager = AliasManager(self, config=self.config)
 
-    def expand_alias(self, line):
-        """ Expand an alias in the command line 
-        
-        Returns the provided command line, possibly with the first word 
-        (command) translated according to alias expansion rules.
-        
-        [ipython]|16> _ip.expand_aliases("np myfile.txt")
-                 <16> 'q:/opt/np/notepad++.exe myfile.txt'
-        """
-        
-        pre,fn,rest = self.split_user_input(line)
-        res = pre + self.expand_aliases(fn, rest)
-        return res
-
-    def expand_aliases(self, fn, rest):
-        """Expand multiple levels of aliases:
-        
-        if:
-        
-        alias foo bar /tmp
-        alias baz foo
-        
-        then:
-        
-        baz huhhahhei -> bar /tmp huhhahhei
-        
-        """
-        line = fn + " " + rest
-        
-        done = set()
-        while 1:
-            pre,fn,rest = prefilter.splitUserInput(line,
-                                                   prefilter.shell_line_split)
-            if fn in self.alias_manager.alias_table:
-                if fn in done:
-                    warn("Cyclic alias definition, repeated '%s'" % fn)
-                    return ""
-                done.add(fn)
-
-                l2 = self.alias_manager.transform_alias(fn, rest)
-                if l2 == line:
-                    break
-                # ls -> ls -F should not recurse forever
-                if l2.split(None,1)[0] == line.split(None,1)[0]:
-                    line = l2
-                    break
-                line=l2
-            else:
-                break
-                
-        return line
-
     #-------------------------------------------------------------------------
     # Things related to the running of code
     #-------------------------------------------------------------------------
@@ -1774,7 +1722,7 b' class InteractiveShell(Component, Magic):'
         This emulates Python's -c option."""
 
         #sys.argv = ['-c']
-        self.push_line(self.prefilter(self.c, False))
+        self.push_line(self.prefilter_manager.prefilter_lines(self.c, False))
         if not self.interactive:
             self.ask_exit()
 
@@ -1807,7 +1755,7 b' class InteractiveShell(Component, Magic):'
         """
         if line.lstrip() == line:
             self.shadowhist.add(line.strip())
-        lineout = self.prefilter(line,self.more)
+        lineout = self.prefilter_manager.prefilter_lines(line,self.more)
 
         if line.strip():
             if self.more:
@@ -2154,7 +2102,7 b' class InteractiveShell(Component, Magic):'
                 if line or more:
                     # push to raw history, so hist line numbers stay in sync
                     self.input_hist_raw.append("# " + line + "\n")
-                    more = self.push_line(self.prefilter(line,more))
+                    more = self.push_line(self.prefilter_manager.prefilter_lines(line,more))
                     # IPython's runsource returns None if there was an error
                     # compiling the code.  This allows us to stop processing right
                     # away, so the user gets the error message at the right place.
@@ -2319,10 +2267,6 b' class InteractiveShell(Component, Magic):'
             else:
                 self.indent_current_nsp = 0
 
-    def split_user_input(self, line):
-        # This is really a hold-over to support ipapi and some extensions
-        return prefilter.splitUserInput(line)
-
     def resetbuffer(self):
         """Reset the input buffer."""
         self.buffer[:] = []
@@ -2340,7 +2284,8 b' class InteractiveShell(Component, Magic):'
           - continue_prompt(False): whether this line is the first one or a
           continuation in a sequence of inputs.
         """
-        growl.notify("raw_input: ", "prompt = %r\ncontinue_prompt = %s" % (prompt, continue_prompt))
+        # growl.notify("raw_input: ", "prompt = %r\ncontinue_prompt = %s" % (prompt, continue_prompt))
+
         # Code run by the user may have modified the readline completer state.
         # We must ensure that our completer is back in place.
 
@@ -2388,7 +2333,7 b' class InteractiveShell(Component, Magic):'
         elif not continue_prompt:
             self.input_hist_raw.append('\n')
         try:
-            lineout = self.prefilter(line,continue_prompt)
+            lineout = self.prefilter_manager.prefilter_lines(line,continue_prompt)
         except:
             # blanket except, in case a user-defined prefilter crashes, so it
             # can't take all of ipython with it.
@@ -2451,293 +2396,8 b' class InteractiveShell(Component, Magic):'
     # Things related to the prefilter
     #-------------------------------------------------------------------------
 
-    def init_handlers(self):
-        # escapes for automatic behavior on the command line
-        self.ESC_SHELL  = '!'
-        self.ESC_SH_CAP = '!!'
-        self.ESC_HELP   = '?'
-        self.ESC_MAGIC  = '%'
-        self.ESC_QUOTE  = ','
-        self.ESC_QUOTE2 = ';'
-        self.ESC_PAREN  = '/'
-
-        # And their associated handlers
-        self.esc_handlers = {self.ESC_PAREN  : self.handle_auto,
-                             self.ESC_QUOTE  : self.handle_auto,
-                             self.ESC_QUOTE2 : self.handle_auto,
-                             self.ESC_MAGIC  : self.handle_magic,
-                             self.ESC_HELP   : self.handle_help,
-                             self.ESC_SHELL  : self.handle_shell_escape,
-                             self.ESC_SH_CAP : self.handle_shell_escape,
-                             }
-
-    def _prefilter(self, line, continue_prompt):
-        """Calls different preprocessors, depending on the form of line."""
-
-        # All handlers *must* return a value, even if it's blank ('').
-
-        # Lines are NOT logged here. Handlers should process the line as
-        # needed, update the cache AND log it (so that the input cache array
-        # stays synced).
-
-        #.....................................................................
-        # Code begins
-
-        #if line.startswith('%crash'): raise RuntimeError,'Crash now!'  # dbg
-
-        # save the line away in case we crash, so the post-mortem handler can
-        # record it
-        growl.notify("_prefilter: ", "line = %s\ncontinue_prompt = %s" % (line, continue_prompt))
-        
-        self._last_input_line = line
-
-        #print '***line: <%s>' % line # dbg
-
-        if not line:
-            # Return immediately on purely empty lines, so that if the user
-            # previously typed some whitespace that started a continuation
-            # prompt, he can break out of that loop with just an empty line.
-            # This is how the default python prompt works.
-
-            # Only return if the accumulated input buffer was just whitespace!
-            if ''.join(self.buffer).isspace():
-                self.buffer[:] = []
-            return ''
-        
-        line_info = prefilter.LineInfo(line, continue_prompt)
-        
-        # the input history needs to track even empty lines
-        stripped = line.strip()
-        
-        if not stripped:
-            if not continue_prompt:
-                self.outputcache.prompt_count -= 1
-            return self.handle_normal(line_info)
-
-        # print '***cont',continue_prompt  # dbg
-        # special handlers are only allowed for single line statements
-        if continue_prompt and not self.multi_line_specials:
-            return self.handle_normal(line_info)
-
-
-        # See whether any pre-existing handler can take care of it  
-        rewritten = self.hooks.input_prefilter(stripped)
-        if rewritten != stripped: # ok, some prefilter did something
-            rewritten = line_info.pre + rewritten  # add indentation
-            return self.handle_normal(prefilter.LineInfo(rewritten,
-                                                         continue_prompt))
-            
-        #print 'pre <%s> iFun <%s> rest <%s>' % (pre,iFun,theRest)  # dbg
-        
-        return prefilter.prefilter(line_info, self)
-
-
-    def _prefilter_dumb(self, line, continue_prompt):
-        """simple prefilter function, for debugging"""
-        return self.handle_normal(line,continue_prompt)
-
-    
-    def multiline_prefilter(self, line, continue_prompt):
-        """ Run _prefilter for each line of input
-        
-        Covers cases where there are multiple lines in the user entry,
-        which is the case when the user goes back to a multiline history
-        entry and presses enter.
-        
-        """
-        growl.notify("multiline_prefilter: ", "%s\n%s" % (line, continue_prompt))
-        out = []
-        for l in line.rstrip('\n').split('\n'):
-            out.append(self._prefilter(l, continue_prompt))
-        growl.notify("multiline_prefilter return: ", '\n'.join(out))
-        return '\n'.join(out)
-    
-    # Set the default prefilter() function (this can be user-overridden)
-    prefilter = multiline_prefilter
-
-    def handle_normal(self, line_info):
-        """Handle normal input lines. Use as a template for handlers."""
-
-        # With autoindent on, we need some way to exit the input loop, and I
-        # don't want to force the user to have to backspace all the way to
-        # clear the line.  The rule will be in this case, that either two
-        # lines of pure whitespace in a row, or a line of pure whitespace but
-        # of a size different to the indent level, will exit the input loop.
-        line = line_info.line
-        continue_prompt = line_info.continue_prompt
-        
-        if (continue_prompt and self.autoindent and line.isspace() and
-            (0 < abs(len(line) - self.indent_current_nsp) <= 2 or
-             (self.buffer[-1]).isspace() )):
-            line = ''
-
-        self.log(line,line,continue_prompt)
-        return line
-
-    def handle_alias(self, line_info):
-        """Handle alias input lines. """
-        tgt = self.alias_manager.alias_table[line_info.iFun]
-        if callable(tgt):
-            if '$' in line_info.line:
-                call_meth = '(_ip, _ip.var_expand(%s))'
-            else:
-                call_meth = '(_ip,%s)'
-            line_out = ("%s_sh.%s" + call_meth) % (line_info.preWhitespace,
-                                         line_info.iFun, 
-            make_quoted_expr(line_info.line))
-        else:
-            transformed = self.expand_aliases(line_info.iFun,line_info.theRest)
-
-            # pre is needed, because it carries the leading whitespace.  Otherwise
-            # aliases won't work in indented sections.
-            line_out = '%s_ip.system(%s)' % (line_info.preWhitespace,
-                                             make_quoted_expr( transformed ))
-        
-        self.log(line_info.line,line_out,line_info.continue_prompt)
-        #print 'line out:',line_out # dbg
-        return line_out
-
-    def handle_shell_escape(self, line_info):
-        """Execute the line in a shell, empty return value"""
-        #print 'line in :', `line` # dbg
-        line = line_info.line
-        if line.lstrip().startswith('!!'):
-            # rewrite LineInfo's line, iFun and theRest to properly hold the
-            # call to %sx and the actual command to be executed, so
-            # handle_magic can work correctly.  Note that this works even if
-            # the line is indented, so it handles multi_line_specials
-            # properly.
-            new_rest = line.lstrip()[2:]
-            line_info.line = '%ssx %s' % (self.ESC_MAGIC,new_rest)
-            line_info.iFun = 'sx'
-            line_info.theRest = new_rest
-            return self.handle_magic(line_info)
-        else:
-            cmd = line.lstrip().lstrip('!')
-            line_out = '%s_ip.system(%s)' % (line_info.preWhitespace,
-                                             make_quoted_expr(cmd))
-        # update cache/log and return
-        self.log(line,line_out,line_info.continue_prompt)
-        return line_out
-
-    def handle_magic(self, line_info):
-        """Execute magic functions."""
-        iFun    = line_info.iFun
-        theRest = line_info.theRest
-        cmd = '%s_ip.magic(%s)' % (line_info.preWhitespace,
-                                   make_quoted_expr(iFun + " " + theRest))
-        self.log(line_info.line,cmd,line_info.continue_prompt)
-        #print 'in handle_magic, cmd=<%s>' % cmd  # dbg
-        return cmd
-
-    def handle_auto(self, line_info):
-        """Hande lines which can be auto-executed, quoting if requested."""
-
-        line    = line_info.line
-        iFun    = line_info.iFun
-        theRest = line_info.theRest
-        pre     = line_info.pre
-        continue_prompt = line_info.continue_prompt
-        obj = line_info.ofind(self)['obj']
-
-        #print 'pre <%s> iFun <%s> rest <%s>' % (pre,iFun,theRest)  # dbg
-
-        # This should only be active for single-line input!
-        if continue_prompt:
-            self.log(line,line,continue_prompt)
-            return line
-
-        force_auto = isinstance(obj, IPyAutocall)
-        auto_rewrite = True
-        
-        if pre == self.ESC_QUOTE:
-            # Auto-quote splitting on whitespace
-            newcmd = '%s("%s")' % (iFun,'", "'.join(theRest.split()) )
-        elif pre == self.ESC_QUOTE2:
-            # Auto-quote whole string
-            newcmd = '%s("%s")' % (iFun,theRest)
-        elif pre == self.ESC_PAREN:
-            newcmd = '%s(%s)' % (iFun,",".join(theRest.split()))
-        else:
-            # Auto-paren.
-            # We only apply it to argument-less calls if the autocall
-            # parameter is set to 2.  We only need to check that autocall is <
-            # 2, since this function isn't called unless it's at least 1.
-            if not theRest and (self.autocall < 2) and not force_auto:
-                newcmd = '%s %s' % (iFun,theRest)
-                auto_rewrite = False
-            else:
-                if not force_auto and theRest.startswith('['):
-                    if hasattr(obj,'__getitem__'):
-                        # Don't autocall in this case: item access for an object
-                        # which is BOTH callable and implements __getitem__.
-                        newcmd = '%s %s' % (iFun,theRest)
-                        auto_rewrite = False
-                    else:
-                        # if the object doesn't support [] access, go ahead and
-                        # autocall
-                        newcmd = '%s(%s)' % (iFun.rstrip(),theRest)
-                elif theRest.endswith(';'):
-                    newcmd = '%s(%s);' % (iFun.rstrip(),theRest[:-1])
-                else:
-                    newcmd = '%s(%s)' % (iFun.rstrip(), theRest)
-
-        if auto_rewrite:
-            rw = self.outputcache.prompt1.auto_rewrite() + newcmd
-            
-            try:
-                # plain ascii works better w/ pyreadline, on some machines, so
-                # we use it and only print uncolored rewrite if we have unicode
-                rw = str(rw)
-                print >>Term.cout, rw
-            except UnicodeEncodeError:
-                print "-------------->" + newcmd
-            
-        # log what is now valid Python, not the actual user input (without the
-        # final newline)
-        self.log(line,newcmd,continue_prompt)
-        return newcmd
-
-    def handle_help(self, line_info):
-        """Try to get some help for the object.
-
-        obj? or ?obj   -> basic information.
-        obj?? or ??obj -> more details.
-        """
-        
-        line = line_info.line
-        # We need to make sure that we don't process lines which would be
-        # otherwise valid python, such as "x=1 # what?"
-        try:
-            codeop.compile_command(line)
-        except SyntaxError:
-            # We should only handle as help stuff which is NOT valid syntax
-            if line[0]==self.ESC_HELP:
-                line = line[1:]
-            elif line[-1]==self.ESC_HELP:
-                line = line[:-1]
-            self.log(line,'#?'+line,line_info.continue_prompt)
-            if line:
-                #print 'line:<%r>' % line  # dbg
-                self.magic_pinfo(line)
-            else:
-                page(self.usage,screen_lines=self.usable_screen_length)
-            return '' # Empty string is needed here!
-        except:
-            # Pass any other exceptions through to the normal handler
-            return self.handle_normal(line_info)
-        else:
-            # If the code compiles ok, we should handle it normally
-            return self.handle_normal(line_info)
-
-    def handle_emacs(self, line_info):
-        """Handle input lines marked by python-mode."""
-
-        # Currently, nothing is done.  Later more functionality can be added
-        # here if needed.
-
-        # The input cache shouldn't be updated
-        return line_info.line
+    def init_prefilter(self):
+        self.prefilter_manager = PrefilterManager(self, config=self.config)
 
     #-------------------------------------------------------------------------
     # Utilities
@@ -2844,6 +2504,3 b' class InteractiveShell(Component, Magic):'
         self.restore_sys_module_state()
 
 
-
-
-

IPython/core/prefilter.py

@@ -1,14 +1,96 b''
-# -*- coding: utf-8 -*-
+#!/usr/bin/env python
+# encoding: utf-8
 """
-Classes and functions for prefiltering (transforming) a line of user input.
-This module is responsible, primarily, for breaking the line up into useful
-pieces and triggering the appropriate handlers in iplib to do the actual
-transforming work.
+Prefiltering components.
+
+Authors:
+
+* Brian Granger
+* Fernando Perez
+* Dan Milstein
 """
-__docformat__ = "restructuredtext en"
 
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2009  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import __builtin__
+import codeop
+import keyword
+import os
 import re
+import sys
+
+from IPython.core.alias import AliasManager
 from IPython.core.autocall import IPyAutocall
+from IPython.core.component import Component
+from IPython.core.splitinput import split_user_input
+
+from IPython.utils.traitlets import List, Int, Any, Str, CBool
+from IPython.utils.genutils import make_quoted_expr
+from IPython.utils.autoattr import auto_attr
+
+#-----------------------------------------------------------------------------
+# Global utilities, errors and constants
+#-----------------------------------------------------------------------------
+
+
+ESC_SHELL  = '!'
+ESC_SH_CAP = '!!'
+ESC_HELP   = '?'
+ESC_MAGIC  = '%'
+ESC_QUOTE  = ','
+ESC_QUOTE2 = ';'
+ESC_PAREN  = '/'
+
+
+class PrefilterError(Exception):
+    pass
+
+
+# RegExp to identify potential function names
+re_fun_name = re.compile(r'[a-zA-Z_]([a-zA-Z0-9_.]*) *$')
+
+# RegExp to exclude strings with this start from autocalling.  In
+# particular, all binary operators should be excluded, so that if foo is
+# callable, foo OP bar doesn't become foo(OP bar), which is invalid.  The
+# characters '!=()' don't need to be checked for, as the checkPythonChars
+# routine explicitely does so, to catch direct calls and rebindings of
+# existing names.
+
+# Warning: the '-' HAS TO BE AT THE END of the first group, otherwise
+# it affects the rest of the group in square brackets.
+re_exclude_auto = re.compile(r'^[,&^\|\*/\+-]'
+                             r'|^is |^not |^in |^and |^or ')
+
+# try to catch also methods for stuff in lists/tuples/dicts: off
+# (experimental). For this to work, the line_split regexp would need
+# to be modified so it wouldn't break things at '['. That line is
+# nasty enough that I shouldn't change it until I can test it _well_.
+#self.re_fun_name = re.compile (r'[a-zA-Z_]([a-zA-Z0-9_.\[\]]*) ?$')
+
+
+# Handler Check Utilities
+def is_shadowed(identifier, ip):
+    """Is the given identifier defined in one of the namespaces which shadow
+    the alias and magic namespaces?  Note that an identifier is different
+    than ifun, because it can not contain a '.' character."""
+    # This is much safer than calling ofind, which can change state
+    return (identifier in ip.user_ns \
+            or identifier in ip.internal_ns \
+            or identifier in ip.ns_table['builtin'])
+
+
+#-----------------------------------------------------------------------------
+# The LineInfo class used throughout
+#-----------------------------------------------------------------------------
 
 
 class LineInfo(object):
@@ -25,39 +107,39 b' class LineInfo(object):'
     pre
       The initial esc character or whitespace.
     
-    preChar
+    pre_char
       The escape character(s) in pre or the empty string if there isn't one.
-      Note that '!!' is a possible value for preChar.  Otherwise it will
+      Note that '!!' is a possible value for pre_char.  Otherwise it will
       always be a single character.
     
-    preWhitespace
-      The leading whitespace from pre if it exists.  If there is a preChar,
+    pre_whitespace
+      The leading whitespace from pre if it exists.  If there is a pre_char,
       this is just ''.
     
-    iFun
+    ifun
       The 'function part', which is basically the maximal initial sequence
       of valid python identifiers and the '.' character.  This is what is
       checked for alias and magic transformations, used for auto-calling,
       etc.
     
-    theRest
+    the_rest
       Everything else on the line.
     """
     def __init__(self, line, continue_prompt):
         self.line            = line
         self.continue_prompt = continue_prompt
-        self.pre, self.iFun, self.theRest = splitUserInput(line)
+        self.pre, self.ifun, self.the_rest = split_user_input(line)
 
-        self.preChar       = self.pre.strip()
-        if self.preChar:
-            self.preWhitespace = '' # No whitespace allowd before esc chars
+        self.pre_char       = self.pre.strip()
+        if self.pre_char:
+            self.pre_whitespace = '' # No whitespace allowd before esc chars
         else: 
-            self.preWhitespace = self.pre
+            self.pre_whitespace = self.pre
 
         self._oinfo = None
 
     def ofind(self, ip):
-        """Do a full, attribute-walking lookup of the iFun in the various
+        """Do a full, attribute-walking lookup of the ifun in the various
         namespaces for the given IPython InteractiveShell instance.
 
         Return a dict with keys: found,obj,ospace,ismagic
@@ -70,252 +152,626 b' class LineInfo(object):'
         without worrying about *further* damaging state.
         """
         if not self._oinfo:
-            self._oinfo = ip._ofind(self.iFun)
+            self._oinfo = ip._ofind(self.ifun)
         return self._oinfo
+
     def __str__(self):                                                         
-        return "Lineinfo [%s|%s|%s]" %(self.pre,self.iFun,self.theRest)        
+        return "Lineinfo [%s|%s|%s]" %(self.pre,self.ifun,self.the_rest) 
 
-def splitUserInput(line, pattern=None):
-    """Split user input into pre-char/whitespace, function part and rest.
 
-    Mostly internal to this module, but also used by iplib.expand_aliases,
-    which passes in a shell pattern.
-    """
-    # It seems to me that the shell splitting should be a separate method.
+#-----------------------------------------------------------------------------
+# Main Prefilter manager
+#-----------------------------------------------------------------------------
     
-    if not pattern:
-        pattern = line_split
-    match = pattern.match(line)
-    if not match:
-        #print "match failed for line '%s'" % line
-        try:
-            iFun,theRest = line.split(None,1)
-        except ValueError:
-            #print "split failed for line '%s'" % line
-            iFun,theRest = line,''
-        pre = re.match('^(\s*)(.*)',line).groups()[0]
-    else:
-        pre,iFun,theRest = match.groups()
 
-    # iFun has to be a valid python identifier, so it better be only pure
-    # ascii, no unicode:
+class PrefilterManager(Component):
+    """Main prefilter component.
+
+    The IPython prefilter is run on all user input before it is run.  The
+    prefilter consumes lines of input and produces transformed lines of 
+    input.  The implementation consists of checkers and handlers.  The 
+    checkers inspect the input line and select which handler will be used
+    to transform the input line.
+    """
+
+    multi_line_specials = CBool(True, config_key='MULTI_LINE_SPECIALS')
+
+    def __init__(self, parent, config=None):
+        super(PrefilterManager, self).__init__(parent, config=config)
+        self.init_handlers()
+        self.init_checkers()
+
+    @auto_attr
+    def shell(self):
+        shell = Component.get_instances(
+            root=self.root,
+            klass='IPython.core.iplib.InteractiveShell'
+        )[0]
+        return shell
+
+    def init_checkers(self):
+        self._checkers = []
+        for checker in _default_checkers:
+            self._checkers.append(checker(self, config=self.config))
+
+    def init_handlers(self):
+        self._handlers = {}
+        self._esc_handlers = {}
+        for handler in _default_handlers:
+            handler(self, config=self.config)
+
+    @property
+    def sorted_checkers(self):
+        """Return a list of checkers, sorted by priority."""
+        return sorted(self._checkers, cmp=lambda x,y: x.priority-y.priority)
+
+    def register_handler(self, name, handler, esc_strings):
+        """Register a handler instance by name with esc_strings."""
+        self._handlers[name] = handler
+        for esc_str in esc_strings:
+            self._esc_handlers[esc_str] = handler
+
+    def unregister_handler(self, name, handler, esc_strings):
+        """Unregister a handler instance by name with esc_strings."""
     try:
-        iFun = iFun.encode('ascii')
-    except UnicodeEncodeError:
-        theRest = iFun + u' ' + theRest
-        iFun = u''
-
-    #print 'line:<%s>' % line # dbg
-    #print 'pre <%s> iFun <%s> rest <%s>' % (pre,iFun.strip(),theRest) # dbg
-    return pre,iFun.strip(),theRest.lstrip()
-
-
-# RegExp for splitting line contents into pre-char//first word-method//rest.
-# For clarity, each group in on one line.
-
-# WARNING: update the regexp if the escapes in iplib are changed, as they
-# are hardwired in.
-
-# Although it's not solely driven by the regex, note that:
-# ,;/% only trigger if they are the first character on the line
-# ! and !! trigger if they are first char(s) *or* follow an indent 
-# ? triggers as first or last char.
-
-# The three parts of the regex are:
-#  1) pre:     pre_char *or* initial whitespace 
-#  2) iFun:    first word/method (mix of \w and '.')
-#  3) theRest: rest of line (separated from iFun by space if non-empty)
-line_split = re.compile(r'^([,;/%?]|!!?|\s*)'
-                        r'\s*([\w\.]+)'
-                        r'(\s+.*$|$)')
-
-shell_line_split = re.compile(r'^(\s*)(\S*\s*)(.*$)')
-
-def prefilter(line_info, ip):
-    """Call one of the passed-in InteractiveShell's handler preprocessors,
-    depending on the form of the line.  Return the results, which must be a
-    value, even if it's a blank ('')."""
-    # Note: the order of these checks does matter. 
-    for check in [ checkEmacs,
-                   checkShellEscape,
-                   checkIPyAutocall,
-                   checkMultiLineMagic,
-                   checkEscChars,
-                   checkAssignment,
-                   checkAutomagic,
-                   checkAlias,
-                   checkPythonOps,
-                   checkAutocall,
-                   ]:
-        handler = check(line_info, ip)
+            del self._handlers[name]
+        except KeyError:
+            pass
+        for esc_str in esc_strings:
+            h = self._esc_handlers.get(esc_str)
+            if h is handler:
+                del self._esc_handlers[esc_str]
+
+    def get_handler_by_name(self, name):
+        """Get a handler by its name."""
+        return self._handlers.get(name)
+
+    def get_handler_by_esc(self, esc_str):
+        """Get a handler by its escape string."""
+        return self._esc_handlers.get(esc_str)
+
+    def prefilter_line_info(self, line_info):
+        """Prefilter a line that has been converted to a LineInfo object."""
+        handler = self.find_handler(line_info)
+        return handler.handle(line_info)
+
+    def find_handler(self, line_info):
+        """Find a handler for the line_info by trying checkers."""
+        for checker in self.sorted_checkers:
+            handler = checker.check(line_info)
         if handler:
-            return handler(line_info)
+                return handler
+        return self.get_handler_by_name('normal')
 
-    return ip.handle_normal(line_info)
+    def prefilter_line(self, line, continue_prompt):
+        """Prefilter a single input line as text."""
+
+        # All handlers *must* return a value, even if it's blank ('').
+
+        # Lines are NOT logged here. Handlers should process the line as
+        # needed, update the cache AND log it (so that the input cache array
+        # stays synced).
+
+        # growl.notify("_prefilter: ", "line = %s\ncontinue_prompt = %s" % (line, continue_prompt))
+
+        # save the line away in case we crash, so the post-mortem handler can
+        # record it        
+        self.shell._last_input_line = line
+
+        if not line:
+            # Return immediately on purely empty lines, so that if the user
+            # previously typed some whitespace that started a continuation
+            # prompt, he can break out of that loop with just an empty line.
+            # This is how the default python prompt works.
+
+            # Only return if the accumulated input buffer was just whitespace!
+            if ''.join(self.shell.buffer).isspace():
+                self.shell.buffer[:] = []
+            return ''
+        
+        line_info = LineInfo(line, continue_prompt)
+        
+        # the input history needs to track even empty lines
+        stripped = line.strip()
+
+        handle_normal = self.get_handler_by_name('normal')        
+        if not stripped:
+            if not continue_prompt:
+                self.shell.outputcache.prompt_count -= 1
+
+            return handle_normal(line_info)
+
+        # special handlers are only allowed for single line statements
+        if continue_prompt and not self.multi_line_specials:
+            return handle_normal(line_info)
+
+        return self.prefilter_line_info(line_info)
+
+    def prefilter_lines(self, lines, continue_prompt):
+        """Prefilter multiple input lines of text.
+
+        Covers cases where there are multiple lines in the user entry,
+        which is the case when the user goes back to a multiline history
+        entry and presses enter.
+        """
+        # growl.notify("multiline_prefilter: ", "%s\n%s" % (line, continue_prompt))
+        out = []
+        for line in lines.rstrip('\n').split('\n'):
+            out.append(self.prefilter_line(line, continue_prompt))
+        # growl.notify("multiline_prefilter return: ", '\n'.join(out))
+        return '\n'.join(out)
+
+
+#-----------------------------------------------------------------------------
+# Prefilter checkers
+#-----------------------------------------------------------------------------
+
+
+class PrefilterChecker(Component):
+    """Inspect an input line and return a handler for that line."""
+
+    priority = Int(100)
+    shell = Any
+    prefilter_manager = Any
+
+    def __init__(self, parent, config=None):
+        super(PrefilterChecker, self).__init__(parent, config=config)
+
+    @auto_attr
+    def shell(self):
+        shell = Component.get_instances(
+            root=self.root,
+            klass='IPython.core.iplib.InteractiveShell'
+        )[0]
+        return shell
+
+    @auto_attr
+    def prefilter_manager(self):
+        return PrefilterManager.get_instances(root=self.root)[0]
+
+    def check(self, line_info):
+        """Inspect line_info and return a handler or None."""
+        return None
 
-# Handler checks
-#
-# All have the same interface: they take a LineInfo object and a ref to the
-# iplib.InteractiveShell object.  They check the line to see if a particular
-# handler should be called, and return either a handler or None.  The
-# handlers which they return are *bound* methods of the InteractiveShell
-# object. 
-#
-# In general, these checks should only take responsibility for their 'own'
-# handler.  If it doesn't get triggered, they should just return None and
-# let the rest of the check sequence run.
 
-def checkShellEscape(l_info,ip):
-    if l_info.line.lstrip().startswith(ip.ESC_SHELL):
-        return ip.handle_shell_escape
+class EmacsChecker(PrefilterChecker):
 
-def checkEmacs(l_info,ip):
+    priority = Int(100)
+
+    def check(self, line_info):
     "Emacs ipython-mode tags certain input lines."
-    if l_info.line.endswith('# PYTHON-MODE'):
-        return ip.handle_emacs
+        if line_info.line.endswith('# PYTHON-MODE'):
+            return self.prefilter_manager.get_handler_by_name('emacs')
     else:
         return None
 
-def checkIPyAutocall(l_info,ip):
+
+class ShellEscapeChecker(PrefilterChecker):
+
+    priority = Int(200)
+
+    def check(self, line_info):
+        if line_info.line.lstrip().startswith(ESC_SHELL):
+            return self.prefilter_manager.get_handler_by_name('shell')
+
+
+class IPyAutocallChecker(PrefilterChecker):
+
+    priority = Int(300)
+
+    def check(self, line_info):
     "Instances of IPyAutocall in user_ns get autocalled immediately"
-    obj = ip.user_ns.get(l_info.iFun, None)
+        obj = self.shell.user_ns.get(line_info.ifun, None)
     if isinstance(obj, IPyAutocall):
-        obj.set_ip(ip)
-        return ip.handle_auto
+            obj.set_ip(self.shell)
+            return self.prefilter_manager.get_handler_by_name('auto')
     else:
         return None
     
     
-def checkMultiLineMagic(l_info,ip):
+class MultiLineMagicChecker(PrefilterChecker):
+
+    priority = Int(400)
+
+    def check(self, line_info):
     "Allow ! and !! in multi-line statements if multi_line_specials is on"
     # Note that this one of the only places we check the first character of
-    # iFun and *not* the preChar.  Also note that the below test matches
+        # ifun and *not* the pre_char.  Also note that the below test matches
     # both ! and !!.    
-    if l_info.continue_prompt \
-        and ip.multi_line_specials:
-            if l_info.iFun.startswith(ip.ESC_MAGIC):
-                return ip.handle_magic
+        if line_info.continue_prompt \
+            and self.prefilter_manager.multi_line_specials:
+                if line_info.ifun.startswith(ESC_MAGIC):
+                    return self.prefilter_manager.get_handler_by_name('magic')
     else:
         return None
 
-def checkEscChars(l_info,ip):
+
+class EscCharsChecker(PrefilterChecker):
+
+    priority = Int(500)
+
+    def check(self, line_info):
     """Check for escape character and return either a handler to handle it,
     or None if there is no escape char."""
-    if l_info.line[-1] == ip.ESC_HELP \
-           and l_info.preChar != ip.ESC_SHELL \
-           and l_info.preChar != ip.ESC_SH_CAP:
+        if line_info.line[-1] == ESC_HELP \
+               and line_info.pre_char != ESC_SHELL \
+               and line_info.pre_char != ESC_SH_CAP:
         # the ? can be at the end, but *not* for either kind of shell escape,
         # because a ? can be a vaild final char in a shell cmd
-        return ip.handle_help
-    elif l_info.preChar in ip.esc_handlers:
-        return ip.esc_handlers[l_info.preChar]
+            return self.prefilter_manager.get_handler_by_name('help')
     else:
-        return None
+            # This returns None like it should if no handler exists
+            return self.prefilter_manager.get_handler_by_esc(line_info.pre_char)
+
 
+class AssignmentChecker(PrefilterChecker):
 
-def checkAssignment(l_info,ip):
+    priority = Int(600)
+
+    def check(self, line_info):
     """Check to see if user is assigning to a var for the first time, in
     which case we want to avoid any sort of automagic / autocall games.
     
     This allows users to assign to either alias or magic names true python
     variables (the magic/alias systems always take second seat to true
     python code).  E.g. ls='hi', or ls,that=1,2"""
-    if l_info.theRest and l_info.theRest[0] in '=,':
-        return ip.handle_normal
+        if line_info.the_rest and line_info.the_rest[0] in '=,':
+            return self.prefilter_manager.get_handler_by_name('normal')
     else:
         return None
 
 
-def checkAutomagic(l_info,ip):
-    """If the iFun is magic, and automagic is on, run it.  Note: normal,
+class AutoMagicChecker(PrefilterChecker):
+
+    priority = Int(700)
+
+    def check(self, line_info):
+        """If the ifun is magic, and automagic is on, run it.  Note: normal,
     non-auto magic would already have been triggered via '%' in
     check_esc_chars. This just checks for automagic.  Also, before
     triggering the magic handler, make sure that there is nothing in the
     user namespace which could shadow it."""
-    if not ip.automagic or not hasattr(ip,'magic_'+l_info.iFun):
+        if not self.shell.automagic or not hasattr(self.shell,'magic_'+line_info.ifun):
         return None
 
     # We have a likely magic method.  Make sure we should actually call it.
-    if l_info.continue_prompt and not ip.multi_line_specials:
+        if line_info.continue_prompt and not self.shell.multi_line_specials:
         return None
 
-    head = l_info.iFun.split('.',1)[0]
-    if isShadowed(head,ip):
+        head = line_info.ifun.split('.',1)[0]
+        if is_shadowed(head, self.shell):
         return None
 
-    return ip.handle_magic
+        return self.prefilter_manager.get_handler_by_name('magic')
+
+
+class AliasChecker(PrefilterChecker):
 
+    priority = Int(800)
         
-def checkAlias(l_info,ip):
+    @auto_attr
+    def alias_manager(self):
+        return AliasManager.get_instances(root=self.root)[0]
+
+    def check(self, line_info):
     "Check if the initital identifier on the line is an alias."
     # Note: aliases can not contain '.'
-    head = l_info.iFun.split('.',1)[0]
-    
-    if l_info.iFun not in ip.alias_manager \
-           or head not in ip.alias_manager \
-           or isShadowed(head,ip): 
+        head = line_info.ifun.split('.',1)[0]
+        if line_info.ifun not in self.alias_manager \
+               or head not in self.alias_manager \
+               or is_shadowed(head, self.shell):
         return None
 
-    return ip.handle_alias
+        return self.prefilter_manager.get_handler_by_name('alias')
+
 
+class PythonOpsChecker(PrefilterChecker):
 
-def checkPythonOps(l_info,ip):
+    priority = Int(900)
+
+    def check(self, line_info):
     """If the 'rest' of the line begins with a function call or pretty much
     any python operator, we should simply execute the line (regardless of
     whether or not there's a possible autocall expansion).  This avoids
     spurious (and very confusing) geattr() accesses."""
-    if l_info.theRest and l_info.theRest[0] in '!=()<>,+*/%^&|':
-        return ip.handle_normal
+        if line_info.the_rest and line_info.the_rest[0] in '!=()<>,+*/%^&|':
+            return self.prefilter_manager.get_handler_by_name('normal')
     else:
         return None
 
 
-def checkAutocall(l_info,ip):
+class AutocallChecker(PrefilterChecker):
+
+    priority = Int(1000)
+
+    def check(self, line_info):
     "Check if the initial word/function is callable and autocall is on."
-    if not ip.autocall:
+        if not self.shell.autocall:
         return None
 
-    oinfo = l_info.ofind(ip) # This can mutate state via getattr
+        oinfo = line_info.ofind(self.shell) # This can mutate state via getattr
     if not oinfo['found']:
         return None
         
     if callable(oinfo['obj']) \
-           and (not re_exclude_auto.match(l_info.theRest)) \
-           and re_fun_name.match(l_info.iFun):
-        #print 'going auto'  # dbg
-        return ip.handle_auto
+               and (not re_exclude_auto.match(line_info.the_rest)) \
+               and re_fun_name.match(line_info.ifun):
+            return self.prefilter_manager.get_handler_by_name('auto')
     else:
-        #print 'was callable?', callable(l_info.oinfo['obj'])  # dbg
         return None
     
-# RegExp to identify potential function names
-re_fun_name = re.compile(r'[a-zA-Z_]([a-zA-Z0-9_.]*) *$')
 
-# RegExp to exclude strings with this start from autocalling.  In
-# particular, all binary operators should be excluded, so that if foo is
-# callable, foo OP bar doesn't become foo(OP bar), which is invalid.  The
-# characters '!=()' don't need to be checked for, as the checkPythonChars
-# routine explicitely does so, to catch direct calls and rebindings of
-# existing names.
+#-----------------------------------------------------------------------------
+# Prefilter handlers
+#-----------------------------------------------------------------------------
+
+
+class PrefilterHandler(Component):
+
+    handler_name = Str('normal')
+    esc_strings = List([])
+    shell = Any
+    prefilter_manager = Any
+
+    def __init__(self, parent, config=None):
+        super(PrefilterHandler, self).__init__(parent, config=config)
+        self.prefilter_manager.register_handler(
+            self.handler_name,
+            self,
+            self.esc_strings
+        )
+
+    @auto_attr
+    def shell(self):
+        shell = Component.get_instances(
+            root=self.root,
+            klass='IPython.core.iplib.InteractiveShell'
+        )[0]
+        return shell
+
+    @auto_attr
+    def prefilter_manager(self):
+        return PrefilterManager.get_instances(root=self.root)[0]
+
+    def handle(self, line_info):
+        """Handle normal input lines. Use as a template for handlers."""
+
+        # With autoindent on, we need some way to exit the input loop, and I
+        # don't want to force the user to have to backspace all the way to
+        # clear the line.  The rule will be in this case, that either two
+        # lines of pure whitespace in a row, or a line of pure whitespace but
+        # of a size different to the indent level, will exit the input loop.
+        line = line_info.line
+        continue_prompt = line_info.continue_prompt
+
+        if (continue_prompt and self.shell.autoindent and line.isspace() and
+            (0 < abs(len(line) - self.shell.indent_current_nsp) <= 2 or
+             (self.shell.buffer[-1]).isspace() )):
+            line = ''
+
+        self.shell.log(line, line, continue_prompt)
+        return line
+
+
+class AliasHandler(PrefilterHandler):
+
+    handler_name = Str('alias')
+    esc_strings = List([])
+
+    @auto_attr
+    def alias_manager(self):
+        return AliasManager.get_instances(root=self.root)[0]
+
+    def handle(self, line_info):
+        """Handle alias input lines. """
+        transformed = self.alias_manager.expand_aliases(line_info.ifun,line_info.the_rest)
+        # pre is needed, because it carries the leading whitespace.  Otherwise
+        # aliases won't work in indented sections.
+        line_out = '%s_ip.system(%s)' % (line_info.pre_whitespace,
+                                         make_quoted_expr(transformed))
+        
+        self.shell.log(line_info.line, line_out, line_info.continue_prompt)
+        return line_out
+
+
+class ShellEscapeHandler(PrefilterHandler):
+
+    handler_name = Str('shell')
+    esc_strings = List([ESC_SHELL, ESC_SH_CAP])
+
+    def handle(self, line_info):
+        """Execute the line in a shell, empty return value"""
+        magic_handler = self.prefilter_manager.get_handler_by_name('magic')
+
+        line = line_info.line
+        if line.lstrip().startswith(ESC_SH_CAP):
+            # rewrite LineInfo's line, ifun and the_rest to properly hold the
+            # call to %sx and the actual command to be executed, so
+            # handle_magic can work correctly.  Note that this works even if
+            # the line is indented, so it handles multi_line_specials
+            # properly.
+            new_rest = line.lstrip()[2:]
+            line_info.line = '%ssx %s' % (ESC_MAGIC, new_rest)
+            line_info.ifun = 'sx'
+            line_info.the_rest = new_rest
+            return magic_handler.handle(line_info)
+        else:
+            cmd = line.lstrip().lstrip(ESC_SHELL)
+            line_out = '%s_ip.system(%s)' % (line_info.pre_whitespace,
+                                             make_quoted_expr(cmd))
+        # update cache/log and return
+        self.shell.log(line, line_out, line_info.continue_prompt)
+        return line_out
+
+
+class MagicHandler(PrefilterHandler):
+
+    handler_name = Str('magic')
+    esc_strings = List(['%'])
+
+    def handle(self, line_info):
+        """Execute magic functions."""
+        ifun    = line_info.ifun
+        the_rest = line_info.the_rest
+        cmd = '%s_ip.magic(%s)' % (line_info.pre_whitespace,
+                                   make_quoted_expr(ifun + " " + the_rest))
+        self.shell.log(line_info.line, cmd, line_info.continue_prompt)
+        return cmd
+
+
+class AutoHandler(PrefilterHandler):
+
+    handler_name = Str('auto')
+    esc_strings = List([ESC_PAREN, ESC_QUOTE, ESC_QUOTE2])
+
+    def handle(self, line_info):
+        """Hande lines which can be auto-executed, quoting if requested."""
+        line    = line_info.line
+        ifun    = line_info.ifun
+        the_rest = line_info.the_rest
+        pre     = line_info.pre
+        continue_prompt = line_info.continue_prompt
+        obj = line_info.ofind(self)['obj']
+
+        #print 'pre <%s> ifun <%s> rest <%s>' % (pre,ifun,the_rest)  # dbg
+
+        # This should only be active for single-line input!
+        if continue_prompt:
+            self.log(line,line,continue_prompt)
+            return line
+
+        force_auto = isinstance(obj, IPyAutocall)
+        auto_rewrite = True
+        
+        if pre == ESC_QUOTE:
+            # Auto-quote splitting on whitespace
+            newcmd = '%s("%s")' % (ifun,'", "'.join(the_rest.split()) )
+        elif pre == ESC_QUOTE2:
+            # Auto-quote whole string
+            newcmd = '%s("%s")' % (ifun,the_rest)
+        elif pre == ESC_PAREN:
+            newcmd = '%s(%s)' % (ifun,",".join(the_rest.split()))
+        else:
+            # Auto-paren.
+            # We only apply it to argument-less calls if the autocall
+            # parameter is set to 2.  We only need to check that autocall is <
+            # 2, since this function isn't called unless it's at least 1.
+            if not the_rest and (self.autocall < 2) and not force_auto:
+                newcmd = '%s %s' % (ifun,the_rest)
+                auto_rewrite = False
+            else:
+                if not force_auto and the_rest.startswith('['):
+                    if hasattr(obj,'__getitem__'):
+                        # Don't autocall in this case: item access for an object
+                        # which is BOTH callable and implements __getitem__.
+                        newcmd = '%s %s' % (ifun,the_rest)
+                        auto_rewrite = False
+                    else:
+                        # if the object doesn't support [] access, go ahead and
+                        # autocall
+                        newcmd = '%s(%s)' % (ifun.rstrip(),the_rest)
+                elif the_rest.endswith(';'):
+                    newcmd = '%s(%s);' % (ifun.rstrip(),the_rest[:-1])
+                else:
+                    newcmd = '%s(%s)' % (ifun.rstrip(), the_rest)
 
-# Warning: the '-' HAS TO BE AT THE END of the first group, otherwise
-# it affects the rest of the group in square brackets.
-re_exclude_auto = re.compile(r'^[,&^\|\*/\+-]'
-                             r'|^is |^not |^in |^and |^or ')
+        if auto_rewrite:
+            rw = self.shell.outputcache.prompt1.auto_rewrite() + newcmd
 
-# try to catch also methods for stuff in lists/tuples/dicts: off
-# (experimental). For this to work, the line_split regexp would need
-# to be modified so it wouldn't break things at '['. That line is
-# nasty enough that I shouldn't change it until I can test it _well_.
-#self.re_fun_name = re.compile (r'[a-zA-Z_]([a-zA-Z0-9_.\[\]]*) ?$')
+            try:
+                # plain ascii works better w/ pyreadline, on some machines, so
+                # we use it and only print uncolored rewrite if we have unicode
+                rw = str(rw)
+                print >>Term.cout, rw
+            except UnicodeEncodeError:
+                print "-------------->" + newcmd
 
-# Handler Check Utilities
-def isShadowed(identifier,ip):
-    """Is the given identifier defined in one of the namespaces which shadow
-    the alias and magic namespaces?  Note that an identifier is different
-    than iFun, because it can not contain a '.' character."""
-    # This is much safer than calling ofind, which can change state
-    return (identifier in ip.user_ns \
-            or identifier in ip.internal_ns \
-            or identifier in ip.ns_table['builtin'])
+        # log what is now valid Python, not the actual user input (without the
+        # final newline)
+        self.shell.log(line,newcmd,continue_prompt)
+        return newcmd
+
+
+class HelpHandler(PrefilterHandler):
+
+    handler_name = Str('help')
+    esc_strings = List([ESC_HELP])
+
+    def handle(self, line_info):
+        """Try to get some help for the object.
+
+        obj? or ?obj   -> basic information.
+        obj?? or ??obj -> more details.
+        """
+        normal_handler = self.prefilter_manager.get_handler_by_name('normal')
+        line = line_info.line
+        # We need to make sure that we don't process lines which would be
+        # otherwise valid python, such as "x=1 # what?"
+        try:
+            codeop.compile_command(line)
+        except SyntaxError:
+            # We should only handle as help stuff which is NOT valid syntax
+            if line[0]==ESC_HELP:
+                line = line[1:]
+            elif line[-1]==ESC_HELP:
+                line = line[:-1]
+            self.shell.log(line, '#?'+line, line_info.continue_prompt)
+            if line:
+                #print 'line:<%r>' % line  # dbg
+                self.shell.magic_pinfo(line)
+            else:
+                page(self.shell.usage, screen_lines=self.shell.usable_screen_length)
+            return '' # Empty string is needed here!
+        except:
+            raise
+            # Pass any other exceptions through to the normal handler
+            return normal_handler.handle(line_info)
+        else:
+            raise
+            # If the code compiles ok, we should handle it normally
+            return normal_handler.handle(line_info)
+
+
+class EmacsHandler(PrefilterHandler):
+
+    handler_name = Str('emacs')
+    esc_strings = List([])
+
+    def handle(self, line_info):
+        """Handle input lines marked by python-mode."""
+
+        # Currently, nothing is done.  Later more functionality can be added
+        # here if needed.
+
+        # The input cache shouldn't be updated
+        return line_info.line
+
+
+#-----------------------------------------------------------------------------
+# Defaults
+#-----------------------------------------------------------------------------
+
+
+_default_checkers = [
+    EmacsChecker,
+    ShellEscapeChecker,
+    IPyAutocallChecker,
+    MultiLineMagicChecker,
+    EscCharsChecker,
+    AssignmentChecker,
+    AutoMagicChecker,
+    AliasChecker,
+    PythonOpsChecker,
+    AutocallChecker
+]
+
+_default_handlers = [
+    PrefilterHandler,
+    AliasHandler,
+    ShellEscapeHandler,
+    MagicHandler,
+    AutoHandler,
+    HelpHandler,
+    EmacsHandler
+]