Improvements in the code that breaks up user input.

IPython/core/alias.py

@@ -38,7 +38,7 b' from IPython.utils.warn import warn, error'
 #-----------------------------------------------------------------------------
 
 # This is used as the pattern for calls to split_user_input.
-shell_line_split = re.compile(r'^(\s*)(\S*\s*)(.*$)')
+shell_line_split = re.compile(r'^(\s*)()(\S+)(.*$)')
 
 def default_aliases():
     """Return list of shell aliases to auto-define.
@@ -222,7 +222,7 b' class AliasManager(Configurable):'
                  <16> 'q:/opt/np/notepad++.exe myfile.txt'
         """
         
-        pre,fn,rest = split_user_input(line)
+        pre,_,fn,rest = split_user_input(line)
         res = pre + self.expand_aliases(fn, rest)
         return res
 
@@ -242,7 +242,7 b' class AliasManager(Configurable):'
         
         done = set()
         while 1:
-            pre,fn,rest = split_user_input(line, shell_line_split)
+            pre,_,fn,rest = split_user_input(line, shell_line_split)
             if fn in self.alias_table:
                 if fn in done:
                     warn("Cyclic alias definition, repeated '%s'" % fn)

IPython/core/oinspect.py

@@ -24,12 +24,16 b' import os'
 import sys
 import types
 from collections import namedtuple
-from itertools import izip_longest
+try:
+    from itertools import izip_longest
+except ImportError:
+    from itertools import zip_longest as izip_longest
 
 # IPython's own
 from IPython.core import page
 from IPython.utils import PyColorize
 from IPython.utils import io
+from IPython.utils import py3compat
 from IPython.utils.text import indent
 from IPython.utils.wildcard import list_namespace
 from IPython.utils.coloransi import *
@@ -249,7 +253,7 b' class Inspector:'
         try:
             # We need a plain string here, NOT unicode!
             hdef = oname + inspect.formatargspec(*getargspec(obj))
-            return hdef.encode('ascii')
+            return py3compat.unicode_to_str(hdef.encode('ascii'))
         except:
             return None
  
@@ -362,7 +366,7 b' class Inspector:'
         except:
             self.noinfo('source',oname)
         else:
-            page.page(self.format(src))
+            page.page(self.format(py3compat.unicode_to_str(src)))
 
     def pfile(self,obj,oname=''):
         """Show the whole file where an object was defined."""

IPython/core/prefilter.py

@@ -131,7 +131,7 b' class LineInfo(object):'
     def __init__(self, line, continue_prompt):
         self.line            = line
         self.continue_prompt = continue_prompt
-        self.pre, self.ifun, self.the_rest = split_user_input(line)
+        self.pre, self.esc, self.ifun, self.the_rest = split_user_input(line)
 
         self.pre_char       = self.pre.strip()
         if self.pre_char:
@@ -630,7 +630,7 b' class MultiLineMagicChecker(PrefilterChecker):'
         # both ! and !!.    
         if line_info.continue_prompt \
             and self.prefilter_manager.multi_line_specials:
-                if line_info.ifun.startswith(ESC_MAGIC):
+                if line_info.esc == ESC_MAGIC:
                     return self.prefilter_manager.get_handler_by_name('magic')
         else:
             return None
@@ -644,14 +644,16 b' class EscCharsChecker(PrefilterChecker):'
         """Check for escape character and return either a handler to handle it,
         or None if there is no escape char."""
         if line_info.line[-1] == ESC_HELP \
-               and line_info.pre_char != ESC_SHELL \
-               and line_info.pre_char != ESC_SH_CAP:
+               and line_info.esc != ESC_SHELL \
+               and line_info.esc != ESC_SH_CAP:
             # the ? can be at the end, but *not* for either kind of shell escape,
             # because a ? can be a vaild final char in a shell cmd
             return self.prefilter_manager.get_handler_by_name('help')
         else:
+            if line_info.pre:
+                return None
             # This returns None like it should if no handler exists
-            return self.prefilter_manager.get_handler_by_esc(line_info.pre_char)
+            return self.prefilter_manager.get_handler_by_esc(line_info.esc)
 
 
 class AssignmentChecker(PrefilterChecker):
@@ -872,6 +874,7 b' class AutoHandler(PrefilterHandler):'
         ifun    = line_info.ifun
         the_rest = line_info.the_rest
         pre     = line_info.pre
+        esc     = line_info.esc
         continue_prompt = line_info.continue_prompt
         obj = line_info.ofind(self)['obj']
         #print 'pre <%s> ifun <%s> rest <%s>' % (pre,ifun,the_rest)  # dbg
@@ -883,13 +886,13 b' class AutoHandler(PrefilterHandler):'
         force_auto = isinstance(obj, IPyAutocall)
         auto_rewrite = getattr(obj, 'rewrite', True)
         
-        if pre == ESC_QUOTE:
+        if esc == ESC_QUOTE:
             # Auto-quote splitting on whitespace
             newcmd = '%s("%s")' % (ifun,'", "'.join(the_rest.split()) )
-        elif pre == ESC_QUOTE2:
+        elif esc == ESC_QUOTE2:
             # Auto-quote whole string
             newcmd = '%s("%s")' % (ifun,the_rest)
-        elif pre == ESC_PAREN:
+        elif esc == ESC_PAREN:
             newcmd = '%s(%s)' % (ifun,",".join(the_rest.split()))
         else:
             # Auto-paren.
@@ -946,7 +949,7 b' class HelpHandler(PrefilterHandler):'
                 line = line[:-1]
             if line:
                 #print 'line:<%r>' % line  # dbg
-                self.shell.magic_pinfo(line)
+                self.shell.magic_pinfo(line_info.ifun)
             else:
                 self.shell.show_usage()
             return '' # Empty string is needed here!

IPython/core/splitinput.py

@@ -40,13 +40,15 b' from IPython.utils import py3compat'
 # ! and !! trigger if they are first char(s) *or* follow an indent 
 # ? triggers as first or last char.
 
-# The three parts of the regex are:
-#  1) pre:     pre_char *or* initial whitespace 
-#  2) ifun:    first word/method (mix of \w and '.')
-#  3) the_rest: rest of line (separated from ifun by space if non-empty)
-line_split = re.compile(r'^([,;/%?]|!!?|\s*)'
+# The four parts of the regex are:
+#  1) pre:     initial whitespace
+#  2) esc:     escape character
+#  3) ifun:    first word/method (mix of \w and '.')
+#  4) the_rest: rest of line (separated from ifun by space if non-empty)
+line_split = re.compile(r'^(\s*)'
+                        r'([,;/%?]|!!?)?'
                         r'\s*([\w\.]+)'
-                        r'(\s+.*$|$)')
+                        r'(.*$|$)')
 
 # r'[\w\.]+'
 # r'\s*=\s*%.*'
@@ -71,19 +73,14 b' def split_user_input(line, pattern=None):'
             # print "split failed for line '%s'" % line
             ifun, the_rest = line, u''
         pre = re.match('^(\s*)(.*)',line).groups()[0]
+        esc = ""
     else:
-        pre,ifun,the_rest = match.groups()
+        pre, esc, ifun, the_rest = match.groups()
     
-    if not py3compat.PY3:
-        # ifun has to be a valid python identifier, so it better encode into
-        # ascii.  We do still make it a unicode string so that we consistently
-        # return unicode, but it will be one that is guaranteed to be pure ascii
-        try:
-            ifun = unicode(ifun.encode('ascii'))
-        except UnicodeEncodeError:
-            the_rest = ifun + u' ' + the_rest
-            ifun = u''
+    if not py3compat.isidentifier(ifun, dotted=True):
+        the_rest = ifun + u' ' + the_rest
+        ifun = u''
 
     #print 'line:<%s>' % line # dbg
     #print 'pre <%s> ifun <%s> rest <%s>' % (pre,ifun.strip(),the_rest) # dbg
-    return pre, ifun.strip(), the_rest.lstrip()
+    return pre, esc, ifun.strip(), the_rest.lstrip()

IPython/core/tests/test_handlers.py

@@ -10,6 +10,7 b' import nose.tools as nt'
 # our own packages
 from IPython.core import autocall
 from IPython.testing import decorators as dec
+from IPython.testing import tools as tt
 from IPython.testing.globalipapp import get_ipython
 
 #-----------------------------------------------------------------------------
@@ -40,15 +41,7 b' def run(tests):'
     """Loop through a list of (pre, post) inputs, where pre is the string
     handed to ipython, and post is how that string looks after it's been
     transformed (i.e. ipython's notion of _i)"""
-    for pre, post in tests:
-        global num_tests
-        num_tests += 1        
-        actual = ip.prefilter_manager.prefilter_lines(pre)
-        if actual != None:
-            actual = actual.rstrip('\n')
-        if actual != post:
-            failures.append('Expected %r to become %r, found %r' % (
-                pre, post, actual))
+    tt.check_pairs(ip.prefilter_manager.prefilter_lines, tests)
 
 
 def test_handlers():