Fix completer crash with certain unicode inputs.

The real bug is in shlex.split in the stdlib
(http://bugs.python.org/issue1170) but we have a workaround, we were
just not using it in the completion code.

IPython/core/completer.py

@@ -83,6 +83,7 from IPython.core.prefilter import ESC_MAGIC
 from IPython.utils import generics
 from IPython.utils import io
 from IPython.utils.dir2 import dir2
+from IPython.utils.process import arg_split
 
 #-----------------------------------------------------------------------------
 # Globals
@@ -473,7 +474,7 class IPCompleter(Completer):
         current (as of Python 2.3) Python readline it's possible to do
         better."""
 
-        #io.rprint('Completer->file_matches: <%s>' % text) # dbg
+        #io.rprint('Completer->file_matches: <%r>' % text) # dbg
 
         # chars that require escaping with backslash - i.e. chars
         # that readline treats incorrectly as delimiters, but we
@@ -488,7 +489,8 class IPCompleter(Completer):
         text_until_cursor = self.text_until_cursor
         open_quotes = 0  # track strings with open quotes
         try:
-            lsplit = shlex.split(text_until_cursor)[-1]
+            # arg_split ~ shlex.split, but with unicode bugs fixed by us
+            lsplit = arg_split(text_until_cursor)[-1]
         except ValueError:
             # typically an unmatched ", or backslash without escaped char.
             if text_until_cursor.count('"')==1:

IPython/core/tests/test_completer.py

@@ -62,6 +62,24 def test_line_split():
          ('cd "some_file/', '', 'some_file/'),
          ]
     check_line_split(sp, t)
+    # Ensure splitting works OK with unicode by re-running the tests with
+    # all inputs turned into unicode
+    check_line_split(sp, [ map(unicode, p) for p in t] )
+
+
+def test_unicode_completions():
+    ip = get_ipython()
+    # Some strings that trigger different types of completion.  Check them both
+    # in str and unicode forms
+    s = ['ru', '%ru', 'cd /', 'floa', 'float(x)/']
+    for t in s + map(unicode, s):
+        # We don't need to check exact completion values (they may change
+        # depending on the state of the namespace, but at least no exceptions
+        # should be thrown and the return value should be a pair of text, list
+        # values.
+        text, matches = ip.complete(t)
+        nt.assert_true(isinstance(text, basestring))
+        nt.assert_true(isinstance(matches, list))
 
 
 class CompletionSplitterTestCase(unittest.TestCase):