From 6f35aed70a76e226bd396e819199c550c8c88a83 2010-07-26 06:33:11
From: Fernando Perez <Fernando.Perez@berkeley.edu>
Date: 2010-07-26 06:33:11
Subject: [PATCH] Backport a bugfix from trunk; works around Python bug 1170.

For more details:
- http://bugs.python.org/issue1170
- http://github.com/ipython/ipython/commit/ccc8b20a12aedb2b65db9c171fcfcf9b05910259

---

diff --git a/IPython/genutils.py b/IPython/genutils.py
index 4fd59f5..7f1ffce 100644
--- a/IPython/genutils.py
+++ b/IPython/genutils.py
@@ -287,13 +287,12 @@ def arg_split(s,posix=False):
     function, but with a default of posix=False for splitting, so that quotes
     in inputs are respected."""
 
-    # XXX - there may be unicode-related problems here!!!  I'm not sure that
-    # shlex is truly unicode-safe, so it might be necessary to do
-    #
-    # s = s.encode(sys.stdin.encoding)
-    #
-    # first, to ensure that shlex gets a normal string.  Input from anyone who
-    # knows more about unicode and shlex than I would be good to have here...
+    # Unfortunately, python's shlex module is buggy with unicode input:
+    # http://bugs.python.org/issue1170
+    # At least encoding the input when it's unicode seems to help, but there
+    # may be more problems lurking.  Apparently this is fixed in python3.
+    if isinstance(s, unicode):
+        s = s.encode(sys.stdin.encoding)
     lex = shlex.shlex(s, posix=posix)
     lex.whitespace_split = True
     return list(lex)