diff --git a/IPython/utils/process.py b/IPython/utils/process.py index af5879d..2017395 100644 --- a/IPython/utils/process.py +++ b/IPython/utils/process.py @@ -114,11 +114,17 @@ def arg_split(s, posix=False): # http://bugs.python.org/issue1170 # At least encoding the input when it's unicode seems to help, but there # may be more problems lurking. Apparently this is fixed in python3. + is_unicode = False if isinstance(s, unicode): - s = s.encode(sys.stdin.encoding) + is_unicode = True + s = s.encode('utf-8') lex = shlex.shlex(s, posix=posix) lex.whitespace_split = True - return list(lex) + tokens = list(lex) + if is_unicode: + # Convert the tokens back to unicode. + tokens = [x.decode('utf-8') for x in tokens] + return tokens def abbrev_cwd(): diff --git a/IPython/utils/tests/test_process.py b/IPython/utils/tests/test_process.py index b89fedf..cbde427 100644 --- a/IPython/utils/tests/test_process.py +++ b/IPython/utils/tests/test_process.py @@ -66,6 +66,9 @@ def test_arg_split(): """Ensure that argument lines are correctly split like in a shell.""" tests = [['hi', ['hi']], [u'hi', [u'hi']], + ['hello there', ['hello', 'there']], + [u'h\N{LATIN SMALL LETTER A WITH CARON}llo', [u'h\N{LATIN SMALL LETTER A WITH CARON}llo']], + ['something "with quotes"', ['something', '"with quotes"']], ] for argstr, argv in tests: nt.assert_equal(arg_split(argstr), argv)