Work around a bug in Python's shlex module with unicode input.

The problem appeared as a broken %popd, but ultimately the real bug is
in Python itself.  I added some more tests that exercise various parts
that were breaking due to the original problem, to at least improve
test coverage.

IPython/core/tests/test_magic.py

@@ -1,274 +1,301 b''
 """Tests for various magic functions.
 
 Needs to be run by nose (to make ipython session available).
 """
 from __future__ import absolute_import
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import sys
 import tempfile
 import types
 from cStringIO import StringIO
 
 import nose.tools as nt
 
 from IPython.utils.path import get_long_path_name
 from IPython.testing import decorators as dec
 from IPython.testing import tools as tt
 
 #-----------------------------------------------------------------------------
 # Test functions begin
 #-----------------------------------------------------------------------------
 def test_rehashx():
     # clear up everything
     _ip = get_ipython()
     _ip.alias_manager.alias_table.clear()
     del _ip.db['syscmdlist']
     
     _ip.magic('rehashx')
     # Practically ALL ipython development systems will have more than 10 aliases
 
     yield (nt.assert_true, len(_ip.alias_manager.alias_table) > 10)
     for key, val in _ip.alias_manager.alias_table.items():
         # we must strip dots from alias names
         nt.assert_true('.' not in key)
 
     # rehashx must fill up syscmdlist
     scoms = _ip.db['syscmdlist']
     yield (nt.assert_true, len(scoms) > 10)
 
 
 def test_magic_parse_options():
     """Test that we don't mangle paths when parsing magic options."""
     ip = get_ipython()
     path = 'c:\\x'
     opts = ip.parse_options('-f %s' % path,'f:')[0]
     # argv splitting is os-dependent
     if os.name == 'posix':
         expected = 'c:x'
     else:
         expected = path
     nt.assert_equals(opts['f'], expected)
 
     
 def doctest_hist_f():
     """Test %hist -f with temporary filename.
 
     In [9]: import tempfile
 
     In [10]: tfile = tempfile.mktemp('.py','tmp-ipython-')
 
     In [11]: %hist -n -f $tfile 3
 
     In [13]: import os; os.unlink(tfile)
     """
 
 
 def doctest_hist_r():
     """Test %hist -r
 
     XXX - This test is not recording the output correctly.  For some reason, in
     testing mode the raw history isn't getting populated.  No idea why.
     Disabling the output checking for now, though at least we do run it.
 
     In [1]: 'hist' in _ip.lsmagic()
     Out[1]: True
 
     In [2]: x=1
 
     In [3]: %hist -r 2
     x=1 # random
     %hist -r 2
     """
 
 def doctest_hist_op():
     """Test %hist -op
 
     In [1]: class b:
        ...:         pass
        ...: 
 
     In [2]: class s(b):
        ...:         def __str__(self):
        ...:             return 's'
        ...: 
 
     In [3]: 
 
     In [4]: class r(b):
        ...:         def __repr__(self):
        ...:             return 'r'
        ...: 
 
     In [5]: class sr(s,r): pass
        ...: 
 
     In [6]: 
 
     In [7]: bb=b()
 
     In [8]: ss=s()
 
     In [9]: rr=r()
 
     In [10]: ssrr=sr()
 
     In [11]: bb
     Out[11]: <...b instance at ...>
 
     In [12]: ss
     Out[12]: <...s instance at ...>
 
     In [13]: 
 
     In [14]: %hist -op
     >>> class b:
     ...     pass
     ... 
     >>> class s(b):
     ...     def __str__(self):
     ...         return 's'
     ... 
     >>> 
     >>> class r(b):
     ...     def __repr__(self):
     ...         return 'r'
     ... 
     >>> class sr(s,r): pass
     >>> 
     >>> bb=b()
     >>> ss=s()
     >>> rr=r()
     >>> ssrr=sr()
     >>> bb
     <...b instance at ...>
     >>> ss
     <...s instance at ...>
     >>> 
     """
 
 def test_shist():
     # Simple tests of ShadowHist class - test generator.
     import os, shutil, tempfile
 
     from IPython.utils import pickleshare
     from IPython.core.history import ShadowHist
     
     tfile = tempfile.mktemp('','tmp-ipython-')
     
     db = pickleshare.PickleShareDB(tfile)
     s = ShadowHist(db)
     s.add('hello')
     s.add('world')
     s.add('hello')
     s.add('hello')
     s.add('karhu')
 
     yield nt.assert_equals,s.all(),[(1, 'hello'), (2, 'world'), (3, 'karhu')]
     
     yield nt.assert_equal,s.get(2),'world'
     
     shutil.rmtree(tfile)
 
     
 # XXX failing for now, until we get clearcmd out of quarantine.  But we should
 # fix this and revert the skip to happen only if numpy is not around.
 #@dec.skipif_not_numpy
 @dec.skipknownfailure
 def test_numpy_clear_array_undec():
     from IPython.extensions import clearcmd
 
     _ip.ex('import numpy as np')
     _ip.ex('a = np.empty(2)')
     yield (nt.assert_true, 'a' in _ip.user_ns)
     _ip.magic('clear array')
     yield (nt.assert_false, 'a' in _ip.user_ns)
     
 
 # Multiple tests for clipboard pasting
 @dec.parametric
 def test_paste():
     _ip = get_ipython()
     def paste(txt, flags='-q'):
         """Paste input text, by default in quiet mode"""
         hooks.clipboard_get = lambda : txt
         _ip.magic('paste '+flags)
 
     # Inject fake clipboard hook but save original so we can restore it later
     hooks = _ip.hooks
     user_ns = _ip.user_ns
     original_clip = hooks.clipboard_get
 
     try:
         # This try/except with an emtpy except clause is here only because
         # try/yield/finally is invalid syntax in Python 2.4.  This will be
         # removed when we drop 2.4-compatibility, and the emtpy except below
         # will be changed to a finally.
 
         # Run tests with fake clipboard function
         user_ns.pop('x', None)
         paste('x=1')
         yield nt.assert_equal(user_ns['x'], 1)
 
         user_ns.pop('x', None)
         paste('>>> x=2')
         yield nt.assert_equal(user_ns['x'], 2)
 
         paste("""
         >>> x = [1,2,3]
         >>> y = []
         >>> for i in x:
         ...     y.append(i**2)
         ...
         """)
         yield nt.assert_equal(user_ns['x'], [1,2,3])
         yield nt.assert_equal(user_ns['y'], [1,4,9])
 
         # Now, test that paste -r works
         user_ns.pop('x', None)
         yield nt.assert_false('x' in user_ns)
         _ip.magic('paste -r')
         yield nt.assert_equal(user_ns['x'], [1,2,3])
 
         # Also test paste echoing, by temporarily faking the writer
         w = StringIO()
         writer = _ip.write
         _ip.write = w.write
         code = """
         a = 100
         b = 200"""
         try:
             paste(code,'')
             out = w.getvalue()
         finally:
             _ip.write = writer
         yield nt.assert_equal(user_ns['a'], 100)
         yield nt.assert_equal(user_ns['b'], 200)
         yield nt.assert_equal(out, code+"\n## -- End pasted text --\n")
         
     finally:
         # This should be in a finally clause, instead of the bare except above.
         # Restore original hook
         hooks.clipboard_get = original_clip
 
 
 def test_time():
     _ip.magic('time None')
 
 
 def doctest_time():
     """
     In [10]: %time None
     CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
     Wall time: 0.00 s
     """
 
+
 def test_doctest_mode():
     "Toggle doctest_mode twice, it should be a no-op and run without error"
     _ip.magic('doctest_mode')
     _ip.magic('doctest_mode')
     
+
+def test_parse_options():
+    """Tests for basic options parsing in magics."""
+    # These are only the most minimal of tests, more should be added later.  At
+    # the very least we check that basic text/unicode calls work OK.
+    nt.assert_equal(_ip.parse_options('foo', '')[1], 'foo')
+    nt.assert_equal(_ip.parse_options(u'foo', '')[1], u'foo')
+
+    
+def test_dirops():
+    """Test various directory handling operations."""
+    curpath = lambda :os.path.splitdrive(os.getcwd())[1].replace('\\','/')
+
+    startdir = os.getcwd()
+    ipdir = _ip.ipython_dir
+    try:
+        _ip.magic('cd "%s"' % ipdir)
+        nt.assert_equal(curpath(), ipdir)
+        _ip.magic('cd -')
+        nt.assert_equal(curpath(), startdir)
+        _ip.magic('pushd "%s"' % ipdir)
+        nt.assert_equal(curpath(), ipdir)
+        _ip.magic('popd')
+        nt.assert_equal(curpath(), startdir)
+    finally:
+        os.chdir(startdir)

IPython/utils/process.py

@@ -1,368 +1,367 b''
 # encoding: utf-8
 """
 Utilities for working with external processes.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import sys
 import shlex
 import subprocess
 
 from IPython.utils.terminal import set_term_title
 
 #-----------------------------------------------------------------------------
 # Code
 #-----------------------------------------------------------------------------
 
 
 class FindCmdError(Exception):
     pass
 
 
 def _find_cmd(cmd):
     """Find the full path to a command using which."""
     return os.popen('which %s' % cmd).read().strip()
 
 
 if os.name == 'posix':
     def _find_cmd(cmd):
         """Find the full path to a command using which."""
         return getoutputerror('/usr/bin/env which %s' % cmd)[0]
 
 
 if sys.platform == 'win32':
     def _find_cmd(cmd):
         """Find the full path to a .bat or .exe using the win32api module."""
         try:
             from win32api import SearchPath
         except ImportError:
             raise ImportError('you need to have pywin32 installed for this to work')
         else:
             PATH = os.environ['PATH']
             extensions = ['.exe', '.com', '.bat', '.py']
             path = None
             for ext in extensions:
                 try:
                     path = SearchPath(PATH,cmd + ext)[0]
                 except:
                     pass
             if path is None:
                 raise OSError("command %r not found" % cmd)
             else:
                 return path
 
 
 def find_cmd(cmd):
     """Find absolute path to executable cmd in a cross platform manner.
     
     This function tries to determine the full path to a command line program
     using `which` on Unix/Linux/OS X and `win32api` on Windows.  Most of the
     time it will use the version that is first on the users `PATH`.  If
     cmd is `python` return `sys.executable`.
 
     Warning, don't use this to find IPython command line programs as there
     is a risk you will find the wrong one.  Instead find those using the
     following code and looking for the application itself::
     
         from IPython.utils.path import get_ipython_module_path
         from IPython.utils.process import pycmd2argv
         argv = pycmd2argv(get_ipython_module_path('IPython.core.ipapp'))
 
     Parameters
     ----------
     cmd : str
         The command line program to look for.
     """
     if cmd == 'python':
         return os.path.abspath(sys.executable)
     try:
         path = _find_cmd(cmd)
     except OSError:
         raise FindCmdError('command could not be found: %s' % cmd)
     # which returns empty if not found
     if path == '':
         raise FindCmdError('command could not be found: %s' % cmd)
     return os.path.abspath(path)
 
 
 def pycmd2argv(cmd):
     r"""Take the path of a python command and return a list (argv-style).
 
     This only works on Python based command line programs and will find the
     location of the ``python`` executable using ``sys.executable`` to make
     sure the right version is used.
 
     For a given path ``cmd``, this returns [cmd] if cmd's extension is .exe,
     .com or .bat, and [, cmd] otherwise.  
 
     Parameters
     ----------
     cmd : string
       The path of the command.
 
     Returns
     -------
     argv-style list.
     """
     ext = os.path.splitext(cmd)[1]
     if ext in ['.exe', '.com', '.bat']:
         return [cmd]
     else:
         if sys.platform == 'win32':
             # The -u option here turns on unbuffered output, which is required
             # on Win32 to prevent wierd conflict and problems with Twisted.
             # Also, use sys.executable to make sure we are picking up the 
             # right python exe.
             return [sys.executable, '-u', cmd]
         else:
             return [sys.executable, cmd]
 
 
 def arg_split(s, posix=False):
     """Split a command line's arguments in a shell-like manner.
 
     This is a modified version of the standard library's shlex.split()
     function, but with a default of posix=False for splitting, so that quotes
     in inputs are respected."""
 
-    # XXX - there may be unicode-related problems here!!!  I'm not sure that
-    # shlex is truly unicode-safe, so it might be necessary to do
-    #
-    # s = s.encode(sys.stdin.encoding)
-    #
-    # first, to ensure that shlex gets a normal string.  Input from anyone who
-    # knows more about unicode and shlex than I would be good to have here...
+    # Unfortunately, python's shlex module is buggy with unicode input:
+    # http://bugs.python.org/issue1170
+    # At least encoding the input when it's unicode seems to help, but there
+    # may be more problems lurking.  Apparently this is fixed in python3.
+    if isinstance(s, unicode):
+        s = s.encode(sys.stdin.encoding)
     lex = shlex.shlex(s, posix=posix)
     lex.whitespace_split = True
     return list(lex)
 
 
 def system(cmd, verbose=0, debug=0, header=''):
     """Execute a system command, return its exit status.
 
     Options:
 
     - verbose (0): print the command to be executed.
 
     - debug (0): only print, do not actually execute.
 
     - header (''): Header to print on screen prior to the executed command (it
     is only prepended to the command, no newlines are added).
 
     Note: a stateful version of this function is available through the
     SystemExec class."""
 
     stat = 0
     if verbose or debug: print header+cmd
     sys.stdout.flush()
     if not debug: stat = os.system(cmd)
     return stat
 
 
 def abbrev_cwd():
     """ Return abbreviated version of cwd, e.g. d:mydir """
     cwd = os.getcwd().replace('\\','/')
     drivepart = ''
     tail = cwd
     if sys.platform == 'win32':
         if len(cwd) < 4:
             return cwd
         drivepart,tail = os.path.splitdrive(cwd)
 
 
     parts = tail.split('/')
     if len(parts) > 2:
         tail = '/'.join(parts[-2:])
 
     return (drivepart + (
         cwd == '/' and '/' or tail))
 
 
 # This function is used by ipython in a lot of places to make system calls.
 # We need it to be slightly different under win32, due to the vagaries of
 # 'network shares'.  A win32 override is below.
 
 def shell(cmd, verbose=0, debug=0, header=''):
     """Execute a command in the system shell, always return None.
 
     Options:
 
     - verbose (0): print the command to be executed.
 
     - debug (0): only print, do not actually execute.
 
     - header (''): Header to print on screen prior to the executed command (it
     is only prepended to the command, no newlines are added).
 
     Note: this is similar to system(), but it returns None so it can
     be conveniently used in interactive loops without getting the return value
     (typically 0) printed many times."""
 
     stat = 0
     if verbose or debug: print header+cmd
     # flush stdout so we don't mangle python's buffering
     sys.stdout.flush()
 
     if not debug:
         set_term_title("IPy " + cmd)
         os.system(cmd)
         set_term_title("IPy " + abbrev_cwd())
 
 # override shell() for win32 to deal with network shares
 if os.name in ('nt','dos'):
 
     shell_ori = shell
 
     def shell(cmd, verbose=0, debug=0, header=''):
         if os.getcwd().startswith(r"\\"):
             path = os.getcwd()
             # change to c drive (cannot be on UNC-share when issuing os.system,
             # as cmd.exe cannot handle UNC addresses)
             os.chdir("c:")
             # issue pushd to the UNC-share and then run the command
             try:
                 shell_ori('"pushd %s&&"'%path+cmd,verbose,debug,header)
             finally:
                 os.chdir(path)
         else:
             shell_ori(cmd,verbose,debug,header)
 
     shell.__doc__ = shell_ori.__doc__
 
     
 def getoutput(cmd, verbose=0, debug=0, header='', split=0):
     """Dummy substitute for perl's backquotes.
 
     Executes a command and returns the output.
 
     Accepts the same arguments as system(), plus:
 
     - split(0): if true, the output is returned as a list split on newlines.
 
     Note: a stateful version of this function is available through the
     SystemExec class.
 
     This is pretty much deprecated and rarely used, getoutputerror may be 
     what you need.
 
     """
 
     if verbose or debug: print header+cmd
     if not debug:
         pipe = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).stdout
         output = pipe.read()
         # stipping last \n is here for backwards compat.
         if output.endswith('\n'):
             output = output[:-1]
         if split:
             return output.split('\n')
         else:
             return output
 
 
 # for compatibility with older naming conventions
 xsys = system
 
 
 def getoutputerror(cmd, verbose=0, debug=0, header='', split=0):
     """Return (standard output,standard error) of executing cmd in a shell.
 
     Accepts the same arguments as system(), plus:
 
     - split(0): if true, each of stdout/err is returned as a list split on
     newlines.
 
     Note: a stateful version of this function is available through the
     SystemExec class."""
 
     if verbose or debug: print header+cmd
     if not cmd:
         if split:
             return [],[]
         else:
             return '',''
     if not debug:
         p = subprocess.Popen(cmd, shell=True,
                              stdin=subprocess.PIPE,
                              stdout=subprocess.PIPE,
                              stderr=subprocess.PIPE,
                              close_fds=True)
         pin, pout, perr = (p.stdin, p.stdout, p.stderr)
 
         tout = pout.read().rstrip()
         terr = perr.read().rstrip()
         pin.close()
         pout.close()
         perr.close()
         if split:
             return tout.split('\n'),terr.split('\n')
         else:
             return tout,terr
 
 
 class SystemExec:
     """Access the system and getoutput functions through a stateful interface.
 
     Note: here we refer to the system and getoutput functions from this
     library, not the ones from the standard python library.
 
     This class offers the system and getoutput functions as methods, but the
     verbose, debug and header parameters can be set for the instance (at
     creation time or later) so that they don't need to be specified on each
     call.
 
     For efficiency reasons, there's no way to override the parameters on a
     per-call basis other than by setting instance attributes. If you need
     local overrides, it's best to directly call system() or getoutput().
 
     The following names are provided as alternate options:
      - xsys: alias to system
      - bq: alias to getoutput
 
     An instance can then be created as:
     >>> sysexec = SystemExec(verbose=1,debug=0,header='Calling: ')
     """
 
     def __init__(self, verbose=0, debug=0, header='', split=0):
         """Specify the instance's values for verbose, debug and header."""
         self.verbose = verbose
         self.debug = debug
         self.header = header
         self.split = split
 
     def system(self, cmd):
         """Stateful interface to system(), with the same keyword parameters."""
 
         system(cmd, self.verbose, self.debug, self.header)
 
     def shell(self, cmd):
         """Stateful interface to shell(), with the same keyword parameters."""
 
         shell(cmd, self.verbose, self.debug, self.header)
 
     xsys = system  # alias
 
     def getoutput(self, cmd):
         """Stateful interface to getoutput()."""
 
         return getoutput(cmd, self.verbose, self.debug, self.header, self.split)
 
     def getoutputerror(self, cmd):
         """Stateful interface to getoutputerror()."""
 
         return getoutputerror(cmd, self.verbose, self.debug, self.header, self.split)
 
     bq = getoutput  # alias
 
 

IPython/utils/tests/test_process.py

@@ -1,62 +1,68 b''
 # encoding: utf-8
 """
 Tests for platutils.py
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import sys
 
 import nose.tools as nt
 
-from IPython.utils.process import find_cmd, FindCmdError
+from IPython.utils.process import find_cmd, FindCmdError, arg_split
 from IPython.testing import decorators as dec
 
 #-----------------------------------------------------------------------------
 # Tests
 #-----------------------------------------------------------------------------
 
 def test_find_cmd_python():
     """Make sure we find sys.exectable for python."""
     nt.assert_equals(find_cmd('python'), sys.executable)
 
     
 @dec.skip_win32
 def test_find_cmd_ls():
     """Make sure we can find the full path to ls."""
     path = find_cmd('ls')
     nt.assert_true(path.endswith('ls'))
 
     
 def has_pywin32():
     try:
         import win32api
     except ImportError:
         return False
     return True
 
 
 @dec.onlyif(has_pywin32, "This test requires win32api to run")
 def test_find_cmd_pythonw():
     """Try to find pythonw on Windows."""
     path = find_cmd('pythonw')
     nt.assert_true(path.endswith('pythonw.exe'))
 
 
 @dec.onlyif(lambda : sys.platform != 'win32' or has_pywin32(),
             "This test runs on posix or in win32 with win32api installed")
 def test_find_cmd_fail():
     """Make sure that FindCmdError is raised if we can't find the cmd."""
     nt.assert_raises(FindCmdError,find_cmd,'asdfasdf')
 
     
-
+def test_arg_split():
+    """Ensure that argument lines are correctly split like in a shell."""
+    tests = [['hi', ['hi']],
+             [u'hi', [u'hi']],
+             ]
+    for argstr, argv in tests:
+        nt.assert_equal(arg_split(argstr), argv)