Merge branch 'rkern-magic-arguments'

IPython/core/magic_arguments.py

@@ -0,0 +1,217 b''
+''' A decorator-based method of constructing IPython magics with `argparse`
+option handling.
+
+New magic functions can be defined like so::
+
+    from IPython.core.magic_arguments import (argument, magic_arguments,
+        parse_argstring)
+
+    @magic_arguments()
+    @argument('-o', '--option', help='An optional argument.')
+    @argument('arg', type=int, help='An integer positional argument.')
+    def magic_cool(self, arg):
+        """ A really cool magic command.
+
+    """
+        args = parse_argstring(magic_cool, arg)
+        ...
+
+The `@magic_arguments` decorator marks the function as having argparse arguments.
+The `@argument` decorator adds an argument using the same syntax as argparse's
+`add_argument()` method. More sophisticated uses may also require the
+`@argument_group` or `@kwds` decorator to customize the formatting and the
+parsing.
+
+Help text for the magic is automatically generated from the docstring and the
+arguments::
+
+    In[1]: %cool?
+        %cool [-o OPTION] arg
+        
+        A really cool magic command.
+        
+        positional arguments:
+          arg                   An integer positional argument.
+        
+        optional arguments:
+          -o OPTION, --option OPTION
+                                An optional argument.
+
+'''
+#-----------------------------------------------------------------------------
+# Copyright (c) 2010, IPython Development Team.
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+#-----------------------------------------------------------------------------
+
+# Our own imports
+from IPython.external import argparse
+from IPython.core.error import UsageError
+from IPython.utils.process import arg_split
+
+
+class MagicArgumentParser(argparse.ArgumentParser):
+    """ An ArgumentParser tweaked for use by IPython magics.
+    """
+    def __init__(self,
+                 prog=None,
+                 usage=None,
+                 description=None,
+                 epilog=None,
+                 version=None,
+                 parents=None,
+                 formatter_class=argparse.HelpFormatter,
+                 prefix_chars='-',
+                 argument_default=None,
+                 conflict_handler='error',
+                 add_help=False):
+        if parents is None:
+            parents = []
+        super(MagicArgumentParser, self).__init__(prog=prog, usage=usage,
+            description=description, epilog=epilog, version=version,
+            parents=parents, formatter_class=formatter_class,
+            prefix_chars=prefix_chars, argument_default=argument_default,
+            conflict_handler=conflict_handler, add_help=add_help)
+
+    def error(self, message):
+        """ Raise a catchable error instead of exiting.
+        """
+        raise UsageError(message)
+
+    def parse_argstring(self, argstring):
+        """ Split a string into an argument list and parse that argument list.
+        """
+        argv = arg_split(argstring)
+        return self.parse_args(argv)
+
+
+def construct_parser(magic_func):
+    """ Construct an argument parser using the function decorations.
+    """
+    kwds = getattr(magic_func, 'argcmd_kwds', {})
+    if 'description' not in kwds:
+        kwds['description'] = getattr(magic_func, '__doc__', None)
+    arg_name = real_name(magic_func)
+    parser = MagicArgumentParser(arg_name, **kwds)
+    # Reverse the list of decorators in order to apply them in the
+    # order in which they appear in the source.
+    group = None
+    for deco in magic_func.decorators[::-1]:
+        result = deco.add_to_parser(parser, group)
+        if result is not None:
+            group = result
+
+    # Replace the starting 'usage: ' with IPython's %.
+    help_text = parser.format_help()
+    if help_text.startswith('usage: '):
+        help_text = help_text.replace('usage: ', '%', 1)
+    else:
+        help_text = '%' + help_text
+
+    # Replace the magic function's docstring with the full help text.
+    magic_func.__doc__ = help_text
+
+    return parser
+
+
+def parse_argstring(magic_func, argstring):
+    """ Parse the string of arguments for the given magic function.
+    """
+    return magic_func.parser.parse_argstring(argstring)
+
+
+def real_name(magic_func):
+    """ Find the real name of the magic.
+    """
+    magic_name = magic_func.__name__
+    if magic_name.startswith('magic_'):
+        magic_name = magic_name[len('magic_'):]
+    return getattr(magic_func, 'argcmd_name', magic_name)
+
+
+class ArgDecorator(object):
+    """ Base class for decorators to add ArgumentParser information to a method.
+    """
+
+    def __call__(self, func):
+        if not getattr(func, 'has_arguments', False):
+            func.has_arguments = True
+            func.decorators = []
+        func.decorators.append(self)
+        return func
+
+    def add_to_parser(self, parser, group):
+        """ Add this object's information to the parser, if necessary.
+        """
+        pass
+
+
+class magic_arguments(ArgDecorator):
+    """ Mark the magic as having argparse arguments and possibly adjust the
+    name.
+    """
+
+    def __init__(self, name=None):
+        self.name = name
+
+    def __call__(self, func):
+        if not getattr(func, 'has_arguments', False):
+            func.has_arguments = True
+            func.decorators = []
+        if self.name is not None:
+            func.argcmd_name = self.name
+        # This should be the first decorator in the list of decorators, thus the
+        # last to execute. Build the parser.
+        func.parser = construct_parser(func)
+        return func
+
+
+class argument(ArgDecorator):
+    """ Store arguments and keywords to pass to add_argument().
+
+    Instances also serve to decorate command methods.
+    """
+    def __init__(self, *args, **kwds):
+        self.args = args
+        self.kwds = kwds
+
+    def add_to_parser(self, parser, group):
+        """ Add this object's information to the parser.
+        """
+        if group is not None:
+            parser = group
+        parser.add_argument(*self.args, **self.kwds)
+        return None
+
+
+class argument_group(ArgDecorator):
+    """ Store arguments and keywords to pass to add_argument_group().
+
+    Instances also serve to decorate command methods.
+    """
+    def __init__(self, *args, **kwds):
+        self.args = args
+        self.kwds = kwds
+
+    def add_to_parser(self, parser, group):
+        """ Add this object's information to the parser.
+        """
+        return parser.add_argument_group(*self.args, **self.kwds)
+
+
+class kwds(ArgDecorator):
+    """ Provide other keywords to the sub-parser constructor.
+    """
+    def __init__(self, **kwds):
+        self.kwds = kwds
+
+    def __call__(self, func):
+        func = super(kwds, self).__call__(func)
+        func.argcmd_kwds = self.kwds
+        return func
+
+
+__all__ = ['magic_arguments', 'argument', 'argument_group', 'kwds',
+    'parse_argstring']

IPython/core/tests/test_magic_arguments.py

@@ -0,0 +1,121 b''
+#-----------------------------------------------------------------------------
+# Copyright (c) 2010, IPython Development Team.
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+#-----------------------------------------------------------------------------
+
+from nose.tools import assert_equal, assert_true
+
+from IPython.external import argparse
+from IPython.core.magic_arguments import (argument, argument_group, kwds,
+    magic_arguments, parse_argstring, real_name)
+from IPython.testing.decorators import parametric
+
+
+@magic_arguments()
+@argument('-f', '--foo', help="an argument")
+def magic_foo1(self, args):
+    """ A docstring.
+    """
+    return parse_argstring(magic_foo1, args)
+
+
+@magic_arguments()
+def magic_foo2(self, args):
+    """ A docstring.
+    """
+    return parse_argstring(magic_foo2, args)
+
+
+@magic_arguments()
+@argument('-f', '--foo', help="an argument")
+@argument_group('Group')
+@argument('-b', '--bar', help="a grouped argument")
+@argument_group('Second Group')
+@argument('-z', '--baz', help="another grouped argument")
+def magic_foo3(self, args):
+    """ A docstring.
+    """
+    return parse_argstring(magic_foo3, args)
+
+
+@magic_arguments()
+@kwds(argument_default=argparse.SUPPRESS)
+@argument('-f', '--foo', help="an argument")
+def magic_foo4(self, args):
+    """ A docstring.
+    """
+    return parse_argstring(magic_foo4, args)
+
+
+@magic_arguments('frobnicate')
+@argument('-f', '--foo', help="an argument")
+def magic_foo5(self, args):
+    """ A docstring.
+    """
+    return parse_argstring(magic_foo5, args)
+
+
+@magic_arguments()
+@argument('-f', '--foo', help="an argument")
+def magic_magic_foo(self, args):
+    """ A docstring.
+    """
+    return parse_argstring(magic_magic_foo, args)
+
+
+@magic_arguments()
+@argument('-f', '--foo', help="an argument")
+def foo(self, args):
+    """ A docstring.
+    """
+    return parse_argstring(foo, args)
+
+
+@parametric
+def test_magic_arguments():
+    # Ideally, these would be doctests, but I could not get it to work.
+    yield assert_equal(magic_foo1.__doc__, '%foo1 [-f FOO]\n\nA docstring.\n\noptional arguments:\n  -f FOO, --foo FOO  an argument\n')
+    yield assert_equal(getattr(magic_foo1, 'argcmd_name', None), None)
+    yield assert_equal(real_name(magic_foo1), 'foo1')
+    yield assert_equal(magic_foo1(None, ''), argparse.Namespace(foo=None))
+    yield assert_true(hasattr(magic_foo1, 'has_arguments'))
+
+    yield assert_equal(magic_foo2.__doc__, '%foo2\n\nA docstring.\n')
+    yield assert_equal(getattr(magic_foo2, 'argcmd_name', None), None)
+    yield assert_equal(real_name(magic_foo2), 'foo2')
+    yield assert_equal(magic_foo2(None, ''), argparse.Namespace())
+    yield assert_true(hasattr(magic_foo2, 'has_arguments'))
+
+    yield assert_equal(magic_foo3.__doc__, '%foo3 [-f FOO] [-b BAR] [-z BAZ]\n\nA docstring.\n\noptional arguments:\n  -f FOO, --foo FOO  an argument\n\nGroup:\n  -b BAR, --bar BAR  a grouped argument\n\nSecond Group:\n  -z BAZ, --baz BAZ  another grouped argument\n')
+    yield assert_equal(getattr(magic_foo3, 'argcmd_name', None), None)
+    yield assert_equal(real_name(magic_foo3), 'foo3')
+    yield assert_equal(magic_foo3(None, ''),
+                       argparse.Namespace(bar=None, baz=None, foo=None))
+    yield assert_true(hasattr(magic_foo3, 'has_arguments'))
+
+    yield assert_equal(magic_foo4.__doc__, '%foo4 [-f FOO]\n\nA docstring.\n\noptional arguments:\n  -f FOO, --foo FOO  an argument\n')
+    yield assert_equal(getattr(magic_foo4, 'argcmd_name', None), None)
+    yield assert_equal(real_name(magic_foo4), 'foo4')
+    yield assert_equal(magic_foo4(None, ''), argparse.Namespace())
+    yield assert_true(hasattr(magic_foo4, 'has_arguments'))
+
+    yield assert_equal(magic_foo5.__doc__, '%frobnicate [-f FOO]\n\nA docstring.\n\noptional arguments:\n  -f FOO, --foo FOO  an argument\n')
+    yield assert_equal(getattr(magic_foo5, 'argcmd_name', None), 'frobnicate')
+    yield assert_equal(real_name(magic_foo5), 'frobnicate')
+    yield assert_equal(magic_foo5(None, ''), argparse.Namespace(foo=None))
+    yield assert_true(hasattr(magic_foo5, 'has_arguments'))
+
+    yield assert_equal(magic_magic_foo.__doc__, '%magic_foo [-f FOO]\n\nA docstring.\n\noptional arguments:\n  -f FOO, --foo FOO  an argument\n')
+    yield assert_equal(getattr(magic_magic_foo, 'argcmd_name', None), None)
+    yield assert_equal(real_name(magic_magic_foo), 'magic_foo')
+    yield assert_equal(magic_magic_foo(None, ''), argparse.Namespace(foo=None))
+    yield assert_true(hasattr(magic_magic_foo, 'has_arguments'))
+
+    yield assert_equal(foo.__doc__, '%foo [-f FOO]\n\nA docstring.\n\noptional arguments:\n  -f FOO, --foo FOO  an argument\n')
+    yield assert_equal(getattr(foo, 'argcmd_name', None), None)
+    yield assert_equal(real_name(foo), 'foo')
+    yield assert_equal(foo(None, ''), argparse.Namespace(foo=None))
+    yield assert_true(hasattr(foo, 'has_arguments'))

IPython/utils/process.py

@@ -1,140 +1,146 b''
 # encoding: utf-8
 """
 Utilities for working with external processes.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
 # Stdlib
 import os
 import sys
 import shlex
 
 # Our own
 if sys.platform == 'win32':
     from ._process_win32 import _find_cmd, system, getoutput, AvoidUNCPath
 else:
     from ._process_posix import _find_cmd, system, getoutput
 
 from ._process_common import getoutputerror
 
 #-----------------------------------------------------------------------------
 # Code
 #-----------------------------------------------------------------------------
 
 
 class FindCmdError(Exception):
     pass
 
 
 def find_cmd(cmd):
     """Find absolute path to executable cmd in a cross platform manner.
     
     This function tries to determine the full path to a command line program
     using `which` on Unix/Linux/OS X and `win32api` on Windows.  Most of the
     time it will use the version that is first on the users `PATH`.  If
     cmd is `python` return `sys.executable`.
 
     Warning, don't use this to find IPython command line programs as there
     is a risk you will find the wrong one.  Instead find those using the
     following code and looking for the application itself::
     
         from IPython.utils.path import get_ipython_module_path
         from IPython.utils.process import pycmd2argv
         argv = pycmd2argv(get_ipython_module_path('IPython.frontend.terminal.ipapp'))
 
     Parameters
     ----------
     cmd : str
         The command line program to look for.
     """
     if cmd == 'python':
         return os.path.abspath(sys.executable)
     try:
         path = _find_cmd(cmd).rstrip()
     except OSError:
         raise FindCmdError('command could not be found: %s' % cmd)
     # which returns empty if not found
     if path == '':
         raise FindCmdError('command could not be found: %s' % cmd)
     return os.path.abspath(path)
 
 
 def pycmd2argv(cmd):
     r"""Take the path of a python command and return a list (argv-style).
 
     This only works on Python based command line programs and will find the
     location of the ``python`` executable using ``sys.executable`` to make
     sure the right version is used.
 
     For a given path ``cmd``, this returns [cmd] if cmd's extension is .exe,
     .com or .bat, and [, cmd] otherwise.  
 
     Parameters
     ----------
     cmd : string
       The path of the command.
 
     Returns
     -------
     argv-style list.
     """
     ext = os.path.splitext(cmd)[1]
     if ext in ['.exe', '.com', '.bat']:
         return [cmd]
     else:
         if sys.platform == 'win32':
             # The -u option here turns on unbuffered output, which is required
             # on Win32 to prevent wierd conflict and problems with Twisted.
             # Also, use sys.executable to make sure we are picking up the 
             # right python exe.
             return [sys.executable, '-u', cmd]
         else:
             return [sys.executable, cmd]
 
 
 def arg_split(s, posix=False):
     """Split a command line's arguments in a shell-like manner.
 
     This is a modified version of the standard library's shlex.split()
     function, but with a default of posix=False for splitting, so that quotes
     in inputs are respected."""
 
     # Unfortunately, python's shlex module is buggy with unicode input:
     # http://bugs.python.org/issue1170
     # At least encoding the input when it's unicode seems to help, but there
     # may be more problems lurking.  Apparently this is fixed in python3.
+    is_unicode = False
     if isinstance(s, unicode):
-        s = s.encode(sys.stdin.encoding)
+        is_unicode = True
+        s = s.encode('utf-8')
     lex = shlex.shlex(s, posix=posix)
     lex.whitespace_split = True
-    return list(lex)
+    tokens = list(lex)
+    if is_unicode:
+        # Convert the tokens back to unicode.
+        tokens = [x.decode('utf-8') for x in tokens]
+    return tokens
 
 
 def abbrev_cwd():
     """ Return abbreviated version of cwd, e.g. d:mydir """
     cwd = os.getcwd().replace('\\','/')
     drivepart = ''
     tail = cwd
     if sys.platform == 'win32':
         if len(cwd) < 4:
             return cwd
         drivepart,tail = os.path.splitdrive(cwd)
 
 
     parts = tail.split('/')
     if len(parts) > 2:
         tail = '/'.join(parts[-2:])
 
     return (drivepart + (
         cwd == '/' and '/' or tail))

IPython/utils/tests/test_process.py

@@ -1,95 +1,98 b''
 # encoding: utf-8
 """
 Tests for platutils.py
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import sys
 from unittest import TestCase
 
 import nose.tools as nt
 
 from IPython.utils.process import (find_cmd, FindCmdError, arg_split,
                                    system, getoutput, getoutputerror)
 from IPython.testing import decorators as dec
 from IPython.testing import tools as tt
 
 #-----------------------------------------------------------------------------
 # Tests
 #-----------------------------------------------------------------------------
 
 def test_find_cmd_python():
     """Make sure we find sys.exectable for python."""
     nt.assert_equals(find_cmd('python'), sys.executable)
 
     
 @dec.skip_win32
 def test_find_cmd_ls():
     """Make sure we can find the full path to ls."""
     path = find_cmd('ls')
     nt.assert_true(path.endswith('ls'))
 
     
 def has_pywin32():
     try:
         import win32api
     except ImportError:
         return False
     return True
 
 
 @dec.onlyif(has_pywin32, "This test requires win32api to run")
 def test_find_cmd_pythonw():
     """Try to find pythonw on Windows."""
     path = find_cmd('pythonw')
     nt.assert_true(path.endswith('pythonw.exe'))
 
 
 @dec.onlyif(lambda : sys.platform != 'win32' or has_pywin32(),
             "This test runs on posix or in win32 with win32api installed")
 def test_find_cmd_fail():
     """Make sure that FindCmdError is raised if we can't find the cmd."""
     nt.assert_raises(FindCmdError,find_cmd,'asdfasdf')
 
     
 def test_arg_split():
     """Ensure that argument lines are correctly split like in a shell."""
     tests = [['hi', ['hi']],
              [u'hi', [u'hi']],
+             ['hello there', ['hello', 'there']],
+             [u'h\N{LATIN SMALL LETTER A WITH CARON}llo', [u'h\N{LATIN SMALL LETTER A WITH CARON}llo']],
+             ['something "with quotes"', ['something', '"with quotes"']],
              ]
     for argstr, argv in tests:
         nt.assert_equal(arg_split(argstr), argv)
 
 
 class SubProcessTestCase(TestCase, tt.TempFileMixin):
     def setUp(self):
         """Make a valid python temp file."""
         lines = ["from __future__ import print_function",
                  "import sys",
                  "print('on stdout', end='', file=sys.stdout)",
                  "print('on stderr', end='', file=sys.stderr)",
                  "sys.stdout.flush()",
                  "sys.stderr.flush()"]
         self.mktmp('\n'.join(lines))
 
     def test_system(self):
         system('python "%s"' % self.fname)
 
     def test_getoutput(self):
         out = getoutput('python "%s"' % self.fname)
         self.assertEquals(out, 'on stdout')
 
     def test_getoutput(self):
         out, err = getoutputerror('python "%s"' % self.fname)
         self.assertEquals(out, 'on stdout')
         self.assertEquals(err, 'on stderr')