Merge pull request #1020 from takluyver/dollar-formatter

Dollar formatter for ! shell calls.

This uses a subclass of `FullEvalFormatter` for filling in variables in shell commands like `!echo $spam`, instead of Itpl.

I've retained the `{foo}` style standard formatting for complex expressions, so it can be picked up by the more powerful built in parser. `$foo` works for names and attribute access.

FullEvalFormatter now always outputs unicode, even if you gave it a bytes string. This avoids problems when you mix 8-bit non-ascii strings and unicode (which e.g. `StringIO.StringIO` suffers from).

Closes  #822 (test was added in 31ab23f).

IPython/core/interactiveshell.py

@@ -73,7 +73,8 b' from IPython.utils.pickleshare import PickleShareDB'
 from IPython.utils.process import system, getoutput
 from IPython.utils.strdispatch import StrDispatch
 from IPython.utils.syspathcontext import prepended_to_syspath
-from IPython.utils.text import num_ini_spaces, format_screen, LSString, SList
+from IPython.utils.text import (num_ini_spaces, format_screen, LSString, SList,
+                                DollarFormatter)
 from IPython.utils.traitlets import (Integer, CBool, CaselessStrEnum, Enum,
                                      List, Unicode, Instance, Type)
 from IPython.utils.warn import warn, error, fatal
@@ -2571,7 +2572,7 b' class InteractiveShell(SingletonConfigurable, Magic):'
     # Utilities
     #-------------------------------------------------------------------------
 
-    def var_expand(self,cmd,depth=0):
+    def var_expand(self, cmd, depth=0, formatter=DollarFormatter()):
         """Expand python variables in a string.
 
         The depth argument indicates how many frames above the caller should
@@ -2580,11 +2581,10 b' class InteractiveShell(SingletonConfigurable, Magic):'
         The global namespace for expansion is always the user's interactive
         namespace.
         """
-        res = ItplNS(cmd, self.user_ns,  # globals
-                          # Skip our own frame in searching for locals:
-                          sys._getframe(depth+1).f_locals # locals
-                          )
-        return py3compat.str_to_unicode(str(res), res.codec)
+        ns = self.user_ns.copy()
+        ns.update(sys._getframe(depth+1).f_locals)
+        ns.pop('self', None)
+        return formatter.format(cmd, **ns)
 
     def mktempfile(self, data=None, prefix='ipython_edit_'):
         """Make a new tempfile and return its filename.

IPython/utils/tests/test_text.py

@@ -44,9 +44,8 b' def test_columnize_long():'
     out = text.columnize(items, displaywidth=size-1)
     nt.assert_equals(out, '\n'.join(items+['']))
 
-def test_eval_formatter():
-    f = text.EvalFormatter()
-    ns = dict(n=12, pi=math.pi, stuff='hello there', os=os)
+def eval_formatter_check(f):
+    ns = dict(n=12, pi=math.pi, stuff='hello there', os=os, u=u"café", b="café")
     s = f.format("{n} {n//4} {stuff.split()[0]}", **ns)
     nt.assert_equals(s, "12 3 hello")
     s = f.format(' '.join(['{n//%i}'%i for i in range(1,8)]), **ns)
@@ -58,12 +57,15 b' def test_eval_formatter():'
     s = f.format("{stuff!r}", **ns)
     nt.assert_equals(s, repr(ns['stuff']))
     
+    # Check with unicode:
+    s = f.format("{u}", **ns)
+    nt.assert_equals(s, ns['u'])
+    # This decodes in a platform dependent manner, but it shouldn't error out
+    s = f.format("{b}", **ns)
+        
     nt.assert_raises(NameError, f.format, '{dne}', **ns)
 
-
-def test_eval_formatter_slicing():
-    f = text.EvalFormatter()
-    f.allow_slicing = True
+def eval_formatter_slicing_check(f):
     ns = dict(n=12, pi=math.pi, stuff='hello there', os=os)
     s = f.format(" {stuff.split()[:]} ", **ns)
     nt.assert_equals(s, " ['hello', 'there'] ")
@@ -75,9 +77,7 b' def test_eval_formatter_slicing():'
     nt.assert_raises(SyntaxError, f.format, "{n:x}", **ns)
     
 
-def test_eval_formatter_no_slicing():
-    f = text.EvalFormatter()
-    f.allow_slicing = False
+def eval_formatter_no_slicing_check(f):
     ns = dict(n=12, pi=math.pi, stuff='hello there', os=os)
     
     s = f.format('{n:x} {pi**2:+f}', **ns)
@@ -85,3 +85,25 b' def test_eval_formatter_no_slicing():'
     
     nt.assert_raises(SyntaxError, f.format, "{a[:]}")
 
+def test_eval_formatter():
+    f = text.EvalFormatter()
+    eval_formatter_check(f)
+    eval_formatter_no_slicing_check(f)
+
+def test_full_eval_formatter():
+    f = text.FullEvalFormatter()
+    eval_formatter_check(f)
+    eval_formatter_slicing_check(f)
+
+def test_dollar_formatter():
+    f = text.DollarFormatter()
+    eval_formatter_check(f)
+    eval_formatter_slicing_check(f)
+    
+    ns = dict(n=12, pi=math.pi, stuff='hello there', os=os)
+    s = f.format("$n", **ns)
+    nt.assert_equals(s, "12")
+    s = f.format("$n.real", **ns)
+    nt.assert_equals(s, "12")
+    s = f.format("$n/{stuff[:5]}", **ns)
+    nt.assert_equals(s, "12/hello")

IPython/utils/text.py

@@ -25,6 +25,7 b' import textwrap'
 from string import Formatter
 
 from IPython.external.path import path
+from IPython.testing.skipdoctest import skip_doctest_py3
 from IPython.utils import py3compat
 from IPython.utils.io import nlprint
 from IPython.utils.data import flatten
@@ -563,33 +564,53 b' def wrap_paragraphs(text, ncols=80):'
     return out_ps
 
 
-
 class EvalFormatter(Formatter):
     """A String Formatter that allows evaluation of simple expressions.
-
-    Any time a format key is not found in the kwargs,
-    it will be tried as an expression in the kwargs namespace.
-
+    
+    Note that this version interprets a : as specifying a format string (as per
+    standard string formatting), so if slicing is required, you must explicitly
+    create a slice.
+    
     This is to be used in templating cases, such as the parallel batch
     script templates, where simple arithmetic on arguments is useful.
 
     Examples
     --------
+    
+    In  [1]: f = EvalFormatter()
+    In  [2]: f.format('{n//4}', n=8)
+    Out [2]: '2'
+    
+    In  [3]: f.format("{greeting[slice(2,4)]}", greeting="Hello")
+    Out [3]: 'll'
+    """
+    def get_field(self, name, args, kwargs):
+        v = eval(name, kwargs)
+        return v, name
 
-    In [1]: f = EvalFormatter()
+@skip_doctest_py3
+class FullEvalFormatter(Formatter):
+    """A String Formatter that allows evaluation of simple expressions.
+    
+    Any time a format key is not found in the kwargs,
+    it will be tried as an expression in the kwargs namespace.
+    
+    Note that this version allows slicing using [1:2], so you cannot specify
+    a format string. Use :class:`EvalFormatter` to permit format strings.
+    
+    Examples
+    --------
+    
+    In [1]: f = FullEvalFormatter()
     In [2]: f.format('{n//4}', n=8)
-    Out[2]: '2'
-
-    In [3]: f.format('{list(range(3))}')
-    Out[3]: '[0, 1, 2]'
+    Out[2]: u'2'
+    
+    In [3]: f.format('{list(range(5))[2:4]}')
+    Out[3]: u'[2, 3]'
 
     In [4]: f.format('{3*2}')
-    Out[4]: '6'
+    Out[4]: u'6'
     """
-
-    # should we allow slicing by disabling the format_spec feature?
-    allow_slicing = True
-
     # copied from Formatter._vformat with minor changes to allow eval
     # and replace the format_spec code with slicing
     def _vformat(self, format_string, args, kwargs, used_args, recursion_depth):
@@ -606,12 +627,11 b' class EvalFormatter(Formatter):'
             # if there's a field, output it
             if field_name is not None:
                 # this is some markup, find the object and do
-                #  the formatting
+                # the formatting
 
-                if self.allow_slicing and format_spec:
+                if format_spec:
                     # override format spec, to allow slicing:
                     field_name = ':'.join([field_name, format_spec])
-                    format_spec = ''
 
                 # eval the contents of the field for the object
                 # to be formatted
@@ -620,14 +640,43 b' class EvalFormatter(Formatter):'
                 # do any conversion on the resulting object
                 obj = self.convert_field(obj, conversion)
 
-                # expand the format spec, if needed
-                format_spec = self._vformat(format_spec, args, kwargs,
-                                            used_args, recursion_depth-1)
-
                 # format the object and append to the result
-                result.append(self.format_field(obj, format_spec))
+                result.append(self.format_field(obj, ''))
+
+        return u''.join(py3compat.cast_unicode(s) for s in result)
 
-        return ''.join(result)
+@skip_doctest_py3
+class DollarFormatter(FullEvalFormatter):
+    """Formatter allowing Itpl style $foo replacement, for names and attribute
+    access only. Standard {foo} replacement also works, and allows full
+    evaluation of its arguments.
+
+    Examples
+    --------
+    In [1]: f = DollarFormatter()
+    In [2]: f.format('{n//4}', n=8)
+    Out[2]: u'2'
+    
+    In [3]: f.format('23 * 76 is $result', result=23*76)
+    Out[3]: u'23 * 76 is 1748'
+    
+    In [4]: f.format('$a or {b}', a=1, b=2)
+    Out[4]: u'1 or 2'
+    """
+    _dollar_pattern = re.compile("(.*)\$([\w\.]+)")
+    def parse(self, fmt_string):
+        for literal_txt, field_name, format_spec, conversion \
+                    in Formatter.parse(self, fmt_string):
+            
+            # Find $foo patterns in the literal text.
+            continue_from = 0
+            for m in self._dollar_pattern.finditer(literal_txt):
+                new_txt, new_field = m.group(1,2)
+                yield (new_txt, new_field, "", None)
+                continue_from = m.end()
+            
+            # Re-yield the {foo} style pattern
+            yield (literal_txt[continue_from:], field_name, format_spec, conversion)
 
 
 def columnize(items, separator='  ', displaywidth=80):