From 09c9952f330f9400d951e3612ab78821f5025586 2011-11-20 22:53:58 From: Fernando Perez Date: 2011-11-20 22:53:58 Subject: [PATCH] Merge pull request #1020 from takluyver/dollar-formatter Dollar formatter for ! shell calls. This uses a subclass of `FullEvalFormatter` for filling in variables in shell commands like `!echo $spam`, instead of Itpl. I've retained the `{foo}` style standard formatting for complex expressions, so it can be picked up by the more powerful built in parser. `$foo` works for names and attribute access. FullEvalFormatter now always outputs unicode, even if you gave it a bytes string. This avoids problems when you mix 8-bit non-ascii strings and unicode (which e.g. `StringIO.StringIO` suffers from). Closes #822 (test was added in 31ab23f). --- diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index ce2ebc3..eea851c 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -73,7 +73,8 @@ from IPython.utils.pickleshare import PickleShareDB from IPython.utils.process import system, getoutput from IPython.utils.strdispatch import StrDispatch from IPython.utils.syspathcontext import prepended_to_syspath -from IPython.utils.text import num_ini_spaces, format_screen, LSString, SList +from IPython.utils.text import (num_ini_spaces, format_screen, LSString, SList, + DollarFormatter) from IPython.utils.traitlets import (Integer, CBool, CaselessStrEnum, Enum, List, Unicode, Instance, Type) from IPython.utils.warn import warn, error, fatal @@ -2571,7 +2572,7 @@ class InteractiveShell(SingletonConfigurable, Magic): # Utilities #------------------------------------------------------------------------- - def var_expand(self,cmd,depth=0): + def var_expand(self, cmd, depth=0, formatter=DollarFormatter()): """Expand python variables in a string. The depth argument indicates how many frames above the caller should @@ -2580,11 +2581,10 @@ class InteractiveShell(SingletonConfigurable, Magic): The global namespace for expansion is always the user's interactive namespace. """ - res = ItplNS(cmd, self.user_ns, # globals - # Skip our own frame in searching for locals: - sys._getframe(depth+1).f_locals # locals - ) - return py3compat.str_to_unicode(str(res), res.codec) + ns = self.user_ns.copy() + ns.update(sys._getframe(depth+1).f_locals) + ns.pop('self', None) + return formatter.format(cmd, **ns) def mktempfile(self, data=None, prefix='ipython_edit_'): """Make a new tempfile and return its filename. diff --git a/IPython/utils/tests/test_text.py b/IPython/utils/tests/test_text.py index ba7f160..1d0b70a 100644 --- a/IPython/utils/tests/test_text.py +++ b/IPython/utils/tests/test_text.py @@ -44,9 +44,8 @@ def test_columnize_long(): out = text.columnize(items, displaywidth=size-1) nt.assert_equals(out, '\n'.join(items+[''])) -def test_eval_formatter(): - f = text.EvalFormatter() - ns = dict(n=12, pi=math.pi, stuff='hello there', os=os) +def eval_formatter_check(f): + ns = dict(n=12, pi=math.pi, stuff='hello there', os=os, u=u"café", b="café") s = f.format("{n} {n//4} {stuff.split()[0]}", **ns) nt.assert_equals(s, "12 3 hello") s = f.format(' '.join(['{n//%i}'%i for i in range(1,8)]), **ns) @@ -58,12 +57,15 @@ def test_eval_formatter(): s = f.format("{stuff!r}", **ns) nt.assert_equals(s, repr(ns['stuff'])) + # Check with unicode: + s = f.format("{u}", **ns) + nt.assert_equals(s, ns['u']) + # This decodes in a platform dependent manner, but it shouldn't error out + s = f.format("{b}", **ns) + nt.assert_raises(NameError, f.format, '{dne}', **ns) - -def test_eval_formatter_slicing(): - f = text.EvalFormatter() - f.allow_slicing = True +def eval_formatter_slicing_check(f): ns = dict(n=12, pi=math.pi, stuff='hello there', os=os) s = f.format(" {stuff.split()[:]} ", **ns) nt.assert_equals(s, " ['hello', 'there'] ") @@ -75,9 +77,7 @@ def test_eval_formatter_slicing(): nt.assert_raises(SyntaxError, f.format, "{n:x}", **ns) -def test_eval_formatter_no_slicing(): - f = text.EvalFormatter() - f.allow_slicing = False +def eval_formatter_no_slicing_check(f): ns = dict(n=12, pi=math.pi, stuff='hello there', os=os) s = f.format('{n:x} {pi**2:+f}', **ns) @@ -85,3 +85,25 @@ def test_eval_formatter_no_slicing(): nt.assert_raises(SyntaxError, f.format, "{a[:]}") +def test_eval_formatter(): + f = text.EvalFormatter() + eval_formatter_check(f) + eval_formatter_no_slicing_check(f) + +def test_full_eval_formatter(): + f = text.FullEvalFormatter() + eval_formatter_check(f) + eval_formatter_slicing_check(f) + +def test_dollar_formatter(): + f = text.DollarFormatter() + eval_formatter_check(f) + eval_formatter_slicing_check(f) + + ns = dict(n=12, pi=math.pi, stuff='hello there', os=os) + s = f.format("$n", **ns) + nt.assert_equals(s, "12") + s = f.format("$n.real", **ns) + nt.assert_equals(s, "12") + s = f.format("$n/{stuff[:5]}", **ns) + nt.assert_equals(s, "12/hello") diff --git a/IPython/utils/text.py b/IPython/utils/text.py index 4deb669..ba16e05 100644 --- a/IPython/utils/text.py +++ b/IPython/utils/text.py @@ -25,6 +25,7 @@ import textwrap from string import Formatter from IPython.external.path import path +from IPython.testing.skipdoctest import skip_doctest_py3 from IPython.utils import py3compat from IPython.utils.io import nlprint from IPython.utils.data import flatten @@ -563,33 +564,53 @@ def wrap_paragraphs(text, ncols=80): return out_ps - class EvalFormatter(Formatter): """A String Formatter that allows evaluation of simple expressions. - - Any time a format key is not found in the kwargs, - it will be tried as an expression in the kwargs namespace. - + + Note that this version interprets a : as specifying a format string (as per + standard string formatting), so if slicing is required, you must explicitly + create a slice. + This is to be used in templating cases, such as the parallel batch script templates, where simple arithmetic on arguments is useful. Examples -------- + + In [1]: f = EvalFormatter() + In [2]: f.format('{n//4}', n=8) + Out [2]: '2' + + In [3]: f.format("{greeting[slice(2,4)]}", greeting="Hello") + Out [3]: 'll' + """ + def get_field(self, name, args, kwargs): + v = eval(name, kwargs) + return v, name - In [1]: f = EvalFormatter() +@skip_doctest_py3 +class FullEvalFormatter(Formatter): + """A String Formatter that allows evaluation of simple expressions. + + Any time a format key is not found in the kwargs, + it will be tried as an expression in the kwargs namespace. + + Note that this version allows slicing using [1:2], so you cannot specify + a format string. Use :class:`EvalFormatter` to permit format strings. + + Examples + -------- + + In [1]: f = FullEvalFormatter() In [2]: f.format('{n//4}', n=8) - Out[2]: '2' - - In [3]: f.format('{list(range(3))}') - Out[3]: '[0, 1, 2]' + Out[2]: u'2' + + In [3]: f.format('{list(range(5))[2:4]}') + Out[3]: u'[2, 3]' In [4]: f.format('{3*2}') - Out[4]: '6' + Out[4]: u'6' """ - - # should we allow slicing by disabling the format_spec feature? - allow_slicing = True - # copied from Formatter._vformat with minor changes to allow eval # and replace the format_spec code with slicing def _vformat(self, format_string, args, kwargs, used_args, recursion_depth): @@ -606,12 +627,11 @@ class EvalFormatter(Formatter): # if there's a field, output it if field_name is not None: # this is some markup, find the object and do - # the formatting + # the formatting - if self.allow_slicing and format_spec: + if format_spec: # override format spec, to allow slicing: field_name = ':'.join([field_name, format_spec]) - format_spec = '' # eval the contents of the field for the object # to be formatted @@ -620,14 +640,43 @@ class EvalFormatter(Formatter): # do any conversion on the resulting object obj = self.convert_field(obj, conversion) - # expand the format spec, if needed - format_spec = self._vformat(format_spec, args, kwargs, - used_args, recursion_depth-1) - # format the object and append to the result - result.append(self.format_field(obj, format_spec)) + result.append(self.format_field(obj, '')) + + return u''.join(py3compat.cast_unicode(s) for s in result) - return ''.join(result) +@skip_doctest_py3 +class DollarFormatter(FullEvalFormatter): + """Formatter allowing Itpl style $foo replacement, for names and attribute + access only. Standard {foo} replacement also works, and allows full + evaluation of its arguments. + + Examples + -------- + In [1]: f = DollarFormatter() + In [2]: f.format('{n//4}', n=8) + Out[2]: u'2' + + In [3]: f.format('23 * 76 is $result', result=23*76) + Out[3]: u'23 * 76 is 1748' + + In [4]: f.format('$a or {b}', a=1, b=2) + Out[4]: u'1 or 2' + """ + _dollar_pattern = re.compile("(.*)\$([\w\.]+)") + def parse(self, fmt_string): + for literal_txt, field_name, format_spec, conversion \ + in Formatter.parse(self, fmt_string): + + # Find $foo patterns in the literal text. + continue_from = 0 + for m in self._dollar_pattern.finditer(literal_txt): + new_txt, new_field = m.group(1,2) + yield (new_txt, new_field, "", None) + continue_from = m.end() + + # Re-yield the {foo} style pattern + yield (literal_txt[continue_from:], field_name, format_spec, conversion) def columnize(items, separator=' ', displaywidth=80):