From bccc0e22b96876185fc6afcad3153953846fe75c 2013-03-31 09:03:54 From: Thomas Kluyver Date: 2013-03-31 09:03:54 Subject: [PATCH] Simplify input transformers More can now be implemented as stateless transformers --- diff --git a/IPython/core/inputsplitter.py b/IPython/core/inputsplitter.py index f6401fe..eebbb7e 100644 --- a/IPython/core/inputsplitter.py +++ b/IPython/core/inputsplitter.py @@ -79,7 +79,7 @@ from IPython.core.inputtransformer import (leading_indent, cellmagic, assemble_logical_lines, help_end, - escaped_transformer, + escaped_commands, assign_from_magic, assign_from_system, assemble_python_lines, @@ -526,7 +526,7 @@ class IPythonInputSplitter(InputSplitter): self.logical_line_transforms = logical_line_transforms or \ [cellmagic(), help_end(), - escaped_transformer(), + escaped_commands(), assign_from_magic(), assign_from_system(), ] diff --git a/IPython/core/inputtransformer.py b/IPython/core/inputtransformer.py index e34f2bb..63bfeaf 100644 --- a/IPython/core/inputtransformer.py +++ b/IPython/core/inputtransformer.py @@ -61,9 +61,6 @@ class InputTransformer(object): """ pass - # Set this to True to allow the transformer to act on lines inside strings. - look_in_string = False - @classmethod def wrap(cls, func): """Can be used by subclasses as a decorator, to return a factory that @@ -71,10 +68,7 @@ class InputTransformer(object): """ @functools.wraps(func) def transformer_factory(): - transformer = cls(func) - if getattr(transformer_factory, 'look_in_string', False): - transformer.look_in_string = True - return transformer + return cls(func) return transformer_factory @@ -214,83 +208,67 @@ def _make_help_call(target, esc, lspace, next_input=None): else: return '%sget_ipython().set_next_input(%r);get_ipython().magic(%r)' % \ (lspace, next_input, arg) - -@CoroutineInputTransformer.wrap -def escaped_transformer(): - """Translate lines beginning with one of IPython's escape characters. - This is stateful to allow magic commands etc. to be continued over several - lines using explicit line continuations (\ at the end of a line). +# These define the transformations for the different escape characters. +def _tr_system(line_info): + "Translate lines escaped with: !" + cmd = line_info.line.lstrip().lstrip(ESC_SHELL) + return '%sget_ipython().system(%r)' % (line_info.pre, cmd) + +def _tr_system2(line_info): + "Translate lines escaped with: !!" + cmd = line_info.line.lstrip()[2:] + return '%sget_ipython().getoutput(%r)' % (line_info.pre, cmd) + +def _tr_help(line_info): + "Translate lines escaped with: ?/??" + # A naked help line should just fire the intro help screen + if not line_info.line[1:]: + return 'get_ipython().show_usage()' + + return _make_help_call(line_info.ifun, line_info.esc, line_info.pre) + +def _tr_magic(line_info): + "Translate lines escaped with: %" + tpl = '%sget_ipython().magic(%r)' + cmd = ' '.join([line_info.ifun, line_info.the_rest]).strip() + return tpl % (line_info.pre, cmd) + +def _tr_quote(line_info): + "Translate lines escaped with: ," + return '%s%s("%s")' % (line_info.pre, line_info.ifun, + '", "'.join(line_info.the_rest.split()) ) + +def _tr_quote2(line_info): + "Translate lines escaped with: ;" + return '%s%s("%s")' % (line_info.pre, line_info.ifun, + line_info.the_rest) + +def _tr_paren(line_info): + "Translate lines escaped with: /" + return '%s%s(%s)' % (line_info.pre, line_info.ifun, + ", ".join(line_info.the_rest.split())) + +tr = { ESC_SHELL : _tr_system, + ESC_SH_CAP : _tr_system2, + ESC_HELP : _tr_help, + ESC_HELP2 : _tr_help, + ESC_MAGIC : _tr_magic, + ESC_QUOTE : _tr_quote, + ESC_QUOTE2 : _tr_quote2, + ESC_PAREN : _tr_paren } + +@StatelessInputTransformer.wrap +def escaped_commands(line): + """Transform escaped commands - %magic, !system, ?help + various autocalls. """ + if not line or line.isspace(): + return line + lineinf = LineInfo(line) + if lineinf.esc not in tr: + return line - # These define the transformations for the different escape characters. - def _tr_system(line_info): - "Translate lines escaped with: !" - cmd = line_info.line.lstrip().lstrip(ESC_SHELL) - return '%sget_ipython().system(%r)' % (line_info.pre, cmd) - - def _tr_system2(line_info): - "Translate lines escaped with: !!" - cmd = line_info.line.lstrip()[2:] - return '%sget_ipython().getoutput(%r)' % (line_info.pre, cmd) - - def _tr_help(line_info): - "Translate lines escaped with: ?/??" - # A naked help line should just fire the intro help screen - if not line_info.line[1:]: - return 'get_ipython().show_usage()' - - return _make_help_call(line_info.ifun, line_info.esc, line_info.pre) - - def _tr_magic(line_info): - "Translate lines escaped with: %" - tpl = '%sget_ipython().magic(%r)' - cmd = ' '.join([line_info.ifun, line_info.the_rest]).strip() - return tpl % (line_info.pre, cmd) - - def _tr_quote(line_info): - "Translate lines escaped with: ," - return '%s%s("%s")' % (line_info.pre, line_info.ifun, - '", "'.join(line_info.the_rest.split()) ) - - def _tr_quote2(line_info): - "Translate lines escaped with: ;" - return '%s%s("%s")' % (line_info.pre, line_info.ifun, - line_info.the_rest) - - def _tr_paren(line_info): - "Translate lines escaped with: /" - return '%s%s(%s)' % (line_info.pre, line_info.ifun, - ", ".join(line_info.the_rest.split())) - - tr = { ESC_SHELL : _tr_system, - ESC_SH_CAP : _tr_system2, - ESC_HELP : _tr_help, - ESC_HELP2 : _tr_help, - ESC_MAGIC : _tr_magic, - ESC_QUOTE : _tr_quote, - ESC_QUOTE2 : _tr_quote2, - ESC_PAREN : _tr_paren } - - line = '' - while True: - line = (yield line) - if not line or line.isspace(): - continue - lineinf = LineInfo(line) - if lineinf.esc not in tr: - continue - - parts = [] - while line is not None: - parts.append(line.rstrip('\\')) - if not line.endswith('\\'): - break - line = (yield None) - - # Output - lineinf = LineInfo(' '.join(parts)) - line = tr[lineinf.esc](lineinf) + return tr[lineinf.esc](lineinf) _initial_space_re = re.compile(r'\s*') @@ -401,8 +379,6 @@ def classic_prompt(): prompt2_re = re.compile(r'^(>>> |^\.\.\. )') return _strip_prompts(prompt1_re, prompt2_re) -classic_prompt.look_in_string = True - @CoroutineInputTransformer.wrap def ipy_prompt(): """Strip IPython's In [1]:/...: prompts.""" @@ -410,8 +386,6 @@ def ipy_prompt(): prompt2_re = re.compile(r'^(In \[\d+\]: |^\ \ \ \.\.\.+: )') return _strip_prompts(prompt1_re, prompt2_re) -ipy_prompt.look_in_string = True - @CoroutineInputTransformer.wrap def leading_indent(): @@ -440,48 +414,27 @@ def leading_indent(): while line is not None: line = (yield line) -leading_indent.look_in_string = True - -def _special_assignment(assignment_re, template): - """Transform assignment from system & magic commands. - - This is stateful so that it can handle magic commands continued on several - lines. - """ - line = '' - while True: - line = (yield line) - if not line or line.isspace(): - continue - - m = assignment_re.match(line) - if not m: - continue - - parts = [] - while line is not None: - parts.append(line.rstrip('\\')) - if not line.endswith('\\'): - break - line = (yield None) - - # Output - whole = assignment_re.match(' '.join(parts)) - line = template % (whole.group('lhs'), whole.group('cmd')) - -@CoroutineInputTransformer.wrap -def assign_from_system(): +assign_system_re = re.compile(r'(?P(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))' + r'\s*=\s*!\s*(?P.*)') +assign_system_template = '%s = get_ipython().getoutput(%r)' +@StatelessInputTransformer.wrap +def assign_from_system(line): """Transform assignment from system commands (e.g. files = !ls)""" - assignment_re = re.compile(r'(?P(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))' - r'\s*=\s*!\s*(?P.*)') - template = '%s = get_ipython().getoutput(%r)' - return _special_assignment(assignment_re, template) + m = assign_system_re.match(line) + if m is None: + return line + + return assign_system_template % m.group('lhs', 'cmd') -@CoroutineInputTransformer.wrap -def assign_from_magic(): +assign_magic_re = re.compile(r'(?P(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))' + r'\s*=\s*%\s*(?P.*)') +assign_magic_template = '%s = get_ipython().magic(%r)' +@StatelessInputTransformer.wrap +def assign_from_magic(line): """Transform assignment from magic commands (e.g. a = %who_ls)""" - assignment_re = re.compile(r'(?P(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))' - r'\s*=\s*%\s*(?P.*)') - template = '%s = get_ipython().magic(%r)' - return _special_assignment(assignment_re, template) + m = assign_magic_re.match(line) + if m is None: + return line + + return assign_magic_template % m.group('lhs', 'cmd') diff --git a/IPython/core/tests/test_inputtransformer.py b/IPython/core/tests/test_inputtransformer.py index aad3abd..48f15bf 100644 --- a/IPython/core/tests/test_inputtransformer.py +++ b/IPython/core/tests/test_inputtransformer.py @@ -254,14 +254,9 @@ syntax_ml = \ def test_assign_system(): tt.check_pairs(transform_and_reset(ipt.assign_from_system), syntax['assign_system']) - for example in syntax_ml['assign_system']: - transform_checker(example, ipt.assign_from_system) def test_assign_magic(): tt.check_pairs(transform_and_reset(ipt.assign_from_magic), syntax['assign_magic']) - for example in syntax_ml['assign_magic']: - transform_checker(example, ipt.assign_from_magic) - def test_classic_prompt(): tt.check_pairs(transform_and_reset(ipt.classic_prompt), syntax['classic_prompt']) @@ -276,39 +271,70 @@ def test_ipy_prompt(): for example in syntax_ml['ipy_prompt']: transform_checker(example, ipt.ipy_prompt) +def test_assemble_logical_lines(): + tests = \ + [ [(u"a = \\", None), + (u"123", u"a = 123"), + ], + [(u"a = \\", None), # Test resetting when within a multi-line string + (u"12 *\\", None), + (None, u"a = 12 *"), + ], + ] + for example in tests: + transform_checker(example, ipt.assemble_logical_lines) + +def test_assemble_python_lines(): + tests = \ + [ [(u"a = '''", None), + (u"abc'''", u"a = '''\nabc'''"), + ], + [(u"a = '''", None), # Test resetting when within a multi-line string + (u"def", None), + (None, u"a = '''\ndef"), + ], + [(u"a = [1,", None), + (u"2]", u"a = [1,\n2]"), + ], + [(u"a = [1,", None), # Test resetting when within a multi-line string + (u"2,", None), + (None, u"a = [1,\n2,"), + ], + ] + for example in tests: + transform_checker(example, ipt.assemble_python_lines) + + def test_help_end(): tt.check_pairs(transform_and_reset(ipt.help_end), syntax['end_help']) def test_escaped_noesc(): - tt.check_pairs(transform_and_reset(ipt.escaped_transformer), syntax['escaped_noesc']) + tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_noesc']) def test_escaped_shell(): - tt.check_pairs(transform_and_reset(ipt.escaped_transformer), syntax['escaped_shell']) + tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_shell']) def test_escaped_help(): - tt.check_pairs(transform_and_reset(ipt.escaped_transformer), syntax['escaped_help']) + tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_help']) def test_escaped_magic(): - tt.check_pairs(transform_and_reset(ipt.escaped_transformer), syntax['escaped_magic']) + tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_magic']) def test_escaped_quote(): - tt.check_pairs(transform_and_reset(ipt.escaped_transformer), syntax['escaped_quote']) + tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_quote']) def test_escaped_quote2(): - tt.check_pairs(transform_and_reset(ipt.escaped_transformer), syntax['escaped_quote2']) + tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_quote2']) def test_escaped_paren(): - tt.check_pairs(transform_and_reset(ipt.escaped_transformer), syntax['escaped_paren']) + tt.check_pairs(transform_and_reset(ipt.escaped_commands), syntax['escaped_paren']) -def test_escaped_multiline(): - for example in syntax_ml['escaped']: - transform_checker(example, ipt.escaped_transformer) def test_cellmagic(): for example in syntax_ml['cellmagic']: