inputtransformer.py
323 lines
| 10.0 KiB
| text/x-python
|
PythonLexer
Thomas Kluyver
|
r10090 | import abc | ||
import re | ||||
Thomas Kluyver
|
r10093 | from StringIO import StringIO | ||
import tokenize | ||||
Thomas Kluyver
|
r10090 | |||
from IPython.core.splitinput import split_user_input, LineInfo | ||||
Thomas Kluyver
|
r10093 | |||
#----------------------------------------------------------------------------- | ||||
# Globals | ||||
#----------------------------------------------------------------------------- | ||||
# The escape sequences that define the syntax transformations IPython will | ||||
# apply to user input. These can NOT be just changed here: many regular | ||||
# expressions and other parts of the code may use their hardcoded values, and | ||||
# for all intents and purposes they constitute the 'IPython syntax', so they | ||||
# should be considered fixed. | ||||
ESC_SHELL = '!' # Send line to underlying system shell | ||||
ESC_SH_CAP = '!!' # Send line to system shell and capture output | ||||
ESC_HELP = '?' # Find information about object | ||||
ESC_HELP2 = '??' # Find extra-detailed information about object | ||||
ESC_MAGIC = '%' # Call magic function | ||||
ESC_MAGIC2 = '%%' # Call cell-magic function | ||||
ESC_QUOTE = ',' # Split args on whitespace, quote each as string and call | ||||
ESC_QUOTE2 = ';' # Quote all args as a single string, call | ||||
ESC_PAREN = '/' # Call first argument with rest of line as arguments | ||||
ESC_SEQUENCES = [ESC_SHELL, ESC_SH_CAP, ESC_HELP ,\ | ||||
ESC_HELP2, ESC_MAGIC, ESC_MAGIC2,\ | ||||
ESC_QUOTE, ESC_QUOTE2, ESC_PAREN ] | ||||
Thomas Kluyver
|
r10090 | |||
class InputTransformer(object): | ||||
__metaclass__ = abc.ABCMeta | ||||
@abc.abstractmethod | ||||
def push(self, line): | ||||
pass | ||||
@abc.abstractmethod | ||||
def reset(self): | ||||
pass | ||||
Thomas Kluyver
|
r10091 | |||
look_in_string = False | ||||
Thomas Kluyver
|
r10090 | |||
class StatelessInputTransformer(InputTransformer): | ||||
"""Decorator for a stateless input transformer implemented as a function.""" | ||||
def __init__(self, func): | ||||
self.func = func | ||||
def push(self, line): | ||||
return self.func(line) | ||||
def reset(self): | ||||
pass | ||||
class CoroutineInputTransformer(InputTransformer): | ||||
"""Decorator for an input transformer implemented as a coroutine.""" | ||||
def __init__(self, coro): | ||||
# Prime it | ||||
self.coro = coro() | ||||
next(self.coro) | ||||
def push(self, line): | ||||
return self.coro.send(line) | ||||
def reset(self): | ||||
Thomas Kluyver
|
r10096 | return self.coro.send(None) | ||
Thomas Kluyver
|
r10090 | |||
Thomas Kluyver
|
r10093 | |||
# Utilities | ||||
def _make_help_call(target, esc, lspace, next_input=None): | ||||
"""Prepares a pinfo(2)/psearch call from a target name and the escape | ||||
(i.e. ? or ??)""" | ||||
method = 'pinfo2' if esc == '??' \ | ||||
else 'psearch' if '*' in target \ | ||||
else 'pinfo' | ||||
arg = " ".join([method, target]) | ||||
if next_input is None: | ||||
return '%sget_ipython().magic(%r)' % (lspace, arg) | ||||
else: | ||||
return '%sget_ipython().set_next_input(%r);get_ipython().magic(%r)' % \ | ||||
(lspace, next_input, arg) | ||||
Thomas Kluyver
|
r10090 | @CoroutineInputTransformer | ||
def escaped_transformer(): | ||||
Thomas Kluyver
|
r10093 | """Translate lines beginning with one of IPython's escape characters.""" | ||
# These define the transformations for the different escape characters. | ||||
def _tr_system(line_info): | ||||
"Translate lines escaped with: !" | ||||
cmd = line_info.line.lstrip().lstrip(ESC_SHELL) | ||||
return '%sget_ipython().system(%r)' % (line_info.pre, cmd) | ||||
def _tr_system2(line_info): | ||||
"Translate lines escaped with: !!" | ||||
cmd = line_info.line.lstrip()[2:] | ||||
return '%sget_ipython().getoutput(%r)' % (line_info.pre, cmd) | ||||
def _tr_help(line_info): | ||||
"Translate lines escaped with: ?/??" | ||||
# A naked help line should just fire the intro help screen | ||||
if not line_info.line[1:]: | ||||
return 'get_ipython().show_usage()' | ||||
return _make_help_call(line_info.ifun, line_info.esc, line_info.pre) | ||||
def _tr_magic(line_info): | ||||
"Translate lines escaped with: %" | ||||
tpl = '%sget_ipython().magic(%r)' | ||||
cmd = ' '.join([line_info.ifun, line_info.the_rest]).strip() | ||||
return tpl % (line_info.pre, cmd) | ||||
def _tr_quote(line_info): | ||||
"Translate lines escaped with: ," | ||||
return '%s%s("%s")' % (line_info.pre, line_info.ifun, | ||||
'", "'.join(line_info.the_rest.split()) ) | ||||
def _tr_quote2(line_info): | ||||
"Translate lines escaped with: ;" | ||||
return '%s%s("%s")' % (line_info.pre, line_info.ifun, | ||||
line_info.the_rest) | ||||
def _tr_paren(line_info): | ||||
"Translate lines escaped with: /" | ||||
return '%s%s(%s)' % (line_info.pre, line_info.ifun, | ||||
", ".join(line_info.the_rest.split())) | ||||
tr = { ESC_SHELL : _tr_system, | ||||
ESC_SH_CAP : _tr_system2, | ||||
ESC_HELP : _tr_help, | ||||
ESC_HELP2 : _tr_help, | ||||
ESC_MAGIC : _tr_magic, | ||||
ESC_QUOTE : _tr_quote, | ||||
ESC_QUOTE2 : _tr_quote2, | ||||
ESC_PAREN : _tr_paren } | ||||
Thomas Kluyver
|
r10090 | line = '' | ||
while True: | ||||
line = (yield line) | ||||
if not line or line.isspace(): | ||||
continue | ||||
lineinf = LineInfo(line) | ||||
Thomas Kluyver
|
r10093 | if lineinf.esc not in tr: | ||
Thomas Kluyver
|
r10090 | continue | ||
parts = [] | ||||
while line is not None: | ||||
parts.append(line.rstrip('\\')) | ||||
if not line.endswith('\\'): | ||||
break | ||||
line = (yield None) | ||||
# Output | ||||
lineinf = LineInfo(' '.join(parts)) | ||||
Thomas Kluyver
|
r10093 | line = tr[lineinf.esc](lineinf) | ||
Thomas Kluyver
|
r10090 | |||
_initial_space_re = re.compile(r'\s*') | ||||
_help_end_re = re.compile(r"""(%{0,2} | ||||
[a-zA-Z_*][\w*]* # Variable name | ||||
(\.[a-zA-Z_*][\w*]*)* # .etc.etc | ||||
) | ||||
(\?\??)$ # ? or ??""", | ||||
re.VERBOSE) | ||||
Thomas Kluyver
|
r10093 | def has_comment(src): | ||
"""Indicate whether an input line has (i.e. ends in, or is) a comment. | ||||
This uses tokenize, so it can distinguish comments from # inside strings. | ||||
Parameters | ||||
---------- | ||||
src : string | ||||
A single line input string. | ||||
Returns | ||||
------- | ||||
Boolean: True if source has a comment. | ||||
""" | ||||
readline = StringIO(src).readline | ||||
toktypes = set() | ||||
try: | ||||
for t in tokenize.generate_tokens(readline): | ||||
toktypes.add(t[0]) | ||||
except tokenize.TokenError: | ||||
pass | ||||
return(tokenize.COMMENT in toktypes) | ||||
Thomas Kluyver
|
r10090 | @StatelessInputTransformer | ||
Thomas Kluyver
|
r10093 | def help_end(line): | ||
Thomas Kluyver
|
r10090 | """Translate lines with ?/?? at the end""" | ||
m = _help_end_re.search(line) | ||||
if m is None or has_comment(line): | ||||
return line | ||||
target = m.group(1) | ||||
esc = m.group(3) | ||||
lspace = _initial_space_re.match(line).group(0) | ||||
# If we're mid-command, put it back on the next prompt for the user. | ||||
next_input = line.rstrip('?') if line.strip() != m.group(0) else None | ||||
return _make_help_call(target, esc, lspace, next_input) | ||||
@CoroutineInputTransformer | ||||
def cellmagic(): | ||||
tpl = 'get_ipython().run_cell_magic(%r, %r, %r)' | ||||
line = '' | ||||
while True: | ||||
line = (yield line) | ||||
Thomas Kluyver
|
r10091 | if (not line) or (not line.startswith(ESC_MAGIC2)): | ||
Thomas Kluyver
|
r10090 | continue | ||
first = line | ||||
body = [] | ||||
line = (yield None) | ||||
while (line is not None) and (line.strip() != ''): | ||||
body.append(line) | ||||
line = (yield None) | ||||
# Output | ||||
magic_name, _, first = first.partition(' ') | ||||
magic_name = magic_name.lstrip(ESC_MAGIC2) | ||||
line = tpl % (magic_name, first, '\n'.join(body)) | ||||
Thomas Kluyver
|
r10091 | |||
def _strip_prompts(prompt1_re, prompt2_re): | ||||
"""Remove matching input prompts from a block of input.""" | ||||
line = '' | ||||
while True: | ||||
line = (yield line) | ||||
if line is None: | ||||
continue | ||||
m = prompt1_re.match(line) | ||||
if m: | ||||
while m: | ||||
line = (yield line[len(m.group(0)):]) | ||||
if line is None: | ||||
break | ||||
m = prompt2_re.match(line) | ||||
else: | ||||
# Prompts not in input - wait for reset | ||||
while line is not None: | ||||
line = (yield line) | ||||
@CoroutineInputTransformer | ||||
def classic_prompt(): | ||||
prompt1_re = re.compile(r'^(>>> )') | ||||
prompt2_re = re.compile(r'^(>>> |^\.\.\. )') | ||||
return _strip_prompts(prompt1_re, prompt2_re) | ||||
classic_prompt.look_in_string = True | ||||
@CoroutineInputTransformer | ||||
def ipy_prompt(): | ||||
prompt1_re = re.compile(r'^In \[\d+\]: ') | ||||
prompt2_re = re.compile(r'^(In \[\d+\]: |^\ \ \ \.\.\.+: )') | ||||
return _strip_prompts(prompt1_re, prompt2_re) | ||||
ipy_prompt.look_in_string = True | ||||
@CoroutineInputTransformer | ||||
def leading_indent(): | ||||
space_re = re.compile(r'^[ \t]+') | ||||
line = '' | ||||
while True: | ||||
line = (yield line) | ||||
if line is None: | ||||
continue | ||||
m = space_re.match(line) | ||||
if m: | ||||
space = m.group(0) | ||||
while line is not None: | ||||
if line.startswith(space): | ||||
line = line[len(space):] | ||||
line = (yield line) | ||||
else: | ||||
# No leading spaces - wait for reset | ||||
while line is not None: | ||||
line = (yield line) | ||||
leading_indent.look_in_string = True | ||||
Thomas Kluyver
|
r10092 | |||
def _special_assignment(assignment_re, template): | ||||
line = '' | ||||
while True: | ||||
line = (yield line) | ||||
if not line or line.isspace(): | ||||
continue | ||||
m = assignment_re.match(line) | ||||
if not m: | ||||
continue | ||||
parts = [] | ||||
while line is not None: | ||||
parts.append(line.rstrip('\\')) | ||||
if not line.endswith('\\'): | ||||
break | ||||
line = (yield None) | ||||
# Output | ||||
whole = assignment_re.match(' '.join(parts)) | ||||
line = template % (whole.group('lhs'), whole.group('cmd')) | ||||
@CoroutineInputTransformer | ||||
def assign_from_system(): | ||||
assignment_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))' | ||||
r'\s*=\s*!\s*(?P<cmd>.*)') | ||||
template = '%s = get_ipython().getoutput(%r)' | ||||
return _special_assignment(assignment_re, template) | ||||
@CoroutineInputTransformer | ||||
def assign_from_magic(): | ||||
assignment_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))' | ||||
r'\s*=\s*%\s*(?P<cmd>.*)') | ||||
template = '%s = get_ipython().magic(%r)' | ||||
return _special_assignment(assignment_re, template) | ||||