diff --git a/IPython/core/inputsplitter.py b/IPython/core/inputsplitter.py
index d836630..87fe9a5 100644
--- a/IPython/core/inputsplitter.py
+++ b/IPython/core/inputsplitter.py
@@ -74,6 +74,7 @@ import tokenize
from StringIO import StringIO
# IPython modules
+from IPython.core.splitinput import split_user_input, LineInfo
from IPython.utils.text import make_quoted_expr
from IPython.utils.py3compat import cast_unicode
@@ -482,132 +483,10 @@ class InputSplitter(object):
# Functions and classes for IPython-specific syntactic support
#-----------------------------------------------------------------------------
-# RegExp for splitting line contents into pre-char//first word-method//rest.
-# For clarity, each group in on one line.
-
-line_split = re.compile("""
- ^(\s*) # any leading space
- ([,;/%]|!!?|\?\??) # escape character or characters
- \s*(%?[\w\.\*]*) # function/method, possibly with leading %
- # to correctly treat things like '?%magic'
- (\s+.*$|$) # rest of line
- """, re.VERBOSE)
-
-
-def split_user_input(line):
- """Split user input into early whitespace, esc-char, function part and rest.
-
- This is currently handles lines with '=' in them in a very inconsistent
- manner.
-
- Examples
- ========
- >>> split_user_input('x=1')
- ('', '', 'x=1', '')
- >>> split_user_input('?')
- ('', '?', '', '')
- >>> split_user_input('??')
- ('', '??', '', '')
- >>> split_user_input(' ?')
- (' ', '?', '', '')
- >>> split_user_input(' ??')
- (' ', '??', '', '')
- >>> split_user_input('??x')
- ('', '??', 'x', '')
- >>> split_user_input('?x=1')
- ('', '', '?x=1', '')
- >>> split_user_input('!ls')
- ('', '!', 'ls', '')
- >>> split_user_input(' !ls')
- (' ', '!', 'ls', '')
- >>> split_user_input('!!ls')
- ('', '!!', 'ls', '')
- >>> split_user_input(' !!ls')
- (' ', '!!', 'ls', '')
- >>> split_user_input(',ls')
- ('', ',', 'ls', '')
- >>> split_user_input(';ls')
- ('', ';', 'ls', '')
- >>> split_user_input(' ;ls')
- (' ', ';', 'ls', '')
- >>> split_user_input('f.g(x)')
- ('', '', 'f.g(x)', '')
- >>> split_user_input('f.g (x)')
- ('', '', 'f.g', '(x)')
- >>> split_user_input('?%hist')
- ('', '?', '%hist', '')
- >>> split_user_input('?x*')
- ('', '?', 'x*', '')
- """
- match = line_split.match(line)
- if match:
- lspace, esc, fpart, rest = match.groups()
- else:
- # print "match failed for line '%s'" % line
- try:
- fpart, rest = line.split(None, 1)
- except ValueError:
- # print "split failed for line '%s'" % line
- fpart, rest = line,''
- lspace = re.match('^(\s*)(.*)', line).groups()[0]
- esc = ''
-
- # fpart has to be a valid python identifier, so it better be only pure
- # ascii, no unicode:
- try:
- fpart = fpart.encode('ascii')
- except UnicodeEncodeError:
- lspace = unicode(lspace)
- rest = fpart + u' ' + rest
- fpart = u''
-
- #print 'line:<%s>' % line # dbg
- #print 'esc <%s> fpart <%s> rest <%s>' % (esc,fpart.strip(),rest) # dbg
- return lspace, esc, fpart.strip(), rest.lstrip()
-
-
# The escaped translators ALL receive a line where their own escape has been
# stripped. Only '?' is valid at the end of the line, all others can only be
# placed at the start.
-class LineInfo(object):
- """A single line of input and associated info.
-
- This is a utility class that mostly wraps the output of
- :func:`split_user_input` into a convenient object to be passed around
- during input transformations.
-
- Includes the following as properties:
-
- line
- The original, raw line
-
- lspace
- Any early whitespace before actual text starts.
-
- esc
- The initial esc character (or characters, for double-char escapes like
- '??' or '!!').
-
- fpart
- The 'function part', which is basically the maximal initial sequence
- of valid python identifiers and the '.' character. This is what is
- checked for alias and magic transformations, used for auto-calling,
- etc.
-
- rest
- Everything else on the line.
- """
- def __init__(self, line):
- self.line = line
- self.lspace, self.esc, self.fpart, self.rest = \
- split_user_input(line)
-
- def __str__(self):
- return "LineInfo [%s|%s|%s|%s]" % (self.lspace, self.esc,
- self.fpart, self.rest)
-
-
# Transformations of the special syntaxes that don't rely on an explicit escape
# character but instead on patterns on the input line
@@ -690,10 +569,10 @@ def _make_help_call(target, esc, lspace, next_input=None):
_initial_space_re = re.compile(r'\s*')
_help_end_re = re.compile(r"""(%?
- [a-zA-Z_*][a-zA-Z0-9_*]* # Variable name
- (\.[a-zA-Z_*][a-zA-Z0-9_*]*)* # .etc.etc
+ [a-zA-Z_*][\w*]* # Variable name
+ (\.[a-zA-Z_*][\w*]*)* # .etc.etc
)
- (\?\??)$ # ? or ??""",
+ (\?\??)$ # ? or ??""",
re.VERBOSE)
def transform_help_end(line):
"""Translate lines with ?/?? at the end"""
@@ -703,7 +582,6 @@ def transform_help_end(line):
target = m.group(1)
esc = m.group(3)
lspace = _initial_space_re.match(line).group(0)
- newline = _make_help_call(target, esc, lspace)
# If we're mid-command, put it back on the next prompt for the user.
next_input = line.rstrip('?') if line.strip() != m.group(0) else None
@@ -731,14 +609,14 @@ class EscapedTransformer(object):
def _tr_system(line_info):
"Translate lines escaped with: !"
cmd = line_info.line.lstrip().lstrip(ESC_SHELL)
- return '%sget_ipython().system(%s)' % (line_info.lspace,
+ return '%sget_ipython().system(%s)' % (line_info.pre,
make_quoted_expr(cmd))
@staticmethod
def _tr_system2(line_info):
"Translate lines escaped with: !!"
cmd = line_info.line.lstrip()[2:]
- return '%sget_ipython().getoutput(%s)' % (line_info.lspace,
+ return '%sget_ipython().getoutput(%s)' % (line_info.pre,
make_quoted_expr(cmd))
@staticmethod
@@ -748,33 +626,33 @@ class EscapedTransformer(object):
if not line_info.line[1:]:
return 'get_ipython().show_usage()'
- return _make_help_call(line_info.fpart, line_info.esc, line_info.lspace)
+ return _make_help_call(line_info.ifun, line_info.esc, line_info.pre)
@staticmethod
def _tr_magic(line_info):
"Translate lines escaped with: %"
tpl = '%sget_ipython().magic(%s)'
- cmd = make_quoted_expr(' '.join([line_info.fpart,
- line_info.rest]).strip())
- return tpl % (line_info.lspace, cmd)
+ cmd = make_quoted_expr(' '.join([line_info.ifun,
+ line_info.the_rest]).strip())
+ return tpl % (line_info.pre, cmd)
@staticmethod
def _tr_quote(line_info):
"Translate lines escaped with: ,"
- return '%s%s("%s")' % (line_info.lspace, line_info.fpart,
- '", "'.join(line_info.rest.split()) )
+ return '%s%s("%s")' % (line_info.pre, line_info.ifun,
+ '", "'.join(line_info.the_rest.split()) )
@staticmethod
def _tr_quote2(line_info):
"Translate lines escaped with: ;"
- return '%s%s("%s")' % (line_info.lspace, line_info.fpart,
- line_info.rest)
+ return '%s%s("%s")' % (line_info.pre, line_info.ifun,
+ line_info.the_rest)
@staticmethod
def _tr_paren(line_info):
"Translate lines escaped with: /"
- return '%s%s(%s)' % (line_info.lspace, line_info.fpart,
- ", ".join(line_info.rest.split()))
+ return '%s%s(%s)' % (line_info.pre, line_info.ifun,
+ ", ".join(line_info.the_rest.split()))
def __call__(self, line):
"""Class to transform lines that are explicitly escaped out.
@@ -843,7 +721,7 @@ class IPythonInputSplitter(InputSplitter):
lines_list = lines.splitlines()
transforms = [transform_ipy_prompt, transform_classic_prompt,
- transform_escaped, transform_help_end,
+ transform_help_end, transform_escaped,
transform_assign_system, transform_assign_magic]
# Transform logic
diff --git a/IPython/core/prefilter.py b/IPython/core/prefilter.py
index cfaf212..c31b34b 100755
--- a/IPython/core/prefilter.py
+++ b/IPython/core/prefilter.py
@@ -32,7 +32,7 @@ from IPython.core.alias import AliasManager
from IPython.core.autocall import IPyAutocall
from IPython.config.configurable import Configurable
from IPython.core.macro import Macro
-from IPython.core.splitinput import split_user_input
+from IPython.core.splitinput import split_user_input, LineInfo
from IPython.core import page
from IPython.utils.traitlets import List, Int, Any, Unicode, CBool, Bool, Instance
@@ -92,78 +92,6 @@ def is_shadowed(identifier, ip):
#-----------------------------------------------------------------------------
-# The LineInfo class used throughout
-#-----------------------------------------------------------------------------
-
-
-class LineInfo(object):
- """A single line of input and associated info.
-
- Includes the following as properties:
-
- line
- The original, raw line
-
- continue_prompt
- Is this line a continuation in a sequence of multiline input?
-
- pre
- The initial esc character or whitespace.
-
- pre_char
- The escape character(s) in pre or the empty string if there isn't one.
- Note that '!!' is a possible value for pre_char. Otherwise it will
- always be a single character.
-
- pre_whitespace
- The leading whitespace from pre if it exists. If there is a pre_char,
- this is just ''.
-
- ifun
- The 'function part', which is basically the maximal initial sequence
- of valid python identifiers and the '.' character. This is what is
- checked for alias and magic transformations, used for auto-calling,
- etc.
-
- the_rest
- Everything else on the line.
- """
- def __init__(self, line, continue_prompt):
- self.line = line
- self.continue_prompt = continue_prompt
- self.pre, self.esc, self.ifun, self.the_rest = split_user_input(line)
-
- self.pre_char = self.pre.strip()
- if self.pre_char:
- self.pre_whitespace = '' # No whitespace allowd before esc chars
- else:
- self.pre_whitespace = self.pre
-
- self._oinfo = None
-
- def ofind(self, ip):
- """Do a full, attribute-walking lookup of the ifun in the various
- namespaces for the given IPython InteractiveShell instance.
-
- Return a dict with keys: found,obj,ospace,ismagic
-
- Note: can cause state changes because of calling getattr, but should
- only be run if autocall is on and if the line hasn't matched any
- other, less dangerous handlers.
-
- Does cache the results of the call, so can be called multiple times
- without worrying about *further* damaging state.
- """
- if not self._oinfo:
- # ip.shell._ofind is actually on the Magic class!
- self._oinfo = ip.shell._ofind(self.ifun)
- return self._oinfo
-
- def __str__(self):
- return "Lineinfo [%s|%s|%s]" %(self.pre, self.ifun, self.the_rest)
-
-
-#-----------------------------------------------------------------------------
# Main Prefilter manager
#-----------------------------------------------------------------------------
diff --git a/IPython/core/splitinput.py b/IPython/core/splitinput.py
index ca5bb02..bd04f6a 100755
--- a/IPython/core/splitinput.py
+++ b/IPython/core/splitinput.py
@@ -1,6 +1,7 @@
# encoding: utf-8
"""
-Simple utility for splitting user input.
+Simple utility for splitting user input. This is used by both inputsplitter and
+prefilter.
Authors:
@@ -28,36 +29,28 @@ from IPython.utils import py3compat
# Main function
#-----------------------------------------------------------------------------
-
# RegExp for splitting line contents into pre-char//first word-method//rest.
# For clarity, each group in on one line.
-# WARNING: update the regexp if the escapes in interactiveshell are changed, as they
-# are hardwired in.
+# WARNING: update the regexp if the escapes in interactiveshell are changed, as
+# they are hardwired in.
# Although it's not solely driven by the regex, note that:
# ,;/% only trigger if they are the first character on the line
# ! and !! trigger if they are first char(s) *or* follow an indent
# ? triggers as first or last char.
-# The four parts of the regex are:
-# 1) pre: initial whitespace
-# 2) esc: escape character
-# 3) ifun: first word/method (mix of \w and '.')
-# 4) the_rest: rest of line (separated from ifun by space if non-empty)
-line_split = re.compile(r'^(\s*)'
- r'([,;/%?]|!!?)?'
- r'\s*([\w\.]+)'
- r'(.*$|$)')
-
-# r'[\w\.]+'
-# r'\s*=\s*%.*'
+line_split = re.compile("""
+ ^(\s*) # any leading space
+ ([,;/%]|!!?|\?\??)? # escape character or characters
+ \s*(%?[\w\.\*]*) # function/method, possibly with leading %
+ # to correctly treat things like '?%magic'
+ (.*?$|$) # rest of line
+ """, re.VERBOSE)
def split_user_input(line, pattern=None):
- """Split user input into pre-char/whitespace, function part and rest.
-
- This is currently handles lines with '=' in them in a very inconsistent
- manner.
+ """Split user input into initial whitespace, escape character, function part
+ and the rest.
"""
# We need to ensure that the rest of this routine deals only with unicode
line = py3compat.cast_unicode(line, sys.stdin.encoding or 'utf-8')
@@ -76,11 +69,70 @@ def split_user_input(line, pattern=None):
esc = ""
else:
pre, esc, ifun, the_rest = match.groups()
-
- if not py3compat.isidentifier(ifun, dotted=True):
- the_rest = ifun + u' ' + the_rest
- ifun = u''
#print 'line:<%s>' % line # dbg
#print 'pre <%s> ifun <%s> rest <%s>' % (pre,ifun.strip(),the_rest) # dbg
- return pre, esc, ifun.strip(), the_rest.lstrip()
+ return pre, esc or '', ifun.strip(), the_rest.lstrip()
+
+class LineInfo(object):
+ """A single line of input and associated info.
+
+ Includes the following as properties:
+
+ line
+ The original, raw line
+
+ continue_prompt
+ Is this line a continuation in a sequence of multiline input?
+
+ pre
+ Any leading whitespace.
+
+ esc
+ The escape character(s) in pre or the empty string if there isn't one.
+ Note that '!!' and '??' are possible values for esc. Otherwise it will
+ always be a single character.
+
+ ifun
+ The 'function part', which is basically the maximal initial sequence
+ of valid python identifiers and the '.' character. This is what is
+ checked for alias and magic transformations, used for auto-calling,
+ etc. In contrast to Python identifiers, it may start with "%" and contain
+ "*".
+
+ the_rest
+ Everything else on the line.
+ """
+ def __init__(self, line, continue_prompt=False):
+ self.line = line
+ self.continue_prompt = continue_prompt
+ self.pre, self.esc, self.ifun, self.the_rest = split_user_input(line)
+
+ self.pre_char = self.pre.strip()
+ if self.pre_char:
+ self.pre_whitespace = '' # No whitespace allowd before esc chars
+ else:
+ self.pre_whitespace = self.pre
+
+ self._oinfo = None
+
+ def ofind(self, ip):
+ """Do a full, attribute-walking lookup of the ifun in the various
+ namespaces for the given IPython InteractiveShell instance.
+
+ Return a dict with keys: found,obj,ospace,ismagic
+
+ Note: can cause state changes because of calling getattr, but should
+ only be run if autocall is on and if the line hasn't matched any
+ other, less dangerous handlers.
+
+ Does cache the results of the call, so can be called multiple times
+ without worrying about *further* damaging state.
+ """
+ if not self._oinfo:
+ # ip.shell._ofind is actually on the Magic class!
+ self._oinfo = ip.shell._ofind(self.ifun)
+ return self._oinfo
+
+ def __str__(self):
+ return "LineInfo [%s|%s|%s|%s]" %(self.pre, self.esc, self.ifun, self.the_rest)
diff --git a/IPython/core/tests/test_inputsplitter.py b/IPython/core/tests/test_inputsplitter.py
index 639a5eb..4335445 100644
--- a/IPython/core/tests/test_inputsplitter.py
+++ b/IPython/core/tests/test_inputsplitter.py
@@ -394,9 +394,8 @@ def test_LineInfo():
def test_split_user_input():
"""Unicode test - split_user_input already has good doctests"""
line = u"Pérez Fernando"
- parts = isp.split_user_input(line)
parts_expected = (u'', u'', u'', line)
- nt.assert_equal(parts, parts_expected)
+ tt.check_pairs(isp.split_user_input, [(line, parts_expected),])
# Transformer tests
@@ -611,7 +610,8 @@ class IPythonInputTestCase(InputSplitterTestCase):
isp.push(raw)
out, out_raw = isp.source_raw_reset()
- self.assertEqual(out.rstrip(), out_t)
+ self.assertEqual(out.rstrip(), out_t,
+ tt.pair_fail_msg.format("inputsplitter",raw, out_t, out))
self.assertEqual(out_raw.rstrip(), raw.rstrip())
def test_syntax_multiline(self):
diff --git a/IPython/core/tests/test_splitinput.py b/IPython/core/tests/test_splitinput.py
new file mode 100644
index 0000000..a5eb828
--- /dev/null
+++ b/IPython/core/tests/test_splitinput.py
@@ -0,0 +1,26 @@
+from IPython.core.splitinput import split_user_input
+from IPython.testing import tools as tt
+
+tests = [
+ ('x=1', ('', '', 'x', '=1')),
+ ('?', ('', '?', '', '')),
+ ('??', ('', '??', '', '')),
+ (' ?', (' ', '?', '', '')),
+ (' ??', (' ', '??', '', '')),
+ ('??x', ('', '??', 'x', '')),
+ ('?x=1', ('', '?', 'x', '=1')),
+ ('!ls', ('', '!', 'ls', '')),
+ (' !ls', (' ', '!', 'ls', '')),
+ ('!!ls', ('', '!!', 'ls', '')),
+ (' !!ls', (' ', '!!', 'ls', '')),
+ (',ls', ('', ',', 'ls', '')),
+ (';ls', ('', ';', 'ls', '')),
+ (' ;ls', (' ', ';', 'ls', '')),
+ ('f.g(x)', ('', '', 'f.g', '(x)')),
+ ('f.g (x)', ('', '', 'f.g', '(x)')),
+ ('?%hist', ('', '?', '%hist', '')),
+ ('?x*', ('', '?', 'x*', '')),
+ ]
+
+def test_split_user_input():
+ return tt.check_pairs(split_user_input, tests)
diff --git a/IPython/testing/tools.py b/IPython/testing/tools.py
index a611814..9750575 100644
--- a/IPython/testing/tools.py
+++ b/IPython/testing/tools.py
@@ -295,7 +295,7 @@ class TempFileMixin(object):
# delete it. I have no clue why
pass
-pair_fail_msg = ("Testing function {0}\n\n"
+pair_fail_msg = ("Testing {0}\n\n"
"In:\n"
" {1!r}\n"
"Expected:\n"
@@ -318,9 +318,10 @@ def check_pairs(func, pairs):
None. Raises an AssertionError if any output does not match the expected
value.
"""
+ name = getattr(func, "func_name", getattr(func, "__name__", "<unknown>"))
for inp, expected in pairs:
out = func(inp)
- assert out == expected, pair_fail_msg.format(func.func_name, inp, expected, out)
+ assert out == expected, pair_fail_msg.format(name, inp, expected, out)
@contextmanager
def mute_warn():
@@ -342,4 +343,3 @@ def make_tempfile(name):
yield
finally:
os.unlink(name)
-