##// END OF EJS Templates
Reuse common code for inputsplitter and prefilter.
Thomas Kluyver -
r4746:76b84816
parent child
Show More
@@ -0,0 +1,26
1 from IPython.core.splitinput import split_user_input
2 from IPython.testing import tools as tt
3
4 tests = [
5 ('x=1', ('', '', 'x', '=1')),
6 ('?', ('', '?', '', '')),
7 ('??', ('', '??', '', '')),
8 (' ?', (' ', '?', '', '')),
9 (' ??', (' ', '??', '', '')),
10 ('??x', ('', '??', 'x', '')),
11 ('?x=1', ('', '?', 'x', '=1')),
12 ('!ls', ('', '!', 'ls', '')),
13 (' !ls', (' ', '!', 'ls', '')),
14 ('!!ls', ('', '!!', 'ls', '')),
15 (' !!ls', (' ', '!!', 'ls', '')),
16 (',ls', ('', ',', 'ls', '')),
17 (';ls', ('', ';', 'ls', '')),
18 (' ;ls', (' ', ';', 'ls', '')),
19 ('f.g(x)', ('', '', 'f.g', '(x)')),
20 ('f.g (x)', ('', '', 'f.g', '(x)')),
21 ('?%hist', ('', '?', '%hist', '')),
22 ('?x*', ('', '?', 'x*', '')),
23 ]
24
25 def test_split_user_input():
26 return tt.check_pairs(split_user_input, tests)
@@ -74,6 +74,7 import tokenize
74 from StringIO import StringIO
74 from StringIO import StringIO
75
75
76 # IPython modules
76 # IPython modules
77 from IPython.core.splitinput import split_user_input, LineInfo
77 from IPython.utils.text import make_quoted_expr
78 from IPython.utils.text import make_quoted_expr
78 from IPython.utils.py3compat import cast_unicode
79 from IPython.utils.py3compat import cast_unicode
79
80
@@ -482,132 +483,10 class InputSplitter(object):
482 # Functions and classes for IPython-specific syntactic support
483 # Functions and classes for IPython-specific syntactic support
483 #-----------------------------------------------------------------------------
484 #-----------------------------------------------------------------------------
484
485
485 # RegExp for splitting line contents into pre-char//first word-method//rest.
486 # For clarity, each group in on one line.
487
488 line_split = re.compile("""
489 ^(\s*) # any leading space
490 ([,;/%]|!!?|\?\??) # escape character or characters
491 \s*(%?[\w\.\*]*) # function/method, possibly with leading %
492 # to correctly treat things like '?%magic'
493 (\s+.*$|$) # rest of line
494 """, re.VERBOSE)
495
496
497 def split_user_input(line):
498 """Split user input into early whitespace, esc-char, function part and rest.
499
500 This is currently handles lines with '=' in them in a very inconsistent
501 manner.
502
503 Examples
504 ========
505 >>> split_user_input('x=1')
506 ('', '', 'x=1', '')
507 >>> split_user_input('?')
508 ('', '?', '', '')
509 >>> split_user_input('??')
510 ('', '??', '', '')
511 >>> split_user_input(' ?')
512 (' ', '?', '', '')
513 >>> split_user_input(' ??')
514 (' ', '??', '', '')
515 >>> split_user_input('??x')
516 ('', '??', 'x', '')
517 >>> split_user_input('?x=1')
518 ('', '', '?x=1', '')
519 >>> split_user_input('!ls')
520 ('', '!', 'ls', '')
521 >>> split_user_input(' !ls')
522 (' ', '!', 'ls', '')
523 >>> split_user_input('!!ls')
524 ('', '!!', 'ls', '')
525 >>> split_user_input(' !!ls')
526 (' ', '!!', 'ls', '')
527 >>> split_user_input(',ls')
528 ('', ',', 'ls', '')
529 >>> split_user_input(';ls')
530 ('', ';', 'ls', '')
531 >>> split_user_input(' ;ls')
532 (' ', ';', 'ls', '')
533 >>> split_user_input('f.g(x)')
534 ('', '', 'f.g(x)', '')
535 >>> split_user_input('f.g (x)')
536 ('', '', 'f.g', '(x)')
537 >>> split_user_input('?%hist')
538 ('', '?', '%hist', '')
539 >>> split_user_input('?x*')
540 ('', '?', 'x*', '')
541 """
542 match = line_split.match(line)
543 if match:
544 lspace, esc, fpart, rest = match.groups()
545 else:
546 # print "match failed for line '%s'" % line
547 try:
548 fpart, rest = line.split(None, 1)
549 except ValueError:
550 # print "split failed for line '%s'" % line
551 fpart, rest = line,''
552 lspace = re.match('^(\s*)(.*)', line).groups()[0]
553 esc = ''
554
555 # fpart has to be a valid python identifier, so it better be only pure
556 # ascii, no unicode:
557 try:
558 fpart = fpart.encode('ascii')
559 except UnicodeEncodeError:
560 lspace = unicode(lspace)
561 rest = fpart + u' ' + rest
562 fpart = u''
563
564 #print 'line:<%s>' % line # dbg
565 #print 'esc <%s> fpart <%s> rest <%s>' % (esc,fpart.strip(),rest) # dbg
566 return lspace, esc, fpart.strip(), rest.lstrip()
567
568
569 # The escaped translators ALL receive a line where their own escape has been
486 # The escaped translators ALL receive a line where their own escape has been
570 # stripped. Only '?' is valid at the end of the line, all others can only be
487 # stripped. Only '?' is valid at the end of the line, all others can only be
571 # placed at the start.
488 # placed at the start.
572
489
573 class LineInfo(object):
574 """A single line of input and associated info.
575
576 This is a utility class that mostly wraps the output of
577 :func:`split_user_input` into a convenient object to be passed around
578 during input transformations.
579
580 Includes the following as properties:
581
582 line
583 The original, raw line
584
585 lspace
586 Any early whitespace before actual text starts.
587
588 esc
589 The initial esc character (or characters, for double-char escapes like
590 '??' or '!!').
591
592 fpart
593 The 'function part', which is basically the maximal initial sequence
594 of valid python identifiers and the '.' character. This is what is
595 checked for alias and magic transformations, used for auto-calling,
596 etc.
597
598 rest
599 Everything else on the line.
600 """
601 def __init__(self, line):
602 self.line = line
603 self.lspace, self.esc, self.fpart, self.rest = \
604 split_user_input(line)
605
606 def __str__(self):
607 return "LineInfo [%s|%s|%s|%s]" % (self.lspace, self.esc,
608 self.fpart, self.rest)
609
610
611 # Transformations of the special syntaxes that don't rely on an explicit escape
490 # Transformations of the special syntaxes that don't rely on an explicit escape
612 # character but instead on patterns on the input line
491 # character but instead on patterns on the input line
613
492
@@ -690,10 +569,10 def _make_help_call(target, esc, lspace, next_input=None):
690
569
691 _initial_space_re = re.compile(r'\s*')
570 _initial_space_re = re.compile(r'\s*')
692 _help_end_re = re.compile(r"""(%?
571 _help_end_re = re.compile(r"""(%?
693 [a-zA-Z_*][a-zA-Z0-9_*]* # Variable name
572 [a-zA-Z_*][\w*]* # Variable name
694 (\.[a-zA-Z_*][a-zA-Z0-9_*]*)* # .etc.etc
573 (\.[a-zA-Z_*][\w*]*)* # .etc.etc
695 )
574 )
696 (\?\??)$ # ? or ??""",
575 (\?\??)$ # ? or ??""",
697 re.VERBOSE)
576 re.VERBOSE)
698 def transform_help_end(line):
577 def transform_help_end(line):
699 """Translate lines with ?/?? at the end"""
578 """Translate lines with ?/?? at the end"""
@@ -703,7 +582,6 def transform_help_end(line):
703 target = m.group(1)
582 target = m.group(1)
704 esc = m.group(3)
583 esc = m.group(3)
705 lspace = _initial_space_re.match(line).group(0)
584 lspace = _initial_space_re.match(line).group(0)
706 newline = _make_help_call(target, esc, lspace)
707
585
708 # If we're mid-command, put it back on the next prompt for the user.
586 # If we're mid-command, put it back on the next prompt for the user.
709 next_input = line.rstrip('?') if line.strip() != m.group(0) else None
587 next_input = line.rstrip('?') if line.strip() != m.group(0) else None
@@ -731,14 +609,14 class EscapedTransformer(object):
731 def _tr_system(line_info):
609 def _tr_system(line_info):
732 "Translate lines escaped with: !"
610 "Translate lines escaped with: !"
733 cmd = line_info.line.lstrip().lstrip(ESC_SHELL)
611 cmd = line_info.line.lstrip().lstrip(ESC_SHELL)
734 return '%sget_ipython().system(%s)' % (line_info.lspace,
612 return '%sget_ipython().system(%s)' % (line_info.pre,
735 make_quoted_expr(cmd))
613 make_quoted_expr(cmd))
736
614
737 @staticmethod
615 @staticmethod
738 def _tr_system2(line_info):
616 def _tr_system2(line_info):
739 "Translate lines escaped with: !!"
617 "Translate lines escaped with: !!"
740 cmd = line_info.line.lstrip()[2:]
618 cmd = line_info.line.lstrip()[2:]
741 return '%sget_ipython().getoutput(%s)' % (line_info.lspace,
619 return '%sget_ipython().getoutput(%s)' % (line_info.pre,
742 make_quoted_expr(cmd))
620 make_quoted_expr(cmd))
743
621
744 @staticmethod
622 @staticmethod
@@ -748,33 +626,33 class EscapedTransformer(object):
748 if not line_info.line[1:]:
626 if not line_info.line[1:]:
749 return 'get_ipython().show_usage()'
627 return 'get_ipython().show_usage()'
750
628
751 return _make_help_call(line_info.fpart, line_info.esc, line_info.lspace)
629 return _make_help_call(line_info.ifun, line_info.esc, line_info.pre)
752
630
753 @staticmethod
631 @staticmethod
754 def _tr_magic(line_info):
632 def _tr_magic(line_info):
755 "Translate lines escaped with: %"
633 "Translate lines escaped with: %"
756 tpl = '%sget_ipython().magic(%s)'
634 tpl = '%sget_ipython().magic(%s)'
757 cmd = make_quoted_expr(' '.join([line_info.fpart,
635 cmd = make_quoted_expr(' '.join([line_info.ifun,
758 line_info.rest]).strip())
636 line_info.the_rest]).strip())
759 return tpl % (line_info.lspace, cmd)
637 return tpl % (line_info.pre, cmd)
760
638
761 @staticmethod
639 @staticmethod
762 def _tr_quote(line_info):
640 def _tr_quote(line_info):
763 "Translate lines escaped with: ,"
641 "Translate lines escaped with: ,"
764 return '%s%s("%s")' % (line_info.lspace, line_info.fpart,
642 return '%s%s("%s")' % (line_info.pre, line_info.ifun,
765 '", "'.join(line_info.rest.split()) )
643 '", "'.join(line_info.the_rest.split()) )
766
644
767 @staticmethod
645 @staticmethod
768 def _tr_quote2(line_info):
646 def _tr_quote2(line_info):
769 "Translate lines escaped with: ;"
647 "Translate lines escaped with: ;"
770 return '%s%s("%s")' % (line_info.lspace, line_info.fpart,
648 return '%s%s("%s")' % (line_info.pre, line_info.ifun,
771 line_info.rest)
649 line_info.the_rest)
772
650
773 @staticmethod
651 @staticmethod
774 def _tr_paren(line_info):
652 def _tr_paren(line_info):
775 "Translate lines escaped with: /"
653 "Translate lines escaped with: /"
776 return '%s%s(%s)' % (line_info.lspace, line_info.fpart,
654 return '%s%s(%s)' % (line_info.pre, line_info.ifun,
777 ", ".join(line_info.rest.split()))
655 ", ".join(line_info.the_rest.split()))
778
656
779 def __call__(self, line):
657 def __call__(self, line):
780 """Class to transform lines that are explicitly escaped out.
658 """Class to transform lines that are explicitly escaped out.
@@ -843,7 +721,7 class IPythonInputSplitter(InputSplitter):
843 lines_list = lines.splitlines()
721 lines_list = lines.splitlines()
844
722
845 transforms = [transform_ipy_prompt, transform_classic_prompt,
723 transforms = [transform_ipy_prompt, transform_classic_prompt,
846 transform_escaped, transform_help_end,
724 transform_help_end, transform_escaped,
847 transform_assign_system, transform_assign_magic]
725 transform_assign_system, transform_assign_magic]
848
726
849 # Transform logic
727 # Transform logic
@@ -32,7 +32,7 from IPython.core.alias import AliasManager
32 from IPython.core.autocall import IPyAutocall
32 from IPython.core.autocall import IPyAutocall
33 from IPython.config.configurable import Configurable
33 from IPython.config.configurable import Configurable
34 from IPython.core.macro import Macro
34 from IPython.core.macro import Macro
35 from IPython.core.splitinput import split_user_input
35 from IPython.core.splitinput import split_user_input, LineInfo
36 from IPython.core import page
36 from IPython.core import page
37
37
38 from IPython.utils.traitlets import List, Int, Any, Unicode, CBool, Bool, Instance
38 from IPython.utils.traitlets import List, Int, Any, Unicode, CBool, Bool, Instance
@@ -92,78 +92,6 def is_shadowed(identifier, ip):
92
92
93
93
94 #-----------------------------------------------------------------------------
94 #-----------------------------------------------------------------------------
95 # The LineInfo class used throughout
96 #-----------------------------------------------------------------------------
97
98
99 class LineInfo(object):
100 """A single line of input and associated info.
101
102 Includes the following as properties:
103
104 line
105 The original, raw line
106
107 continue_prompt
108 Is this line a continuation in a sequence of multiline input?
109
110 pre
111 The initial esc character or whitespace.
112
113 pre_char
114 The escape character(s) in pre or the empty string if there isn't one.
115 Note that '!!' is a possible value for pre_char. Otherwise it will
116 always be a single character.
117
118 pre_whitespace
119 The leading whitespace from pre if it exists. If there is a pre_char,
120 this is just ''.
121
122 ifun
123 The 'function part', which is basically the maximal initial sequence
124 of valid python identifiers and the '.' character. This is what is
125 checked for alias and magic transformations, used for auto-calling,
126 etc.
127
128 the_rest
129 Everything else on the line.
130 """
131 def __init__(self, line, continue_prompt):
132 self.line = line
133 self.continue_prompt = continue_prompt
134 self.pre, self.esc, self.ifun, self.the_rest = split_user_input(line)
135
136 self.pre_char = self.pre.strip()
137 if self.pre_char:
138 self.pre_whitespace = '' # No whitespace allowd before esc chars
139 else:
140 self.pre_whitespace = self.pre
141
142 self._oinfo = None
143
144 def ofind(self, ip):
145 """Do a full, attribute-walking lookup of the ifun in the various
146 namespaces for the given IPython InteractiveShell instance.
147
148 Return a dict with keys: found,obj,ospace,ismagic
149
150 Note: can cause state changes because of calling getattr, but should
151 only be run if autocall is on and if the line hasn't matched any
152 other, less dangerous handlers.
153
154 Does cache the results of the call, so can be called multiple times
155 without worrying about *further* damaging state.
156 """
157 if not self._oinfo:
158 # ip.shell._ofind is actually on the Magic class!
159 self._oinfo = ip.shell._ofind(self.ifun)
160 return self._oinfo
161
162 def __str__(self):
163 return "Lineinfo [%s|%s|%s]" %(self.pre, self.ifun, self.the_rest)
164
165
166 #-----------------------------------------------------------------------------
167 # Main Prefilter manager
95 # Main Prefilter manager
168 #-----------------------------------------------------------------------------
96 #-----------------------------------------------------------------------------
169
97
@@ -1,6 +1,7
1 # encoding: utf-8
1 # encoding: utf-8
2 """
2 """
3 Simple utility for splitting user input.
3 Simple utility for splitting user input. This is used by both inputsplitter and
4 prefilter.
4
5
5 Authors:
6 Authors:
6
7
@@ -28,36 +29,28 from IPython.utils import py3compat
28 # Main function
29 # Main function
29 #-----------------------------------------------------------------------------
30 #-----------------------------------------------------------------------------
30
31
31
32 # RegExp for splitting line contents into pre-char//first word-method//rest.
32 # RegExp for splitting line contents into pre-char//first word-method//rest.
33 # For clarity, each group in on one line.
33 # For clarity, each group in on one line.
34
34
35 # WARNING: update the regexp if the escapes in interactiveshell are changed, as they
35 # WARNING: update the regexp if the escapes in interactiveshell are changed, as
36 # are hardwired in.
36 # they are hardwired in.
37
37
38 # Although it's not solely driven by the regex, note that:
38 # Although it's not solely driven by the regex, note that:
39 # ,;/% only trigger if they are the first character on the line
39 # ,;/% only trigger if they are the first character on the line
40 # ! and !! trigger if they are first char(s) *or* follow an indent
40 # ! and !! trigger if they are first char(s) *or* follow an indent
41 # ? triggers as first or last char.
41 # ? triggers as first or last char.
42
42
43 # The four parts of the regex are:
43 line_split = re.compile("""
44 # 1) pre: initial whitespace
44 ^(\s*) # any leading space
45 # 2) esc: escape character
45 ([,;/%]|!!?|\?\??)? # escape character or characters
46 # 3) ifun: first word/method (mix of \w and '.')
46 \s*(%?[\w\.\*]*) # function/method, possibly with leading %
47 # 4) the_rest: rest of line (separated from ifun by space if non-empty)
47 # to correctly treat things like '?%magic'
48 line_split = re.compile(r'^(\s*)'
48 (.*?$|$) # rest of line
49 r'([,;/%?]|!!?)?'
49 """, re.VERBOSE)
50 r'\s*([\w\.]+)'
51 r'(.*$|$)')
52
53 # r'[\w\.]+'
54 # r'\s*=\s*%.*'
55
50
56 def split_user_input(line, pattern=None):
51 def split_user_input(line, pattern=None):
57 """Split user input into pre-char/whitespace, function part and rest.
52 """Split user input into initial whitespace, escape character, function part
58
53 and the rest.
59 This is currently handles lines with '=' in them in a very inconsistent
60 manner.
61 """
54 """
62 # We need to ensure that the rest of this routine deals only with unicode
55 # We need to ensure that the rest of this routine deals only with unicode
63 line = py3compat.cast_unicode(line, sys.stdin.encoding or 'utf-8')
56 line = py3compat.cast_unicode(line, sys.stdin.encoding or 'utf-8')
@@ -76,11 +69,70 def split_user_input(line, pattern=None):
76 esc = ""
69 esc = ""
77 else:
70 else:
78 pre, esc, ifun, the_rest = match.groups()
71 pre, esc, ifun, the_rest = match.groups()
79
80 if not py3compat.isidentifier(ifun, dotted=True):
81 the_rest = ifun + u' ' + the_rest
82 ifun = u''
83
72
84 #print 'line:<%s>' % line # dbg
73 #print 'line:<%s>' % line # dbg
85 #print 'pre <%s> ifun <%s> rest <%s>' % (pre,ifun.strip(),the_rest) # dbg
74 #print 'pre <%s> ifun <%s> rest <%s>' % (pre,ifun.strip(),the_rest) # dbg
86 return pre, esc, ifun.strip(), the_rest.lstrip()
75 return pre, esc or '', ifun.strip(), the_rest.lstrip()
76
77 class LineInfo(object):
78 """A single line of input and associated info.
79
80 Includes the following as properties:
81
82 line
83 The original, raw line
84
85 continue_prompt
86 Is this line a continuation in a sequence of multiline input?
87
88 pre
89 Any leading whitespace.
90
91 esc
92 The escape character(s) in pre or the empty string if there isn't one.
93 Note that '!!' and '??' are possible values for esc. Otherwise it will
94 always be a single character.
95
96 ifun
97 The 'function part', which is basically the maximal initial sequence
98 of valid python identifiers and the '.' character. This is what is
99 checked for alias and magic transformations, used for auto-calling,
100 etc. In contrast to Python identifiers, it may start with "%" and contain
101 "*".
102
103 the_rest
104 Everything else on the line.
105 """
106 def __init__(self, line, continue_prompt=False):
107 self.line = line
108 self.continue_prompt = continue_prompt
109 self.pre, self.esc, self.ifun, self.the_rest = split_user_input(line)
110
111 self.pre_char = self.pre.strip()
112 if self.pre_char:
113 self.pre_whitespace = '' # No whitespace allowd before esc chars
114 else:
115 self.pre_whitespace = self.pre
116
117 self._oinfo = None
118
119 def ofind(self, ip):
120 """Do a full, attribute-walking lookup of the ifun in the various
121 namespaces for the given IPython InteractiveShell instance.
122
123 Return a dict with keys: found,obj,ospace,ismagic
124
125 Note: can cause state changes because of calling getattr, but should
126 only be run if autocall is on and if the line hasn't matched any
127 other, less dangerous handlers.
128
129 Does cache the results of the call, so can be called multiple times
130 without worrying about *further* damaging state.
131 """
132 if not self._oinfo:
133 # ip.shell._ofind is actually on the Magic class!
134 self._oinfo = ip.shell._ofind(self.ifun)
135 return self._oinfo
136
137 def __str__(self):
138 return "LineInfo [%s|%s|%s|%s]" %(self.pre, self.esc, self.ifun, self.the_rest)
@@ -394,9 +394,8 def test_LineInfo():
394 def test_split_user_input():
394 def test_split_user_input():
395 """Unicode test - split_user_input already has good doctests"""
395 """Unicode test - split_user_input already has good doctests"""
396 line = u"Pérez Fernando"
396 line = u"Pérez Fernando"
397 parts = isp.split_user_input(line)
398 parts_expected = (u'', u'', u'', line)
397 parts_expected = (u'', u'', u'', line)
399 nt.assert_equal(parts, parts_expected)
398 tt.check_pairs(isp.split_user_input, [(line, parts_expected),])
400
399
401
400
402 # Transformer tests
401 # Transformer tests
@@ -611,7 +610,8 class IPythonInputTestCase(InputSplitterTestCase):
611
610
612 isp.push(raw)
611 isp.push(raw)
613 out, out_raw = isp.source_raw_reset()
612 out, out_raw = isp.source_raw_reset()
614 self.assertEqual(out.rstrip(), out_t)
613 self.assertEqual(out.rstrip(), out_t,
614 tt.pair_fail_msg.format("inputsplitter",raw, out_t, out))
615 self.assertEqual(out_raw.rstrip(), raw.rstrip())
615 self.assertEqual(out_raw.rstrip(), raw.rstrip())
616
616
617 def test_syntax_multiline(self):
617 def test_syntax_multiline(self):
@@ -295,7 +295,7 class TempFileMixin(object):
295 # delete it. I have no clue why
295 # delete it. I have no clue why
296 pass
296 pass
297
297
298 pair_fail_msg = ("Testing function {0}\n\n"
298 pair_fail_msg = ("Testing {0}\n\n"
299 "In:\n"
299 "In:\n"
300 " {1!r}\n"
300 " {1!r}\n"
301 "Expected:\n"
301 "Expected:\n"
@@ -318,9 +318,10 def check_pairs(func, pairs):
318 None. Raises an AssertionError if any output does not match the expected
318 None. Raises an AssertionError if any output does not match the expected
319 value.
319 value.
320 """
320 """
321 name = getattr(func, "func_name", getattr(func, "__name__", "<unknown>"))
321 for inp, expected in pairs:
322 for inp, expected in pairs:
322 out = func(inp)
323 out = func(inp)
323 assert out == expected, pair_fail_msg.format(func.func_name, inp, expected, out)
324 assert out == expected, pair_fail_msg.format(name, inp, expected, out)
324
325
325 @contextmanager
326 @contextmanager
326 def mute_warn():
327 def mute_warn():
@@ -342,4 +343,3 def make_tempfile(name):
342 yield
343 yield
343 finally:
344 finally:
344 os.unlink(name)
345 os.unlink(name)
345
General Comments 0
You need to be logged in to leave comments. Login now