##// END OF EJS Templates
Change get_source() api as per code review, to source_reset()....
Fernando Perez -
Show More
@@ -1,266 +1,260 b''
1 1 """Analysis of text input into executable blocks.
2 2
3 3 This is a simple example of how an interactive terminal-based client can use
4 4 this tool::
5 5
6 6 bb = BlockBreaker()
7 7 while not bb.interactive_block_ready():
8 8 bb.push(raw_input('>>> '))
9 9 print 'Input source was:\n', bb.source,
10 10 """
11 11 #-----------------------------------------------------------------------------
12 12 # Copyright (C) 2010 The IPython Development Team
13 13 #
14 14 # Distributed under the terms of the BSD License. The full license is in
15 15 # the file COPYING, distributed as part of this software.
16 16 #-----------------------------------------------------------------------------
17 17
18 18 #-----------------------------------------------------------------------------
19 19 # Imports
20 20 #-----------------------------------------------------------------------------
21 21 # stdlib
22 22 import codeop
23 23 import re
24 24 import sys
25 25
26 26 #-----------------------------------------------------------------------------
27 27 # Utilities
28 28 #-----------------------------------------------------------------------------
29 29
30 30 # FIXME: move these utilities to the general ward...
31 31
32 32 # compiled regexps for autoindent management
33 33 dedent_re = re.compile(r'^\s+raise|^\s+return|^\s+pass')
34 34 ini_spaces_re = re.compile(r'^([ \t\r\f\v]+)')
35 35
36 36
37 37 def num_ini_spaces(s):
38 38 """Return the number of initial spaces in a string.
39 39
40 40 Note that tabs are counted as a single space. For now, we do *not* support
41 41 mixing of tabs and spaces in the user's input.
42 42
43 43 Parameters
44 44 ----------
45 45 s : string
46 46 """
47 47
48 48 ini_spaces = ini_spaces_re.match(s)
49 49 if ini_spaces:
50 50 return ini_spaces.end()
51 51 else:
52 52 return 0
53 53
54 54
55 55 def remove_comments(src):
56 56 """Remove all comments from input source.
57 57
58 58 Note: comments are NOT recognized inside of strings!
59 59
60 60 Parameters
61 61 ----------
62 62 src : string
63 63 A single or multiline input string.
64 64
65 65 Returns
66 66 -------
67 67 String with all Python comments removed.
68 68 """
69 69
70 70 return re.sub('#.*', '', src)
71 71
72 72
73 73 def get_input_encoding():
74 74 """Return the default standard input encoding."""
75 75 return getattr(sys.stdin, 'encoding', 'ascii')
76 76
77 77 #-----------------------------------------------------------------------------
78 78 # Classes and functions
79 79 #-----------------------------------------------------------------------------
80 80
81 81 class BlockBreaker(object):
82 82 # Command compiler
83 83 compile = None
84 84 # Number of spaces of indentation
85 85 indent_spaces = 0
86 86 # String, indicating the default input encoding
87 87 encoding = ''
88 88 # String where the current full source input is stored, properly encoded
89 89 source = ''
90 90 # Code object corresponding to the current source
91 91 code = None
92 92 # Boolean indicating whether the current block is complete
93 93 is_complete = None
94 94 # Input mode
95 95 input_mode = 'append'
96 96
97 97 # Private attributes
98 98
99 99 # List
100 100 _buffer = None
101 101
102 102 def __init__(self, input_mode=None):
103 103 """Create a new BlockBreaker instance.
104 104
105 105 Parameters
106 106 ----------
107 107 input_mode : str
108 108
109 109 One of 'append', 'replace', default is 'append'. This controls how
110 110 new inputs are used: in 'append' mode, they are appended to the
111 111 existing buffer and the whole buffer is compiled; in 'replace' mode,
112 112 each new input completely replaces all prior inputs. Replace mode is
113 113 thus equivalent to prepending a full reset() to every push() call.
114 114
115 115 In practice, line-oriented clients likely want to use 'append' mode
116 116 while block-oriented ones will want to use 'replace'.
117 117 """
118 118 self._buffer = []
119 119 self.compile = codeop.CommandCompiler()
120 120 self.encoding = get_input_encoding()
121 121 self.input_mode = BlockBreaker.input_mode if input_mode is None \
122 122 else input_mode
123 123
124 124 def reset(self):
125 125 """Reset the input buffer and associated state."""
126 126 self.indent_spaces = 0
127 127 self._buffer[:] = []
128 128 self.source = ''
129 129 self.code = None
130 130
131 def get_source(self, reset=False):
132 """Return the input source.
133
134 Parameters
135 ----------
136 reset : boolean
137 If true, all state is reset and prior input forgotten.
131 def source_reset(self):
132 """Return the input source and perform a full reset.
138 133 """
139 134 out = self.source
140 if reset:
141 135 self.reset()
142 136 return out
143 137
144 138 def push(self, lines):
145 139 """Push one ore more lines of input.
146 140
147 141 This stores the given lines and returns a status code indicating
148 142 whether the code forms a complete Python block or not.
149 143
150 144 Any exceptions generated in compilation are allowed to propagate.
151 145
152 146 Parameters
153 147 ----------
154 148 lines : string
155 149 One or more lines of Python input.
156 150
157 151 Returns
158 152 -------
159 153 is_complete : boolean
160 154 True if the current input source (the result of the current input
161 155 plus prior inputs) forms a complete Python execution block. Note that
162 156 this value is also stored as an attribute so it can be queried at any
163 157 time.
164 158 """
165 159 if self.input_mode == 'replace':
166 160 self.reset()
167 161
168 162 # If the source code has leading blanks, add 'if 1:\n' to it
169 163 # this allows execution of indented pasted code. It is tempting
170 164 # to add '\n' at the end of source to run commands like ' a=1'
171 165 # directly, but this fails for more complicated scenarios
172 166 if not self._buffer and lines[:1] in [' ', '\t']:
173 167 lines = 'if 1:\n%s' % lines
174 168
175 169 self._store(lines)
176 170 source = self.source
177 171
178 172 # Before calling compile(), reset the code object to None so that if an
179 173 # exception is raised in compilation, we don't mislead by having
180 174 # inconsistent code/source attributes.
181 175 self.code, self.is_complete = None, None
182 176 try:
183 177 self.code = self.compile(source)
184 178 # Invalid syntax can produce any of a number of different errors from
185 179 # inside the compiler, so we have to catch them all. Syntax errors
186 180 # immediately produce a 'ready' block, so the invalid Python can be
187 181 # sent to the kernel for evaluation with possible ipython
188 182 # special-syntax conversion.
189 183 except (SyntaxError, OverflowError, ValueError, TypeError, MemoryError):
190 184 self.is_complete = True
191 185 else:
192 186 # Compilation didn't produce any exceptions (though it may not have
193 187 # given a complete code object)
194 188 self.is_complete = self.code is not None
195 189 self._update_indent(lines)
196 190
197 191 return self.is_complete
198 192
199 193 def interactive_block_ready(self):
200 194 """Return whether a block of interactive input is ready for execution.
201 195
202 196 This method is meant to be used by line-oriented frontends, who need to
203 197 guess whether a block is complete or not based solely on prior and
204 198 current input lines. The BlockBreaker considers it has a complete
205 199 interactive block when *all* of the following are true:
206 200
207 201 1. The input compiles to a complete statement.
208 202
209 203 2. The indentation level is flush-left (because if we are indented,
210 204 like inside a function definition or for loop, we need to keep
211 205 reading new input).
212 206
213 207 3. There is one extra line consisting only of whitespace.
214 208
215 209 Because of condition #3, this method should be used only by
216 210 *line-oriented* frontends, since it means that intermediate blank lines
217 211 are not allowed in function definitions (or any other indented block).
218 212
219 213 Block-oriented frontends that have a separate keyboard event to
220 214 indicate execution should use the :meth:`split_blocks` method instead.
221 215 """
222 216 if not self.is_complete:
223 217 return False
224 218 if self.indent_spaces==0:
225 219 return True
226 220 last_line = self.source.splitlines()[-1]
227 221 if not last_line or last_line.isspace():
228 222 return True
229 223 else:
230 224 return False
231 225
232 226 def split_blocks(self, lines):
233 227 """Split a multiline string into multiple input blocks"""
234 228 raise NotImplementedError
235 229
236 230 #------------------------------------------------------------------------
237 231 # Private interface
238 232 #------------------------------------------------------------------------
239 233
240 234 def _update_indent(self, lines):
241 235 """Keep track of the indent level."""
242 236
243 237 for line in remove_comments(lines).splitlines():
244 238
245 239 if line and not line.isspace():
246 240 if self.code is not None:
247 241 inisp = num_ini_spaces(line)
248 242 if inisp < self.indent_spaces:
249 243 self.indent_spaces = inisp
250 244
251 245 if line[-1] == ':':
252 246 self.indent_spaces += 4
253 247 elif dedent_re.match(line):
254 248 self.indent_spaces -= 4
255 249
256 250 def _store(self, lines):
257 251 """Store one or more lines of input.
258 252
259 253 If input lines are not newline-terminated, a newline is automatically
260 254 appended."""
261 255
262 256 if lines.endswith('\n'):
263 257 self._buffer.append(lines)
264 258 else:
265 259 self._buffer.append(lines+'\n')
266 260 self.source = ''.join(self._buffer).encode(self.encoding)
@@ -1,191 +1,189 b''
1 1 """Tests for the blockbreaker module.
2 2 """
3 3 #-----------------------------------------------------------------------------
4 4 # Copyright (C) 2010 The IPython Development Team
5 5 #
6 6 # Distributed under the terms of the BSD License. The full license is in
7 7 # the file COPYING, distributed as part of this software.
8 8 #-----------------------------------------------------------------------------
9 9
10 10 #-----------------------------------------------------------------------------
11 11 # Imports
12 12 #-----------------------------------------------------------------------------
13 13 # stdlib
14 14 import unittest
15 15
16 16 # Third party
17 17 import nose.tools as nt
18 18
19 19 # Our own
20 20 from IPython.core import blockbreaker as BB
21 21
22 22 #-----------------------------------------------------------------------------
23 23 # Tests
24 24 #-----------------------------------------------------------------------------
25 25 def test_spaces():
26 26 tests = [('', 0),
27 27 (' ', 1),
28 28 ('\n', 0),
29 29 (' \n', 1),
30 30 ('x', 0),
31 31 (' x', 1),
32 32 (' x',2),
33 33 (' x',4),
34 34 # Note: tabs are counted as a single whitespace!
35 35 ('\tx', 1),
36 36 ('\t x', 2),
37 37 ]
38 38
39 39 for s, nsp in tests:
40 40 nt.assert_equal(BB.num_ini_spaces(s), nsp)
41 41
42 42
43 43 def test_remove_comments():
44 44 tests = [('text', 'text'),
45 45 ('text # comment', 'text '),
46 46 ('text # comment\n', 'text \n'),
47 47 ('text # comment \n', 'text \n'),
48 48 ('line # c \nline\n','line \nline\n'),
49 49 ('line # c \nline#c2 \nline\nline #c\n\n',
50 50 'line \nline\nline\nline \n\n'),
51 51 ]
52 52
53 53 for inp, out in tests:
54 54 nt.assert_equal(BB.remove_comments(inp), out)
55 55
56 56
57 57 def test_get_input_encoding():
58 58 encoding = BB.get_input_encoding()
59 59 nt.assert_true(isinstance(encoding, basestring))
60 60 # simple-minded check that at least encoding a simple string works with the
61 61 # encoding we got.
62 62 nt.assert_equal('test'.encode(encoding), 'test')
63 63
64 64
65 65 class BlockBreakerTestCase(unittest.TestCase):
66 66 def setUp(self):
67 67 self.bb = BB.BlockBreaker()
68 68
69 69 def test_reset(self):
70 70 bb = self.bb
71 71 bb.push('x=1')
72 72 bb.reset()
73 73 self.assertEqual(bb._buffer, [])
74 74 self.assertEqual(bb.indent_spaces, 0)
75 self.assertEqual(bb.get_source(), '')
75 self.assertEqual(bb.source, '')
76 76 self.assertEqual(bb.code, None)
77 77
78 78 def test_source(self):
79 79 self.bb._store('1')
80 80 self.bb._store('2')
81 out = self.bb.get_source()
82 self.assertEqual(out, '1\n2\n')
83 out = self.bb.get_source(reset=True)
84 self.assertEqual(out, '1\n2\n')
81 self.assertEqual(self.bb.source, '1\n2\n')
82 self.assertTrue(len(self.bb._buffer)>0)
83 self.assertEqual(self.bb.source_reset(), '1\n2\n')
85 84 self.assertEqual(self.bb._buffer, [])
86 out = self.bb.get_source()
87 self.assertEqual(out, '')
85 self.assertEqual(self.bb.source, '')
88 86
89 87 def test_indent(self):
90 88 bb = self.bb # shorthand
91 89 bb.push('x=1')
92 90 self.assertEqual(bb.indent_spaces, 0)
93 91 bb.push('if 1:\n x=1')
94 92 self.assertEqual(bb.indent_spaces, 4)
95 93 bb.push('y=2\n')
96 94 self.assertEqual(bb.indent_spaces, 0)
97 95 bb.push('if 1:')
98 96 self.assertEqual(bb.indent_spaces, 4)
99 97 bb.push(' x=1')
100 98 self.assertEqual(bb.indent_spaces, 4)
101 99 # Blank lines shouldn't change the indent level
102 100 bb.push(' '*2)
103 101 self.assertEqual(bb.indent_spaces, 4)
104 102
105 103 def test_indent2(self):
106 104 bb = self.bb
107 105 # When a multiline statement contains parens or multiline strings, we
108 106 # shouldn't get confused.
109 107 bb.push("if 1:")
110 108 bb.push(" x = (1+\n 2)")
111 109 self.assertEqual(bb.indent_spaces, 4)
112 110
113 111 def test_dedent(self):
114 112 bb = self.bb # shorthand
115 113 bb.push('if 1:')
116 114 self.assertEqual(bb.indent_spaces, 4)
117 115 bb.push(' pass')
118 116 self.assertEqual(bb.indent_spaces, 0)
119 117
120 118 def test_push(self):
121 119 bb = self.bb
122 120 bb.push('x=1')
123 121 self.assertTrue(bb.is_complete)
124 122
125 123 def test_push2(self):
126 124 bb = self.bb
127 125 bb.push('if 1:')
128 126 self.assertFalse(bb.is_complete)
129 127 for line in [' x=1', '# a comment', ' y=2']:
130 128 bb.push(line)
131 129 self.assertTrue(bb.is_complete)
132 130
133 131 def test_push3(self):
134 132 """Test input with leading whitespace"""
135 133 bb = self.bb
136 134 bb.push(' x=1')
137 135 bb.push(' y=2')
138 136 self.assertEqual(bb.source, 'if 1:\n x=1\n y=2\n')
139 137
140 138 def test_replace_mode(self):
141 139 bb = self.bb
142 140 bb.input_mode = 'replace'
143 141 bb.push('x=1')
144 142 self.assertEqual(bb.source, 'x=1\n')
145 143 bb.push('x=2')
146 144 self.assertEqual(bb.source, 'x=2\n')
147 145
148 146 def test_interactive_block_ready(self):
149 147 bb = self.bb
150 148 bb.push('x=1')
151 149 self.assertTrue(bb.interactive_block_ready())
152 150
153 151 def test_interactive_block_ready2(self):
154 152 bb = self.bb
155 153 bb.push('if 1:')
156 154 self.assertFalse(bb.interactive_block_ready())
157 155 bb.push(' x=1')
158 156 self.assertFalse(bb.interactive_block_ready())
159 157 bb.push('')
160 158 self.assertTrue(bb.interactive_block_ready())
161 159
162 160 def test_interactive_block_ready3(self):
163 161 bb = self.bb
164 162 bb.push("x = (2+\n3)")
165 163 self.assertTrue(bb.interactive_block_ready())
166 164
167 165 def test_interactive_block_ready4(self):
168 166 bb = self.bb
169 167 # When a multiline statement contains parens or multiline strings, we
170 168 # shouldn't get confused.
171 169 # FIXME: we should be able to better handle de-dents in statements like
172 170 # multiline strings and multiline expressions (continued with \ or
173 171 # parens). Right now we aren't handling the indentation tracking quite
174 172 # correctly with this, though in practice it may not be too much of a
175 173 # problem. We'll need to see.
176 174 bb.push("if 1:")
177 175 bb.push(" x = (2+")
178 176 bb.push(" 3)")
179 177 self.assertFalse(bb.interactive_block_ready())
180 178 bb.push(" y = 3")
181 179 self.assertFalse(bb.interactive_block_ready())
182 180 bb.push('')
183 181 self.assertTrue(bb.interactive_block_ready())
184 182
185 183 def test_syntax_error(self):
186 184 bb = self.bb
187 185 # Syntax errors immediately produce a 'ready' block, so the invalid
188 186 # Python can be sent to the kernel for evaluation with possible ipython
189 187 # special-syntax conversion.
190 188 bb.push('run foo')
191 189 self.assertTrue(bb.interactive_block_ready())
General Comments 0
You need to be logged in to leave comments. Login now