##// END OF EJS Templates
Fix up so tests pass again. input_splitter now uses ast module instead of compiler, bringing it closer to the Python 3 implementation.
Thomas Kluyver -
Show More
@@ -66,6 +66,7 b' from __future__ import print_function'
66 66 # Imports
67 67 #-----------------------------------------------------------------------------
68 68 # stdlib
69 import ast
69 70 import codeop
70 71 import re
71 72 import sys
@@ -185,9 +186,6 b' def split_blocks(python):'
185 186 commands : list of str
186 187 Separate commands that can be exec'ed independently.
187 188 """
188
189 import compiler
190
191 189 # compiler.parse treats trailing spaces after a newline as a
192 190 # SyntaxError. This is different than codeop.CommandCompiler, which
193 191 # will compile the trailng spaces just fine. We simply strip any
@@ -197,22 +195,15 b' def split_blocks(python):'
197 195 python_ori = python # save original in case we bail on error
198 196 python = python.strip()
199 197
200 # The compiler module does not like unicode. We need to convert
201 # it encode it:
202 if isinstance(python, unicode):
203 # Use the utf-8-sig BOM so the compiler detects this a UTF-8
204 # encode string.
205 python = '\xef\xbb\xbf' + python.encode('utf-8')
206
207 198 # The compiler module will parse the code into an abstract syntax tree.
208 199 # This has a bug with str("a\nb"), but not str("""a\nb""")!!!
209 200 try:
210 ast = compiler.parse(python)
201 code_ast = ast.parse(python)
211 202 except:
212 203 return [python_ori]
213 204
214 205 # Uncomment to help debug the ast tree
215 # for n in ast.node:
206 # for n in code_ast.body:
216 207 # print n.lineno,'->',n
217 208
218 209 # Each separate command is available by iterating over ast.node. The
@@ -223,14 +214,7 b' def split_blocks(python):'
223 214 # other situations that cause Discard nodes that shouldn't be discarded.
224 215 # We might eventually discover other cases where lineno is None and have
225 216 # to put in a more sophisticated test.
226 linenos = [x.lineno-1 for x in ast.node if x.lineno is not None]
227
228 # When we have a bare string as the first statement, it does not end up as
229 # a Discard Node in the AST as we might expect. Instead, it gets interpreted
230 # as the docstring of the module. Check for this case and prepend 0 (the
231 # first line number) to the list of linenos to account for it.
232 if ast.doc is not None:
233 linenos.insert(0, 0)
217 linenos = [x.lineno-1 for x in code_ast.body if x.lineno is not None]
234 218
235 219 # When we finally get the slices, we will need to slice all the way to
236 220 # the end even though we don't have a line number for it. Fortunately,
@@ -616,7 +600,7 b' class InputSplitter(object):'
616 600 setattr(self, store, self._set_source(buffer))
617 601
618 602 def _set_source(self, buffer):
619 return ''.join(buffer).encode(self.encoding)
603 return ''.join(buffer)
620 604
621 605
622 606 #-----------------------------------------------------------------------------
@@ -1550,12 +1550,14 b' class InteractiveShell(Configurable, Magic):'
1550 1550 # otherwise we end up with a monster history after a while:
1551 1551 readline.set_history_length(self.history_length)
1552 1552
1553 stdin_encoding = sys.stdin.encoding or "utf-8"
1554
1553 1555 # Load the last 1000 lines from history
1554 1556 for _, _, cell in self.history_manager.get_tail(1000,
1555 1557 include_latest=True):
1556 1558 if cell.strip(): # Ignore blank lines
1557 1559 for line in cell.splitlines():
1558 readline.add_history(line)
1560 readline.add_history(line.encode(stdin_encoding))
1559 1561
1560 1562 # Configure auto-indent for all platforms
1561 1563 self.set_autoindent(self.autoindent)
@@ -2106,7 +2108,6 b' class InteractiveShell(Configurable, Magic):'
2106 2108 cell = self.prefilter_manager.prefilter_line(blocks[0])
2107 2109 blocks = self.input_splitter.split_blocks(cell)
2108 2110
2109
2110 2111 # Store the 'ipython' version of the cell as well, since that's what
2111 2112 # needs to go into the translated history and get executed (the
2112 2113 # original cell may contain non-python syntax).
@@ -2246,7 +2247,7 b' class InteractiveShell(Configurable, Magic):'
2246 2247 else:
2247 2248 usource = source
2248 2249
2249 if 0: # dbg
2250 if False: # dbg
2250 2251 print 'Source:', repr(source) # dbg
2251 2252 print 'USource:', repr(usource) # dbg
2252 2253 print 'type:', type(source) # dbg
@@ -2063,7 +2063,8 b' Currently the magic system has the following functions:\\n"""'
2063 2063 return
2064 2064 cmds = self.extract_input_lines(ranges, 'r' in opts)
2065 2065 with open(fname,'w') as f:
2066 f.write(cmds)
2066 f.write("# coding: utf-8\n")
2067 f.write(cmds.encode("utf-8"))
2067 2068 print 'The following commands were written to file `%s`:' % fname
2068 2069 print cmds
2069 2070
@@ -83,7 +83,8 b' def test_history():'
83 83 testfilename = os.path.realpath(os.path.join(tmpdir, "test.py"))
84 84 ip.magic_save(testfilename + " ~1/1-3")
85 85 testfile = open(testfilename, "r")
86 nt.assert_equal(testfile.read(), "\n".join(hist))
86 nt.assert_equal(testfile.read().decode("utf-8"),
87 "# coding: utf-8\n" + "\n".join(hist))
87 88
88 89 # Duplicate line numbers - check that it doesn't crash, and
89 90 # gets a new session
General Comments 0
You need to be logged in to leave comments. Login now