From dceccd93b8a81ab920529f4f433018fbe9ee5f4b 2010-10-06 04:47:21
From: Fernando Perez <Fernando.Perez@berkeley.edu>
Date: 2010-10-06 04:47:21
Subject: [PATCH] Add support for accessing raw data to inputsplitter.
This is necessary to be able to use inputsplitter systematically
across all frontends, including plain terminal one.
---
diff --git a/IPython/core/history.py b/IPython/core/history.py
index b92e16b..490643f 100644
--- a/IPython/core/history.py
+++ b/IPython/core/history.py
@@ -61,13 +61,12 @@ class HistoryManager(object):
histfname = 'history'
self.hist_file = os.path.join(shell.ipython_dir, histfname)
- # Fill the history zero entry, user counter starts at 1
- self.input_hist.append('\n')
- self.input_hist_raw.append('\n')
-
# Objects related to shadow history management
self._init_shadow_hist()
+ # Fill the history zero entry, user counter starts at 1
+ self.store_inputs('\n', '\n')
+
# For backwards compatibility, we must put these back in the shell
# object, until we've removed all direct uses of the history objects in
# the shell itself.
@@ -154,10 +153,42 @@ class HistoryManager(object):
hist[i] = (input_hist[i], output_hist.get(i))
else:
hist[i] = input_hist[i]
- if len(hist)==0:
+ if not hist:
raise IndexError('No history for range of indices: %r' % index)
return hist
+ def store_inputs(self, source, source_raw=None):
+ """Store source and raw input in history.
+
+ Parameters
+ ----------
+ source : str
+ Python input.
+
+ source_raw : str, optional
+ If given, this is the raw input without any IPython transformations
+ applied to it. If not given, ``source`` is used.
+ """
+ if source_raw is None:
+ source_raw = source
+ self.input_hist.append(source)
+ self.input_hist_raw.append(source_raw)
+ self.shadow_hist.add(source)
+
+ def sync_inputs(self):
+ """Ensure raw and translated histories have same length."""
+ if len(self.input_hist) != len (self.input_hist_raw):
+ self.input_hist_raw = InputList(self.input_hist)
+
+
+ def reset(self):
+ """Clear all histories managed by this object."""
+ self.input_hist[:] = []
+ self.input_hist_raw[:] = []
+ self.output_hist.clear()
+ # The directory history can't be completely empty
+ self.dir_hist[:] = [os.getcwd()]
+
def magic_history(self, parameter_s = ''):
"""Print input history (_i<n> variables), with most recent last.
diff --git a/IPython/core/inputsplitter.py b/IPython/core/inputsplitter.py
index b53d6bc..d1ce545 100644
--- a/IPython/core/inputsplitter.py
+++ b/IPython/core/inputsplitter.py
@@ -600,20 +600,23 @@ class InputSplitter(object):
if line and not line.isspace():
self.indent_spaces, self._full_dedent = self._find_indent(line)
- def _store(self, lines):
+ def _store(self, lines, buffer=None, store='source'):
"""Store one or more lines of input.
If input lines are not newline-terminated, a newline is automatically
appended."""
+ if buffer is None:
+ buffer = self._buffer
+
if lines.endswith('\n'):
- self._buffer.append(lines)
+ buffer.append(lines)
else:
- self._buffer.append(lines+'\n')
- self._set_source()
+ buffer.append(lines+'\n')
+ setattr(self, store, self._set_source(buffer))
- def _set_source(self):
- self.source = ''.join(self._buffer).encode(self.encoding)
+ def _set_source(self, buffer):
+ return ''.join(buffer).encode(self.encoding)
#-----------------------------------------------------------------------------
@@ -933,6 +936,32 @@ transform_escaped = EscapedTransformer()
class IPythonInputSplitter(InputSplitter):
"""An input splitter that recognizes all of IPython's special syntax."""
+ # String with raw, untransformed input.
+ source_raw = ''
+
+ # Private attributes
+
+ # List with lines of raw input accumulated so far.
+ _buffer_raw = None
+
+ def __init__(self, input_mode=None):
+ InputSplitter.__init__(self, input_mode)
+ self._buffer_raw = []
+
+ def reset(self):
+ """Reset the input buffer and associated state."""
+ InputSplitter.reset(self)
+ self._buffer_raw[:] = []
+ self.source_raw = ''
+
+ def source_raw_reset(self):
+ """Return input and raw source and perform a full reset.
+ """
+ out = self.source
+ out_r = self.source_raw
+ self.reset()
+ return out, out_r
+
def push(self, lines):
"""Push one or more lines of IPython input.
"""
@@ -964,13 +993,18 @@ class IPythonInputSplitter(InputSplitter):
# by one. Note that this only matters if the input has more than one
# line.
changed_input_mode = False
-
- if len(lines_list)>1 and self.input_mode == 'cell':
+
+ if self.input_mode == 'cell':
self.reset()
changed_input_mode = True
saved_input_mode = 'cell'
self.input_mode = 'line'
+ # Store raw source before applying any transformations to it. Note
+ # that this must be done *after* the reset() call that would otherwise
+ # flush the buffer.
+ self._store(lines, self._buffer_raw, 'source_raw')
+
try:
push = super(IPythonInputSplitter, self).push
for line in lines_list:
@@ -983,5 +1017,4 @@ class IPythonInputSplitter(InputSplitter):
finally:
if changed_input_mode:
self.input_mode = saved_input_mode
-
return out
diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py
index 94cbec7..7ed856a 100644
--- a/IPython/core/interactiveshell.py
+++ b/IPython/core/interactiveshell.py
@@ -218,7 +218,7 @@ class InteractiveShell(Configurable, Magic):
plugin_manager = Instance('IPython.core.plugin.PluginManager')
payload_manager = Instance('IPython.core.payload.PayloadManager')
history_manager = Instance('IPython.core.history.HistoryManager')
-
+
# Private interface
_post_execute = set()
@@ -369,8 +369,9 @@ class InteractiveShell(Configurable, Magic):
# command compiler
self.compile = codeop.CommandCompiler()
- # User input buffer
+ # User input buffers
self.buffer = []
+ self.buffer_raw = []
# Make an empty namespace, which extension writers can rely on both
# existing and NEVER being used by ipython itself. This gives them a
@@ -965,30 +966,25 @@ class InteractiveShell(Configurable, Magic):
# Finally, update the real user's namespace
self.user_ns.update(ns)
-
def reset(self):
"""Clear all internal namespaces.
Note that this is much more aggressive than %reset, since it clears
fully all namespaces, as well as all input/output lists.
"""
- for ns in self.ns_refs_table:
- ns.clear()
-
- self.alias_manager.clear_aliases()
-
- # Clear input and output histories
- self.input_hist[:] = []
- self.input_hist_raw[:] = []
- self.output_hist.clear()
+ # Clear histories
+ self.history_manager.reset()
# Reset counter used to index all histories
self.execution_count = 0
# Restore the user namespaces to minimal usability
+ for ns in self.ns_refs_table:
+ ns.clear()
self.init_user_ns()
# Restore the default and user aliases
+ self.alias_manager.clear_aliases()
self.alias_manager.init_aliases()
def reset_selective(self, regex=None):
@@ -2103,9 +2099,7 @@ class InteractiveShell(Configurable, Magic):
self.execution_count += 1
# Store raw and processed history
- self.input_hist_raw.append(cell)
- self.input_hist.append(ipy_cell)
-
+ self.history_manager.store_inputs(ipy_cell, cell)
# dbg code!!!
def myapp(self, val): # dbg
@@ -2194,7 +2188,13 @@ class InteractiveShell(Configurable, Magic):
# interactive IPython session (via a magic, for example).
self.resetbuffer()
lines = lines.splitlines()
- more = 0
+
+ # Since we will prefilter all lines, store the user's raw input too
+ # before we apply any transformations
+ self.buffer_raw[:] = [ l+'\n' for l in lines]
+
+ more = False
+ prefilter_lines = self.prefilter_manager.prefilter_lines
with nested(self.builtin_trap, self.display_trap):
for line in lines:
# skip blank lines so we don't mess up the prompt counter, but
@@ -2202,19 +2202,13 @@ class InteractiveShell(Configurable, Magic):
# is true)
if line or more:
- # push to raw history, so hist line numbers stay in sync
- self.input_hist_raw.append(line + '\n')
- prefiltered = self.prefilter_manager.prefilter_lines(line,
- more)
- more = self.push_line(prefiltered)
+ more = self.push_line(prefilter_lines(line, more))
# IPython's runsource returns None if there was an error
# compiling the code. This allows us to stop processing
# right away, so the user gets the error message at the
# right place.
if more is None:
break
- else:
- self.input_hist_raw.append("\n")
# final newline in case the input didn't have it, so that the code
# actually does get executed
if more:
@@ -2370,8 +2364,11 @@ class InteractiveShell(Configurable, Magic):
for subline in line.splitlines():
self._autoindent_update(subline)
self.buffer.append(line)
- more = self.runsource('\n'.join(self.buffer), self.filename)
+ full_source = '\n'.join(self.buffer)
+ more = self.runsource(full_source, self.filename)
if not more:
+ self.history_manager.store_inputs('\n'.join(self.buffer_raw),
+ full_source)
self.resetbuffer()
self.execution_count += 1
return more
@@ -2379,6 +2376,7 @@ class InteractiveShell(Configurable, Magic):
def resetbuffer(self):
"""Reset the input buffer."""
self.buffer[:] = []
+ self.buffer_raw[:] = []
def _is_secondary_block_start(self, s):
if not s.endswith(':'):
diff --git a/IPython/core/logger.py b/IPython/core/logger.py
index 1df2165..e142650 100644
--- a/IPython/core/logger.py
+++ b/IPython/core/logger.py
@@ -194,7 +194,8 @@ which already exists. But you must first start the logging process with
# add blank lines if the input cache fell out of sync.
if out_cache.do_full_cache and \
out_cache.prompt_count +1 > len(input_hist):
- input_hist.extend(['\n'] * (out_cache.prompt_count - len(input_hist)))
+ pass
+ #input_hist.extend(['\n'] * (out_cache.prompt_count - len(input_hist)))
if not continuation and line_mod:
self._iii = self._ii
@@ -203,7 +204,7 @@ which already exists. But you must first start the logging process with
# put back the final \n of every input line
self._i00 = line_mod+'\n'
#print 'Logging input:<%s>' % line_mod # dbg
- input_hist.append(self._i00)
+ #input_hist.append(self._i00)
#print '---[%s]' % (len(input_hist)-1,) # dbg
# hackish access to top-level namespace to create _i1,_i2... dynamically
@@ -222,7 +223,7 @@ which already exists. But you must first start the logging process with
new_i = '_i%s' % in_num
if continuation:
self._i00 = '%s%s\n' % (self.shell.user_ns[new_i],line_mod)
- input_hist[in_num] = self._i00
+ #input_hist[in_num] = self._i00
to_main[new_i] = self._i00
self.shell.user_ns.update(to_main)
diff --git a/IPython/core/prefilter.py b/IPython/core/prefilter.py
index e427091..dc1d1dc 100755
--- a/IPython/core/prefilter.py
+++ b/IPython/core/prefilter.py
@@ -428,7 +428,7 @@ class PrefilterManager(Configurable):
which is the case when the user goes back to a multiline history
entry and presses enter.
"""
- llines = lines.rstrip('\n').split('\n')
+ llines = lines.rstrip('\n').splitlines()
# We can get multiple lines in one shot, where multiline input 'blends'
# into one line, in cases like recalling from the readline history
# buffer. We need to make sure that in such cases, we correctly
diff --git a/IPython/core/tests/test_inputsplitter.py b/IPython/core/tests/test_inputsplitter.py
index fd2dbfc..3a40449 100644
--- a/IPython/core/tests/test_inputsplitter.py
+++ b/IPython/core/tests/test_inputsplitter.py
@@ -563,6 +563,8 @@ class IPythonInputTestCase(InputSplitterTestCase):
In addition, this runs the tests over the syntax and syntax_ml dicts that
were tested by individual functions, as part of the OO interface.
+
+ It also makes some checks on the raw buffer storage.
"""
def setUp(self):
@@ -577,21 +579,26 @@ class IPythonInputTestCase(InputSplitterTestCase):
continue
isp.push(raw)
- out = isp.source_reset().rstrip()
- self.assertEqual(out, out_t)
+ out, out_raw = isp.source_raw_reset()
+ self.assertEqual(out.rstrip(), out_t)
+ self.assertEqual(out_raw.rstrip(), raw.rstrip())
def test_syntax_multiline(self):
isp = self.isp
for example in syntax_ml.itervalues():
out_t_parts = []
+ raw_parts = []
for line_pairs in example:
- for raw, out_t_part in line_pairs:
- isp.push(raw)
+ for lraw, out_t_part in line_pairs:
+ isp.push(lraw)
out_t_parts.append(out_t_part)
+ raw_parts.append(lraw)
- out = isp.source_reset().rstrip()
+ out, out_raw = isp.source_raw_reset()
out_t = '\n'.join(out_t_parts).rstrip()
- self.assertEqual(out, out_t)
+ raw = '\n'.join(raw_parts).rstrip()
+ self.assertEqual(out.rstrip(), out_t)
+ self.assertEqual(out_raw.rstrip(), raw)
class BlockIPythonInputTestCase(IPythonInputTestCase):
@@ -616,9 +623,10 @@ class BlockIPythonInputTestCase(IPythonInputTestCase):
out_t = '\n'.join(out_t_parts)
isp.push(raw)
- out = isp.source_reset()
+ out, out_raw = isp.source_raw_reset()
# Match ignoring trailing whitespace
self.assertEqual(out.rstrip(), out_t.rstrip())
+ self.assertEqual(out_raw.rstrip(), raw.rstrip())
#-----------------------------------------------------------------------------
@@ -652,7 +660,8 @@ if __name__ == '__main__':
# Here we just return input so we can use it in a test suite, but a
# real interpreter would instead send it for execution somewhere.
#src = isp.source; raise EOFError # dbg
- src = isp.source_reset()
+ src, raw = isp.source_raw_reset()
print 'Input source was:\n', src
+ print 'Raw source was:\n', raw
except EOFError:
print 'Bye'
diff --git a/IPython/frontend/terminal/interactiveshell.py b/IPython/frontend/terminal/interactiveshell.py
index 80f5a92..d4c9df8 100644
--- a/IPython/frontend/terminal/interactiveshell.py
+++ b/IPython/frontend/terminal/interactiveshell.py
@@ -191,8 +191,7 @@ class TerminalInteractiveShell(InteractiveShell):
# if you run stuff with -c <cmd>, raw hist is not updated
# ensure that it's in sync
- if len(self.input_hist) != len (self.input_hist_raw):
- self.input_hist_raw = InputList(self.input_hist)
+ self.history_manager.sync_inputs()
while 1:
try:
@@ -218,7 +217,7 @@ class TerminalInteractiveShell(InteractiveShell):
if display_banner:
self.show_banner()
- more = 0
+ more = False
# Mark activity in the builtins
__builtin__.__dict__['__IPYTHON__active'] += 1
@@ -231,7 +230,7 @@ class TerminalInteractiveShell(InteractiveShell):
# Before showing any prompts, if the counter is at zero, we execute an
# empty line to ensure the user only sees prompts starting at one.
if self.execution_count == 0:
- self.push_line('\n')
+ self.execution_count += 1
while not self.exit_now:
self.hooks.pre_prompt_hook()
@@ -249,7 +248,7 @@ class TerminalInteractiveShell(InteractiveShell):
except:
self.showtraceback()
try:
- line = self.raw_input(prompt, more)
+ line = self.raw_input(prompt)
if self.exit_now:
# quick exit on sys.std[in|out] close
break
@@ -266,7 +265,7 @@ class TerminalInteractiveShell(InteractiveShell):
if self.autoindent:
self.indent_current_nsp = 0
- more = 0
+ more = False
except KeyboardInterrupt:
pass
except EOFError:
@@ -286,18 +285,22 @@ class TerminalInteractiveShell(InteractiveShell):
# asynchronously by signal handlers, for example.
self.showtraceback()
else:
- more = self.push_line(line)
+ #more = self.push_line(line)
+ self.input_splitter.push(line)
+ more = self.input_splitter.push_accepts_more()
if (self.SyntaxTB.last_syntax_error and
self.autoedit_syntax):
self.edit_syntax_error()
-
+ if not more:
+ pass
+
# We are off again...
__builtin__.__dict__['__IPYTHON__active'] -= 1
# Turn off the exit flag, so the mainloop can be restarted if desired
self.exit_now = False
- def raw_input(self,prompt='',continue_prompt=False):
+ def raw_input(self, prompt='', continue_prompt=False):
"""Write a prompt and read a line.
The returned line does not include the trailing newline.
@@ -310,8 +313,6 @@ class TerminalInteractiveShell(InteractiveShell):
- continue_prompt(False): whether this line is the first one or a
continuation in a sequence of inputs.
"""
- # growl.notify("raw_input: ", "prompt = %r\ncontinue_prompt = %s" % (prompt, continue_prompt))
-
# Code run by the user may have modified the readline completer state.
# We must ensure that our completer is back in place.
@@ -329,8 +330,6 @@ class TerminalInteractiveShell(InteractiveShell):
# Try to be reasonably smart about not re-indenting pasted input more
# than necessary. We do this by trimming out the auto-indent initial
# spaces, if the user's actual input started itself with whitespace.
- #debugx('self.buffer[-1]')
-
if self.autoindent:
if num_ini_spaces(line) > self.indent_current_nsp:
line = line[self.indent_current_nsp:]
@@ -340,22 +339,15 @@ class TerminalInteractiveShell(InteractiveShell):
# it.
if line.strip():
if continue_prompt:
- self.input_hist_raw[-1] += '%s\n' % line
if self.has_readline and self.readline_use:
- try:
- histlen = self.readline.get_current_history_length()
- if histlen > 1:
- newhist = self.input_hist_raw[-1].rstrip()
- self.readline.remove_history_item(histlen-1)
- self.readline.replace_history_item(histlen-2,
- newhist.encode(self.stdin_encoding))
- except AttributeError:
- pass # re{move,place}_history_item are new in 2.4.
+ histlen = self.readline.get_current_history_length()
+ if histlen > 1:
+ newhist = self.input_hist_raw[-1].rstrip()
+ self.readline.remove_history_item(histlen-1)
+ self.readline.replace_history_item(histlen-2,
+ newhist.encode(self.stdin_encoding))
else:
self.input_hist_raw.append('%s\n' % line)
- # only entries starting at first column go to shadow history
- if line.lstrip() == line:
- self.shadowhist.add(line.strip())
elif not continue_prompt:
self.input_hist_raw.append('\n')
try:
@@ -368,6 +360,45 @@ class TerminalInteractiveShell(InteractiveShell):
else:
return lineout
+
+ def raw_input(self, prompt=''):
+ """Write a prompt and read a line.
+
+ The returned line does not include the trailing newline.
+ When the user enters the EOF key sequence, EOFError is raised.
+
+ Optional inputs:
+
+ - prompt(''): a string to be printed to prompt the user.
+
+ - continue_prompt(False): whether this line is the first one or a
+ continuation in a sequence of inputs.
+ """
+ # Code run by the user may have modified the readline completer state.
+ # We must ensure that our completer is back in place.
+
+ if self.has_readline:
+ self.set_readline_completer()
+
+ try:
+ line = raw_input_original(prompt).decode(self.stdin_encoding)
+ except ValueError:
+ warn("\n********\nYou or a %run:ed script called sys.stdin.close()"
+ " or sys.stdout.close()!\nExiting IPython!")
+ self.ask_exit()
+ return ""
+
+ # Try to be reasonably smart about not re-indenting pasted input more
+ # than necessary. We do this by trimming out the auto-indent initial
+ # spaces, if the user's actual input started itself with whitespace.
+ if self.autoindent:
+ if num_ini_spaces(line) > self.indent_current_nsp:
+ line = line[self.indent_current_nsp:]
+ self.indent_current_nsp = 0
+
+ return line
+
+
# TODO: The following three methods are an early attempt to refactor
# the main code execution logic. We don't use them, but they may be
# helpful when we refactor the code execution logic further.