From dceccd93b8a81ab920529f4f433018fbe9ee5f4b 2010-10-06 04:47:21 From: Fernando Perez Date: 2010-10-06 04:47:21 Subject: [PATCH] Add support for accessing raw data to inputsplitter. This is necessary to be able to use inputsplitter systematically across all frontends, including plain terminal one. --- diff --git a/IPython/core/history.py b/IPython/core/history.py index b92e16b..490643f 100644 --- a/IPython/core/history.py +++ b/IPython/core/history.py @@ -61,13 +61,12 @@ class HistoryManager(object): histfname = 'history' self.hist_file = os.path.join(shell.ipython_dir, histfname) - # Fill the history zero entry, user counter starts at 1 - self.input_hist.append('\n') - self.input_hist_raw.append('\n') - # Objects related to shadow history management self._init_shadow_hist() + # Fill the history zero entry, user counter starts at 1 + self.store_inputs('\n', '\n') + # For backwards compatibility, we must put these back in the shell # object, until we've removed all direct uses of the history objects in # the shell itself. @@ -154,10 +153,42 @@ class HistoryManager(object): hist[i] = (input_hist[i], output_hist.get(i)) else: hist[i] = input_hist[i] - if len(hist)==0: + if not hist: raise IndexError('No history for range of indices: %r' % index) return hist + def store_inputs(self, source, source_raw=None): + """Store source and raw input in history. + + Parameters + ---------- + source : str + Python input. + + source_raw : str, optional + If given, this is the raw input without any IPython transformations + applied to it. If not given, ``source`` is used. + """ + if source_raw is None: + source_raw = source + self.input_hist.append(source) + self.input_hist_raw.append(source_raw) + self.shadow_hist.add(source) + + def sync_inputs(self): + """Ensure raw and translated histories have same length.""" + if len(self.input_hist) != len (self.input_hist_raw): + self.input_hist_raw = InputList(self.input_hist) + + + def reset(self): + """Clear all histories managed by this object.""" + self.input_hist[:] = [] + self.input_hist_raw[:] = [] + self.output_hist.clear() + # The directory history can't be completely empty + self.dir_hist[:] = [os.getcwd()] + def magic_history(self, parameter_s = ''): """Print input history (_i variables), with most recent last. diff --git a/IPython/core/inputsplitter.py b/IPython/core/inputsplitter.py index b53d6bc..d1ce545 100644 --- a/IPython/core/inputsplitter.py +++ b/IPython/core/inputsplitter.py @@ -600,20 +600,23 @@ class InputSplitter(object): if line and not line.isspace(): self.indent_spaces, self._full_dedent = self._find_indent(line) - def _store(self, lines): + def _store(self, lines, buffer=None, store='source'): """Store one or more lines of input. If input lines are not newline-terminated, a newline is automatically appended.""" + if buffer is None: + buffer = self._buffer + if lines.endswith('\n'): - self._buffer.append(lines) + buffer.append(lines) else: - self._buffer.append(lines+'\n') - self._set_source() + buffer.append(lines+'\n') + setattr(self, store, self._set_source(buffer)) - def _set_source(self): - self.source = ''.join(self._buffer).encode(self.encoding) + def _set_source(self, buffer): + return ''.join(buffer).encode(self.encoding) #----------------------------------------------------------------------------- @@ -933,6 +936,32 @@ transform_escaped = EscapedTransformer() class IPythonInputSplitter(InputSplitter): """An input splitter that recognizes all of IPython's special syntax.""" + # String with raw, untransformed input. + source_raw = '' + + # Private attributes + + # List with lines of raw input accumulated so far. + _buffer_raw = None + + def __init__(self, input_mode=None): + InputSplitter.__init__(self, input_mode) + self._buffer_raw = [] + + def reset(self): + """Reset the input buffer and associated state.""" + InputSplitter.reset(self) + self._buffer_raw[:] = [] + self.source_raw = '' + + def source_raw_reset(self): + """Return input and raw source and perform a full reset. + """ + out = self.source + out_r = self.source_raw + self.reset() + return out, out_r + def push(self, lines): """Push one or more lines of IPython input. """ @@ -964,13 +993,18 @@ class IPythonInputSplitter(InputSplitter): # by one. Note that this only matters if the input has more than one # line. changed_input_mode = False - - if len(lines_list)>1 and self.input_mode == 'cell': + + if self.input_mode == 'cell': self.reset() changed_input_mode = True saved_input_mode = 'cell' self.input_mode = 'line' + # Store raw source before applying any transformations to it. Note + # that this must be done *after* the reset() call that would otherwise + # flush the buffer. + self._store(lines, self._buffer_raw, 'source_raw') + try: push = super(IPythonInputSplitter, self).push for line in lines_list: @@ -983,5 +1017,4 @@ class IPythonInputSplitter(InputSplitter): finally: if changed_input_mode: self.input_mode = saved_input_mode - return out diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index 94cbec7..7ed856a 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -218,7 +218,7 @@ class InteractiveShell(Configurable, Magic): plugin_manager = Instance('IPython.core.plugin.PluginManager') payload_manager = Instance('IPython.core.payload.PayloadManager') history_manager = Instance('IPython.core.history.HistoryManager') - + # Private interface _post_execute = set() @@ -369,8 +369,9 @@ class InteractiveShell(Configurable, Magic): # command compiler self.compile = codeop.CommandCompiler() - # User input buffer + # User input buffers self.buffer = [] + self.buffer_raw = [] # Make an empty namespace, which extension writers can rely on both # existing and NEVER being used by ipython itself. This gives them a @@ -965,30 +966,25 @@ class InteractiveShell(Configurable, Magic): # Finally, update the real user's namespace self.user_ns.update(ns) - def reset(self): """Clear all internal namespaces. Note that this is much more aggressive than %reset, since it clears fully all namespaces, as well as all input/output lists. """ - for ns in self.ns_refs_table: - ns.clear() - - self.alias_manager.clear_aliases() - - # Clear input and output histories - self.input_hist[:] = [] - self.input_hist_raw[:] = [] - self.output_hist.clear() + # Clear histories + self.history_manager.reset() # Reset counter used to index all histories self.execution_count = 0 # Restore the user namespaces to minimal usability + for ns in self.ns_refs_table: + ns.clear() self.init_user_ns() # Restore the default and user aliases + self.alias_manager.clear_aliases() self.alias_manager.init_aliases() def reset_selective(self, regex=None): @@ -2103,9 +2099,7 @@ class InteractiveShell(Configurable, Magic): self.execution_count += 1 # Store raw and processed history - self.input_hist_raw.append(cell) - self.input_hist.append(ipy_cell) - + self.history_manager.store_inputs(ipy_cell, cell) # dbg code!!! def myapp(self, val): # dbg @@ -2194,7 +2188,13 @@ class InteractiveShell(Configurable, Magic): # interactive IPython session (via a magic, for example). self.resetbuffer() lines = lines.splitlines() - more = 0 + + # Since we will prefilter all lines, store the user's raw input too + # before we apply any transformations + self.buffer_raw[:] = [ l+'\n' for l in lines] + + more = False + prefilter_lines = self.prefilter_manager.prefilter_lines with nested(self.builtin_trap, self.display_trap): for line in lines: # skip blank lines so we don't mess up the prompt counter, but @@ -2202,19 +2202,13 @@ class InteractiveShell(Configurable, Magic): # is true) if line or more: - # push to raw history, so hist line numbers stay in sync - self.input_hist_raw.append(line + '\n') - prefiltered = self.prefilter_manager.prefilter_lines(line, - more) - more = self.push_line(prefiltered) + more = self.push_line(prefilter_lines(line, more)) # IPython's runsource returns None if there was an error # compiling the code. This allows us to stop processing # right away, so the user gets the error message at the # right place. if more is None: break - else: - self.input_hist_raw.append("\n") # final newline in case the input didn't have it, so that the code # actually does get executed if more: @@ -2370,8 +2364,11 @@ class InteractiveShell(Configurable, Magic): for subline in line.splitlines(): self._autoindent_update(subline) self.buffer.append(line) - more = self.runsource('\n'.join(self.buffer), self.filename) + full_source = '\n'.join(self.buffer) + more = self.runsource(full_source, self.filename) if not more: + self.history_manager.store_inputs('\n'.join(self.buffer_raw), + full_source) self.resetbuffer() self.execution_count += 1 return more @@ -2379,6 +2376,7 @@ class InteractiveShell(Configurable, Magic): def resetbuffer(self): """Reset the input buffer.""" self.buffer[:] = [] + self.buffer_raw[:] = [] def _is_secondary_block_start(self, s): if not s.endswith(':'): diff --git a/IPython/core/logger.py b/IPython/core/logger.py index 1df2165..e142650 100644 --- a/IPython/core/logger.py +++ b/IPython/core/logger.py @@ -194,7 +194,8 @@ which already exists. But you must first start the logging process with # add blank lines if the input cache fell out of sync. if out_cache.do_full_cache and \ out_cache.prompt_count +1 > len(input_hist): - input_hist.extend(['\n'] * (out_cache.prompt_count - len(input_hist))) + pass + #input_hist.extend(['\n'] * (out_cache.prompt_count - len(input_hist))) if not continuation and line_mod: self._iii = self._ii @@ -203,7 +204,7 @@ which already exists. But you must first start the logging process with # put back the final \n of every input line self._i00 = line_mod+'\n' #print 'Logging input:<%s>' % line_mod # dbg - input_hist.append(self._i00) + #input_hist.append(self._i00) #print '---[%s]' % (len(input_hist)-1,) # dbg # hackish access to top-level namespace to create _i1,_i2... dynamically @@ -222,7 +223,7 @@ which already exists. But you must first start the logging process with new_i = '_i%s' % in_num if continuation: self._i00 = '%s%s\n' % (self.shell.user_ns[new_i],line_mod) - input_hist[in_num] = self._i00 + #input_hist[in_num] = self._i00 to_main[new_i] = self._i00 self.shell.user_ns.update(to_main) diff --git a/IPython/core/prefilter.py b/IPython/core/prefilter.py index e427091..dc1d1dc 100755 --- a/IPython/core/prefilter.py +++ b/IPython/core/prefilter.py @@ -428,7 +428,7 @@ class PrefilterManager(Configurable): which is the case when the user goes back to a multiline history entry and presses enter. """ - llines = lines.rstrip('\n').split('\n') + llines = lines.rstrip('\n').splitlines() # We can get multiple lines in one shot, where multiline input 'blends' # into one line, in cases like recalling from the readline history # buffer. We need to make sure that in such cases, we correctly diff --git a/IPython/core/tests/test_inputsplitter.py b/IPython/core/tests/test_inputsplitter.py index fd2dbfc..3a40449 100644 --- a/IPython/core/tests/test_inputsplitter.py +++ b/IPython/core/tests/test_inputsplitter.py @@ -563,6 +563,8 @@ class IPythonInputTestCase(InputSplitterTestCase): In addition, this runs the tests over the syntax and syntax_ml dicts that were tested by individual functions, as part of the OO interface. + + It also makes some checks on the raw buffer storage. """ def setUp(self): @@ -577,21 +579,26 @@ class IPythonInputTestCase(InputSplitterTestCase): continue isp.push(raw) - out = isp.source_reset().rstrip() - self.assertEqual(out, out_t) + out, out_raw = isp.source_raw_reset() + self.assertEqual(out.rstrip(), out_t) + self.assertEqual(out_raw.rstrip(), raw.rstrip()) def test_syntax_multiline(self): isp = self.isp for example in syntax_ml.itervalues(): out_t_parts = [] + raw_parts = [] for line_pairs in example: - for raw, out_t_part in line_pairs: - isp.push(raw) + for lraw, out_t_part in line_pairs: + isp.push(lraw) out_t_parts.append(out_t_part) + raw_parts.append(lraw) - out = isp.source_reset().rstrip() + out, out_raw = isp.source_raw_reset() out_t = '\n'.join(out_t_parts).rstrip() - self.assertEqual(out, out_t) + raw = '\n'.join(raw_parts).rstrip() + self.assertEqual(out.rstrip(), out_t) + self.assertEqual(out_raw.rstrip(), raw) class BlockIPythonInputTestCase(IPythonInputTestCase): @@ -616,9 +623,10 @@ class BlockIPythonInputTestCase(IPythonInputTestCase): out_t = '\n'.join(out_t_parts) isp.push(raw) - out = isp.source_reset() + out, out_raw = isp.source_raw_reset() # Match ignoring trailing whitespace self.assertEqual(out.rstrip(), out_t.rstrip()) + self.assertEqual(out_raw.rstrip(), raw.rstrip()) #----------------------------------------------------------------------------- @@ -652,7 +660,8 @@ if __name__ == '__main__': # Here we just return input so we can use it in a test suite, but a # real interpreter would instead send it for execution somewhere. #src = isp.source; raise EOFError # dbg - src = isp.source_reset() + src, raw = isp.source_raw_reset() print 'Input source was:\n', src + print 'Raw source was:\n', raw except EOFError: print 'Bye' diff --git a/IPython/frontend/terminal/interactiveshell.py b/IPython/frontend/terminal/interactiveshell.py index 80f5a92..d4c9df8 100644 --- a/IPython/frontend/terminal/interactiveshell.py +++ b/IPython/frontend/terminal/interactiveshell.py @@ -191,8 +191,7 @@ class TerminalInteractiveShell(InteractiveShell): # if you run stuff with -c , raw hist is not updated # ensure that it's in sync - if len(self.input_hist) != len (self.input_hist_raw): - self.input_hist_raw = InputList(self.input_hist) + self.history_manager.sync_inputs() while 1: try: @@ -218,7 +217,7 @@ class TerminalInteractiveShell(InteractiveShell): if display_banner: self.show_banner() - more = 0 + more = False # Mark activity in the builtins __builtin__.__dict__['__IPYTHON__active'] += 1 @@ -231,7 +230,7 @@ class TerminalInteractiveShell(InteractiveShell): # Before showing any prompts, if the counter is at zero, we execute an # empty line to ensure the user only sees prompts starting at one. if self.execution_count == 0: - self.push_line('\n') + self.execution_count += 1 while not self.exit_now: self.hooks.pre_prompt_hook() @@ -249,7 +248,7 @@ class TerminalInteractiveShell(InteractiveShell): except: self.showtraceback() try: - line = self.raw_input(prompt, more) + line = self.raw_input(prompt) if self.exit_now: # quick exit on sys.std[in|out] close break @@ -266,7 +265,7 @@ class TerminalInteractiveShell(InteractiveShell): if self.autoindent: self.indent_current_nsp = 0 - more = 0 + more = False except KeyboardInterrupt: pass except EOFError: @@ -286,18 +285,22 @@ class TerminalInteractiveShell(InteractiveShell): # asynchronously by signal handlers, for example. self.showtraceback() else: - more = self.push_line(line) + #more = self.push_line(line) + self.input_splitter.push(line) + more = self.input_splitter.push_accepts_more() if (self.SyntaxTB.last_syntax_error and self.autoedit_syntax): self.edit_syntax_error() - + if not more: + pass + # We are off again... __builtin__.__dict__['__IPYTHON__active'] -= 1 # Turn off the exit flag, so the mainloop can be restarted if desired self.exit_now = False - def raw_input(self,prompt='',continue_prompt=False): + def raw_input(self, prompt='', continue_prompt=False): """Write a prompt and read a line. The returned line does not include the trailing newline. @@ -310,8 +313,6 @@ class TerminalInteractiveShell(InteractiveShell): - continue_prompt(False): whether this line is the first one or a continuation in a sequence of inputs. """ - # growl.notify("raw_input: ", "prompt = %r\ncontinue_prompt = %s" % (prompt, continue_prompt)) - # Code run by the user may have modified the readline completer state. # We must ensure that our completer is back in place. @@ -329,8 +330,6 @@ class TerminalInteractiveShell(InteractiveShell): # Try to be reasonably smart about not re-indenting pasted input more # than necessary. We do this by trimming out the auto-indent initial # spaces, if the user's actual input started itself with whitespace. - #debugx('self.buffer[-1]') - if self.autoindent: if num_ini_spaces(line) > self.indent_current_nsp: line = line[self.indent_current_nsp:] @@ -340,22 +339,15 @@ class TerminalInteractiveShell(InteractiveShell): # it. if line.strip(): if continue_prompt: - self.input_hist_raw[-1] += '%s\n' % line if self.has_readline and self.readline_use: - try: - histlen = self.readline.get_current_history_length() - if histlen > 1: - newhist = self.input_hist_raw[-1].rstrip() - self.readline.remove_history_item(histlen-1) - self.readline.replace_history_item(histlen-2, - newhist.encode(self.stdin_encoding)) - except AttributeError: - pass # re{move,place}_history_item are new in 2.4. + histlen = self.readline.get_current_history_length() + if histlen > 1: + newhist = self.input_hist_raw[-1].rstrip() + self.readline.remove_history_item(histlen-1) + self.readline.replace_history_item(histlen-2, + newhist.encode(self.stdin_encoding)) else: self.input_hist_raw.append('%s\n' % line) - # only entries starting at first column go to shadow history - if line.lstrip() == line: - self.shadowhist.add(line.strip()) elif not continue_prompt: self.input_hist_raw.append('\n') try: @@ -368,6 +360,45 @@ class TerminalInteractiveShell(InteractiveShell): else: return lineout + + def raw_input(self, prompt=''): + """Write a prompt and read a line. + + The returned line does not include the trailing newline. + When the user enters the EOF key sequence, EOFError is raised. + + Optional inputs: + + - prompt(''): a string to be printed to prompt the user. + + - continue_prompt(False): whether this line is the first one or a + continuation in a sequence of inputs. + """ + # Code run by the user may have modified the readline completer state. + # We must ensure that our completer is back in place. + + if self.has_readline: + self.set_readline_completer() + + try: + line = raw_input_original(prompt).decode(self.stdin_encoding) + except ValueError: + warn("\n********\nYou or a %run:ed script called sys.stdin.close()" + " or sys.stdout.close()!\nExiting IPython!") + self.ask_exit() + return "" + + # Try to be reasonably smart about not re-indenting pasted input more + # than necessary. We do this by trimming out the auto-indent initial + # spaces, if the user's actual input started itself with whitespace. + if self.autoindent: + if num_ini_spaces(line) > self.indent_current_nsp: + line = line[self.indent_current_nsp:] + self.indent_current_nsp = 0 + + return line + + # TODO: The following three methods are an early attempt to refactor # the main code execution logic. We don't use them, but they may be # helpful when we refactor the code execution logic further.