History refactored and saved to json file

- stopped using reading and writing history. history saved to json file and terminal readline data is populated from it
- removed duplicate attributes of history manager being stored in shell
  - only kept db as a property (it was widely used)
- more refactoring of history manager
  - stopped using InputList (errorprone semantics) for history and switched to regular lists
- added test

IPython/core/tests/test_history.py

@@ -0,0 +1,48 b''
+"""Tests for the IPython tab-completion machinery.
+"""
+#-----------------------------------------------------------------------------
+# Module imports
+#-----------------------------------------------------------------------------
+
+# stdlib
+import os
+import sys
+import unittest
+
+# third party
+import nose.tools as nt
+
+# our own packages
+from IPython.utils.tempdir import TemporaryDirectory
+from IPython.core.history import HistoryManager
+
+def test_history():
+
+    ip = get_ipython()
+    with TemporaryDirectory() as tmpdir:
+        #tmpdir = '/software/temp'
+        histfile = os.path.realpath(os.path.join(tmpdir, 'history.json'))
+        # Ensure that we restore the history management that we mess with in
+        # this test doesn't affect the IPython instance used by the test suite
+        # beyond this test.
+        hist_manager_ori = ip.history_manager
+        try:
+            ip.history_manager = HistoryManager(ip)
+            ip.history_manager.hist_file = histfile
+            print 'test',histfile
+            hist = ['a=1\n', 'def f():\n    test = 1\n    return test\n', 'b=2\n']
+            # test save and load
+            ip.history_manager.input_hist_raw[:] = []
+            for h in hist:
+                ip.history_manager.input_hist_raw.append(h)
+            ip.save_history()
+            ip.history_manager.input_hist_raw[:] = []
+            ip.reload_history()
+            print type(ip.history_manager.input_hist_raw)
+            print ip.history_manager.input_hist_raw
+            nt.assert_equal(len(ip.history_manager.input_hist_raw), len(hist))
+            for i,h in enumerate(hist):
+                nt.assert_equal(hist[i], ip.history_manager.input_hist_raw[i])
+        finally:
+            # Restore history manager
+            ip.history_manager = hist_manager_ori

IPython/config/default/ipython_config.py

@@ -90,6 +90,8 b' c = get_config()'
 
 # c.InteractiveShell.quiet = False
 
+# c.InteractiveShell.history_length = 10000
+
 # Readline 
 # c.InteractiveShell.readline_use = True
 

IPython/core/displayhook.py

@@ -173,7 +173,7 b' class DisplayHook(Configurable):'
         """Should we silence the display hook because of ';'?"""
         # do not print output if input ends in ';'
         try:
-            if self.shell.input_hist[self.prompt_count].endswith(';\n'):
+            if self.shell.history_manager.input_hist_parsed[self.prompt_count].endswith(';\n'):
                 return True
         except IndexError:
             # some uses of ipshellembed may fail here

IPython/core/history.py

@@ -14,6 +14,7 b' from __future__ import print_function'
 
 # Stdlib imports
 import fnmatch
+import json
 import os
 import sys
 
@@ -37,7 +38,7 b' class HistoryManager(object):'
     # An instance of the IPython shell we are attached to
     shell = None
     # An InputList instance to hold processed history
-    input_hist = None
+    input_hist_parsed = None
     # An InputList instance to hold raw history (as typed by user)
     input_hist_raw = None
     # A list of directories visited during session
@@ -64,11 +65,11 b' class HistoryManager(object):'
         self.shell = shell
         
         # List of input with multi-line handling.
-        self.input_hist = InputList()
+        self.input_hist_parsed = []
         # This one will hold the 'raw' input history, without any
         # pre-processing.  This will allow users to retrieve the input just as
         # it was exactly typed in by the user, with %hist -r.
-        self.input_hist_raw = InputList()
+        self.input_hist_raw = []
 
         # list of visited directories
         try:
@@ -84,7 +85,7 b' class HistoryManager(object):'
             histfname = 'history-%s' % shell.profile
         else:
             histfname = 'history'
-        self.hist_file = os.path.join(shell.ipython_dir, histfname)
+        self.hist_file = os.path.join(shell.ipython_dir, histfname + '.json')
 
         # Objects related to shadow history management
         self._init_shadow_hist()
@@ -95,18 +96,7 b' class HistoryManager(object):'
         
         # Fill the history zero entry, user counter starts at 1
         self.store_inputs('\n', '\n')
-
-        # For backwards compatibility, we must put these back in the shell
-        # object, until we've removed all direct uses of the history objects in
-        # the shell itself.
-        shell.input_hist = self.input_hist
-        shell.input_hist_raw = self.input_hist_raw
-        shell.output_hist = self.output_hist
-        shell.dir_hist = self.dir_hist
-        shell.histfile = self.hist_file
-        shell.shadowhist = self.shadow_hist
-        shell.db = self.shadow_db
-
+ 
     def _init_shadow_hist(self):
         try:
             self.shadow_db = PickleShareDB(os.path.join(
@@ -119,24 +109,41 b' class HistoryManager(object):'
             sys.exit()
         self.shadow_hist = ShadowHist(self.shadow_db, self.shell)
         
-    def save_hist(self):
-        """Save input history to a file (via readline library)."""
+    def populate_readline_history(self):
+        """Populate the readline history from the raw history.
 
-        try:
-            self.shell.readline.write_history_file(self.hist_file)
-        except:
-            print('Unable to save IPython command history to file: ' + 
-                  `self.hist_file`)
-
-    def reload_hist(self):
-        """Reload the input history from disk file."""
+        We only store one copy of the raw history, which is persisted to a json
+        file on disk.  The readline history is repopulated from the contents of
+        this file."""
 
         try:
             self.shell.readline.clear_history()
-            self.shell.readline.read_history_file(self.hist_file)
         except AttributeError:
             pass
+        else:
+            for h in self.input_hist_raw:
+                if not h.isspace():
+                    for line in h.splitlines():
+                        self.shell.readline.add_history(line)
+
+    def save_history(self):
+        """Save input history to a file (via readline library)."""
+        hist = dict(raw=self.input_hist_raw, #[-self.shell.history_length:],
+                    parsed=self.input_hist_parsed) #[-self.shell.history_length:])
+        with open(self.hist_file,'wt') as hfile:
+            json.dump(hist, hfile,
+                      sort_keys=True, indent=4)
+        
+    def reload_history(self):
+        """Reload the input history from disk file."""
 
+        with open(self.hist_file,'rt') as hfile:
+            hist = json.load(hfile)
+            self.input_hist_parsed = hist['parsed']
+            self.input_hist_raw = hist['raw']
+            if self.shell.has_readline:
+                self.populate_readline_history()
+        
     def get_history(self, index=None, raw=False, output=True):
         """Get the history list.
 
@@ -163,7 +170,7 b' class HistoryManager(object):'
         if raw:
             input_hist = self.input_hist_raw
         else:
-            input_hist = self.input_hist
+            input_hist = self.input_hist_parsed
         if output:
             output_hist = self.output_hist
         n = len(input_hist)
@@ -201,7 +208,7 b' class HistoryManager(object):'
         """
         if source_raw is None:
             source_raw = source
-        self.input_hist.append(source)
+        self.input_hist_parsed.append(source)
         self.input_hist_raw.append(source_raw)
         self.shadow_hist.add(source)
 
@@ -221,12 +228,12 b' class HistoryManager(object):'
 
     def sync_inputs(self):
         """Ensure raw and translated histories have same length."""
-        if len(self.input_hist) != len (self.input_hist_raw):
-            self.input_hist_raw = InputList(self.input_hist)
+        if len(self.input_hist_parsed) != len (self.input_hist_raw):
+            self.input_hist_raw = InputList(self.input_hist_parsed)
 
     def reset(self):
         """Clear all histories managed by this object."""
-        self.input_hist[:] = []
+        self.input_hist_parsed[:] = []
         self.input_hist_raw[:] = []
         self.output_hist.clear()
         # The directory history can't be completely empty
@@ -299,12 +306,12 b" def magic_history(self, parameter_s = ''):"
         close_at_end = True
 
     if 't' in opts:
-        input_hist = self.shell.input_hist
+        input_hist = self.shell.history_manager.input_hist_parsed
     elif 'r' in opts:
-        input_hist = self.shell.input_hist_raw
+        input_hist = self.shell.history_manager.input_hist_raw
     else:
         # Raw history is the default
-        input_hist = self.shell.input_hist_raw
+        input_hist = self.shell.history_manager.input_hist_raw
             
     default_length = 40
     pattern = None
@@ -337,7 +344,7 b" def magic_history(self, parameter_s = ''):"
     
     found = False
     if pattern is not None:
-        sh = self.shell.shadowhist.all()
+        sh = self.shell.history_manager.shadowhist.all()
         for idx, s in sh:
             if fnmatch.fnmatch(s, pattern):
                 print("0%d: %s" %(idx, s.expandtabs(4)), file=outfile)
@@ -373,7 +380,7 b" def magic_history(self, parameter_s = ''):"
         else:
             print(inline, end='', file=outfile)
         if print_outputs:
-            output = self.shell.output_hist.get(in_num)
+            output = self.shell.history_manager.output_hist.get(in_num)
             if output is not None:
                 print(repr(output), file=outfile)
 

IPython/core/interactiveshell.py

@@ -45,7 +45,6 b' from IPython.core.error import TryNext, UsageError'
 from IPython.core.extensions import ExtensionManager
 from IPython.core.fakemodule import FakeModule, init_fakemod_dict
 from IPython.core.history import HistoryManager
-from IPython.core.inputlist import InputList
 from IPython.core.inputsplitter import IPythonInputSplitter
 from IPython.core.logger import Logger
 from IPython.core.magic import Magic
@@ -176,6 +175,8 b' class InteractiveShell(Configurable, Magic):'
     prompts_pad_left = CBool(True, config=True)
     quiet = CBool(False, config=True)
 
+    history_length = Int(10000, config=True)
+    
     # The readline stuff will eventually be moved to the terminal subclass
     # but for now, we can't do that as readline is welded in everywhere.
     readline_use = CBool(True, config=True)
@@ -293,6 +294,14 b' class InteractiveShell(Configurable, Magic):'
         self.hooks.late_startup_hook()
         atexit.register(self.atexit_operations)
 
+    # While we're trying to have each part of the code directly access what it
+    # needs without keeping redundant references to objects, we have too much
+    # legacy code that expects ip.db to exist, so let's make it a property that
+    # retrieves the underlying object from our new history manager.
+    @property
+    def db(self):
+        return self.history_manager.shadow_db
+    
     @classmethod
     def instance(cls, *args, **kwargs):
         """Returns a global InteractiveShell instance."""
@@ -947,16 +956,16 b' class InteractiveShell(Configurable, Magic):'
             warn('help() not available - check site.py')
 
         # make global variables for user access to the histories
-        ns['_ih'] = self.input_hist
-        ns['_oh'] = self.output_hist
-        ns['_dh'] = self.dir_hist
+        ns['_ih'] = self.history_manager.input_hist_parsed
+        ns['_oh'] = self.history_manager.output_hist
+        ns['_dh'] = self.history_manager.dir_hist
 
         ns['_sh'] = shadowns
 
         # user aliases to input and output histories.  These shouldn't show up
         # in %who, as they can have very large reprs.
-        ns['In']  = self.input_hist
-        ns['Out'] = self.output_hist
+        ns['In']  = self.history_manager.input_hist_parsed
+        ns['Out'] = self.history_manager.output_hist
 
         # Store myself as the public api!!!
         ns['get_ipython'] = self.get_ipython
@@ -1228,19 +1237,13 b' class InteractiveShell(Configurable, Magic):'
     def init_history(self):
         self.history_manager = HistoryManager(shell=self)
 
-    def save_hist(self):
+    def save_history(self):
         """Save input history to a file (via readline library)."""
-        self.history_manager.save_hist()
+        self.history_manager.save_history()
 
-    # For backwards compatibility
-    savehist = save_hist
-        
-    def reload_hist(self):
+    def reload_history(self):
         """Reload the input history from disk file."""
-        self.history_manager.reload_hist()
-
-    # For backwards compatibility
-    reloadhist = reload_hist
+        self.history_manager.reload_history()
 
     def history_saving_wrapper(self, func):
         """ Wrap func for readline history saving
@@ -1254,11 +1257,11 b' class InteractiveShell(Configurable, Magic):'
             return func
 
         def wrapper():
-            self.save_hist()
+            self.save_history()
             try:
                 func()
             finally:
-                readline.read_history_file(self.histfile)
+                self.reload_history()
         return wrapper
 
     #-------------------------------------------------------------------------
@@ -1488,8 +1491,6 b' class InteractiveShell(Configurable, Magic):'
             self.has_readline = False
             self.readline = None
             # Set a number of methods that depend on readline to be no-op
-            self.save_hist = no_op
-            self.reload_hist = no_op
             self.set_readline_completer = no_op
             self.set_custom_completer = no_op
             self.set_completer_frame = no_op
@@ -1541,17 +1542,13 b' class InteractiveShell(Configurable, Magic):'
             delims = delims.replace(ESC_MAGIC, '')
             readline.set_completer_delims(delims)
             # otherwise we end up with a monster history after a while:
-            readline.set_history_length(1000)
+            readline.set_history_length(self.history_length)
             try:
                 #print '*** Reading readline history'  # dbg
-                readline.read_history_file(self.histfile)
+                self.reload_history() 
             except IOError:
                 pass  # It doesn't exist yet.
 
-            # If we have readline, we want our history saved upon ipython
-            # exiting. 
-            atexit.register(self.save_hist)
-
         # Configure auto-indent for all platforms
         self.set_autoindent(self.autoindent)
 
@@ -2109,7 +2106,8 b' class InteractiveShell(Configurable, Magic):'
                 list.append(self, val)
 
             import new
-            self.input_hist.append = types.MethodType(myapp, self.input_hist)
+            self.history_manager.input_hist_parsed.append = types.MethodType(myapp,
+                                                                            self.history_manager.input_hist_parsed)
         # End dbg
 
         # All user code execution must happen with our context managers active
@@ -2521,6 +2519,9 b' class InteractiveShell(Configurable, Magic):'
             except OSError:
                 pass
 
+        
+        self.save_history()
+
         # Clear all user namespaces to release all references cleanly.
         self.reset()
 

IPython/core/magic.py

@@ -186,9 +186,9 b' python-profiler package from non-free.""")'
         N-M -> include items N..M (closed endpoint)."""
 
         if raw:
-            hist = self.shell.input_hist_raw
+            hist = self.shell.history_manager.input_hist_raw
         else:
-            hist = self.shell.input_hist
+            hist = self.shell.history_manager.input_hist_parsed
 
         cmds = []
         for chunk in slices:
@@ -200,7 +200,7 b' python-profiler package from non-free.""")'
             else:
                 ini = int(chunk)
                 fin = ini+1
-            cmds.append(hist[ini:fin])
+            cmds.append(''.join(hist[ini:fin]))
         return cmds
             
     def arg_err(self,func):
@@ -1106,19 +1106,19 b' Currently the magic system has the following functions:\\n"""'
                 logger.timestamp = False
 
             if log_raw_input:
-                input_hist = self.shell.input_hist_raw
+                input_hist = self.shell.history_manager.input_hist_raw
             else:
-                input_hist = self.shell.input_hist
+                input_hist = self.shell.history_manager.input_hist_parsed
                 
             if log_output:
                 log_write = logger.log_write
-                output_hist = self.shell.output_hist
+                output_hist = self.shell.history_manager.output_hist
                 for n in range(1,len(input_hist)-1):
                     log_write(input_hist[n].rstrip())
                     if n in output_hist:
                         log_write(repr(output_hist[n]),'output')
             else:
-                logger.log_write(input_hist[1:])
+                logger.log_write(''.join(input_hist[1:]))
             if timestamp:
                 # re-enable timestamping
                 logger.timestamp = True
@@ -1551,7 +1551,7 b' Currently the magic system has the following functions:\\n"""'
         
         stats = None
         try:
-            self.shell.save_hist()
+            self.shell.save_history()
 
             if opts.has_key('p'):
                 stats = self.magic_prun('',0,opts,arg_lst,prog_ns)
@@ -1670,7 +1670,7 b' Currently the magic system has the following functions:\\n"""'
                 # contained therein.
                 del sys.modules[main_mod_name]
 
-            self.shell.reload_hist()
+            self.shell.reload_history()
                 
         return stats
 
@@ -2980,8 +2980,8 b' Defaulting color scheme to \'NoColor\'"""'
         else:
             start_magic = start
         # Look through the input history in reverse
-        for n in range(len(self.shell.input_hist)-2,0,-1):
-            input = self.shell.input_hist[n]
+        for n in range(len(self.shell.history_manager.input_hist_parsed)-2,0,-1):
+            input = self.shell.history_manager.input_hist_parsed[n]
             # skip plain 'r' lines so we don't recurse to infinity
             if input != '_ip.magic("r")\n' and \
                    (input.startswith(start) or input.startswith(start_magic)):

IPython/frontend/terminal/interactiveshell.py

@@ -332,14 +332,14 b' class TerminalInteractiveShell(InteractiveShell):'
                 if self.has_readline and self.readline_use:
                     histlen = self.readline.get_current_history_length()
                     if histlen > 1:
-                        newhist = self.input_hist_raw[-1].rstrip()
+                        newhist = self.history_manager.input_hist_raw[-1].rstrip()
                         self.readline.remove_history_item(histlen-1)
                         self.readline.replace_history_item(histlen-2,
                                         newhist.encode(self.stdin_encoding))
             else:
-                self.input_hist_raw.append('%s\n' % line)                
+                self.history_manager.input_hist_raw.append('%s\n' % line)                
         elif not continue_prompt:
-            self.input_hist_raw.append('\n')
+            self.history_manager.input_hist_raw.append('\n')
         try:
             lineout = self.prefilter_manager.prefilter_lines(line,continue_prompt)
         except:

IPython/quarantine/clearcmd.py

@@ -50,8 +50,8 b' def clear_f(self,arg):'
                     del user_ns[key]
                 except: pass
             # must be done in-place
-            self.input_hist[:] = ['\n'] * pc 
-            self.input_hist_raw[:] = ['\n'] * pc
+            self.history_manager.input_hist_parsed[:] = ['\n'] * pc 
+            self.history_manager.input_hist_raw[:] = ['\n'] * pc
 
         elif target == 'array':
             # Support cleaning up numpy arrays

IPython/quarantine/ipy_jot.py

@@ -118,7 +118,7 b" def jot_obj(self, obj, name, comment=''):"
 
     # which one works better? 
     #all = ip.shadowhist.all()
-    all = ip.shell.input_hist
+    all = ip.shell.history_manager.input_hist_parsed
 
     # We may actually want to make snapshot of files that are run-ned.