BUG: use a :memory: history DB for testing. Refactor the initialization of the HistoryManager to support this.

IPython/core/history.py

@@ -1,740 +1,748 b''
 """ History related magics and functionality """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010 The IPython Development Team.
 #
 #  Distributed under the terms of the BSD License.
 #
 #  The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
 # Stdlib imports
 import datetime
 import json
 import os
 import re
 import sqlite3
 
 from collections import defaultdict
 
 # Our own packages
 from IPython.config.configurable import Configurable
 import IPython.utils.io
 
 from IPython.testing import decorators as testdec
 from IPython.utils.io import ask_yes_no
 from IPython.utils.traitlets import Bool, Dict, Instance, Int, List, Unicode
 from IPython.utils.warn import warn
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
 class HistoryManager(Configurable):
     """A class to organize all history-related functionality in one place.
     """
     # Public interface
 
     # An instance of the IPython shell we are attached to
     shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
     # Lists to hold processed and raw history. These start with a blank entry
     # so that we can index them starting from 1
     input_hist_parsed = List([""])
     input_hist_raw = List([""])
     # A list of directories visited during session
     dir_hist = List()
+    def _dir_hist_default(self):
+        try:
+            return [os.getcwd()]
+        except OSError:
+            return []
+
     # A dict of output history, keyed with ints from the shell's
     # execution count. If there are several outputs from one command,
     # only the last one is stored.
     output_hist = Dict()
     # Contains all outputs, in lists of reprs.
-    output_hist_reprs = Instance(defaultdict)
-    
+    output_hist_reprs = Instance(defaultdict, args=(list,))
+
     # String holding the path to the history file
-    hist_file = Unicode()
+    hist_file = Unicode(config=True)
+
     # The SQLite database
     db = Instance(sqlite3.Connection)
     # The number of the current session in the history database
     session_number = Int()
     # Should we log output to the database? (default no)
     db_log_output = Bool(False, config=True)
     # Write to database every x commands (higher values save disk access & power)
     #  Values of 1 or less effectively disable caching. 
     db_cache_size = Int(0, config=True)
     # The input and output caches
     db_input_cache = List()
     db_output_cache = List()
     
     # Private interface
     # Variables used to store the three last inputs from the user.  On each new
     # history update, we populate the user's namespace with these, shifted as
     # necessary.
-    _i00, _i, _ii, _iii = '','','',''
+    _i00 = Unicode(u'')
+    _i = Unicode(u'')
+    _ii = Unicode(u'')
+    _iii = Unicode(u'')
 
     # A set with all forms of the exit command, so that we don't store them in
     # the history (it's annoying to rewind the first entry and land on an exit
     # call).
-    _exit_commands = None
-    
-    def __init__(self, shell, config=None):
+    _exit_commands = Instance(set, args=(['Quit', 'quit', 'Exit', 'exit',
+        '%Quit', '%quit', '%Exit', '%exit'],))
+
+    def __init__(self, shell, config=None, **traits):
         """Create a new history manager associated with a shell instance.
         """
         # We need a pointer back to the shell for various tasks.
-        super(HistoryManager, self).__init__(shell=shell, config=config)
+        super(HistoryManager, self).__init__(shell=shell, config=config,
+            **traits)
 
-        # list of visited directories
-        try:
-            self.dir_hist = [os.getcwd()]
-        except OSError:
-            self.dir_hist = []
+        if self.hist_file == u'':
+            # No one has set the hist_file, yet.
+            if shell.profile:
+                histfname = 'history-%s' % shell.profile
+            else:
+                histfname = 'history'
+            self.hist_file = os.path.join(shell.ipython_dir, histfname + '.sqlite')
 
-        # Now the history file
-        if shell.profile:
-            histfname = 'history-%s' % shell.profile
-        else:
-            histfname = 'history'
-        self.hist_file = os.path.join(shell.ipython_dir, histfname + '.sqlite')
         try:
             self.init_db()
         except sqlite3.DatabaseError:
-            newpath = os.path.join(self.shell.ipython_dir, "hist-corrupt.sqlite")
-            os.rename(self.hist_file, newpath)
-            print("ERROR! History file wasn't a valid SQLite database.",
-            "It was moved to %s" % newpath, "and a new file created.")
-            self.init_db()
-        
+            if os.path.isfile(self.hist_file):
+                # Try to move the file out of the way.
+                newpath = os.path.join(self.shell.ipython_dir, "hist-corrupt.sqlite")
+                os.rename(self.hist_file, newpath)
+                print("ERROR! History file wasn't a valid SQLite database.",
+                "It was moved to %s" % newpath, "and a new file created.")
+                self.init_db()
+            else:
+                # The hist_file is probably :memory: or something else.
+                raise
+
         self.new_session()
-    
-        self._i00, self._i, self._ii, self._iii = '','','',''
-        self.output_hist_reprs = defaultdict(list)
 
-        self._exit_commands = set(['Quit', 'quit', 'Exit', 'exit', '%Quit',
-                                   '%quit', '%Exit', '%exit'])
         
     def init_db(self):
         """Connect to the database, and create tables if necessary."""
         self.db = sqlite3.connect(self.hist_file)
         self.db.execute("""CREATE TABLE IF NOT EXISTS sessions (session integer
                         primary key autoincrement, start timestamp,
                         end timestamp, num_cmds integer, remark text)""")
         self.db.execute("""CREATE TABLE IF NOT EXISTS history 
                 (session integer, line integer, source text, source_raw text,
                 PRIMARY KEY (session, line))""")
         # Output history is optional, but ensure the table's there so it can be
         # enabled later.
         self.db.execute("""CREATE TABLE IF NOT EXISTS output_history
                         (session integer, line integer, output text,
                         PRIMARY KEY (session, line))""")
         self.db.commit()
     
     def new_session(self):
         """Get a new session number."""
         with self.db:
             cur = self.db.execute("""INSERT INTO sessions VALUES (NULL, ?, NULL,
                             NULL, "") """, (datetime.datetime.now(),))
             self.session_number = cur.lastrowid
             
     def end_session(self):
         """Close the database session, filling in the end time and line count."""
         self.writeout_cache()
         with self.db:
             self.db.execute("""UPDATE sessions SET end=?, num_cmds=? WHERE
                             session==?""", (datetime.datetime.now(),
                             len(self.input_hist_parsed)-1, self.session_number))
         self.session_number = 0
                             
     def name_session(self, name):
         """Give the current session a name in the history database."""
         with self.db:
             self.db.execute("UPDATE sessions SET remark=? WHERE session==?",
                             (name, self.session_number))
                             
     def reset(self, new_session=True):
         """Clear the session history, releasing all object references, and
         optionally open a new session."""
         if self.session_number:
             self.end_session()
         self.input_hist_parsed[:] = [""]
         self.input_hist_raw[:] = [""]
         self.output_hist.clear()
         # The directory history can't be completely empty
         self.dir_hist[:] = [os.getcwd()]
         
         if new_session:
             self.new_session()
     
     ## -------------------------------
     ## Methods for retrieving history:
     ## -------------------------------
     def _run_sql(self, sql, params, raw=True, output=False):
         """Prepares and runs an SQL query for the history database.
         
         Parameters
         ----------
         sql : str
           Any filtering expressions to go after SELECT ... FROM ...
         params : tuple
           Parameters passed to the SQL query (to replace "?")
         raw, output : bool
           See :meth:`get_range`
         
         Returns
         -------
         Tuples as :meth:`get_range`
         """
         toget = 'source_raw' if raw else 'source'
         sqlfrom = "history"
         if output:
             sqlfrom = "history LEFT JOIN output_history USING (session, line)"
             toget = "history.%s, output_history.output" % toget
         cur = self.db.execute("SELECT session, line, %s FROM %s " %\
                                 (toget, sqlfrom) + sql, params)
         if output:    # Regroup into 3-tuples, and parse JSON
             loads = lambda out: json.loads(out) if out else None
             return ((ses, lin, (inp, loads(out))) \
                                         for ses, lin, inp, out in cur)
         return cur
         
     
     def get_tail(self, n=10, raw=True, output=False, include_latest=False):
         """Get the last n lines from the history database.
         
         Parameters
         ----------
         n : int
           The number of lines to get
         raw, output : bool
           See :meth:`get_range`
         include_latest : bool
           If False (default), n+1 lines are fetched, and the latest one
           is discarded. This is intended to be used where the function
           is called by a user command, which it should not return.
         
         Returns
         -------
         Tuples as :meth:`get_range`
         """
         self.writeout_cache()
         if not include_latest:
             n += 1
         cur = self._run_sql("ORDER BY session DESC, line DESC LIMIT ?",
                                 (n,), raw=raw, output=output)
         if not include_latest:
             return reversed(list(cur)[1:])
         return reversed(list(cur))
         
     def search(self, pattern="*", raw=True, search_raw=True,
                                                         output=False):
         """Search the database using unix glob-style matching (wildcards
         * and ?).
         
         Parameters
         ----------
         pattern : str
           The wildcarded pattern to match when searching
         search_raw : bool
           If True, search the raw input, otherwise, the parsed input
         raw, output : bool
           See :meth:`get_range`
         
         Returns
         -------
         Tuples as :meth:`get_range`
         """
         tosearch = "source_raw" if search_raw else "source"
         if output:
             tosearch = "history." + tosearch
         self.writeout_cache()
         return self._run_sql("WHERE %s GLOB ?" % tosearch, (pattern,),
                                     raw=raw, output=output)
                                 
     def _get_range_session(self, start=1, stop=None, raw=True, output=False):
         """Get input and output history from the current session. Called by
         get_range, and takes similar parameters."""
         input_hist = self.input_hist_raw if raw else self.input_hist_parsed
             
         n = len(input_hist)
         if start < 0:
             start += n
         if not stop:
             stop = n
         elif stop < 0:
             stop += n
         
         for i in range(start, stop):
             if output:
                 line = (input_hist[i], self.output_hist_reprs.get(i))
             else:
                 line = input_hist[i]
             yield (0, i, line)
             
     def get_range(self, session=0, start=1, stop=None, raw=True,output=False):
         """Retrieve input by session.
         
         Parameters
         ----------
         session : int
             Session number to retrieve. The current session is 0, and negative
             numbers count back from current session, so -1 is previous session.
         start : int
             First line to retrieve.
         stop : int
             End of line range (excluded from output itself). If None, retrieve
             to the end of the session.
         raw : bool
             If True, return untranslated input
         output : bool
             If True, attempt to include output. This will be 'real' Python
             objects for the current session, or text reprs from previous
             sessions if db_log_output was enabled at the time. Where no output
             is found, None is used.
             
         Returns
         -------
         An iterator over the desired lines. Each line is a 3-tuple, either
         (session, line, input) if output is False, or
         (session, line, (input, output)) if output is True.
         """
         if session == 0 or session==self.session_number:   # Current session
             return self._get_range_session(start, stop, raw, output)
         if session < 0:
             session += self.session_number
             
         if stop:
             lineclause = "line >= ? AND line < ?"
             params = (session, start, stop)
         else:
             lineclause = "line>=?"
             params = (session, start)
         
         return self._run_sql("WHERE session==? AND %s""" % lineclause,
                                     params, raw=raw, output=output)
         
     def get_range_by_str(self, rangestr, raw=True, output=False):
         """Get lines of history from a string of ranges, as used by magic
         commands %hist, %save, %macro, etc.
         
         Parameters
         ----------
         rangestr : str
           A string specifying ranges, e.g. "5 ~2/1-4". See
           :func:`magic_history` for full details.
         raw, output : bool
           As :meth:`get_range`
           
         Returns
         -------
         Tuples as :meth:`get_range`
         """
         for sess, s, e in extract_hist_ranges(rangestr):
             for line in self.get_range(sess, s, e, raw=raw, output=output):
                 yield line
     
     ## ----------------------------
     ## Methods for storing history:
     ## ----------------------------
     def store_inputs(self, line_num, source, source_raw=None):
         """Store source and raw input in history and create input cache
         variables _i*.
         
         Parameters
         ----------
         line_num : int
           The prompt number of this input.
         
         source : str
           Python input.
 
         source_raw : str, optional
           If given, this is the raw input without any IPython transformations
           applied to it.  If not given, ``source`` is used.
         """
         if source_raw is None:
             source_raw = source
         source = source.rstrip('\n')
         source_raw = source_raw.rstrip('\n')
             
         # do not store exit/quit commands
         if source_raw.strip() in self._exit_commands:
             return
         
         self.input_hist_parsed.append(source)
         self.input_hist_raw.append(source_raw)
         
         self.db_input_cache.append((line_num, source, source_raw))
         # Trigger to flush cache and write to DB.
         if len(self.db_input_cache) >= self.db_cache_size:
             self.writeout_cache()
 
         # update the auto _i variables
         self._iii = self._ii
         self._ii = self._i
         self._i = self._i00
         self._i00 = source_raw
 
         # hackish access to user namespace to create _i1,_i2... dynamically
         new_i = '_i%s' % line_num
         to_main = {'_i': self._i,
                    '_ii': self._ii,
                    '_iii': self._iii,
                    new_i : self._i00 }
         self.shell.user_ns.update(to_main)
         
     def store_output(self, line_num):
         """If database output logging is enabled, this saves all the
         outputs from the indicated prompt number to the database. It's
         called by run_cell after code has been executed.
         
         Parameters
         ----------
         line_num : int
           The line number from which to save outputs
         """
         if (not self.db_log_output) or not self.output_hist_reprs[line_num]:
             return
         output = json.dumps(self.output_hist_reprs[line_num])
         
         self.db_output_cache.append((line_num, output))
         if self.db_cache_size <= 1:
             self.writeout_cache()
         
     def _writeout_input_cache(self):
         for line in self.db_input_cache:
             with self.db:
                 self.db.execute("INSERT INTO history VALUES (?, ?, ?, ?)",
                                 (self.session_number,)+line)
     
     def _writeout_output_cache(self):
         for line in self.db_output_cache:
             with self.db:
                 self.db.execute("INSERT INTO output_history VALUES (?, ?, ?)",
                                 (self.session_number,)+line)
     
     def writeout_cache(self):
         """Write any entries in the cache to the database."""
         try:
             self._writeout_input_cache()
         except sqlite3.IntegrityError:
             self.new_session()
             print("ERROR! Session/line number was not unique in",
                   "database. History logging moved to new session",
                                             self.session_number)
             try: # Try writing to the new session. If this fails, don't recurse
                 self.writeout_cache()
             except sqlite3.IntegrityError:
                 pass
         finally:
             self.db_input_cache = []
             
         try:
             self._writeout_output_cache()
         except sqlite3.IntegrityError:
             print("!! Session/line number for output was not unique",
                   "in database. Output will not be stored.")
         finally:
             self.db_output_cache = []
 
         
 # To match, e.g. ~5/8-~2/3
 range_re = re.compile(r"""
 ((?P<startsess>~?\d+)/)?
 (?P<start>\d+)                    # Only the start line num is compulsory
 ((?P<sep>[\-:])
  ((?P<endsess>~?\d+)/)?
  (?P<end>\d+))?
 """, re.VERBOSE)
 
 def extract_hist_ranges(ranges_str):
     """Turn a string of history ranges into 3-tuples of (session, start, stop).
     
     Examples
     --------
     list(extract_input_ranges("~8/5-~7/4 2"))
     [(-8, 5, None), (-7, 1, 4), (0, 2, 3)]
     """
     for range_str in ranges_str.split():
         rmatch = range_re.match(range_str)
         if not rmatch:
             continue
         start = int(rmatch.group("start"))
         end = rmatch.group("end")
         end = int(end) if end else start+1   # If no end specified, get (a, a+1)
         if rmatch.group("sep") == "-":       # 1-3 == 1:4 --> [1, 2, 3]
             end += 1
         startsess = rmatch.group("startsess") or "0"
         endsess = rmatch.group("endsess") or startsess
         startsess = int(startsess.replace("~","-"))
         endsess = int(endsess.replace("~","-"))
         assert endsess >= startsess
 
         if endsess == startsess:
             yield (startsess, start, end)
             continue
         # Multiple sessions in one range:
         yield (startsess, start, None)
         for sess in range(startsess+1, endsess):
             yield (sess, 1, None)
         yield (endsess, 1, end)
 
 def _format_lineno(session, line):
     """Helper function to format line numbers properly."""
     if session == 0:
         return str(line)
     return "%s#%s" % (session, line)
 
 @testdec.skip_doctest
 def magic_history(self, parameter_s = ''):
     """Print input history (_i<n> variables), with most recent last.
     
     %history       -> print at most 40 inputs (some may be multi-line)\\
     %history n     -> print at most n inputs\\
     %history n1 n2 -> print inputs between n1 and n2 (n2 not included)\\
 
     By default, input history is printed without line numbers so it can be
     directly pasted into an editor. Use -n to show them.
 
     Ranges of history can be indicated using the syntax:
     4      : Line 4, current session
     4-6    : Lines 4-6, current session
     243/1-5: Lines 1-5, session 243
     ~2/7   : Line 7, session 2 before current
     ~8/1-~6/5 : From the first line of 8 sessions ago, to the fifth line
                 of 6 sessions ago.
     Multiple ranges can be entered, separated by spaces
     
     The same syntax is used by %macro, %save, %edit, %rerun
 
     Options:
 
       -n: print line numbers for each input.
       This feature is only available if numbered prompts are in use.
 
       -o: also print outputs for each input.
 
       -p: print classic '>>>' python prompts before each input.  This is useful
        for making documentation, and in conjunction with -o, for producing
        doctest-ready output.
 
       -r: (default) print the 'raw' history, i.e. the actual commands you typed.
       
       -t: print the 'translated' history, as IPython understands it.  IPython
       filters your input and converts it all into valid Python source before
       executing it (things like magics or aliases are turned into function
       calls, for example). With this option, you'll see the native history
       instead of the user-entered version: '%cd /' will be seen as
       'get_ipython().magic("%cd /")' instead of '%cd /'.
       
       -g: treat the arg as a pattern to grep for in (full) history.
       This includes the saved history (almost all commands ever written).
       Use '%hist -g' to show full saved history (may be very long).
       
       -l: get the last n lines from all sessions. Specify n as a single arg, or
       the default is the last 10 lines.
 
       -f FILENAME: instead of printing the output to the screen, redirect it to
        the given file.  The file is always overwritten, though IPython asks for
        confirmation first if it already exists.
        
     Examples
     --------
     ::
     
       In [6]: %hist -n 4 6
       4:a = 12
       5:print a**2
 
     """
 
     if not self.shell.displayhook.do_full_cache:
         print('This feature is only available if numbered prompts are in use.')
         return
     opts,args = self.parse_options(parameter_s,'noprtglf:',mode='string')
     
     # For brevity
     history_manager = self.shell.history_manager
     
     def _format_lineno(session, line):
         """Helper function to format line numbers properly."""
         if session in (0, history_manager.session_number):
             return str(line)
         return "%s/%s" % (session, line)
 
     # Check if output to specific file was requested.
     try:
         outfname = opts['f']
     except KeyError:
         outfile = IPython.utils.io.Term.cout  # default
         # We don't want to close stdout at the end!
         close_at_end = False
     else:
         if os.path.exists(outfname):
             if not ask_yes_no("File %r exists. Overwrite?" % outfname): 
                 print('Aborting.')
                 return
 
         outfile = open(outfname,'w')
         close_at_end = True
     
     print_nums = 'n' in opts
     get_output = 'o' in opts
     pyprompts = 'p' in opts
     # Raw history is the default
     raw = not('t' in opts)
             
     default_length = 40
     pattern = None
     
     if 'g' in opts:         # Glob search
         pattern = "*" + args + "*" if args else "*"
         hist = history_manager.search(pattern, raw=raw, output=get_output)
     elif 'l' in opts:       # Get 'tail'
         try:
             n = int(args)
         except ValueError, IndexError:
             n = 10
         hist = history_manager.get_tail(n, raw=raw, output=get_output)
     else:
         if args:            # Get history by ranges
             hist = history_manager.get_range_by_str(args, raw, get_output)
         else:               # Just get history for the current session
             hist = history_manager.get_range(raw=raw, output=get_output)
     
     # We could be displaying the entire history, so let's not try to pull it 
     # into a list in memory. Anything that needs more space will just misalign.
     width = 4
         
     for session, lineno, inline in hist:
         # Print user history with tabs expanded to 4 spaces.  The GUI clients
         # use hard tabs for easier usability in auto-indented code, but we want
         # to produce PEP-8 compliant history for safe pasting into an editor.
         if get_output:
             inline, output = inline
         inline = inline.expandtabs(4).rstrip()
             
         multiline = "\n" in inline
         line_sep = '\n' if multiline else ' '
         if print_nums:
             print('%s:%s' % (_format_lineno(session, lineno).rjust(width),
                     line_sep),  file=outfile, end='')
         if pyprompts:
             print(">>> ", end="", file=outfile)
             if multiline:
                 inline = "\n... ".join(inline.splitlines()) + "\n..."
         print(inline, file=outfile)
         if get_output and output:
             print("\n".join(output), file=outfile)
 
     if close_at_end:
         outfile.close()
 
 
 def magic_rep(self, arg):
     r""" Repeat a command, or get command to input line for editing
 
     - %rep (no arguments):
     
     Place a string version of last computation result (stored in the special '_'
     variable) to the next input prompt. Allows you to create elaborate command
     lines without using copy-paste::
     
          In[1]: l = ["hei", "vaan"]
          In[2]: "".join(l)
         Out[2]: heivaan
          In[3]: %rep
          In[4]: heivaan_ <== cursor blinking
     
     %rep 45
     
     Place history line 45 on the next input prompt. Use %hist to find
     out the number.
     
     %rep 1-4
     
     Combine the specified lines into one cell, and place it on the next
     input prompt. See %history for the slice syntax.
     
     %rep foo+bar
     
     If foo+bar can be evaluated in the user namespace, the result is
     placed at the next input prompt. Otherwise, the history is searched
     for lines which contain that substring, and the most recent one is
     placed at the next input prompt.
     """
     if not arg:                 # Last output
         self.set_next_input(str(self.shell.user_ns["_"]))
         return
                                 # Get history range
     histlines = self.history_manager.get_range_by_str(arg)
     cmd = "\n".join(x[2] for x in histlines)
     if cmd:
         self.set_next_input(cmd.rstrip())
         return
 
     try:                        # Variable in user namespace
         cmd = str(eval(arg, self.shell.user_ns))
     except Exception:           # Search for term in history
         histlines = self.history_manager.search("*"+arg+"*")
         for h in reversed([x[2] for x in histlines]):
             if 'rep' in h:
                 continue
             self.set_next_input(h.rstrip())
             return
     else:
         self.set_next_input(cmd.rstrip())
     print("Couldn't evaluate or find in history:", arg)
         
 def magic_rerun(self, parameter_s=''):
     """Re-run previous input
     
     By default, you can specify ranges of input history to be repeated
     (as with %history). With no arguments, it will repeat the last line.
     
     Options:
     
       -l <n> : Repeat the last n lines of input, not including the
       current command.
       
       -g foo : Repeat the most recent line which contains foo
     """
     opts, args = self.parse_options(parameter_s, 'l:g:', mode='string')
     if "l" in opts:         # Last n lines
         n = int(opts['l'])
         hist = self.history_manager.get_tail(n)
     elif "g" in opts:       # Search
         p = "*"+opts['g']+"*"
         hist = list(self.history_manager.search(p))
         for l in reversed(hist):
             if "rerun" not in l[2]:
                 hist = [l]     # The last match which isn't a %rerun
                 break
         else:
             hist = []          # No matches except %rerun
     elif args:              # Specify history ranges
         hist = self.history_manager.get_range_by_str(args)
     else:                   # Last line
         hist = self.history_manager.get_tail(1)
     hist = [x[2] for x in hist]
     if not hist:
         print("No lines in history match specification")
         return
     histlines = "\n".join(hist)
     print("=== Executing: ===")
     print(histlines)
     print("=== Output: ===")
     self.run_cell("\n".join(hist), store_history=False)
 
 
 def init_ipython(ip):
     ip.define_magic("rep", magic_rep) 
     ip.define_magic("recall", magic_rep)
     ip.define_magic("rerun", magic_rerun)
     ip.define_magic("hist",magic_history)    # Alternative name
     ip.define_magic("history",magic_history)
 
     # XXX - ipy_completers are in quarantine, need to be updated to new apis
     #import ipy_completers
     #ipy_completers.quick_completer('%hist' ,'-g -t -r -n')

IPython/core/tests/test_history.py

@@ -1,119 +1,111 b''
 # coding: utf-8
 """Tests for the IPython tab-completion machinery.
 """
 #-----------------------------------------------------------------------------
 # Module imports
 #-----------------------------------------------------------------------------
 
 # stdlib
 import os
 import sys
 import unittest
 
 # third party
 import nose.tools as nt
 
 # our own packages
 from IPython.utils.tempdir import TemporaryDirectory
 from IPython.core.history import HistoryManager, extract_hist_ranges
 
 def setUp():
     nt.assert_equal(sys.getdefaultencoding(), "ascii")
 
 def test_history():
     ip = get_ipython()
     with TemporaryDirectory() as tmpdir:
-        #tmpdir = '/software/temp'
-        histfile = os.path.realpath(os.path.join(tmpdir, 'history.sqlite'))
-        # Ensure that we restore the history management that we mess with in
-        # this test doesn't affect the IPython instance used by the test suite
-        # beyond this test.
+        # Make a new :memory: DB.
         hist_manager_ori = ip.history_manager
         try:
-            ip.history_manager = HistoryManager(shell=ip)
-            ip.history_manager.hist_file = histfile
-            ip.history_manager.init_db()  # Has to be called after changing file
-            ip.history_manager.reset()
-            print 'test',histfile
+            ip.history_manager = HistoryManager(shell=ip, hist_file=':memory:')
             hist = ['a=1', 'def f():\n    test = 1\n    return test', u"b='€Æ¾÷ß'"]
             for i, h in enumerate(hist, start=1):
                 ip.history_manager.store_inputs(i, h)
             
             ip.history_manager.db_log_output = True
             # Doesn't match the input, but we'll just check it's stored.
             ip.history_manager.output_hist_reprs[3].append("spam")
             ip.history_manager.store_output(3)
             
             nt.assert_equal(ip.history_manager.input_hist_raw, [''] + hist)
             
             # Check lines were written to DB
             c = ip.history_manager.db.execute("SELECT source_raw FROM history")
             nt.assert_equal([x for x, in c], hist)
               
             # New session
             ip.history_manager.reset()
             newcmds = ["z=5","class X(object):\n    pass", "k='p'"]
             for i, cmd in enumerate(newcmds, start=1):
                 ip.history_manager.store_inputs(i, cmd)
             gothist = ip.history_manager.get_range(start=1, stop=4)
             nt.assert_equal(list(gothist), zip([0,0,0],[1,2,3], newcmds))
             # Previous session:
             gothist = ip.history_manager.get_range(-1, 1, 4)
             nt.assert_equal(list(gothist), zip([1,1,1],[1,2,3], hist))
             
             # Check get_hist_tail
             gothist = ip.history_manager.get_tail(4, output=True,
                                                     include_latest=True)
             expected = [(1, 3, (hist[-1], ["spam"])),
                         (2, 1, (newcmds[0], None)),
                         (2, 2, (newcmds[1], None)),
                         (2, 3, (newcmds[2], None)),]
             nt.assert_equal(list(gothist), expected)
             
             gothist = ip.history_manager.get_tail(2)
             expected = [(2, 1, newcmds[0]),
                         (2, 2, newcmds[1])]
             nt.assert_equal(list(gothist), expected)
             
             # Check get_hist_search
             gothist = ip.history_manager.search("*test*")
             nt.assert_equal(list(gothist), [(1,2,hist[1])] )
             gothist = ip.history_manager.search("b*", output=True)
             nt.assert_equal(list(gothist), [(1,3,(hist[2],["spam"]))] )
             
             # Cross testing: check that magic %save can get previous session.
             testfilename = os.path.realpath(os.path.join(tmpdir, "test.py"))
             ip.magic_save(testfilename + " ~1/1-3")
             testfile = open(testfilename, "r")
             nt.assert_equal(testfile.read().decode("utf-8"),
                     "# coding: utf-8\n" + "\n".join(hist))
             
             # Duplicate line numbers - check that it doesn't crash, and
             # gets a new session
             ip.history_manager.store_inputs(1, "rogue")
             nt.assert_equal(ip.history_manager.session_number, 3)
         finally:
             # Restore history manager
             ip.history_manager = hist_manager_ori
 
 
 def test_extract_hist_ranges():
     instr = "1 2/3 ~4/5-6 ~4/7-~4/9 ~9/2-~7/5"
     expected = [(0, 1, 2),  # 0 == current session
                 (2, 3, 4),
                 (-4, 5, 7),
                 (-4, 7, 10),
                 (-9, 2, None),  # None == to end
                 (-8, 1, None),
                 (-7, 1, 6)]
     actual = list(extract_hist_ranges(instr))
     nt.assert_equal(actual, expected)
     
 def test_magic_rerun():
     """Simple test for %rerun (no args -> rerun last line)"""
     ip = get_ipython()
     ip.run_cell("a = 10")
     ip.run_cell("a += 1")
     nt.assert_equal(ip.user_ns["a"], 11)
     ip.run_cell("%rerun")
     nt.assert_equal(ip.user_ns["a"], 12)

IPython/testing/tools.py

@@ -1,277 +1,278 b''
 """Generic testing tools that do NOT depend on Twisted.
 
 In particular, this module exposes a set of top-level assert* functions that
 can be used in place of nose.tools.assert* in method generators (the ones in
 nose can not, at least as of nose 0.10.4).
 
 Note: our testing package contains testing.util, which does depend on Twisted
 and provides utilities for tests that manage Deferreds.  All testing support
 tools that only depend on nose, IPython or the standard library should go here
 instead.
 
 
 Authors
 -------
 - Fernando Perez <[e-mail]>
 """
 
 from __future__ import absolute_import
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import re
 import sys
 
 try:
     # These tools are used by parts of the runtime, so we make the nose
     # dependency optional at this point.  Nose is a hard dependency to run the
     # test suite, but NOT to use ipython itself.
     import nose.tools as nt
     has_nose = True
 except ImportError:
     has_nose = False
 
 from IPython.config.loader import Config
 from IPython.utils.process import find_cmd, getoutputerror
 from IPython.utils.text import list_strings
 from IPython.utils.io import temp_pyfile
 
 from . import decorators as dec
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 # Make a bunch of nose.tools assert wrappers that can be used in test
 # generators.  This will expose an assert* function for each one in nose.tools.
 
 _tpl = """
 def %(name)s(*a,**kw):
     return nt.%(name)s(*a,**kw)
 """
 
 if has_nose:
     for _x in [a for a in dir(nt) if a.startswith('assert')]:
         exec _tpl % dict(name=_x)
 
 #-----------------------------------------------------------------------------
 # Functions and classes
 #-----------------------------------------------------------------------------
 
 # The docstring for full_path doctests differently on win32 (different path
 # separator) so just skip the doctest there.  The example remains informative.
 doctest_deco = dec.skip_doctest if sys.platform == 'win32' else dec.null_deco
 
 @doctest_deco
 def full_path(startPath,files):
     """Make full paths for all the listed files, based on startPath.
 
     Only the base part of startPath is kept, since this routine is typically
     used with a script's __file__ variable as startPath.  The base of startPath
     is then prepended to all the listed files, forming the output list.
 
     Parameters
     ----------
       startPath : string
         Initial path to use as the base for the results.  This path is split
       using os.path.split() and only its first component is kept.
 
       files : string or list
         One or more files.
 
     Examples
     --------
 
     >>> full_path('/foo/bar.py',['a.txt','b.txt'])
     ['/foo/a.txt', '/foo/b.txt']
 
     >>> full_path('/foo',['a.txt','b.txt'])
     ['/a.txt', '/b.txt']
 
     If a single file is given, the output is still a list:
     >>> full_path('/foo','a.txt')
     ['/a.txt']
     """
 
     files = list_strings(files)
     base = os.path.split(startPath)[0]
     return [ os.path.join(base,f) for f in files ]
 
 
 def parse_test_output(txt):
     """Parse the output of a test run and return errors, failures.
 
     Parameters
     ----------
     txt : str
       Text output of a test run, assumed to contain a line of one of the
       following forms::
         'FAILED (errors=1)'
         'FAILED (failures=1)'
         'FAILED (errors=1, failures=1)'
 
     Returns
     -------
     nerr, nfail: number of errors and failures.
     """
 
     err_m = re.search(r'^FAILED \(errors=(\d+)\)', txt, re.MULTILINE)
     if err_m:
         nerr = int(err_m.group(1))
         nfail = 0
         return  nerr, nfail
     
     fail_m = re.search(r'^FAILED \(failures=(\d+)\)', txt, re.MULTILINE)
     if fail_m:
         nerr = 0
         nfail = int(fail_m.group(1))
         return  nerr, nfail
 
     both_m = re.search(r'^FAILED \(errors=(\d+), failures=(\d+)\)', txt,
                        re.MULTILINE)
     if both_m:
         nerr = int(both_m.group(1))
         nfail = int(both_m.group(2))
         return  nerr, nfail
     
     # If the input didn't match any of these forms, assume no error/failures
     return 0, 0
 
 
 # So nose doesn't think this is a test
 parse_test_output.__test__ = False
 
 
 def default_argv():
     """Return a valid default argv for creating testing instances of ipython"""
 
     return ['--quick', # so no config file is loaded
             # Other defaults to minimize side effects on stdout
             '--colors=NoColor', '--no-term-title','--no-banner',
             '--autocall=0']
 
 
 def default_config():
     """Return a config object with good defaults for testing."""
     config = Config()
     config.TerminalInteractiveShell.colors = 'NoColor'
     config.TerminalTerminalInteractiveShell.term_title = False,
     config.TerminalInteractiveShell.autocall = 0
+    config.HistoryManager.hist_file = u':memory:'
     return config
 
 
 def ipexec(fname, options=None):
     """Utility to call 'ipython filename'.
 
     Starts IPython witha minimal and safe configuration to make startup as fast
     as possible.
 
     Note that this starts IPython in a subprocess!
 
     Parameters
     ----------
     fname : str
       Name of file to be executed (should have .py or .ipy extension).
 
     options : optional, list
       Extra command-line flags to be passed to IPython.
 
     Returns
     -------
     (stdout, stderr) of ipython subprocess.
     """
     if options is None: options = []
     
     # For these subprocess calls, eliminate all prompt printing so we only see
     # output from script execution
     prompt_opts = ['--prompt-in1=""', '--prompt-in2=""', '--prompt-out=""']
     cmdargs = ' '.join(default_argv() + prompt_opts + options)
     
     _ip = get_ipython()
     test_dir = os.path.dirname(__file__)
 
     ipython_cmd = find_cmd('ipython')
     # Absolute path for filename
     full_fname = os.path.join(test_dir, fname)
     full_cmd = '%s %s %s' % (ipython_cmd, cmdargs, full_fname)
     #print >> sys.stderr, 'FULL CMD:', full_cmd # dbg
     return getoutputerror(full_cmd)
 
 
 def ipexec_validate(fname, expected_out, expected_err='',
                     options=None):
     """Utility to call 'ipython filename' and validate output/error.
 
     This function raises an AssertionError if the validation fails.
 
     Note that this starts IPython in a subprocess!
 
     Parameters
     ----------
     fname : str
       Name of the file to be executed (should have .py or .ipy extension).
 
     expected_out : str
       Expected stdout of the process.
 
     expected_err : optional, str
       Expected stderr of the process.
 
     options : optional, list
       Extra command-line flags to be passed to IPython.
 
     Returns
     -------
     None
     """
 
     import nose.tools as nt
     
     out, err = ipexec(fname)
     #print 'OUT', out  # dbg
     #print 'ERR', err  # dbg
     # If there are any errors, we must check those befor stdout, as they may be
     # more informative than simply having an empty stdout.
     if err:
         if expected_err:
             nt.assert_equals(err.strip(), expected_err.strip())
         else:
             raise ValueError('Running file %r produced error: %r' %
                              (fname, err))
     # If no errors or output on stderr was expected, match stdout
     nt.assert_equals(out.strip(), expected_out.strip())
 
 
 class TempFileMixin(object):
     """Utility class to create temporary Python/IPython files.
 
     Meant as a mixin class for test cases."""
     
     def mktmp(self, src, ext='.py'):
         """Make a valid python temp file."""
         fname, f = temp_pyfile(src, ext)
         self.tmpfile = f
         self.fname = fname
 
     def tearDown(self):
         if hasattr(self, 'tmpfile'):
             # If the tmpfile wasn't made because of skipped tests, like in
             # win32, there's nothing to cleanup.
             self.tmpfile.close()
             try:
                 os.unlink(self.fname)
             except:
                 # On Windows, even though we close the file, we still can't
                 # delete it.  I have no clue why
                 pass