Update history.py to use pathlib

Tests were returning exeption in atexit_operations
AttributeError: 'str' object has no attribute 'unlink'

This commit embraces usage of pathlib in history.py and related
tests, fixing aforementioned problem.

IPython/core/history.py

@@ -1,881 +1,885
 """ History related magics and functionality """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 
 import atexit
 import datetime
-import os
+from pathlib import Path
 import re
 import sqlite3
 import threading
 
 from traitlets.config.configurable import LoggingConfigurable
 from decorator import decorator
 from IPython.utils.decorators import undoc
 from IPython.paths import locate_profile
 from traitlets import (
-    Any, Bool, Dict, Instance, Integer, List, Unicode, TraitError,
-    default, observe,
+    Any, Bool, Dict, Instance, Integer, List, Unicode, Union, TraitError,
+    default, observe
 )
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
 @undoc
 class DummyDB(object):
     """Dummy DB that will act as a black hole for history.
-    
+
     Only used in the absence of sqlite"""
     def execute(*args, **kwargs):
         return []
-    
+
     def commit(self, *args, **kwargs):
         pass
-    
+
     def __enter__(self, *args, **kwargs):
         pass
-    
+
     def __exit__(self, *args, **kwargs):
         pass
 
 
 @decorator
 def only_when_enabled(f, self, *a, **kw):
     """Decorator: return an empty list in the absence of sqlite."""
     if not self.enabled:
         return []
     else:
         return f(self, *a, **kw)
 
 
 # use 16kB as threshold for whether a corrupt history db should be saved
 # that should be at least 100 entries or so
 _SAVE_DB_SIZE = 16384
 
 @decorator
 def catch_corrupt_db(f, self, *a, **kw):
     """A decorator which wraps HistoryAccessor method calls to catch errors from
     a corrupt SQLite database, move the old database out of the way, and create
     a new one.
 
     We avoid clobbering larger databases because this may be triggered due to filesystem issues,
     not just a corrupt file.
     """
     try:
         return f(self, *a, **kw)
     except (sqlite3.DatabaseError, sqlite3.OperationalError) as e:
         self._corrupt_db_counter += 1
         self.log.error("Failed to open SQLite history %s (%s).", self.hist_file, e)
         if self.hist_file != ':memory:':
             if self._corrupt_db_counter > self._corrupt_db_limit:
                 self.hist_file = ':memory:'
                 self.log.error("Failed to load history too many times, history will not be saved.")
-            elif os.path.isfile(self.hist_file):
+            elif self.hist_file.is_file():
                 # move the file out of the way
-                base, ext = os.path.splitext(self.hist_file)
-                size = os.stat(self.hist_file).st_size
+                base = str(self.hist_file.parent / self.hist_file.stem)
+                ext = self.hist_file.suffix
+                size = self.hist_file.stat().st_size
                 if size >= _SAVE_DB_SIZE:
                     # if there's significant content, avoid clobbering
                     now = datetime.datetime.now().isoformat().replace(':', '.')
                     newpath = base + '-corrupt-' + now + ext
                     # don't clobber previous corrupt backups
                     for i in range(100):
-                        if not os.path.isfile(newpath):
+                        if not Path(newpath).exists():
                             break
                         else:
                             newpath = base + '-corrupt-' + now + (u'-%i' % i) + ext
                 else:
                     # not much content, possibly empty; don't worry about clobbering
                     # maybe we should just delete it?
                     newpath = base + '-corrupt' + ext
-                os.rename(self.hist_file, newpath)
+                self.hist_file.rename(newpath)
                 self.log.error("History file was moved to %s and a new file created.", newpath)
             self.init_db()
             return []
         else:
             # Failed with :memory:, something serious is wrong
             raise
-        
+
 class HistoryAccessorBase(LoggingConfigurable):
     """An abstract class for History Accessors """
 
     def get_tail(self, n=10, raw=True, output=False, include_latest=False):
         raise NotImplementedError
 
     def search(self, pattern="*", raw=True, search_raw=True,
                output=False, n=None, unique=False):
         raise NotImplementedError
 
     def get_range(self, session, start=1, stop=None, raw=True,output=False):
         raise NotImplementedError
 
     def get_range_by_str(self, rangestr, raw=True, output=False):
         raise NotImplementedError
 
 
 class HistoryAccessor(HistoryAccessorBase):
     """Access the history database without adding to it.
-    
+
     This is intended for use by standalone history tools. IPython shells use
     HistoryManager, below, which is a subclass of this."""
 
     # counter for init_db retries, so we don't keep trying over and over
     _corrupt_db_counter = 0
      # after two failures, fallback on :memory:
     _corrupt_db_limit = 2
 
     # String holding the path to the history file
-    hist_file = Unicode(
+    hist_file = Union([Instance(Path), Unicode()],
         help="""Path to file to use for SQLite history database.
-        
+
         By default, IPython will put the history database in the IPython
         profile directory.  If you would rather share one history among
         profiles, you can set this value in each, so that they are consistent.
-        
+
         Due to an issue with fcntl, SQLite is known to misbehave on some NFS
         mounts.  If you see IPython hanging, try setting this to something on a
         local disk, e.g::
-        
+
             ipython --HistoryManager.hist_file=/tmp/ipython_hist.sqlite
 
         you can also use the specific value `:memory:` (including the colon
         at both end but not the back ticks), to avoid creating an history file.
-        
-        """).tag(config=True)
-    
+
+        """
+    ).tag(config=True)
+
     enabled = Bool(True,
         help="""enable the SQLite history
-        
+
         set enabled=False to disable the SQLite history,
         in which case there will be no stored history, no SQLite connection,
         and no background saving thread.  This may be necessary in some
         threaded environments where IPython is embedded.
         """
     ).tag(config=True)
-    
+
     connection_options = Dict(
         help="""Options for configuring the SQLite connection
-        
+
         These options are passed as keyword args to sqlite3.connect
         when establishing database connections.
         """
     ).tag(config=True)
 
     # The SQLite database
     db = Any()
     @observe('db')
     def _db_changed(self, change):
         """validate the db, since it can be an Instance of two different types"""
         new = change['new']
         connection_types = (DummyDB, sqlite3.Connection)
         if not isinstance(new, connection_types):
             msg = "%s.db must be sqlite3 Connection or DummyDB, not %r" % \
                     (self.__class__.__name__, new)
             raise TraitError(msg)
-    
-    def __init__(self, profile='default', hist_file=u'', **traits):
+
+    def __init__(self, profile='default', hist_file="", **traits):
         """Create a new history accessor.
-        
+
         Parameters
         ----------
         profile : str
           The name of the profile from which to open history.
         hist_file : str
           Path to an SQLite history database stored by IPython. If specified,
           hist_file overrides profile.
         config : :class:`~traitlets.config.loader.Config`
           Config object. hist_file can also be set through this.
         """
         # We need a pointer back to the shell for various tasks.
         super(HistoryAccessor, self).__init__(**traits)
         # defer setting hist_file from kwarg until after init,
         # otherwise the default kwarg value would clobber any value
         # set by config
         if hist_file:
             self.hist_file = hist_file
-        
-        if self.hist_file == u'':
+
+        try:
+            self.hist_file 
+        except TraitError:
             # No one has set the hist_file, yet.
             self.hist_file = self._get_hist_file_name(profile)
-        
+
         self.init_db()
-    
+
     def _get_hist_file_name(self, profile='default'):
         """Find the history file for the given profile name.
-        
+
         This is overridden by the HistoryManager subclass, to use the shell's
         active profile.
-        
+
         Parameters
         ----------
         profile : str
           The name of a profile which has a history file.
         """
-        return os.path.join(locate_profile(profile), 'history.sqlite')
-    
+        return Path(locate_profile(profile)) / 'history.sqlite'
+
     @catch_corrupt_db
     def init_db(self):
         """Connect to the database, and create tables if necessary."""
         if not self.enabled:
             self.db = DummyDB()
             return
-        
+
         # use detect_types so that timestamps return datetime objects
         kwargs = dict(detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES)
         kwargs.update(self.connection_options)
         self.db = sqlite3.connect(self.hist_file, **kwargs)
         self.db.execute("""CREATE TABLE IF NOT EXISTS sessions (session integer
                         primary key autoincrement, start timestamp,
                         end timestamp, num_cmds integer, remark text)""")
         self.db.execute("""CREATE TABLE IF NOT EXISTS history
                 (session integer, line integer, source text, source_raw text,
                 PRIMARY KEY (session, line))""")
         # Output history is optional, but ensure the table's there so it can be
         # enabled later.
         self.db.execute("""CREATE TABLE IF NOT EXISTS output_history
                         (session integer, line integer, output text,
                         PRIMARY KEY (session, line))""")
         self.db.commit()
         # success! reset corrupt db count
         self._corrupt_db_counter = 0
 
     def writeout_cache(self):
         """Overridden by HistoryManager to dump the cache before certain
         database lookups."""
         pass
 
     ## -------------------------------
     ## Methods for retrieving history:
     ## -------------------------------
     def _run_sql(self, sql, params, raw=True, output=False):
         """Prepares and runs an SQL query for the history database.
 
         Parameters
         ----------
         sql : str
           Any filtering expressions to go after SELECT ... FROM ...
         params : tuple
           Parameters passed to the SQL query (to replace "?")
         raw, output : bool
           See :meth:`get_range`
 
         Returns
         -------
         Tuples as :meth:`get_range`
         """
         toget = 'source_raw' if raw else 'source'
         sqlfrom = "history"
         if output:
             sqlfrom = "history LEFT JOIN output_history USING (session, line)"
             toget = "history.%s, output_history.output" % toget
         cur = self.db.execute("SELECT session, line, %s FROM %s " %\
                                 (toget, sqlfrom) + sql, params)
         if output:    # Regroup into 3-tuples, and parse JSON
             return ((ses, lin, (inp, out)) for ses, lin, inp, out in cur)
         return cur
 
     @only_when_enabled
     @catch_corrupt_db
     def get_session_info(self, session):
         """Get info about a session.
 
         Parameters
         ----------
 
         session : int
             Session number to retrieve.
 
         Returns
         -------
-        
+
         session_id : int
            Session ID number
         start : datetime
            Timestamp for the start of the session.
         end : datetime
            Timestamp for the end of the session, or None if IPython crashed.
         num_cmds : int
            Number of commands run, or None if IPython crashed.
         remark : unicode
            A manually set description.
         """
         query = "SELECT * from sessions where session == ?"
         return self.db.execute(query, (session,)).fetchone()
 
     @catch_corrupt_db
     def get_last_session_id(self):
         """Get the last session ID currently in the database.
-        
+
         Within IPython, this should be the same as the value stored in
         :attr:`HistoryManager.session_number`.
         """
         for record in self.get_tail(n=1, include_latest=True):
             return record[0]
 
     @catch_corrupt_db
     def get_tail(self, n=10, raw=True, output=False, include_latest=False):
         """Get the last n lines from the history database.
 
         Parameters
         ----------
         n : int
           The number of lines to get
         raw, output : bool
           See :meth:`get_range`
         include_latest : bool
           If False (default), n+1 lines are fetched, and the latest one
           is discarded. This is intended to be used where the function
           is called by a user command, which it should not return.
 
         Returns
         -------
         Tuples as :meth:`get_range`
         """
         self.writeout_cache()
         if not include_latest:
             n += 1
         cur = self._run_sql("ORDER BY session DESC, line DESC LIMIT ?",
                                 (n,), raw=raw, output=output)
         if not include_latest:
             return reversed(list(cur)[1:])
         return reversed(list(cur))
 
     @catch_corrupt_db
     def search(self, pattern="*", raw=True, search_raw=True,
                output=False, n=None, unique=False):
         """Search the database using unix glob-style matching (wildcards
         * and ?).
 
         Parameters
         ----------
         pattern : str
           The wildcarded pattern to match when searching
         search_raw : bool
           If True, search the raw input, otherwise, the parsed input
         raw, output : bool
           See :meth:`get_range`
         n : None or int
           If an integer is given, it defines the limit of
           returned entries.
         unique : bool
           When it is true, return only unique entries.
 
         Returns
         -------
         Tuples as :meth:`get_range`
         """
         tosearch = "source_raw" if search_raw else "source"
         if output:
             tosearch = "history." + tosearch
         self.writeout_cache()
         sqlform = "WHERE %s GLOB ?" % tosearch
         params = (pattern,)
         if unique:
             sqlform += ' GROUP BY {0}'.format(tosearch)
         if n is not None:
             sqlform += " ORDER BY session DESC, line DESC LIMIT ?"
             params += (n,)
         elif unique:
             sqlform += " ORDER BY session, line"
         cur = self._run_sql(sqlform, params, raw=raw, output=output)
         if n is not None:
             return reversed(list(cur))
         return cur
-    
+
     @catch_corrupt_db
     def get_range(self, session, start=1, stop=None, raw=True,output=False):
         """Retrieve input by session.
 
         Parameters
         ----------
         session : int
             Session number to retrieve.
         start : int
             First line to retrieve.
         stop : int
             End of line range (excluded from output itself). If None, retrieve
             to the end of the session.
         raw : bool
             If True, return untranslated input
         output : bool
             If True, attempt to include output. This will be 'real' Python
             objects for the current session, or text reprs from previous
             sessions if db_log_output was enabled at the time. Where no output
             is found, None is used.
 
         Returns
         -------
         entries
           An iterator over the desired lines. Each line is a 3-tuple, either
           (session, line, input) if output is False, or
           (session, line, (input, output)) if output is True.
         """
         if stop:
             lineclause = "line >= ? AND line < ?"
             params = (session, start, stop)
         else:
             lineclause = "line>=?"
             params = (session, start)
 
         return self._run_sql("WHERE session==? AND %s" % lineclause,
                                     params, raw=raw, output=output)
 
     def get_range_by_str(self, rangestr, raw=True, output=False):
         """Get lines of history from a string of ranges, as used by magic
         commands %hist, %save, %macro, etc.
 
         Parameters
         ----------
         rangestr : str
           A string specifying ranges, e.g. "5 ~2/1-4". See
           :func:`magic_history` for full details.
         raw, output : bool
           As :meth:`get_range`
 
         Returns
         -------
         Tuples as :meth:`get_range`
         """
         for sess, s, e in extract_hist_ranges(rangestr):
             for line in self.get_range(sess, s, e, raw=raw, output=output):
                 yield line
 
 
 class HistoryManager(HistoryAccessor):
     """A class to organize all history-related functionality in one place.
     """
     # Public interface
 
     # An instance of the IPython shell we are attached to
     shell = Instance('IPython.core.interactiveshell.InteractiveShellABC',
                      allow_none=True)
     # Lists to hold processed and raw history. These start with a blank entry
     # so that we can index them starting from 1
     input_hist_parsed = List([""])
     input_hist_raw = List([""])
     # A list of directories visited during session
     dir_hist = List()
     @default('dir_hist')
     def _dir_hist_default(self):
         try:
-            return [os.getcwd()]
+            return [Path.cwd()]
         except OSError:
             return []
 
     # A dict of output history, keyed with ints from the shell's
     # execution count.
     output_hist = Dict()
     # The text/plain repr of outputs.
     output_hist_reprs = Dict()
 
     # The number of the current session in the history database
     session_number = Integer()
-    
+
     db_log_output = Bool(False,
         help="Should the history database include output? (default: no)"
     ).tag(config=True)
     db_cache_size = Integer(0,
         help="Write to database every x commands (higher values save disk access & power).\n"
         "Values of 1 or less effectively disable caching."
     ).tag(config=True)
     # The input and output caches
     db_input_cache = List()
     db_output_cache = List()
-    
+
     # History saving in separate thread
     save_thread = Instance('IPython.core.history.HistorySavingThread',
                            allow_none=True)
     save_flag = Instance(threading.Event, allow_none=True)
-    
+
     # Private interface
     # Variables used to store the three last inputs from the user.  On each new
     # history update, we populate the user's namespace with these, shifted as
     # necessary.
     _i00 = Unicode(u'')
     _i = Unicode(u'')
     _ii = Unicode(u'')
     _iii = Unicode(u'')
 
     # A regex matching all forms of the exit command, so that we don't store
     # them in the history (it's annoying to rewind the first entry and land on
     # an exit call).
     _exit_re = re.compile(r"(exit|quit)(\s*\(.*\))?$")
 
     def __init__(self, shell=None, config=None, **traits):
         """Create a new history manager associated with a shell instance.
         """
         # We need a pointer back to the shell for various tasks.
         super(HistoryManager, self).__init__(shell=shell, config=config,
             **traits)
         self.save_flag = threading.Event()
         self.db_input_cache_lock = threading.Lock()
         self.db_output_cache_lock = threading.Lock()
-        
+
         try:
             self.new_session()
         except sqlite3.OperationalError:
             self.log.error("Failed to create history session in %s. History will not be saved.",
                 self.hist_file, exc_info=True)
             self.hist_file = ':memory:'
-        
+
         if self.enabled and self.hist_file != ':memory:':
             self.save_thread = HistorySavingThread(self)
             self.save_thread.start()
 
     def _get_hist_file_name(self, profile=None):
         """Get default history file name based on the Shell's profile.
-        
+
         The profile parameter is ignored, but must exist for compatibility with
         the parent class."""
         profile_dir = self.shell.profile_dir.location
-        return os.path.join(profile_dir, 'history.sqlite')
-    
+        return Path(profile_dir)/'history.sqlite'
+
     @only_when_enabled
     def new_session(self, conn=None):
         """Get a new session number."""
         if conn is None:
             conn = self.db
-        
+
         with conn:
             cur = conn.execute("""INSERT INTO sessions VALUES (NULL, ?, NULL,
                             NULL, "") """, (datetime.datetime.now(),))
             self.session_number = cur.lastrowid
-            
+
     def end_session(self):
         """Close the database session, filling in the end time and line count."""
         self.writeout_cache()
         with self.db:
             self.db.execute("""UPDATE sessions SET end=?, num_cmds=? WHERE
                             session==?""", (datetime.datetime.now(),
                             len(self.input_hist_parsed)-1, self.session_number))
         self.session_number = 0
-                            
+
     def name_session(self, name):
         """Give the current session a name in the history database."""
         with self.db:
             self.db.execute("UPDATE sessions SET remark=? WHERE session==?",
                             (name, self.session_number))
-                            
+
     def reset(self, new_session=True):
         """Clear the session history, releasing all object references, and
         optionally open a new session."""
         self.output_hist.clear()
         # The directory history can't be completely empty
-        self.dir_hist[:] = [os.getcwd()]
-        
+        self.dir_hist[:] = [Path.cwd()]
+
         if new_session:
             if self.session_number:
                 self.end_session()
             self.input_hist_parsed[:] = [""]
             self.input_hist_raw[:] = [""]
             self.new_session()
 
     # ------------------------------
     # Methods for retrieving history
     # ------------------------------
     def get_session_info(self, session=0):
         """Get info about a session.
 
         Parameters
         ----------
 
         session : int
             Session number to retrieve. The current session is 0, and negative
             numbers count back from current session, so -1 is the previous session.
 
         Returns
         -------
-        
+
         session_id : int
            Session ID number
         start : datetime
            Timestamp for the start of the session.
         end : datetime
            Timestamp for the end of the session, or None if IPython crashed.
         num_cmds : int
            Number of commands run, or None if IPython crashed.
         remark : unicode
            A manually set description.
         """
         if session <= 0:
             session += self.session_number
 
         return super(HistoryManager, self).get_session_info(session=session)
 
     def _get_range_session(self, start=1, stop=None, raw=True, output=False):
         """Get input and output history from the current session. Called by
         get_range, and takes similar parameters."""
         input_hist = self