Grand Persistence Overhaul, featuring PickleShare. startup

and shutdown hooks added.

IPython/Extensions/pickleshare.py

@@ -0,0 +1,267 b''
+#!/usr/bin/env python
+
+""" PickleShare - a small 'shelve' like datastore with concurrency support
+
+Like shelve, a PickleShareDB object acts like a normal dictionary. Unlike 
+shelve, many processes can access the database simultaneously. Changing a 
+value in database is immediately visible to other processes accessing the 
+same database.
+
+Concurrency is possible because the values are stored in separate files. Hence
+the "database" is a directory where *all* files are governed by PickleShare.
+
+Example usage::
+    
+    from pickleshare import *
+    db = PickleShareDB('~/testpickleshare')
+    db.clear()
+    print "Should be empty:",db.items()
+    db['hello'] = 15
+    db['aku ankka'] = [1,2,313]
+    db['paths/are/ok/key'] = [1,(5,46)]
+    print db.keys()
+    del db['aku ankka']
+
+This module is certainly not ZODB, but can be used for low-load 
+(non-mission-critical) situations where tiny code size trumps the 
+advanced features of a "real" object database.
+
+Installation guide: easy_install pickleshare
+
+Author: Ville Vainio <[e-mail]>
+License: MIT open source license.
+
+"""
+
+from path import path as Path
+import os,stat,time
+import cPickle as pickle
+import UserDict
+import warnings
+import glob
+
+class PickleShareDB(UserDict.DictMixin):
+    """ The main 'connection' object for PickleShare database """
+    def __init__(self,root):
+        """ Return a db object that will manage the specied directory"""
+        self.root = Path(root).expanduser().abspath()
+        if not self.root.isdir():
+            self.root.makedirs()
+        # cache has { 'key' : (obj, orig_mod_time) }
+        self.cache = {}
+
+    def __getitem__(self,key):
+        """ db['key'] reading """
+        fil = self.root / key
+        try:
+            mtime = (fil.stat()[stat.ST_MTIME])
+        except OSError:
+            raise KeyError(key)
+
+        if fil in self.cache and mtime == self.cache[fil][1]:
+            return self.cache[fil][0]
+        try:
+            # The cached item has expired, need to read
+            obj = pickle.load(fil.open())
+        except:
+            raise KeyError(key)
+            
+        self.cache[fil] = (obj,mtime)
+        return obj
+    
+    def __setitem__(self,key,value):
+        """ db['key'] = 5 """
+        fil = self.root / key
+        parent = fil.parent
+        if parent and not parent.isdir():
+            parent.makedirs()
+        pickled = pickle.dump(value,fil.open('w'))
+        try:
+            self.cache[fil] = (value,fil.mtime)
+        except OSError,e:
+            if e.errno != 2:
+                raise
+    
+    def __delitem__(self,key):
+        """ del db["key"] """
+        fil = self.root / key
+        self.cache.pop(fil,None)
+        try:
+            fil.remove()
+        except OSError:
+            # notfound and permission denied are ok - we
+            # lost, the other process wins the conflict
+            pass
+        
+    def _normalized(self, p):
+        """ Make a key suitable for user's eyes """
+        return str(self.root.relpathto(p)).replace('\\','/')
+    
+    def keys(self, globpat = None):
+        """ All keys in DB, or all keys matching a glob"""
+        
+        if globpat is None:
+            files = self.root.walkfiles()
+        else:
+            files = [Path(p) for p in glob.glob(self.root/globpat)]
+        return [self._normalized(p) for p in files if p.isfile()]
+
+    def uncache(self,*items):
+        """ Removes all, or specified items from cache
+        
+        Use this after reading a large amount of large objects
+        to free up memory, when you won't be needing the objects
+        for a while.
+         
+        """
+        if not items:
+            self.cache = {}
+        for it in items:
+            self.cache.pop(it,None)
+            
+    def waitget(self,key, maxwaittime = 60 ):
+        """ Wait (poll) for a key to get a value
+        
+        Will wait for `maxwaittime` seconds before raising a KeyError.
+        The call exits normally if the `key` field in db gets a value
+        within the timeout period.
+        
+        Use this for synchronizing different processes or for ensuring
+        that an unfortunately timed "db['key'] = newvalue" operation 
+        in another process (which causes all 'get' operation to cause a 
+        KeyError for the duration of pickling) won't screw up your program 
+        logic. 
+        """
+        
+        wtimes = [0.2] * 3 + [0.5] * 2 + [1]
+        tries = 0
+        waited = 0
+        while 1:
+            try:
+                val = self[key]
+                return val
+            except KeyError:
+                pass
+            
+            if waited > maxwaittime:
+                raise KeyError(key)
+            
+            time.sleep(wtimes[tries])
+            waited+=wtimes[tries]
+            if tries < len(wtimes) -1:
+                tries+=1
+    
+    def getlink(self,folder):
+        """ Get a convenient link for accessing items  """
+        return PickleShareLink(self, folder)
+    
+    def __repr__(self):
+        return "PickleShareDB('%s')" % self.root
+        
+        
+                
+class PickleShareLink:
+    """ A shortdand for accessing nested PickleShare data conveniently.
+
+    Created through PickleShareDB.getlink(), example::
+
+        lnk = db.getlink('myobjects/test')
+        lnk.foo = 2
+        lnk.bar = lnk.foo + 5
+    
+    """
+    def __init__(self, db, keydir ):    
+        self.__dict__.update(locals())
+        
+    def __getattr__(self,key):
+        return self.__dict__['db'][self.__dict__['keydir']+'/' + key]
+    def __setattr__(self,key,val):
+        self.db[self.keydir+'/' + key] = val
+    def __repr__(self):
+        db = self.__dict__['db']
+        keys = db.keys( self.__dict__['keydir'] +"/*")
+        return "<PickleShareLink '%s': %s>" % (
+            self.__dict__['keydir'],
+            ";".join([Path(k).basename() for k in keys]))
+            
+        
+def test():
+    db = PickleShareDB('~/testpickleshare')
+    db.clear()
+    print "Should be empty:",db.items()
+    db['hello'] = 15
+    db['aku ankka'] = [1,2,313]
+    db['paths/nest/ok/keyname'] = [1,(5,46)]
+    print db.keys()
+    print db.keys('paths/nest/ok/k*')
+    print dict(db) # snapsot of whole db
+    db.uncache() # frees memory, causes re-reads later
+
+    # shorthand for accessing deeply nested files
+    lnk = db.getlink('myobjects/test')
+    lnk.foo = 2
+    lnk.bar = lnk.foo + 5
+    print lnk.bar # 7
+
+def stress():
+    db = PickleShareDB('~/fsdbtest')
+    import time,sys
+    for i in range(1000):
+        for j in range(300):
+            if i % 15 == 0 and i < 200:
+                if str(j) in db:
+                    del db[str(j)]
+                continue
+
+            if j%33 == 0:
+                time.sleep(0.02)
+            
+            db[str(j)] = db.get(str(j), []) + [(i,j,"proc %d" % os.getpid())]
+        print i,
+        sys.stdout.flush()
+        if i % 10 == 0:
+            db.uncache()
+    
+def main():
+    import textwrap
+    usage = textwrap.dedent("""\
+    pickleshare - manage PickleShare databases 
+    
+    Usage:
+        
+        pickleshare dump /path/to/db > dump.txt
+        pickleshare load /path/to/db < dump.txt
+        pickleshare test /path/to/db
+    """)
+    DB = PickleShareDB
+    import sys
+    if len(sys.argv) < 2:
+        print usage
+        return
+        
+    cmd = sys.argv[1]
+    args = sys.argv[2:]
+    if cmd == 'dump':
+        if not args: args= ['.']
+        db = DB(args[0])
+        import pprint
+        pprint.pprint(db.items())
+    elif cmd == 'load':
+        cont = sys.stdin.read()
+        db = DB(args[0])
+        data = eval(cont)
+        db.clear()
+        for k,v in db.items():
+            db[k] = v
+    elif cmd == 'testwait':
+        db = DB(args[0])
+        db.clear()
+        print db.waitget('250')
+    elif cmd == 'test':
+        test()
+        stress()
+    
+if __name__== "__main__":
+    main()
+    
+    
  No newline at end of file

IPython/Extensions/pspersistence.py

@@ -0,0 +1,149 b''
+import IPython.ipapi
+ip = IPython.ipapi.get()
+
+import pickleshare
+
+import inspect,pickle,os,textwrap
+from IPython.FakeModule import FakeModule
+
+def refresh_variables(ip):
+    db = ip.getdb()
+    for key in db.keys('autorestore/*'):
+        # strip autorestore
+        justkey = os.path.basename(key)
+        try:
+            obj = db[key]
+        except KeyError:
+            print "Unable to restore variable '%s', ignoring (use %%store -d to forget!)" % justkey
+            print "The error was:",sys.exc_info()[0]
+        else:
+            #print "restored",justkey,"=",obj #dbg
+            ip.user_ns()[justkey] = obj
+    
+    
+
+def restore_data(self):
+    #o = ip.options()
+    #self.db = pickleshare.PickleShareDB(o.ipythondir + "/db")
+    #print "restoring ps data" # dbg
+
+    ip = self.getapi()
+    refresh_variables(ip)
+    raise IPython.ipapi.TryNext
+    
+    
+ip.set_hook('late_startup_hook', restore_data)
+
+def magic_store(self, parameter_s=''):
+    """Lightweight persistence for python variables.
+
+    Example:
+    
+    ville@badger[~]|1> A = ['hello',10,'world']\\
+    ville@badger[~]|2> %store A\\
+    ville@badger[~]|3> Exit
+    
+    (IPython session is closed and started again...)
+    
+    ville@badger:~$ ipython -p pysh\\
+    ville@badger[~]|1> print A
+    
+    ['hello', 10, 'world']
+    
+    Usage:
+    
+    %store          - Show list of all variables and their current values\\
+    %store <var>    - Store the *current* value of the variable to disk\\
+    %store -d <var> - Remove the variable and its value from storage\\
+    %store -z       - Remove all variables from storage\\
+    %store -r       - Refresh all variables from store (delete current vals)\\
+    %store foo >a.txt  - Store value of foo to new file a.txt\\
+    %store foo >>a.txt - Append value of foo to file a.txt\\   
+    
+    It should be noted that if you change the value of a variable, you
+    need to %store it again if you want to persist the new value.
+    
+    Note also that the variables will need to be pickleable; most basic
+    python types can be safely %stored.
+    """
+    
+    opts,argsl = self.parse_options(parameter_s,'drz',mode='string')
+    args = argsl.split(None,1)
+    ip = self.getapi()
+    # delete
+    if opts.has_key('d'):
+        try:
+            todel = args[0]
+        except IndexError:
+            error('You must provide the variable to forget')
+        else:
+            try:
+                del self.db['autorestore/' + todel]
+            except:
+                error("Can't delete variable '%s'" % todel)
+    # reset
+    elif opts.has_key('z'):
+        for k in self.db.keys('autorestore/*'):
+            del self.db[k]
+
+    elif opts.has_key('r'):
+        refresh_variables(ip)
+
+    
+    # run without arguments -> list variables & values
+    elif not args:
+        vars = self.db.keys('autorestore/*')
+        vars.sort()            
+        if vars:
+            size = max(map(len,vars))
+        else:
+            size = 0
+            
+        print 'Stored variables and their in-db values:'
+        fmt = '%-'+str(size)+'s -> %s'
+        get = self.db.get
+        for var in vars:
+            justkey = os.path.basename(var)
+            # print 30 first characters from every var
+            print fmt % (justkey,repr(get(var,'<unavailable>'))[:50])
+    
+    # default action - store the variable
+    else:
+        # %store foo >file.txt or >>file.txt
+        if len(args) > 1 and args[1].startswith('>'):
+            fnam = os.path.expanduser(args[1].lstrip('>').lstrip())
+            if args[1].startswith('>>'):
+                fil = open(fnam,'a')
+            else:
+                fil = open(fnam,'w')
+            obj = ip.ev(args[0])
+            print "Writing '%s' (%s) to file '%s'." % (args[0],
+              obj.__class__.__name__, fnam)
+
+            
+            if not isinstance (obj,basestring):
+                pprint(obj,fil)
+            else:
+                fil.write(obj)
+                if not obj.endswith('\n'):
+                    fil.write('\n')
+            
+            fil.close()
+            return
+        
+        # %store foo
+        obj = ip.ev(args[0])
+        if isinstance(inspect.getmodule(obj), FakeModule):
+            print textwrap.dedent("""\
+            Warning:%s is %s 
+            Proper storage of interactively declared classes (or instances
+            of those classes) is not possible! Only instances
+            of classes in real modules on file system can be %%store'd.
+            """ % (args[0], obj) ) 
+            return
+        #pickled = pickle.dumps(obj)
+        self.db[ 'autorestore/' + args[0] ] = obj
+        print "Stored '%s' (%s)" % (args[0], obj.__class__.__name__)
+
+ip.expose_magic('store',magic_store)
+    
  No newline at end of file

IPython/Extensions/ipy_system_conf.py

@@ -16,3 +16,4 b' import sys'
 
 import ext_rehashdir # %rehashdir magic
 import ext_rescapture # var = !ls and var = %magic
+import pspersistence # %store magic
  No newline at end of file

IPython/Magic.py

@@ -1,7 +1,7 b''
 # -*- coding: utf-8 -*-
 """Magic functions for InteractiveShell.
 
-$Id: Magic.py 1099 2006-01-29 21:05:57Z vivainio $"""
+$Id: Magic.py 1107 2006-01-30 19:02:20Z vivainio $"""
 
 #*****************************************************************************
 #       Copyright (C) 2001 Janko Hauser <[e-mail]> and
@@ -2301,7 +2301,7 b' Defaulting color scheme to \'NoColor\'"""'
         !command runs is immediately discarded after executing 'command'."""
 
         parameter_s = parameter_s.strip()
-        bkms = self.shell.persist.get("bookmarks",{})
+        #bkms = self.shell.persist.get("bookmarks",{})
 
         numcd = re.match(r'(-)(\d+)$',parameter_s)
         # jump in directory history by number
@@ -2326,18 +2326,19 b' Defaulting color scheme to \'NoColor\'"""'
             except IndexError:
                 print 'No previous directory to change to.'
                 return
-        # jump to bookmark
-        elif opts.has_key('b') or (bkms.has_key(ps) and not os.path.isdir(ps)):
+        # jump to bookmark if needed
+        else:
+            if not os.path.isdir(ps) or opts.has_key('b'):
+                bkms = self.db.get('bookmarks', {})
+            
             if bkms.has_key(ps):
                 target = bkms[ps]
                 print '(bookmark:%s) -> %s' % (ps,target)
                 ps = target
             else:
-                if bkms:
+                    if opts.has_key('b'):
                     error("Bookmark '%s' not found.  "
                           "Use '%%bookmark -l' to see your bookmarks." % ps)
-                else:
-                    print "Bookmarks not set - use %bookmark <bookmarkname>"
                 return
             
         # at this point ps should point to the target dir
@@ -2634,112 +2635,6 b' Defaulting color scheme to \'NoColor\'"""'
         
         self.shell.jobs.new(parameter_s,self.shell.user_ns)
         
-    def magic_store(self, parameter_s=''):
-        """Lightweight persistence for python variables.
-
-        Example:
-        
-        ville@badger[~]|1> A = ['hello',10,'world']\\
-        ville@badger[~]|2> %store A\\
-        ville@badger[~]|3> Exit
-        
-        (IPython session is closed and started again...)
-        
-        ville@badger:~$ ipython -p pysh\\
-        ville@badger[~]|1> print A
-        
-        ['hello', 10, 'world']
-        
-        Usage:
-        
-        %store          - Show list of all variables and their current values\\
-        %store <var>    - Store the *current* value of the variable to disk\\
-        %store -d <var> - Remove the variable and its value from storage\\
-        %store -r       - Remove all variables from storage\\
-        %store foo >a.txt  - Store value of foo to new file a.txt\\
-        %store foo >>a.txt - Append value of foo to file a.txt\\   
-        
-        It should be noted that if you change the value of a variable, you
-        need to %store it again if you want to persist the new value.
-        
-        Note also that the variables will need to be pickleable; most basic
-        python types can be safely %stored.
-        """
-        
-        opts,argsl = self.parse_options(parameter_s,'dr',mode='string')
-        args = argsl.split(None,1)
-        ip = self.getapi()
-        # delete
-        if opts.has_key('d'):
-            try:
-                todel = args[0]
-            except IndexError:
-                error('You must provide the variable to forget')
-            else:
-                try:
-                    del self.shell.persist['S:' + todel]
-                except:
-                    error("Can't delete variable '%s'" % todel)
-        # reset
-        elif opts.has_key('r'):
-            for k in self.shell.persist.keys():
-                if k.startswith('S:'):
-                    del self.shell.persist[k]
-        
-        # run without arguments -> list variables & values
-        elif not args:
-            vars = [v[2:] for v in self.shell.persist.keys()
-                    if v.startswith('S:')]
-            vars.sort()            
-            if vars:
-                size = max(map(len,vars))
-            else:
-                size = 0
-                
-            print 'Stored variables and their in-memory values:'
-            fmt = '%-'+str(size)+'s -> %s'
-            get = self.shell.user_ns.get
-            for var in vars:
-                # print 30 first characters from every var
-                print fmt % (var,repr(get(var,'<unavailable>'))[:50])
-        
-        # default action - store the variable
-        else:
-            # %store foo >file.txt or >>file.txt
-            if len(args) > 1 and args[1].startswith('>'):
-                fnam = os.path.expanduser(args[1].lstrip('>').lstrip())
-                if args[1].startswith('>>'):
-                    fil = open(fnam,'a')
-                else:
-                    fil = open(fnam,'w')
-                obj = ip.ev(args[0])
-                print "Writing '%s' (%s) to file '%s'." % (args[0],
-                  obj.__class__.__name__, fnam)
-
-                
-                if not isinstance (obj,basestring):
-                    pprint(obj,fil)
-                else:
-                    fil.write(obj)
-                    if not obj.endswith('\n'):
-                        fil.write('\n')
-                
-                fil.close()
-                return
-            
-            # %store foo
-            obj = self.shell.user_ns[args[0] ]
-            if isinstance(inspect.getmodule(obj), FakeModule):
-                print textwrap.dedent("""\
-                Warning:%s is %s 
-                Proper storage of interactively declared classes (or instances
-                of those classes) is not possible! Only instances
-                of classes in real modules on file system can be %%store'd.
-                """ % (args[0], obj) ) 
-                return
-            pickled = pickle.dumps(obj)
-            self.shell.persist[ 'S:' + args[0] ] = pickled
-            print "Stored '%s' (%s, %d bytes)" % (args[0], obj.__class__.__name__,len(pickled))
     
     def magic_bookmark(self, parameter_s=''):
         """Manage IPython's bookmark system.
@@ -2763,7 +2658,7 b' Defaulting color scheme to \'NoColor\'"""'
             error('You can only give at most two arguments')
             return
 
-        bkms = self.shell.persist.get('bookmarks',{})
+        bkms = self.db.get('bookmarks',{})
             
         if opts.has_key('d'):
             try:
@@ -2795,7 +2690,7 b' Defaulting color scheme to \'NoColor\'"""'
                 bkms[args[0]] = os.getcwd()
             elif len(args)==2:
                 bkms[args[0]] = args[1]
-        self.shell.persist['bookmarks'] = bkms
+        self.db['bookmarks'] = bkms
 
     def magic_pycat(self, parameter_s=''):
         """Show a syntax-highlighted file through a pager.

IPython/hooks.py

@@ -32,7 +32,7 b" ip.set_hook('editor', calljed)"
 You can then enable the functionality by doing 'import myiphooks'
 somewhere in your configuration files or ipython command line.
 
-$Id: hooks.py 1095 2006-01-28 19:43:56Z vivainio $"""
+$Id: hooks.py 1107 2006-01-30 19:02:20Z vivainio $"""
 
 #*****************************************************************************
 #       Copyright (C) 2005 Fernando Perez. <[e-mail]>
@@ -54,7 +54,7 b' from pprint import pformat'
 # List here all the default hooks.  For now it's just the editor functions
 # but over time we'll move here all the public API for user-accessible things.
 __all__ = ['editor', 'fix_error_editor', 'result_display',
-           'input_prefilter']
+           'input_prefilter', 'shutdown_hook', 'late_startup_hook']
 
 def editor(self,filename, linenum=None):
     """Open the default editor at the given filename and linenumber.
@@ -167,3 +167,18 b' def input_prefilter(self,line):'
     """
     #print "attempt to rewrite",line #dbg
     return line
+
+def shutdown_hook(self):
+    """ default shutdown hook
+    
+    Typically, shotdown hooks should raise TryNext so all shutdown ops are done
+    """
+    
+    #print "default shutdown hook ok" # dbg
+    return
+
+def late_startup_hook(self):
+    """ Executed after ipython has been constructed and configured 
+    
+    """
+    #print "default startup hook ok" # dbg
  No newline at end of file

IPython/ipapi.py

@@ -149,6 +149,13 b' class IPApi:'
         """
         return self.IP.meta
         
+    def getdb(self):
+        """ Return a handle to persistent dict-like database
+        
+        Return a PickleShareDB object.
+        """
+        return self.IP.db
+        
 
 def launch_new_instance(user_ns = None):
     """ Create and start a new ipython instance.

IPython/iplib.py

@@ -6,7 +6,7 b' Requires Python 2.3 or newer.'
 
 This file contains all the classes and helper functions specific to IPython.
 
-$Id: iplib.py 1102 2006-01-30 06:08:16Z fperez $
+$Id: iplib.py 1107 2006-01-30 19:02:20Z vivainio $
 """
 
 #*****************************************************************************
@@ -58,6 +58,7 b' import sys'
 import tempfile
 import traceback
 import types
+import pickleshare
 
 from pprint import pprint, pformat
 
@@ -191,6 +192,7 b' class InteractiveShell(object,Magic):'
                  user_ns = None,user_global_ns=None,banner2='',
                  custom_exceptions=((),None),embedded=False):
 
+        
         # log system
         self.logger = Logger(self,logfname='ipython_log.py',logmode='rotate')
 
@@ -607,6 +609,7 b' class InteractiveShell(object,Magic):'
 
         rc = self.rc
         
+        self.db = pickleshare.PickleShareDB(rc.ipythondir + "/db")
         # Load readline proper
         if rc.readline:
             self.init_readline()
@@ -648,32 +651,9 b' class InteractiveShell(object,Magic):'
         # Load user aliases
         for alias in rc.alias:
             self.magic_alias(alias)
-
-        # dynamic data that survives through sessions
-        # XXX make the filename a config option?
-        persist_base = 'persist'
-        if rc.profile:
-            persist_base += '_%s' % rc.profile
-        self.persist_fname =  os.path.join(rc.ipythondir,persist_base)
-
-        try:
-            self.persist = pickle.load(file(self.persist_fname))
-        except:
-            self.persist = {}
-        
-        
-        for (key, value) in [(k[2:],v) for (k,v) in self.persist.items() if k.startswith('S:')]:
-            try:
-                obj = pickle.loads(value)
-            except:
-                
-                print "Unable to restore variable '%s', ignoring (use %%store -d to forget!)" % key
-                print "The error was:",sys.exc_info()[0]
-                continue
+        self.hooks.late_startup_hook()
                 
             
-            self.user_ns[key] = obj
-
     def add_builtins(self):
         """Store ipython references into the builtin namespace.
 
@@ -1147,10 +1127,7 b' want to merge them back into the new files.""" % locals()'
                 pass
 
         # save the "persistent data" catch-all dictionary
-        try:
-            pickle.dump(self.persist, open(self.persist_fname,"w"))
-        except:
-            print "*** ERROR *** persistent data saving failed."
+        self.hooks.shutdown_hook()
         
     def savehist(self):
         """Save input history to a file (via readline library)."""

doc/ChangeLog

@@ -1,3 +1,16 b''
[phone]  Ville Vainio  <[e-mail]>  
+
+	* pickleshare,pspersistence,ipapi,Magic: persistence overhaul.
+	Now %store and bookmarks work through PickleShare, meaning that 
+	concurrent access is possible and all ipython sessions see the 
+	same database situation all the time, instead of snapshot of
+	the situation when the session was started. Hence, %bookmark
+	results are immediately accessible from othes sessions. The database
+	is also available for use by user extensions. See:
+	http://www.python.org/pypi/pickleshare
+	
+	* hooks.py: Two new hooks, 'shutdown_hook' and 'late_startup_hook'.
+
 2006-01-29  Fernando Perez  <[e-mail]>
 
 	* IPython/iplib.py (interact): Fix that we were not catching