From fff926bd56726702a408992ea3997db157af9865 2006-01-30 19:02:20 From: vivainio Date: 2006-01-30 19:02:20 Subject: [PATCH] Grand Persistence Overhaul, featuring PickleShare. startup and shutdown hooks added. --- diff --git a/IPython/Extensions/ipy_system_conf.py b/IPython/Extensions/ipy_system_conf.py index a180611..0de956d 100644 --- a/IPython/Extensions/ipy_system_conf.py +++ b/IPython/Extensions/ipy_system_conf.py @@ -16,3 +16,4 @@ import sys import ext_rehashdir # %rehashdir magic import ext_rescapture # var = !ls and var = %magic +import pspersistence # %store magic \ No newline at end of file diff --git a/IPython/Extensions/pickleshare.py b/IPython/Extensions/pickleshare.py new file mode 100644 index 0000000..ae24270 --- /dev/null +++ b/IPython/Extensions/pickleshare.py @@ -0,0 +1,267 @@ +#!/usr/bin/env python + +""" PickleShare - a small 'shelve' like datastore with concurrency support + +Like shelve, a PickleShareDB object acts like a normal dictionary. Unlike +shelve, many processes can access the database simultaneously. Changing a +value in database is immediately visible to other processes accessing the +same database. + +Concurrency is possible because the values are stored in separate files. Hence +the "database" is a directory where *all* files are governed by PickleShare. + +Example usage:: + + from pickleshare import * + db = PickleShareDB('~/testpickleshare') + db.clear() + print "Should be empty:",db.items() + db['hello'] = 15 + db['aku ankka'] = [1,2,313] + db['paths/are/ok/key'] = [1,(5,46)] + print db.keys() + del db['aku ankka'] + +This module is certainly not ZODB, but can be used for low-load +(non-mission-critical) situations where tiny code size trumps the +advanced features of a "real" object database. + +Installation guide: easy_install pickleshare + +Author: Ville Vainio +License: MIT open source license. + +""" + +from path import path as Path +import os,stat,time +import cPickle as pickle +import UserDict +import warnings +import glob + +class PickleShareDB(UserDict.DictMixin): + """ The main 'connection' object for PickleShare database """ + def __init__(self,root): + """ Return a db object that will manage the specied directory""" + self.root = Path(root).expanduser().abspath() + if not self.root.isdir(): + self.root.makedirs() + # cache has { 'key' : (obj, orig_mod_time) } + self.cache = {} + + def __getitem__(self,key): + """ db['key'] reading """ + fil = self.root / key + try: + mtime = (fil.stat()[stat.ST_MTIME]) + except OSError: + raise KeyError(key) + + if fil in self.cache and mtime == self.cache[fil][1]: + return self.cache[fil][0] + try: + # The cached item has expired, need to read + obj = pickle.load(fil.open()) + except: + raise KeyError(key) + + self.cache[fil] = (obj,mtime) + return obj + + def __setitem__(self,key,value): + """ db['key'] = 5 """ + fil = self.root / key + parent = fil.parent + if parent and not parent.isdir(): + parent.makedirs() + pickled = pickle.dump(value,fil.open('w')) + try: + self.cache[fil] = (value,fil.mtime) + except OSError,e: + if e.errno != 2: + raise + + def __delitem__(self,key): + """ del db["key"] """ + fil = self.root / key + self.cache.pop(fil,None) + try: + fil.remove() + except OSError: + # notfound and permission denied are ok - we + # lost, the other process wins the conflict + pass + + def _normalized(self, p): + """ Make a key suitable for user's eyes """ + return str(self.root.relpathto(p)).replace('\\','/') + + def keys(self, globpat = None): + """ All keys in DB, or all keys matching a glob""" + + if globpat is None: + files = self.root.walkfiles() + else: + files = [Path(p) for p in glob.glob(self.root/globpat)] + return [self._normalized(p) for p in files if p.isfile()] + + def uncache(self,*items): + """ Removes all, or specified items from cache + + Use this after reading a large amount of large objects + to free up memory, when you won't be needing the objects + for a while. + + """ + if not items: + self.cache = {} + for it in items: + self.cache.pop(it,None) + + def waitget(self,key, maxwaittime = 60 ): + """ Wait (poll) for a key to get a value + + Will wait for `maxwaittime` seconds before raising a KeyError. + The call exits normally if the `key` field in db gets a value + within the timeout period. + + Use this for synchronizing different processes or for ensuring + that an unfortunately timed "db['key'] = newvalue" operation + in another process (which causes all 'get' operation to cause a + KeyError for the duration of pickling) won't screw up your program + logic. + """ + + wtimes = [0.2] * 3 + [0.5] * 2 + [1] + tries = 0 + waited = 0 + while 1: + try: + val = self[key] + return val + except KeyError: + pass + + if waited > maxwaittime: + raise KeyError(key) + + time.sleep(wtimes[tries]) + waited+=wtimes[tries] + if tries < len(wtimes) -1: + tries+=1 + + def getlink(self,folder): + """ Get a convenient link for accessing items """ + return PickleShareLink(self, folder) + + def __repr__(self): + return "PickleShareDB('%s')" % self.root + + + +class PickleShareLink: + """ A shortdand for accessing nested PickleShare data conveniently. + + Created through PickleShareDB.getlink(), example:: + + lnk = db.getlink('myobjects/test') + lnk.foo = 2 + lnk.bar = lnk.foo + 5 + + """ + def __init__(self, db, keydir ): + self.__dict__.update(locals()) + + def __getattr__(self,key): + return self.__dict__['db'][self.__dict__['keydir']+'/' + key] + def __setattr__(self,key,val): + self.db[self.keydir+'/' + key] = val + def __repr__(self): + db = self.__dict__['db'] + keys = db.keys( self.__dict__['keydir'] +"/*") + return "" % ( + self.__dict__['keydir'], + ";".join([Path(k).basename() for k in keys])) + + +def test(): + db = PickleShareDB('~/testpickleshare') + db.clear() + print "Should be empty:",db.items() + db['hello'] = 15 + db['aku ankka'] = [1,2,313] + db['paths/nest/ok/keyname'] = [1,(5,46)] + print db.keys() + print db.keys('paths/nest/ok/k*') + print dict(db) # snapsot of whole db + db.uncache() # frees memory, causes re-reads later + + # shorthand for accessing deeply nested files + lnk = db.getlink('myobjects/test') + lnk.foo = 2 + lnk.bar = lnk.foo + 5 + print lnk.bar # 7 + +def stress(): + db = PickleShareDB('~/fsdbtest') + import time,sys + for i in range(1000): + for j in range(300): + if i % 15 == 0 and i < 200: + if str(j) in db: + del db[str(j)] + continue + + if j%33 == 0: + time.sleep(0.02) + + db[str(j)] = db.get(str(j), []) + [(i,j,"proc %d" % os.getpid())] + print i, + sys.stdout.flush() + if i % 10 == 0: + db.uncache() + +def main(): + import textwrap + usage = textwrap.dedent("""\ + pickleshare - manage PickleShare databases + + Usage: + + pickleshare dump /path/to/db > dump.txt + pickleshare load /path/to/db < dump.txt + pickleshare test /path/to/db + """) + DB = PickleShareDB + import sys + if len(sys.argv) < 2: + print usage + return + + cmd = sys.argv[1] + args = sys.argv[2:] + if cmd == 'dump': + if not args: args= ['.'] + db = DB(args[0]) + import pprint + pprint.pprint(db.items()) + elif cmd == 'load': + cont = sys.stdin.read() + db = DB(args[0]) + data = eval(cont) + db.clear() + for k,v in db.items(): + db[k] = v + elif cmd == 'testwait': + db = DB(args[0]) + db.clear() + print db.waitget('250') + elif cmd == 'test': + test() + stress() + +if __name__== "__main__": + main() + + \ No newline at end of file diff --git a/IPython/Extensions/pspersistence.py b/IPython/Extensions/pspersistence.py new file mode 100644 index 0000000..bde8b9c --- /dev/null +++ b/IPython/Extensions/pspersistence.py @@ -0,0 +1,149 @@ +import IPython.ipapi +ip = IPython.ipapi.get() + +import pickleshare + +import inspect,pickle,os,textwrap +from IPython.FakeModule import FakeModule + +def refresh_variables(ip): + db = ip.getdb() + for key in db.keys('autorestore/*'): + # strip autorestore + justkey = os.path.basename(key) + try: + obj = db[key] + except KeyError: + print "Unable to restore variable '%s', ignoring (use %%store -d to forget!)" % justkey + print "The error was:",sys.exc_info()[0] + else: + #print "restored",justkey,"=",obj #dbg + ip.user_ns()[justkey] = obj + + + +def restore_data(self): + #o = ip.options() + #self.db = pickleshare.PickleShareDB(o.ipythondir + "/db") + #print "restoring ps data" # dbg + + ip = self.getapi() + refresh_variables(ip) + raise IPython.ipapi.TryNext + + +ip.set_hook('late_startup_hook', restore_data) + +def magic_store(self, parameter_s=''): + """Lightweight persistence for python variables. + + Example: + + ville@badger[~]|1> A = ['hello',10,'world']\\ + ville@badger[~]|2> %store A\\ + ville@badger[~]|3> Exit + + (IPython session is closed and started again...) + + ville@badger:~$ ipython -p pysh\\ + ville@badger[~]|1> print A + + ['hello', 10, 'world'] + + Usage: + + %store - Show list of all variables and their current values\\ + %store - Store the *current* value of the variable to disk\\ + %store -d - Remove the variable and its value from storage\\ + %store -z - Remove all variables from storage\\ + %store -r - Refresh all variables from store (delete current vals)\\ + %store foo >a.txt - Store value of foo to new file a.txt\\ + %store foo >>a.txt - Append value of foo to file a.txt\\ + + It should be noted that if you change the value of a variable, you + need to %store it again if you want to persist the new value. + + Note also that the variables will need to be pickleable; most basic + python types can be safely %stored. + """ + + opts,argsl = self.parse_options(parameter_s,'drz',mode='string') + args = argsl.split(None,1) + ip = self.getapi() + # delete + if opts.has_key('d'): + try: + todel = args[0] + except IndexError: + error('You must provide the variable to forget') + else: + try: + del self.db['autorestore/' + todel] + except: + error("Can't delete variable '%s'" % todel) + # reset + elif opts.has_key('z'): + for k in self.db.keys('autorestore/*'): + del self.db[k] + + elif opts.has_key('r'): + refresh_variables(ip) + + + # run without arguments -> list variables & values + elif not args: + vars = self.db.keys('autorestore/*') + vars.sort() + if vars: + size = max(map(len,vars)) + else: + size = 0 + + print 'Stored variables and their in-db values:' + fmt = '%-'+str(size)+'s -> %s' + get = self.db.get + for var in vars: + justkey = os.path.basename(var) + # print 30 first characters from every var + print fmt % (justkey,repr(get(var,''))[:50]) + + # default action - store the variable + else: + # %store foo >file.txt or >>file.txt + if len(args) > 1 and args[1].startswith('>'): + fnam = os.path.expanduser(args[1].lstrip('>').lstrip()) + if args[1].startswith('>>'): + fil = open(fnam,'a') + else: + fil = open(fnam,'w') + obj = ip.ev(args[0]) + print "Writing '%s' (%s) to file '%s'." % (args[0], + obj.__class__.__name__, fnam) + + + if not isinstance (obj,basestring): + pprint(obj,fil) + else: + fil.write(obj) + if not obj.endswith('\n'): + fil.write('\n') + + fil.close() + return + + # %store foo + obj = ip.ev(args[0]) + if isinstance(inspect.getmodule(obj), FakeModule): + print textwrap.dedent("""\ + Warning:%s is %s + Proper storage of interactively declared classes (or instances + of those classes) is not possible! Only instances + of classes in real modules on file system can be %%store'd. + """ % (args[0], obj) ) + return + #pickled = pickle.dumps(obj) + self.db[ 'autorestore/' + args[0] ] = obj + print "Stored '%s' (%s)" % (args[0], obj.__class__.__name__) + +ip.expose_magic('store',magic_store) + \ No newline at end of file diff --git a/IPython/Magic.py b/IPython/Magic.py index a17475b..e3611a9 100644 --- a/IPython/Magic.py +++ b/IPython/Magic.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """Magic functions for InteractiveShell. -$Id: Magic.py 1099 2006-01-29 21:05:57Z vivainio $""" +$Id: Magic.py 1107 2006-01-30 19:02:20Z vivainio $""" #***************************************************************************** # Copyright (C) 2001 Janko Hauser and @@ -2301,7 +2301,7 @@ Defaulting color scheme to 'NoColor'""" !command runs is immediately discarded after executing 'command'.""" parameter_s = parameter_s.strip() - bkms = self.shell.persist.get("bookmarks",{}) + #bkms = self.shell.persist.get("bookmarks",{}) numcd = re.match(r'(-)(\d+)$',parameter_s) # jump in directory history by number @@ -2326,19 +2326,20 @@ Defaulting color scheme to 'NoColor'""" except IndexError: print 'No previous directory to change to.' return - # jump to bookmark - elif opts.has_key('b') or (bkms.has_key(ps) and not os.path.isdir(ps)): - if bkms.has_key(ps): - target = bkms[ps] - print '(bookmark:%s) -> %s' % (ps,target) - ps = target - else: - if bkms: - error("Bookmark '%s' not found. " - "Use '%%bookmark -l' to see your bookmarks." % ps) + # jump to bookmark if needed + else: + if not os.path.isdir(ps) or opts.has_key('b'): + bkms = self.db.get('bookmarks', {}) + + if bkms.has_key(ps): + target = bkms[ps] + print '(bookmark:%s) -> %s' % (ps,target) + ps = target else: - print "Bookmarks not set - use %bookmark " - return + if opts.has_key('b'): + error("Bookmark '%s' not found. " + "Use '%%bookmark -l' to see your bookmarks." % ps) + return # at this point ps should point to the target dir if ps: @@ -2634,112 +2635,6 @@ Defaulting color scheme to 'NoColor'""" self.shell.jobs.new(parameter_s,self.shell.user_ns) - def magic_store(self, parameter_s=''): - """Lightweight persistence for python variables. - - Example: - - ville@badger[~]|1> A = ['hello',10,'world']\\ - ville@badger[~]|2> %store A\\ - ville@badger[~]|3> Exit - - (IPython session is closed and started again...) - - ville@badger:~$ ipython -p pysh\\ - ville@badger[~]|1> print A - - ['hello', 10, 'world'] - - Usage: - - %store - Show list of all variables and their current values\\ - %store - Store the *current* value of the variable to disk\\ - %store -d - Remove the variable and its value from storage\\ - %store -r - Remove all variables from storage\\ - %store foo >a.txt - Store value of foo to new file a.txt\\ - %store foo >>a.txt - Append value of foo to file a.txt\\ - - It should be noted that if you change the value of a variable, you - need to %store it again if you want to persist the new value. - - Note also that the variables will need to be pickleable; most basic - python types can be safely %stored. - """ - - opts,argsl = self.parse_options(parameter_s,'dr',mode='string') - args = argsl.split(None,1) - ip = self.getapi() - # delete - if opts.has_key('d'): - try: - todel = args[0] - except IndexError: - error('You must provide the variable to forget') - else: - try: - del self.shell.persist['S:' + todel] - except: - error("Can't delete variable '%s'" % todel) - # reset - elif opts.has_key('r'): - for k in self.shell.persist.keys(): - if k.startswith('S:'): - del self.shell.persist[k] - - # run without arguments -> list variables & values - elif not args: - vars = [v[2:] for v in self.shell.persist.keys() - if v.startswith('S:')] - vars.sort() - if vars: - size = max(map(len,vars)) - else: - size = 0 - - print 'Stored variables and their in-memory values:' - fmt = '%-'+str(size)+'s -> %s' - get = self.shell.user_ns.get - for var in vars: - # print 30 first characters from every var - print fmt % (var,repr(get(var,''))[:50]) - - # default action - store the variable - else: - # %store foo >file.txt or >>file.txt - if len(args) > 1 and args[1].startswith('>'): - fnam = os.path.expanduser(args[1].lstrip('>').lstrip()) - if args[1].startswith('>>'): - fil = open(fnam,'a') - else: - fil = open(fnam,'w') - obj = ip.ev(args[0]) - print "Writing '%s' (%s) to file '%s'." % (args[0], - obj.__class__.__name__, fnam) - - - if not isinstance (obj,basestring): - pprint(obj,fil) - else: - fil.write(obj) - if not obj.endswith('\n'): - fil.write('\n') - - fil.close() - return - - # %store foo - obj = self.shell.user_ns[args[0] ] - if isinstance(inspect.getmodule(obj), FakeModule): - print textwrap.dedent("""\ - Warning:%s is %s - Proper storage of interactively declared classes (or instances - of those classes) is not possible! Only instances - of classes in real modules on file system can be %%store'd. - """ % (args[0], obj) ) - return - pickled = pickle.dumps(obj) - self.shell.persist[ 'S:' + args[0] ] = pickled - print "Stored '%s' (%s, %d bytes)" % (args[0], obj.__class__.__name__,len(pickled)) def magic_bookmark(self, parameter_s=''): """Manage IPython's bookmark system. @@ -2763,7 +2658,7 @@ Defaulting color scheme to 'NoColor'""" error('You can only give at most two arguments') return - bkms = self.shell.persist.get('bookmarks',{}) + bkms = self.db.get('bookmarks',{}) if opts.has_key('d'): try: @@ -2795,7 +2690,7 @@ Defaulting color scheme to 'NoColor'""" bkms[args[0]] = os.getcwd() elif len(args)==2: bkms[args[0]] = args[1] - self.shell.persist['bookmarks'] = bkms + self.db['bookmarks'] = bkms def magic_pycat(self, parameter_s=''): """Show a syntax-highlighted file through a pager. diff --git a/IPython/hooks.py b/IPython/hooks.py index 29e0527..28f68cf 100644 --- a/IPython/hooks.py +++ b/IPython/hooks.py @@ -32,7 +32,7 @@ ip.set_hook('editor', calljed) You can then enable the functionality by doing 'import myiphooks' somewhere in your configuration files or ipython command line. -$Id: hooks.py 1095 2006-01-28 19:43:56Z vivainio $""" +$Id: hooks.py 1107 2006-01-30 19:02:20Z vivainio $""" #***************************************************************************** # Copyright (C) 2005 Fernando Perez. @@ -54,7 +54,7 @@ from pprint import pformat # List here all the default hooks. For now it's just the editor functions # but over time we'll move here all the public API for user-accessible things. __all__ = ['editor', 'fix_error_editor', 'result_display', - 'input_prefilter'] + 'input_prefilter', 'shutdown_hook', 'late_startup_hook'] def editor(self,filename, linenum=None): """Open the default editor at the given filename and linenumber. @@ -166,4 +166,19 @@ def input_prefilter(self,line): """ #print "attempt to rewrite",line #dbg - return line \ No newline at end of file + return line + +def shutdown_hook(self): + """ default shutdown hook + + Typically, shotdown hooks should raise TryNext so all shutdown ops are done + """ + + #print "default shutdown hook ok" # dbg + return + +def late_startup_hook(self): + """ Executed after ipython has been constructed and configured + + """ + #print "default startup hook ok" # dbg \ No newline at end of file diff --git a/IPython/ipapi.py b/IPython/ipapi.py index ed629a8..56b3f00 100644 --- a/IPython/ipapi.py +++ b/IPython/ipapi.py @@ -148,6 +148,13 @@ class IPApi: data that should persist through the ipython session. """ return self.IP.meta + + def getdb(self): + """ Return a handle to persistent dict-like database + + Return a PickleShareDB object. + """ + return self.IP.db def launch_new_instance(user_ns = None): diff --git a/IPython/iplib.py b/IPython/iplib.py index 2dfa6b9..a513674 100644 --- a/IPython/iplib.py +++ b/IPython/iplib.py @@ -6,7 +6,7 @@ Requires Python 2.3 or newer. This file contains all the classes and helper functions specific to IPython. -$Id: iplib.py 1102 2006-01-30 06:08:16Z fperez $ +$Id: iplib.py 1107 2006-01-30 19:02:20Z vivainio $ """ #***************************************************************************** @@ -58,6 +58,7 @@ import sys import tempfile import traceback import types +import pickleshare from pprint import pprint, pformat @@ -191,6 +192,7 @@ class InteractiveShell(object,Magic): user_ns = None,user_global_ns=None,banner2='', custom_exceptions=((),None),embedded=False): + # log system self.logger = Logger(self,logfname='ipython_log.py',logmode='rotate') @@ -607,6 +609,7 @@ class InteractiveShell(object,Magic): rc = self.rc + self.db = pickleshare.PickleShareDB(rc.ipythondir + "/db") # Load readline proper if rc.readline: self.init_readline() @@ -648,31 +651,8 @@ class InteractiveShell(object,Magic): # Load user aliases for alias in rc.alias: self.magic_alias(alias) - - # dynamic data that survives through sessions - # XXX make the filename a config option? - persist_base = 'persist' - if rc.profile: - persist_base += '_%s' % rc.profile - self.persist_fname = os.path.join(rc.ipythondir,persist_base) - - try: - self.persist = pickle.load(file(self.persist_fname)) - except: - self.persist = {} - + self.hooks.late_startup_hook() - for (key, value) in [(k[2:],v) for (k,v) in self.persist.items() if k.startswith('S:')]: - try: - obj = pickle.loads(value) - except: - - print "Unable to restore variable '%s', ignoring (use %%store -d to forget!)" % key - print "The error was:",sys.exc_info()[0] - continue - - - self.user_ns[key] = obj def add_builtins(self): """Store ipython references into the builtin namespace. @@ -1147,10 +1127,7 @@ want to merge them back into the new files.""" % locals() pass # save the "persistent data" catch-all dictionary - try: - pickle.dump(self.persist, open(self.persist_fname,"w")) - except: - print "*** ERROR *** persistent data saving failed." + self.hooks.shutdown_hook() def savehist(self): """Save input history to a file (via readline library).""" diff --git a/doc/ChangeLog b/doc/ChangeLog index ac24f4f..2fc6659 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,3 +1,16 @@ +2006-01-30 Ville Vainio + + * pickleshare,pspersistence,ipapi,Magic: persistence overhaul. + Now %store and bookmarks work through PickleShare, meaning that + concurrent access is possible and all ipython sessions see the + same database situation all the time, instead of snapshot of + the situation when the session was started. Hence, %bookmark + results are immediately accessible from othes sessions. The database + is also available for use by user extensions. See: + http://www.python.org/pypi/pickleshare + + * hooks.py: Two new hooks, 'shutdown_hook' and 'late_startup_hook'. + 2006-01-29 Fernando Perez * IPython/iplib.py (interact): Fix that we were not catching