appendfile.py
161 lines
| 5.5 KiB
| text/x-python
|
PythonLexer
/ mercurial / appendfile.py
|
r1999 | # appendfile.py - special classes to make repo updates atomic | ||
# | ||||
# Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com> | ||||
# | ||||
# This software may be used and distributed according to the terms | ||||
# of the GNU General Public License, incorporated herein by reference. | ||||
|
r3877 | import cStringIO, changelog, errno, manifest, os, tempfile, util | ||
|
r1999 | |||
# writes to metadata files are ordered. reads: changelog, manifest, | ||||
# normal files. writes: normal files, manifest, changelog. | ||||
# manifest contains pointers to offsets in normal files. changelog | ||||
# contains pointers to offsets in manifest. if reader reads old | ||||
# changelog while manifest or normal files are written, it has no | ||||
# pointers into new parts of those files that are maybe not consistent | ||||
# yet, so will not read them. | ||||
# localrepo.addchangegroup thinks it writes changelog first, then | ||||
# manifest, then normal files (this is order they are available, and | ||||
# needed for computing linkrev fields), but uses appendfile to hide | ||||
# updates from readers. data not written to manifest or changelog | ||||
# until all normal files updated. write manifest first, then | ||||
# changelog. | ||||
# with this write ordering, readers cannot see inconsistent view of | ||||
# repo during update. | ||||
class appendfile(object): | ||||
'''implement enough of file protocol to append to revlog file. | ||||
appended data is written to temp file. reads and seeks span real | ||||
file and temp file. readers cannot see appended data until | ||||
writedata called.''' | ||||
|
r2089 | def __init__(self, fp, tmpname): | ||
if tmpname: | ||||
self.tmpname = tmpname | ||||
|
r2176 | self.tmpfp = util.posixfile(self.tmpname, 'ab+') | ||
|
r2089 | else: | ||
|
r2165 | fd, self.tmpname = tempfile.mkstemp(prefix="hg-appendfile-") | ||
|
r2176 | os.close(fd) | ||
self.tmpfp = util.posixfile(self.tmpname, 'ab+') | ||||
|
r1999 | self.realfp = fp | ||
|
r2010 | self.offset = fp.tell() | ||
|
r1999 | # real file is not written by anyone else. cache its size so | ||
# seek and read can be fast. | ||||
|
r2176 | self.realsize = util.fstat(fp).st_size | ||
self.name = fp.name | ||||
|
r1999 | |||
|
r2075 | def end(self): | ||
self.tmpfp.flush() # make sure the stat is correct | ||||
|
r2176 | return self.realsize + util.fstat(self.tmpfp).st_size | ||
|
r2089 | |||
def tell(self): | ||||
return self.offset | ||||
def flush(self): | ||||
self.tmpfp.flush() | ||||
def close(self): | ||||
self.realfp.close() | ||||
self.tmpfp.close() | ||||
|
r2075 | |||
def seek(self, offset, whence=0): | ||||
|
r1999 | '''virtual file offset spans real file and temp file.''' | ||
|
r2075 | if whence == 0: | ||
self.offset = offset | ||||
elif whence == 1: | ||||
self.offset += offset | ||||
elif whence == 2: | ||||
self.offset = self.end() + offset | ||||
|
r2089 | if self.offset < self.realsize: | ||
|
r1999 | self.realfp.seek(self.offset) | ||
else: | ||||
|
r2089 | self.tmpfp.seek(self.offset - self.realsize) | ||
|
r1999 | |||
def read(self, count=-1): | ||||
'''only trick here is reads that span real file and temp file.''' | ||||
fp = cStringIO.StringIO() | ||||
old_offset = self.offset | ||||
|
r2089 | if self.offset < self.realsize: | ||
|
r1999 | s = self.realfp.read(count) | ||
fp.write(s) | ||||
self.offset += len(s) | ||||
if count > 0: | ||||
count -= len(s) | ||||
if count != 0: | ||||
if old_offset != self.offset: | ||||
|
r2089 | self.tmpfp.seek(self.offset - self.realsize) | ||
|
r1999 | s = self.tmpfp.read(count) | ||
fp.write(s) | ||||
self.offset += len(s) | ||||
return fp.getvalue() | ||||
def write(self, s): | ||||
'''append to temp file.''' | ||||
|
r2027 | self.tmpfp.seek(0, 2) | ||
|
r1999 | self.tmpfp.write(s) | ||
# all writes are appends, so offset must go to end of file. | ||||
|
r2089 | self.offset = self.realsize + self.tmpfp.tell() | ||
|
r1999 | |||
class appendopener(object): | ||||
'''special opener for files that only read or append.''' | ||||
def __init__(self, opener): | ||||
self.realopener = opener | ||||
|
r2089 | # key: file name, value: appendfile name | ||
self.tmpnames = {} | ||||
|
r1999 | |||
def __call__(self, name, mode='r'): | ||||
|
r2089 | '''open file.''' | ||
|
r1999 | |||
|
r2075 | assert mode in 'ra+' | ||
|
r2089 | try: | ||
realfp = self.realopener(name, 'r') | ||||
except IOError, err: | ||||
if err.errno != errno.ENOENT: raise | ||||
realfp = self.realopener(name, 'w+') | ||||
tmpname = self.tmpnames.get(name) | ||||
fp = appendfile(realfp, tmpname) | ||||
if tmpname is None: | ||||
self.tmpnames[name] = fp.tmpname | ||||
return fp | ||||
|
r1999 | |||
def writedata(self): | ||||
'''copy data from temp files to real files.''' | ||||
# write .d file before .i file. | ||||
|
r2089 | tmpnames = self.tmpnames.items() | ||
tmpnames.sort() | ||||
for name, tmpname in tmpnames: | ||||
|
r2236 | ifp = open(tmpname, 'rb') | ||
ofp = self.realopener(name, 'a') | ||||
for chunk in util.filechunkiter(ifp): | ||||
ofp.write(chunk) | ||||
ifp.close() | ||||
|
r2102 | os.unlink(tmpname) | ||
|
r2232 | del self.tmpnames[name] | ||
|
r2236 | ofp.close() | ||
|
r2089 | |||
|
r2232 | def cleanup(self): | ||
'''delete temp files (this discards unwritten data!)''' | ||||
for tmpname in self.tmpnames.values(): | ||||
os.unlink(tmpname) | ||||
|
r1999 | # files for changelog and manifest are in different appendopeners, so | ||
# not mixed up together. | ||||
class appendchangelog(changelog.changelog, appendopener): | ||||
|
r2082 | def __init__(self, opener, version): | ||
|
r1999 | appendopener.__init__(self, opener) | ||
|
r2082 | changelog.changelog.__init__(self, self, version) | ||
|
r2075 | def checkinlinesize(self, fp, tr): | ||
return | ||||
|
r1999 | |||
class appendmanifest(manifest.manifest, appendopener): | ||||
|
r2082 | def __init__(self, opener, version): | ||
|
r1999 | appendopener.__init__(self, opener) | ||
|
r2082 | manifest.manifest.__init__(self, self, version) | ||
|
r2075 | def checkinlinesize(self, fp, tr): | ||
return | ||||