store.py
538 lines
| 16.5 KiB
| text/x-python
|
PythonLexer
/ mercurial / store.py
|
r6839 | # store.py - repository store handling for Mercurial | ||
# | ||||
# Copyright 2008 Matt Mackall <mpm@selenic.com> | ||||
# | ||||
|
r8225 | # This software may be used and distributed according to the terms of the | ||
|
r10263 | # GNU General Public License version 2 or any later version. | ||
|
r6839 | |||
|
r7229 | from i18n import _ | ||
|
r17747 | import scmutil, util, parsers | ||
|
r17374 | import os, stat, errno | ||
|
r6840 | |||
|
r7229 | _sha = util.sha1 | ||
|
r8531 | # This avoids a collision between a file named foo and a dir named | ||
# foo.i or foo.d | ||||
|
r17607 | def _encodedir(path): | ||
|
r13949 | ''' | ||
|
r17607 | >>> _encodedir('data/foo.i') | ||
|
r13949 | 'data/foo.i' | ||
|
r17607 | >>> _encodedir('data/foo.i/bla.i') | ||
|
r13949 | 'data/foo.i.hg/bla.i' | ||
|
r17607 | >>> _encodedir('data/foo.i.hg/bla.i') | ||
|
r13949 | 'data/foo.i.hg.hg/bla.i' | ||
|
r17607 | >>> _encodedir('data/foo.i\\ndata/foo.i/bla.i\\ndata/foo.i.hg/bla.i\\n') | ||
|
r17605 | 'data/foo.i\\ndata/foo.i.hg/bla.i\\ndata/foo.i.hg.hg/bla.i\\n' | ||
|
r13949 | ''' | ||
|
r8531 | return (path | ||
.replace(".hg/", ".hg.hg/") | ||||
.replace(".i/", ".i.hg/") | ||||
.replace(".d/", ".d.hg/")) | ||||
|
r17607 | encodedir = getattr(parsers, 'encodedir', _encodedir) | ||
|
r8531 | def decodedir(path): | ||
|
r13949 | ''' | ||
>>> decodedir('data/foo.i') | ||||
'data/foo.i' | ||||
>>> decodedir('data/foo.i.hg/bla.i') | ||||
'data/foo.i/bla.i' | ||||
>>> decodedir('data/foo.i.hg.hg/bla.i') | ||||
'data/foo.i.hg/bla.i' | ||||
''' | ||||
|
r17586 | if ".hg/" not in path: | ||
|
r8531 | return path | ||
return (path | ||||
.replace(".d.hg/", ".d/") | ||||
.replace(".i.hg/", ".i/") | ||||
.replace(".hg.hg/", ".hg/")) | ||||
|
r6839 | def _buildencodefun(): | ||
|
r13949 | ''' | ||
>>> enc, dec = _buildencodefun() | ||||
>>> enc('nothing/special.txt') | ||||
'nothing/special.txt' | ||||
>>> dec('nothing/special.txt') | ||||
'nothing/special.txt' | ||||
>>> enc('HELLO') | ||||
'_h_e_l_l_o' | ||||
>>> dec('_h_e_l_l_o') | ||||
'HELLO' | ||||
>>> enc('hello:world?') | ||||
'hello~3aworld~3f' | ||||
>>> dec('hello~3aworld~3f') | ||||
'hello:world?' | ||||
>>> enc('the\x07quick\xADshot') | ||||
'the~07quick~adshot' | ||||
>>> dec('the~07quick~adshot') | ||||
'the\\x07quick\\xadshot' | ||||
''' | ||||
|
r6839 | e = '_' | ||
|
r14288 | winreserved = [ord(x) for x in '\\:*?"<>|'] | ||
|
r10282 | cmap = dict([(chr(x), chr(x)) for x in xrange(127)]) | ||
|
r14288 | for x in (range(32) + range(126, 256) + winreserved): | ||
|
r6839 | cmap[chr(x)] = "~%02x" % x | ||
for x in range(ord("A"), ord("Z")+1) + [ord(e)]: | ||||
cmap[chr(x)] = e + chr(x).lower() | ||||
dmap = {} | ||||
for k, v in cmap.iteritems(): | ||||
dmap[v] = k | ||||
def decode(s): | ||||
i = 0 | ||||
while i < len(s): | ||||
for l in xrange(1, 4): | ||||
try: | ||||
|
r10282 | yield dmap[s[i:i + l]] | ||
|
r6839 | i += l | ||
break | ||||
except KeyError: | ||||
pass | ||||
else: | ||||
raise KeyError | ||||
|
r17608 | return (lambda s: ''.join([cmap[c] for c in s]), | ||
lambda s: ''.join(list(decode(s)))) | ||||
_encodefname, _decodefname = _buildencodefun() | ||||
|
r6839 | |||
|
r17608 | def encodefilename(s): | ||
''' | ||||
>>> encodefilename('foo.i/bar.d/bla.hg/hi:world?/HELLO') | ||||
'foo.i.hg/bar.d.hg/bla.hg.hg/hi~3aworld~3f/_h_e_l_l_o' | ||||
''' | ||||
return _encodefname(encodedir(s)) | ||||
def decodefilename(s): | ||||
''' | ||||
>>> decodefilename('foo.i.hg/bar.d.hg/bla.hg.hg/hi~3aworld~3f/_h_e_l_l_o') | ||||
'foo.i/bar.d/bla.hg/hi:world?/HELLO' | ||||
''' | ||||
return decodedir(_decodefname(s)) | ||||
|
r6839 | |||
|
r14288 | def _buildlowerencodefun(): | ||
|
r13949 | ''' | ||
|
r14288 | >>> f = _buildlowerencodefun() | ||
|
r13949 | >>> f('nothing/special.txt') | ||
'nothing/special.txt' | ||||
>>> f('HELLO') | ||||
'hello' | ||||
>>> f('hello:world?') | ||||
'hello~3aworld~3f' | ||||
>>> f('the\x07quick\xADshot') | ||||
'the~07quick~adshot' | ||||
''' | ||||
|
r14288 | winreserved = [ord(x) for x in '\\:*?"<>|'] | ||
|
r10282 | cmap = dict([(chr(x), chr(x)) for x in xrange(127)]) | ||
|
r14288 | for x in (range(32) + range(126, 256) + winreserved): | ||
|
r7229 | cmap[chr(x)] = "~%02x" % x | ||
for x in range(ord("A"), ord("Z")+1): | ||||
cmap[chr(x)] = chr(x).lower() | ||||
return lambda s: "".join([cmap[c] for c in s]) | ||||
|
r14288 | lowerencode = _buildlowerencodefun() | ||
|
r7229 | |||
|
r17570 | # Windows reserved names: con, prn, aux, nul, com1..com9, lpt1..lpt9 | ||
_winres3 = ('aux', 'con', 'prn', 'nul') # length 3 | ||||
_winres4 = ('com', 'lpt') # length 4 (with trailing 1..9) | ||||
|
r12687 | def _auxencode(path, dotencode): | ||
|
r13949 | ''' | ||
Encodes filenames containing names reserved by Windows or which end in | ||||
period or space. Does not touch other single reserved characters c. | ||||
Specifically, c in '\\:*?"<>|' or ord(c) <= 31 are *not* encoded here. | ||||
Additionally encodes space or period at the beginning, if dotencode is | ||||
|
r17569 | True. Parameter path is assumed to be all lowercase. | ||
A segment only needs encoding if a reserved name appears as a | ||||
basename (e.g. "aux", "aux.foo"). A directory or file named "foo.aux" | ||||
doesn't need encoding. | ||||
|
r13949 | |||
|
r17589 | >>> s = '.foo/aux.txt/txt.aux/con/prn/nul/foo.' | ||
>>> _auxencode(s.split('/'), True) | ||||
|
r17574 | ['~2efoo', 'au~78.txt', 'txt.aux', 'co~6e', 'pr~6e', 'nu~6c', 'foo~2e'] | ||
|
r17589 | >>> s = '.com1com2/lpt9.lpt4.lpt1/conprn/com0/lpt0/foo.' | ||
>>> _auxencode(s.split('/'), False) | ||||
|
r17574 | ['.com1com2', 'lp~749.lpt4.lpt1', 'conprn', 'com0', 'lpt0', 'foo~2e'] | ||
|
r17589 | >>> _auxencode(['foo. '], True) | ||
|
r17574 | ['foo.~20'] | ||
|
r17589 | >>> _auxencode([' .foo'], True) | ||
|
r17574 | ['~20.foo'] | ||
|
r13949 | ''' | ||
|
r17589 | for i, n in enumerate(path): | ||
|
r17572 | if not n: | ||
continue | ||||
if dotencode and n[0] in '. ': | ||||
n = "~%02x" % ord(n[0]) + n[1:] | ||||
|
r17589 | path[i] = n | ||
|
r17572 | else: | ||
l = n.find('.') | ||||
if l == -1: | ||||
l = len(n) | ||||
if ((l == 3 and n[:3] in _winres3) or | ||||
(l == 4 and n[3] <= '9' and n[3] >= '1' | ||||
and n[:3] in _winres4)): | ||||
# encode third letter ('aux' -> 'au~78') | ||||
ec = "~%02x" % ord(n[2]) | ||||
n = n[0:2] + ec + n[3:] | ||||
|
r17589 | path[i] = n | ||
|
r17572 | if n[-1] in '. ': | ||
# encode last period or space ('foo...' -> 'foo..~2e') | ||||
|
r17589 | path[i] = n[:-1] + "~%02x" % ord(n[-1]) | ||
return path | ||||
|
r7229 | |||
|
r14288 | _maxstorepathlen = 120 | ||
_dirprefixlen = 8 | ||||
_maxshortdirslen = 8 * (_dirprefixlen + 1) - 4 | ||||
|
r17610 | |||
def _hashencode(path, dotencode): | ||||
digest = _sha(path).hexdigest() | ||||
le = lowerencode(path).split('/')[1:] | ||||
parts = _auxencode(le, dotencode) | ||||
basename = parts[-1] | ||||
_root, ext = os.path.splitext(basename) | ||||
sdirs = [] | ||||
sdirslen = 0 | ||||
for p in parts[:-1]: | ||||
d = p[:_dirprefixlen] | ||||
if d[-1] in '. ': | ||||
# Windows can't access dirs ending in period or space | ||||
d = d[:-1] + '_' | ||||
if sdirslen == 0: | ||||
t = len(d) | ||||
else: | ||||
t = sdirslen + 1 + len(d) | ||||
if t > _maxshortdirslen: | ||||
break | ||||
sdirs.append(d) | ||||
sdirslen = t | ||||
dirs = '/'.join(sdirs) | ||||
if len(dirs) > 0: | ||||
dirs += '/' | ||||
res = 'dh/' + dirs + digest + ext | ||||
spaceleft = _maxstorepathlen - len(res) | ||||
if spaceleft > 0: | ||||
filler = basename[:spaceleft] | ||||
res = 'dh/' + dirs + filler + digest + ext | ||||
return res | ||||
|
r17590 | def _hybridencode(path, dotencode): | ||
|
r7229 | '''encodes path with a length limit | ||
Encodes all paths that begin with 'data/', according to the following. | ||||
Default encoding (reversible): | ||||
Encodes all uppercase letters 'X' as '_x'. All reserved or illegal | ||||
characters are encoded as '~xx', where xx is the two digit hex code | ||||
of the character (see encodefilename). | ||||
Relevant path components consisting of Windows reserved filenames are | ||||
|
r17738 | masked by encoding the third character ('aux' -> 'au~78', see _auxencode). | ||
|
r7229 | |||
Hashed encoding (not reversible): | ||||
|
r14288 | If the default-encoded path is longer than _maxstorepathlen, a | ||
|
r7229 | non-reversible hybrid hashing of the path is done instead. | ||
|
r14288 | This encoding uses up to _dirprefixlen characters of all directory | ||
|
r7229 | levels of the lowerencoded path, but not more levels than can fit into | ||
|
r14288 | _maxshortdirslen. | ||
|
r7229 | Then follows the filler followed by the sha digest of the full path. | ||
The filler is the beginning of the basename of the lowerencoded path | ||||
(the basename is everything after the last path separator). The filler | ||||
is as long as possible, filling in characters from the basename until | ||||
|
r14288 | the encoded path has _maxstorepathlen characters (or all chars of the | ||
basename have been taken). | ||||
|
r7229 | The extension (e.g. '.i' or '.d') is preserved. | ||
The string 'data/' at the beginning is replaced with 'dh/', if the hashed | ||||
encoding was used. | ||||
''' | ||||
|
r17609 | path = encodedir(path) | ||
ef = _encodefname(path).split('/') | ||||
|
r17590 | res = '/'.join(_auxencode(ef, dotencode)) | ||
|
r14288 | if len(res) > _maxstorepathlen: | ||
|
r17610 | res = _hashencode(path, dotencode) | ||
|
r7229 | return res | ||
|
r17624 | def _pathencode(path): | ||
|
r17693 | if len(path) > _maxstorepathlen: | ||
return None | ||||
|
r17624 | ef = _encodefname(encodedir(path)).split('/') | ||
res = '/'.join(_auxencode(ef, True)) | ||||
if len(res) > _maxstorepathlen: | ||||
return None | ||||
return res | ||||
_pathencode = getattr(parsers, 'pathencode', _pathencode) | ||||
def _dothybridencode(f): | ||||
ef = _pathencode(f) | ||||
if ef is None: | ||||
return _hashencode(encodedir(f), True) | ||||
return ef | ||||
|
r17623 | def _plainhybridencode(f): | ||
return _hybridencode(f, False) | ||||
|
r17726 | def _calcmode(vfs): | ||
|
r6898 | try: | ||
# files in .hg/ will be created using this mode | ||||
|
r17726 | mode = vfs.stat().st_mode | ||
|
r6898 | # avoid some useless chmods | ||
|
r7890 | if (0777 & ~util.umask) == (0777 & mode): | ||
|
r6898 | mode = None | ||
except OSError: | ||||
mode = None | ||||
return mode | ||||
|
r17249 | _data = ('data 00manifest.d 00manifest.i 00changelog.d 00changelog.i' | ||
' phaseroots obsstore') | ||||
|
r6903 | |||
|
r8778 | class basicstore(object): | ||
|
r6840 | '''base class for local repository stores''' | ||
|
r17651 | def __init__(self, path, vfstype): | ||
|
r17724 | vfs = vfstype(path) | ||
self.path = vfs.base | ||||
|
r17726 | self.createmode = _calcmode(vfs) | ||
|
r17652 | vfs.createmode = self.createmode | ||
|
r17728 | self.rawvfs = vfs | ||
|
r17653 | self.vfs = scmutil.filtervfs(vfs, encodedir) | ||
self.opener = self.vfs | ||||
|
r6840 | |||
def join(self, f): | ||||
|
r13426 | return self.path + '/' + encodedir(f) | ||
|
r6840 | |||
|
r6899 | def _walk(self, relpath, recurse): | ||
|
r6900 | '''yields (unencoded, encoded, size)''' | ||
|
r13426 | path = self.path | ||
if relpath: | ||||
path += '/' + relpath | ||||
striplen = len(self.path) + 1 | ||||
|
r6899 | l = [] | ||
|
r17728 | if self.rawvfs.isdir(path): | ||
|
r6899 | visit = [path] | ||
|
r17747 | readdir = self.rawvfs.readdir | ||
|
r6899 | while visit: | ||
p = visit.pop() | ||||
|
r17747 | for f, kind, st in readdir(p, stat=True): | ||
|
r13426 | fp = p + '/' + f | ||
|
r6899 | if kind == stat.S_IFREG and f[-2:] in ('.d', '.i'): | ||
|
r6900 | n = util.pconvert(fp[striplen:]) | ||
|
r8531 | l.append((decodedir(n), n, st.st_size)) | ||
|
r6899 | elif kind == stat.S_IFDIR and recurse: | ||
visit.append(fp) | ||||
|
r17054 | l.sort() | ||
return l | ||||
|
r6840 | |||
|
r6900 | def datafiles(self): | ||
|
r6899 | return self._walk('data', True) | ||
|
r6840 | |||
def walk(self): | ||||
|
r6900 | '''yields (unencoded, encoded, size)''' | ||
|
r6840 | # yield data files first | ||
|
r6892 | for x in self.datafiles(): | ||
|
r6840 | yield x | ||
# yield manifest before changelog | ||||
|
r8210 | for x in reversed(self._walk('', False)): | ||
|
r6840 | yield x | ||
|
r6903 | def copylist(self): | ||
return ['requires'] + _data.split() | ||||
|
r13391 | def write(self): | ||
pass | ||||
|
r17744 | def __contains__(self, path): | ||
'''Checks if the store contains path''' | ||||
path = "/".join(("data", path)) | ||||
# file? | ||||
if os.path.exists(self.join(path + ".i")): | ||||
return True | ||||
# dir? | ||||
if not path.endswith("/"): | ||||
path = path + "/" | ||||
return os.path.exists(self.join(path)) | ||||
|
r6898 | class encodedstore(basicstore): | ||
|
r17651 | def __init__(self, path, vfstype): | ||
|
r17724 | vfs = vfstype(path + '/store') | ||
self.path = vfs.base | ||||
|
r17726 | self.createmode = _calcmode(vfs) | ||
|
r17652 | vfs.createmode = self.createmode | ||
|
r17728 | self.rawvfs = vfs | ||
|
r17653 | self.vfs = scmutil.filtervfs(vfs, encodefilename) | ||
self.opener = self.vfs | ||||
|
r6840 | |||
|
r6900 | def datafiles(self): | ||
for a, b, size in self._walk('data', True): | ||||
|
r6892 | try: | ||
|
r6900 | a = decodefilename(a) | ||
|
r6892 | except KeyError: | ||
|
r6900 | a = None | ||
yield a, b, size | ||||
|
r6840 | |||
def join(self, f): | ||||
|
r13426 | return self.path + '/' + encodefilename(f) | ||
|
r6840 | |||
|
r6903 | def copylist(self): | ||
return (['requires', '00changelog.i'] + | ||||
|
r13426 | ['store/' + f for f in _data.split()]) | ||
|
r6903 | |||
|
r8530 | class fncache(object): | ||
|
r8531 | # the filename used to be partially encoded | ||
# hence the encodedir/decodedir dance | ||||
|
r17722 | def __init__(self, vfs): | ||
self.vfs = vfs | ||||
|
r7229 | self.entries = None | ||
|
r13391 | self._dirty = False | ||
|
r7229 | |||
|
r8530 | def _load(self): | ||
'''fill the entries from the fncache file''' | ||||
|
r13391 | self._dirty = False | ||
|
r8530 | try: | ||
|
r17722 | fp = self.vfs('fncache', mode='rb') | ||
|
r8530 | except IOError: | ||
# skip nonexistent file | ||||
|
r16404 | self.entries = set() | ||
|
r8530 | return | ||
|
r17604 | self.entries = set(decodedir(fp.read()).splitlines()) | ||
|
r16404 | if '' in self.entries: | ||
fp.seek(0) | ||||
for n, line in enumerate(fp): | ||||
if not line.rstrip('\n'): | ||||
t = _('invalid entry in fncache, line %s') % (n + 1) | ||||
raise util.Abort(t) | ||||
|
r8530 | fp.close() | ||
|
r7229 | |||
|
r16404 | def _write(self, files, atomictemp): | ||
|
r17722 | fp = self.vfs('fncache', mode='wb', atomictemp=atomictemp) | ||
|
r16404 | if files: | ||
|
r17592 | fp.write(encodedir('\n'.join(files) + '\n')) | ||
|
r16404 | fp.close() | ||
self._dirty = False | ||||
|
r8530 | def rewrite(self, files): | ||
|
r16404 | self._write(files, False) | ||
|
r8530 | self.entries = set(files) | ||
|
r13391 | |||
def write(self): | ||||
|
r16404 | if self._dirty: | ||
self._write(self.entries, True) | ||||
|
r8530 | |||
def add(self, fn): | ||||
if self.entries is None: | ||||
self._load() | ||||
|
r10577 | if fn not in self.entries: | ||
|
r13391 | self._dirty = True | ||
|
r10577 | self.entries.add(fn) | ||
|
r8530 | |||
|
r17782 | def __contains__(self, fn): | ||
|
r8530 | if self.entries is None: | ||
self._load() | ||||
|
r17782 | return fn in self.entries | ||
|
r8530 | |||
def __iter__(self): | ||||
if self.entries is None: | ||||
self._load() | ||||
return iter(self.entries) | ||||
|
r7229 | |||
|
r17845 | class _fncachevfs(scmutil.abstractvfs, scmutil.auditvfs): | ||
|
r17721 | def __init__(self, vfs, fnc, encode): | ||
|
r17845 | scmutil.auditvfs.__init__(self, vfs) | ||
|
r14194 | self.fncache = fnc | ||
self.encode = encode | ||||
def __call__(self, path, mode='r', *args, **kw): | ||||
if mode not in ('r', 'rb') and path.startswith('data/'): | ||||
self.fncache.add(path) | ||||
|
r17722 | return self.vfs(self.encode(path), mode, *args, **kw) | ||
|
r14194 | |||
|
r17725 | def join(self, path): | ||
if path: | ||||
return self.vfs.join(self.encode(path)) | ||||
else: | ||||
return self.vfs.join(path) | ||||
|
r7229 | class fncachestore(basicstore): | ||
|
r17651 | def __init__(self, path, vfstype, dotencode): | ||
|
r17591 | if dotencode: | ||
encode = _dothybridencode | ||||
else: | ||||
encode = _plainhybridencode | ||||
|
r12687 | self.encode = encode | ||
|
r17724 | vfs = vfstype(path + '/store') | ||
self.path = vfs.base | ||||
|
r17562 | self.pathsep = self.path + '/' | ||
|
r17726 | self.createmode = _calcmode(vfs) | ||
|
r17652 | vfs.createmode = self.createmode | ||
|
r17727 | self.rawvfs = vfs | ||
|
r17652 | fnc = fncache(vfs) | ||
|
r9133 | self.fncache = fnc | ||
|
r17653 | self.vfs = _fncachevfs(vfs, fnc, encode) | ||
self.opener = self.vfs | ||||
|
r7229 | |||
def join(self, f): | ||||
|
r17562 | return self.pathsep + self.encode(f) | ||
|
r7229 | |||
|
r17731 | def getsize(self, path): | ||
return self.rawvfs.stat(path).st_size | ||||
|
r7229 | def datafiles(self): | ||
rewrite = False | ||||
existing = [] | ||||
|
r17373 | for f in sorted(self.fncache): | ||
|
r12687 | ef = self.encode(f) | ||
|
r7229 | try: | ||
|
r17731 | yield f, ef, self.getsize(ef) | ||
|
r7229 | existing.append(f) | ||
|
r17374 | except OSError, err: | ||
if err.errno != errno.ENOENT: | ||||
raise | ||||
|
r7229 | # nonexistent entry | ||
rewrite = True | ||||
if rewrite: | ||||
# rewrite fncache to remove nonexistent entries | ||||
# (may be caused by rollback / strip) | ||||
|
r8530 | self.fncache.rewrite(existing) | ||
|
r7229 | |||
def copylist(self): | ||||
|
r17249 | d = ('data dh fncache phaseroots obsstore' | ||
|
r15742 | ' 00manifest.d 00manifest.i 00changelog.d 00changelog.i') | ||
|
r7229 | return (['requires', '00changelog.i'] + | ||
|
r13426 | ['store/' + f for f in d.split()]) | ||
|
r7229 | |||
|
r13391 | def write(self): | ||
self.fncache.write() | ||||
|
r17783 | def _exists(self, f): | ||
ef = self.encode(f) | ||||
try: | ||||
self.getsize(ef) | ||||
return True | ||||
except OSError, err: | ||||
if err.errno != errno.ENOENT: | ||||
raise | ||||
# nonexistent entry | ||||
return False | ||||
|
r17745 | def __contains__(self, path): | ||
'''Checks if the store contains path''' | ||||
path = "/".join(("data", path)) | ||||
|
r17782 | # check for files (exact match) | ||
|
r17784 | e = path + '.i' | ||
if e in self.fncache and self._exists(e): | ||||
|
r17782 | return True | ||
# now check for directories (prefix match) | ||||
if not path.endswith('/'): | ||||
path += '/' | ||||
for e in self.fncache: | ||||
|
r17784 | if e.startswith(path) and self._exists(e): | ||
|
r17782 | return True | ||
return False | ||||
|
r17745 | |||
|
r17651 | def store(requirements, path, vfstype): | ||
|
r6898 | if 'store' in requirements: | ||
|
r7229 | if 'fncache' in requirements: | ||
|
r17651 | return fncachestore(path, vfstype, 'dotencode' in requirements) | ||
return encodedstore(path, vfstype) | ||||
return basicstore(path, vfstype) | ||||