store.py
430 lines
| 13.5 KiB
| text/x-python
|
PythonLexer
/ mercurial / store.py
|
r6839 | # store.py - repository store handling for Mercurial | ||
# | ||||
# Copyright 2008 Matt Mackall <mpm@selenic.com> | ||||
# | ||||
|
r8225 | # This software may be used and distributed according to the terms of the | ||
|
r10263 | # GNU General Public License version 2 or any later version. | ||
|
r6839 | |||
|
r7229 | from i18n import _ | ||
|
r14090 | import osutil, scmutil, util | ||
|
r8312 | import os, stat | ||
|
r6840 | |||
|
r7229 | _sha = util.sha1 | ||
|
r8531 | # This avoids a collision between a file named foo and a dir named | ||
# foo.i or foo.d | ||||
def encodedir(path): | ||||
|
r13949 | ''' | ||
>>> encodedir('data/foo.i') | ||||
'data/foo.i' | ||||
>>> encodedir('data/foo.i/bla.i') | ||||
'data/foo.i.hg/bla.i' | ||||
>>> encodedir('data/foo.i.hg/bla.i') | ||||
'data/foo.i.hg.hg/bla.i' | ||||
''' | ||||
|
r8531 | if not path.startswith('data/'): | ||
return path | ||||
return (path | ||||
.replace(".hg/", ".hg.hg/") | ||||
.replace(".i/", ".i.hg/") | ||||
.replace(".d/", ".d.hg/")) | ||||
def decodedir(path): | ||||
|
r13949 | ''' | ||
>>> decodedir('data/foo.i') | ||||
'data/foo.i' | ||||
>>> decodedir('data/foo.i.hg/bla.i') | ||||
'data/foo.i/bla.i' | ||||
>>> decodedir('data/foo.i.hg.hg/bla.i') | ||||
'data/foo.i.hg/bla.i' | ||||
''' | ||||
|
r11790 | if not path.startswith('data/') or ".hg/" not in path: | ||
|
r8531 | return path | ||
return (path | ||||
.replace(".d.hg/", ".d/") | ||||
.replace(".i.hg/", ".i/") | ||||
.replace(".hg.hg/", ".hg/")) | ||||
|
r6839 | def _buildencodefun(): | ||
|
r13949 | ''' | ||
>>> enc, dec = _buildencodefun() | ||||
>>> enc('nothing/special.txt') | ||||
'nothing/special.txt' | ||||
>>> dec('nothing/special.txt') | ||||
'nothing/special.txt' | ||||
>>> enc('HELLO') | ||||
'_h_e_l_l_o' | ||||
>>> dec('_h_e_l_l_o') | ||||
'HELLO' | ||||
>>> enc('hello:world?') | ||||
'hello~3aworld~3f' | ||||
>>> dec('hello~3aworld~3f') | ||||
'hello:world?' | ||||
>>> enc('the\x07quick\xADshot') | ||||
'the~07quick~adshot' | ||||
>>> dec('the~07quick~adshot') | ||||
'the\\x07quick\\xadshot' | ||||
''' | ||||
|
r6839 | e = '_' | ||
|
r14288 | winreserved = [ord(x) for x in '\\:*?"<>|'] | ||
|
r10282 | cmap = dict([(chr(x), chr(x)) for x in xrange(127)]) | ||
|
r14288 | for x in (range(32) + range(126, 256) + winreserved): | ||
|
r6839 | cmap[chr(x)] = "~%02x" % x | ||
for x in range(ord("A"), ord("Z")+1) + [ord(e)]: | ||||
cmap[chr(x)] = e + chr(x).lower() | ||||
dmap = {} | ||||
for k, v in cmap.iteritems(): | ||||
dmap[v] = k | ||||
def decode(s): | ||||
i = 0 | ||||
while i < len(s): | ||||
for l in xrange(1, 4): | ||||
try: | ||||
|
r10282 | yield dmap[s[i:i + l]] | ||
|
r6839 | i += l | ||
break | ||||
except KeyError: | ||||
pass | ||||
else: | ||||
raise KeyError | ||||
|
r8531 | return (lambda s: "".join([cmap[c] for c in encodedir(s)]), | ||
lambda s: decodedir("".join(list(decode(s))))) | ||||
|
r6839 | |||
encodefilename, decodefilename = _buildencodefun() | ||||
|
r14288 | def _buildlowerencodefun(): | ||
|
r13949 | ''' | ||
|
r14288 | >>> f = _buildlowerencodefun() | ||
|
r13949 | >>> f('nothing/special.txt') | ||
'nothing/special.txt' | ||||
>>> f('HELLO') | ||||
'hello' | ||||
>>> f('hello:world?') | ||||
'hello~3aworld~3f' | ||||
>>> f('the\x07quick\xADshot') | ||||
'the~07quick~adshot' | ||||
''' | ||||
|
r14288 | winreserved = [ord(x) for x in '\\:*?"<>|'] | ||
|
r10282 | cmap = dict([(chr(x), chr(x)) for x in xrange(127)]) | ||
|
r14288 | for x in (range(32) + range(126, 256) + winreserved): | ||
|
r7229 | cmap[chr(x)] = "~%02x" % x | ||
for x in range(ord("A"), ord("Z")+1): | ||||
cmap[chr(x)] = chr(x).lower() | ||||
return lambda s: "".join([cmap[c] for c in s]) | ||||
|
r14288 | lowerencode = _buildlowerencodefun() | ||
|
r7229 | |||
|
r14288 | _winreservednames = '''con prn aux nul | ||
|
r7229 | com1 com2 com3 com4 com5 com6 com7 com8 com9 | ||
lpt1 lpt2 lpt3 lpt4 lpt5 lpt6 lpt7 lpt8 lpt9'''.split() | ||||
|
r12687 | def _auxencode(path, dotencode): | ||
|
r13949 | ''' | ||
Encodes filenames containing names reserved by Windows or which end in | ||||
period or space. Does not touch other single reserved characters c. | ||||
Specifically, c in '\\:*?"<>|' or ord(c) <= 31 are *not* encoded here. | ||||
Additionally encodes space or period at the beginning, if dotencode is | ||||
True. | ||||
path is assumed to be all lowercase. | ||||
>>> _auxencode('.foo/aux.txt/txt.aux/con/prn/nul/foo.', True) | ||||
'~2efoo/au~78.txt/txt.aux/co~6e/pr~6e/nu~6c/foo~2e' | ||||
>>> _auxencode('.com1com2/lpt9.lpt4.lpt1/conprn/foo.', False) | ||||
'.com1com2/lp~749.lpt4.lpt1/conprn/foo~2e' | ||||
>>> _auxencode('foo. ', True) | ||||
'foo.~20' | ||||
>>> _auxencode(' .foo', True) | ||||
'~20.foo' | ||||
''' | ||||
|
r7229 | res = [] | ||
for n in path.split('/'): | ||||
if n: | ||||
base = n.split('.')[0] | ||||
|
r14288 | if base and (base in _winreservednames): | ||
|
r7229 | # encode third letter ('aux' -> 'au~78') | ||
ec = "~%02x" % ord(n[2]) | ||||
n = n[0:2] + ec + n[3:] | ||||
|
r7515 | if n[-1] in '. ': | ||
# encode last period or space ('foo...' -> 'foo..~2e') | ||||
n = n[:-1] + "~%02x" % ord(n[-1]) | ||||
|
r12687 | if dotencode and n[0] in '. ': | ||
n = "~%02x" % ord(n[0]) + n[1:] | ||||
|
r7229 | res.append(n) | ||
return '/'.join(res) | ||||
|
r14288 | _maxstorepathlen = 120 | ||
_dirprefixlen = 8 | ||||
_maxshortdirslen = 8 * (_dirprefixlen + 1) - 4 | ||||
|
r12687 | def _hybridencode(path, auxencode): | ||
|
r7229 | '''encodes path with a length limit | ||
Encodes all paths that begin with 'data/', according to the following. | ||||
Default encoding (reversible): | ||||
Encodes all uppercase letters 'X' as '_x'. All reserved or illegal | ||||
characters are encoded as '~xx', where xx is the two digit hex code | ||||
of the character (see encodefilename). | ||||
Relevant path components consisting of Windows reserved filenames are | ||||
masked by encoding the third character ('aux' -> 'au~78', see auxencode). | ||||
Hashed encoding (not reversible): | ||||
|
r14288 | If the default-encoded path is longer than _maxstorepathlen, a | ||
|
r7229 | non-reversible hybrid hashing of the path is done instead. | ||
|
r14288 | This encoding uses up to _dirprefixlen characters of all directory | ||
|
r7229 | levels of the lowerencoded path, but not more levels than can fit into | ||
|
r14288 | _maxshortdirslen. | ||
|
r7229 | Then follows the filler followed by the sha digest of the full path. | ||
The filler is the beginning of the basename of the lowerencoded path | ||||
(the basename is everything after the last path separator). The filler | ||||
is as long as possible, filling in characters from the basename until | ||||
|
r14288 | the encoded path has _maxstorepathlen characters (or all chars of the | ||
basename have been taken). | ||||
|
r7229 | The extension (e.g. '.i' or '.d') is preserved. | ||
The string 'data/' at the beginning is replaced with 'dh/', if the hashed | ||||
encoding was used. | ||||
''' | ||||
if not path.startswith('data/'): | ||||
return path | ||||
|
r8531 | # escape directories ending with .i and .d | ||
path = encodedir(path) | ||||
|
r7229 | ndpath = path[len('data/'):] | ||
res = 'data/' + auxencode(encodefilename(ndpath)) | ||||
|
r14288 | if len(res) > _maxstorepathlen: | ||
|
r7229 | digest = _sha(path).hexdigest() | ||
aep = auxencode(lowerencode(ndpath)) | ||||
_root, ext = os.path.splitext(aep) | ||||
parts = aep.split('/') | ||||
basename = parts[-1] | ||||
sdirs = [] | ||||
for p in parts[:-1]: | ||||
|
r14288 | d = p[:_dirprefixlen] | ||
|
r7514 | if d[-1] in '. ': | ||
# Windows can't access dirs ending in period or space | ||||
d = d[:-1] + '_' | ||||
|
r7229 | t = '/'.join(sdirs) + '/' + d | ||
|
r14288 | if len(t) > _maxshortdirslen: | ||
|
r7229 | break | ||
sdirs.append(d) | ||||
dirs = '/'.join(sdirs) | ||||
if len(dirs) > 0: | ||||
dirs += '/' | ||||
res = 'dh/' + dirs + digest + ext | ||||
|
r14288 | spaceleft = _maxstorepathlen - len(res) | ||
if spaceleft > 0: | ||||
filler = basename[:spaceleft] | ||||
|
r7229 | res = 'dh/' + dirs + filler + digest + ext | ||
return res | ||||
|
r6898 | def _calcmode(path): | ||
try: | ||||
# files in .hg/ will be created using this mode | ||||
mode = os.stat(path).st_mode | ||||
# avoid some useless chmods | ||||
|
r7890 | if (0777 & ~util.umask) == (0777 & mode): | ||
|
r6898 | mode = None | ||
except OSError: | ||||
mode = None | ||||
return mode | ||||
|
r17249 | _data = ('data 00manifest.d 00manifest.i 00changelog.d 00changelog.i' | ||
' phaseroots obsstore') | ||||
|
r6903 | |||
|
r8778 | class basicstore(object): | ||
|
r6840 | '''base class for local repository stores''' | ||
|
r14092 | def __init__(self, path, openertype): | ||
|
r6840 | self.path = path | ||
|
r6898 | self.createmode = _calcmode(path) | ||
|
r14092 | op = openertype(self.path) | ||
|
r8633 | op.createmode = self.createmode | ||
|
r14090 | self.opener = scmutil.filteropener(op, encodedir) | ||
|
r6840 | |||
def join(self, f): | ||||
|
r13426 | return self.path + '/' + encodedir(f) | ||
|
r6840 | |||
|
r6899 | def _walk(self, relpath, recurse): | ||
|
r6900 | '''yields (unencoded, encoded, size)''' | ||
|
r13426 | path = self.path | ||
if relpath: | ||||
path += '/' + relpath | ||||
striplen = len(self.path) + 1 | ||||
|
r6899 | l = [] | ||
if os.path.isdir(path): | ||||
visit = [path] | ||||
while visit: | ||||
p = visit.pop() | ||||
for f, kind, st in osutil.listdir(p, stat=True): | ||||
|
r13426 | fp = p + '/' + f | ||
|
r6899 | if kind == stat.S_IFREG and f[-2:] in ('.d', '.i'): | ||
|
r6900 | n = util.pconvert(fp[striplen:]) | ||
|
r8531 | l.append((decodedir(n), n, st.st_size)) | ||
|
r6899 | elif kind == stat.S_IFDIR and recurse: | ||
visit.append(fp) | ||||
|
r17054 | l.sort() | ||
return l | ||||
|
r6840 | |||
|
r6900 | def datafiles(self): | ||
|
r6899 | return self._walk('data', True) | ||
|
r6840 | |||
def walk(self): | ||||
|
r6900 | '''yields (unencoded, encoded, size)''' | ||
|
r6840 | # yield data files first | ||
|
r6892 | for x in self.datafiles(): | ||
|
r6840 | yield x | ||
# yield manifest before changelog | ||||
|
r8210 | for x in reversed(self._walk('', False)): | ||
|
r6840 | yield x | ||
|
r6903 | def copylist(self): | ||
return ['requires'] + _data.split() | ||||
|
r13391 | def write(self): | ||
pass | ||||
|
r6898 | class encodedstore(basicstore): | ||
|
r14092 | def __init__(self, path, openertype): | ||
|
r13426 | self.path = path + '/store' | ||
|
r6898 | self.createmode = _calcmode(self.path) | ||
|
r14092 | op = openertype(self.path) | ||
|
r6840 | op.createmode = self.createmode | ||
|
r14090 | self.opener = scmutil.filteropener(op, encodefilename) | ||
|
r6840 | |||
|
r6900 | def datafiles(self): | ||
for a, b, size in self._walk('data', True): | ||||
|
r6892 | try: | ||
|
r6900 | a = decodefilename(a) | ||
|
r6892 | except KeyError: | ||
|
r6900 | a = None | ||
yield a, b, size | ||||
|
r6840 | |||
def join(self, f): | ||||
|
r13426 | return self.path + '/' + encodefilename(f) | ||
|
r6840 | |||
|
r6903 | def copylist(self): | ||
return (['requires', '00changelog.i'] + | ||||
|
r13426 | ['store/' + f for f in _data.split()]) | ||
|
r6903 | |||
|
r8530 | class fncache(object): | ||
|
r8531 | # the filename used to be partially encoded | ||
# hence the encodedir/decodedir dance | ||||
|
r7229 | def __init__(self, opener): | ||
self.opener = opener | ||||
self.entries = None | ||||
|
r13391 | self._dirty = False | ||
|
r7229 | |||
|
r8530 | def _load(self): | ||
'''fill the entries from the fncache file''' | ||||
|
r13391 | self._dirty = False | ||
|
r8530 | try: | ||
fp = self.opener('fncache', mode='rb') | ||||
except IOError: | ||||
# skip nonexistent file | ||||
|
r16404 | self.entries = set() | ||
|
r8530 | return | ||
|
r16404 | self.entries = set(map(decodedir, fp.read().splitlines())) | ||
if '' in self.entries: | ||||
fp.seek(0) | ||||
for n, line in enumerate(fp): | ||||
if not line.rstrip('\n'): | ||||
t = _('invalid entry in fncache, line %s') % (n + 1) | ||||
raise util.Abort(t) | ||||
|
r8530 | fp.close() | ||
|
r7229 | |||
|
r16404 | def _write(self, files, atomictemp): | ||
fp = self.opener('fncache', mode='wb', atomictemp=atomictemp) | ||||
if files: | ||||
fp.write('\n'.join(map(encodedir, files)) + '\n') | ||||
fp.close() | ||||
self._dirty = False | ||||
|
r8530 | def rewrite(self, files): | ||
|
r16404 | self._write(files, False) | ||
|
r8530 | self.entries = set(files) | ||
|
r13391 | |||
def write(self): | ||||
|
r16404 | if self._dirty: | ||
self._write(self.entries, True) | ||||
|
r8530 | |||
def add(self, fn): | ||||
if self.entries is None: | ||||
self._load() | ||||
|
r10577 | if fn not in self.entries: | ||
|
r13391 | self._dirty = True | ||
|
r10577 | self.entries.add(fn) | ||
|
r8530 | |||
def __contains__(self, fn): | ||||
if self.entries is None: | ||||
self._load() | ||||
return fn in self.entries | ||||
def __iter__(self): | ||||
if self.entries is None: | ||||
self._load() | ||||
return iter(self.entries) | ||||
|
r7229 | |||
|
r14194 | class _fncacheopener(scmutil.abstractopener): | ||
def __init__(self, op, fnc, encode): | ||||
self.opener = op | ||||
self.fncache = fnc | ||||
self.encode = encode | ||||
def __call__(self, path, mode='r', *args, **kw): | ||||
if mode not in ('r', 'rb') and path.startswith('data/'): | ||||
self.fncache.add(path) | ||||
return self.opener(self.encode(path), mode, *args, **kw) | ||||
|
r7229 | class fncachestore(basicstore): | ||
|
r14092 | def __init__(self, path, openertype, encode): | ||
|
r12687 | self.encode = encode | ||
|
r13426 | self.path = path + '/store' | ||
|
r7229 | self.createmode = _calcmode(self.path) | ||
|
r14194 | op = openertype(self.path) | ||
|
r9133 | op.createmode = self.createmode | ||
fnc = fncache(op) | ||||
self.fncache = fnc | ||||
|
r14194 | self.opener = _fncacheopener(op, fnc, encode) | ||
|
r7229 | |||
def join(self, f): | ||||
|
r13426 | return self.path + '/' + self.encode(f) | ||
|
r7229 | |||
|
r17154 | def getsize(self, path): | ||
return os.stat(self.path + '/' + path).st_size | ||||
|
r7229 | def datafiles(self): | ||
rewrite = False | ||||
existing = [] | ||||
|
r8530 | for f in self.fncache: | ||
|
r12687 | ef = self.encode(f) | ||
|
r7229 | try: | ||
|
r17154 | yield f, ef, self.getsize(ef) | ||
|
r7229 | existing.append(f) | ||
except OSError: | ||||
# nonexistent entry | ||||
rewrite = True | ||||
if rewrite: | ||||
# rewrite fncache to remove nonexistent entries | ||||
# (may be caused by rollback / strip) | ||||
|
r8530 | self.fncache.rewrite(existing) | ||
|
r7229 | |||
def copylist(self): | ||||
|
r17249 | d = ('data dh fncache phaseroots obsstore' | ||
|
r15742 | ' 00manifest.d 00manifest.i 00changelog.d 00changelog.i') | ||
|
r7229 | return (['requires', '00changelog.i'] + | ||
|
r13426 | ['store/' + f for f in d.split()]) | ||
|
r7229 | |||
|
r13391 | def write(self): | ||
self.fncache.write() | ||||
|
r14092 | def store(requirements, path, openertype): | ||
|
r6898 | if 'store' in requirements: | ||
|
r7229 | if 'fncache' in requirements: | ||
|
r12687 | auxencode = lambda f: _auxencode(f, 'dotencode' in requirements) | ||
encode = lambda f: _hybridencode(f, auxencode) | ||||
|
r14092 | return fncachestore(path, openertype, encode) | ||
return encodedstore(path, openertype) | ||||
return basicstore(path, openertype) | ||||