store.py
730 lines
| 21.4 KiB
| text/x-python
|
PythonLexer
/ mercurial / store.py
|
r6839 | # store.py - repository store handling for Mercurial | ||
# | ||||
# Copyright 2008 Matt Mackall <mpm@selenic.com> | ||||
# | ||||
|
r8225 | # This software may be used and distributed according to the terms of the | ||
|
r10263 | # GNU General Public License version 2 or any later version. | ||
|
r6839 | |||
|
r27480 | from __future__ import absolute_import | ||
import errno | ||||
|
r42144 | import functools | ||
|
r27480 | import os | ||
import stat | ||||
from .i18n import _ | ||||
|
r43359 | from .pycompat import getattr | ||
|
r27480 | from . import ( | ||
|
r43175 | changelog, | ||
|
r27480 | error, | ||
|
r43175 | manifest, | ||
|
r35600 | node, | ||
|
r32372 | policy, | ||
|
r30077 | pycompat, | ||
|
r27480 | util, | ||
|
r31234 | vfs as vfsmod, | ||
|
r27480 | ) | ||
|
r44517 | from .utils import hashutil | ||
|
r6840 | |||
|
r43906 | parsers = policy.importmod('parsers') | ||
|
r42144 | # how much bytes should be read from fncache in one read | ||
# It is done to prevent loading large fncache files into memory | ||||
fncache_chunksize = 10 ** 6 | ||||
|
r32372 | |||
|
r43346 | |||
|
r40529 | def _matchtrackedpath(path, matcher): | ||
"""parses a fncache entry and returns whether the entry is tracking a path | ||||
matched by matcher or not. | ||||
If matcher is None, returns True""" | ||||
if matcher is None: | ||||
return True | ||||
path = decodedir(path) | ||||
|
r43347 | if path.startswith(b'data/'): | ||
return matcher(path[len(b'data/') : -len(b'.i')]) | ||||
elif path.startswith(b'meta/'): | ||||
return matcher.visitdir(path[len(b'meta/') : -len(b'/00manifest.i')]) | ||||
|
r40529 | |||
|
r43347 | raise error.ProgrammingError(b"cannot decode path %s" % path) | ||
|
r40658 | |||
|
r43346 | |||
|
r8531 | # This avoids a collision between a file named foo and a dir named | ||
# foo.i or foo.d | ||||
|
r17607 | def _encodedir(path): | ||
|
r13949 | ''' | ||
|
r34133 | >>> _encodedir(b'data/foo.i') | ||
|
r13949 | 'data/foo.i' | ||
|
r34133 | >>> _encodedir(b'data/foo.i/bla.i') | ||
|
r13949 | 'data/foo.i.hg/bla.i' | ||
|
r34133 | >>> _encodedir(b'data/foo.i.hg/bla.i') | ||
|
r13949 | 'data/foo.i.hg.hg/bla.i' | ||
|
r34133 | >>> _encodedir(b'data/foo.i\\ndata/foo.i/bla.i\\ndata/foo.i.hg/bla.i\\n') | ||
|
r17605 | 'data/foo.i\\ndata/foo.i.hg/bla.i\\ndata/foo.i.hg.hg/bla.i\\n' | ||
|
r13949 | ''' | ||
|
r43346 | return ( | ||
|
r43347 | path.replace(b".hg/", b".hg.hg/") | ||
.replace(b".i/", b".i.hg/") | ||||
.replace(b".d/", b".d.hg/") | ||||
|
r43346 | ) | ||
|
r8531 | |||
|
r17607 | encodedir = getattr(parsers, 'encodedir', _encodedir) | ||
|
r43346 | |||
|
r8531 | def decodedir(path): | ||
|
r13949 | ''' | ||
|
r34133 | >>> decodedir(b'data/foo.i') | ||
|
r13949 | 'data/foo.i' | ||
|
r34133 | >>> decodedir(b'data/foo.i.hg/bla.i') | ||
|
r13949 | 'data/foo.i/bla.i' | ||
|
r34133 | >>> decodedir(b'data/foo.i.hg.hg/bla.i') | ||
|
r13949 | 'data/foo.i.hg/bla.i' | ||
''' | ||||
|
r43347 | if b".hg/" not in path: | ||
|
r8531 | return path | ||
|
r43346 | return ( | ||
|
r43347 | path.replace(b".d.hg/", b".d/") | ||
.replace(b".i.hg/", b".i/") | ||||
.replace(b".hg.hg/", b".hg/") | ||||
|
r43346 | ) | ||
|
r8531 | |||
|
r29071 | def _reserved(): | ||
''' characters that are problematic for filesystems | ||||
* ascii escapes (0..31) | ||||
* ascii hi (126..255) | ||||
* windows specials | ||||
these characters will be escaped by encodefunctions | ||||
''' | ||||
|
r30076 | winreserved = [ord(x) for x in u'\\:*?"<>|'] | ||
|
r29071 | for x in range(32): | ||
yield x | ||||
for x in range(126, 256): | ||||
yield x | ||||
for x in winreserved: | ||||
yield x | ||||
|
r43346 | |||
|
r6839 | def _buildencodefun(): | ||
|
r13949 | ''' | ||
>>> enc, dec = _buildencodefun() | ||||
|
r34133 | >>> enc(b'nothing/special.txt') | ||
|
r13949 | 'nothing/special.txt' | ||
|
r34133 | >>> dec(b'nothing/special.txt') | ||
|
r13949 | 'nothing/special.txt' | ||
|
r34133 | >>> enc(b'HELLO') | ||
|
r13949 | '_h_e_l_l_o' | ||
|
r34133 | >>> dec(b'_h_e_l_l_o') | ||
|
r13949 | 'HELLO' | ||
|
r34133 | >>> enc(b'hello:world?') | ||
|
r13949 | 'hello~3aworld~3f' | ||
|
r34133 | >>> dec(b'hello~3aworld~3f') | ||
|
r13949 | 'hello:world?' | ||
|
r34138 | >>> enc(b'the\\x07quick\\xADshot') | ||
|
r13949 | 'the~07quick~adshot' | ||
|
r34133 | >>> dec(b'the~07quick~adshot') | ||
|
r13949 | 'the\\x07quick\\xadshot' | ||
''' | ||||
|
r43347 | e = b'_' | ||
|
r31253 | xchr = pycompat.bytechr | ||
asciistr = list(map(xchr, range(127))) | ||||
|
r43347 | capitals = list(range(ord(b"A"), ord(b"Z") + 1)) | ||
|
r30077 | |||
|
r30108 | cmap = dict((x, x) for x in asciistr) | ||
|
r29071 | for x in _reserved(): | ||
|
r43347 | cmap[xchr(x)] = b"~%02x" % x | ||
|
r30077 | for x in capitals + [ord(e)]: | ||
cmap[xchr(x)] = e + xchr(x).lower() | ||||
|
r6839 | dmap = {} | ||
|
r43376 | for k, v in pycompat.iteritems(cmap): | ||
|
r6839 | dmap[v] = k | ||
|
r43346 | |||
|
r6839 | def decode(s): | ||
i = 0 | ||||
while i < len(s): | ||||
|
r38806 | for l in pycompat.xrange(1, 4): | ||
|
r6839 | try: | ||
|
r43346 | yield dmap[s[i : i + l]] | ||
|
r6839 | i += l | ||
break | ||||
except KeyError: | ||||
pass | ||||
else: | ||||
raise KeyError | ||||
|
r43346 | |||
return ( | ||||
|
r43347 | lambda s: b''.join( | ||
|
r43346 | [cmap[s[c : c + 1]] for c in pycompat.xrange(len(s))] | ||
), | ||||
|
r43347 | lambda s: b''.join(list(decode(s))), | ||
|
r43346 | ) | ||
|
r17608 | |||
_encodefname, _decodefname = _buildencodefun() | ||||
|
r6839 | |||
|
r43346 | |||
|
r17608 | def encodefilename(s): | ||
''' | ||||
|
r34133 | >>> encodefilename(b'foo.i/bar.d/bla.hg/hi:world?/HELLO') | ||
|
r17608 | 'foo.i.hg/bar.d.hg/bla.hg.hg/hi~3aworld~3f/_h_e_l_l_o' | ||
''' | ||||
return _encodefname(encodedir(s)) | ||||
|
r43346 | |||
|
r17608 | def decodefilename(s): | ||
''' | ||||
|
r34133 | >>> decodefilename(b'foo.i.hg/bar.d.hg/bla.hg.hg/hi~3aworld~3f/_h_e_l_l_o') | ||
|
r17608 | 'foo.i/bar.d/bla.hg/hi:world?/HELLO' | ||
''' | ||||
return decodedir(_decodefname(s)) | ||||
|
r6839 | |||
|
r43346 | |||
|
r14288 | def _buildlowerencodefun(): | ||
|
r13949 | ''' | ||
|
r14288 | >>> f = _buildlowerencodefun() | ||
|
r34133 | >>> f(b'nothing/special.txt') | ||
|
r13949 | 'nothing/special.txt' | ||
|
r34133 | >>> f(b'HELLO') | ||
|
r13949 | 'hello' | ||
|
r34133 | >>> f(b'hello:world?') | ||
|
r13949 | 'hello~3aworld~3f' | ||
|
r34138 | >>> f(b'the\\x07quick\\xADshot') | ||
|
r13949 | 'the~07quick~adshot' | ||
''' | ||||
|
r34211 | xchr = pycompat.bytechr | ||
|
r38806 | cmap = dict([(xchr(x), xchr(x)) for x in pycompat.xrange(127)]) | ||
|
r29071 | for x in _reserved(): | ||
|
r43347 | cmap[xchr(x)] = b"~%02x" % x | ||
for x in range(ord(b"A"), ord(b"Z") + 1): | ||||
|
r34211 | cmap[xchr(x)] = xchr(x).lower() | ||
|
r43346 | |||
|
r34210 | def lowerencode(s): | ||
|
r43347 | return b"".join([cmap[c] for c in pycompat.iterbytestr(s)]) | ||
|
r43346 | |||
|
r34210 | return lowerencode | ||
|
r7229 | |||
|
r43346 | |||
|
r18430 | lowerencode = getattr(parsers, 'lowerencode', None) or _buildlowerencodefun() | ||
|
r7229 | |||
|
r17570 | # Windows reserved names: con, prn, aux, nul, com1..com9, lpt1..lpt9 | ||
|
r43347 | _winres3 = (b'aux', b'con', b'prn', b'nul') # length 3 | ||
_winres4 = (b'com', b'lpt') # length 4 (with trailing 1..9) | ||||
|
r43346 | |||
|
r12687 | def _auxencode(path, dotencode): | ||
|
r13949 | ''' | ||
Encodes filenames containing names reserved by Windows or which end in | ||||
period or space. Does not touch other single reserved characters c. | ||||
Specifically, c in '\\:*?"<>|' or ord(c) <= 31 are *not* encoded here. | ||||
Additionally encodes space or period at the beginning, if dotencode is | ||||
|
r17569 | True. Parameter path is assumed to be all lowercase. | ||
A segment only needs encoding if a reserved name appears as a | ||||
basename (e.g. "aux", "aux.foo"). A directory or file named "foo.aux" | ||||
doesn't need encoding. | ||||
|
r13949 | |||
|
r34133 | >>> s = b'.foo/aux.txt/txt.aux/con/prn/nul/foo.' | ||
>>> _auxencode(s.split(b'/'), True) | ||||
|
r17574 | ['~2efoo', 'au~78.txt', 'txt.aux', 'co~6e', 'pr~6e', 'nu~6c', 'foo~2e'] | ||
|
r34133 | >>> s = b'.com1com2/lpt9.lpt4.lpt1/conprn/com0/lpt0/foo.' | ||
>>> _auxencode(s.split(b'/'), False) | ||||
|
r17574 | ['.com1com2', 'lp~749.lpt4.lpt1', 'conprn', 'com0', 'lpt0', 'foo~2e'] | ||
|
r34133 | >>> _auxencode([b'foo. '], True) | ||
|
r17574 | ['foo.~20'] | ||
|
r34133 | >>> _auxencode([b' .foo'], True) | ||
|
r17574 | ['~20.foo'] | ||
|
r13949 | ''' | ||
|
r17589 | for i, n in enumerate(path): | ||
|
r17572 | if not n: | ||
continue | ||||
|
r43347 | if dotencode and n[0] in b'. ': | ||
n = b"~%02x" % ord(n[0:1]) + n[1:] | ||||
|
r17589 | path[i] = n | ||
|
r17572 | else: | ||
|
r43347 | l = n.find(b'.') | ||
|
r17572 | if l == -1: | ||
l = len(n) | ||||
|
r43346 | if (l == 3 and n[:3] in _winres3) or ( | ||
|
r43347 | l == 4 | ||
and n[3:4] <= b'9' | ||||
and n[3:4] >= b'1' | ||||
and n[:3] in _winres4 | ||||
|
r43346 | ): | ||
|
r17572 | # encode third letter ('aux' -> 'au~78') | ||
|
r43347 | ec = b"~%02x" % ord(n[2:3]) | ||
|
r17572 | n = n[0:2] + ec + n[3:] | ||
|
r17589 | path[i] = n | ||
|
r43347 | if n[-1] in b'. ': | ||
|
r17572 | # encode last period or space ('foo...' -> 'foo..~2e') | ||
|
r43347 | path[i] = n[:-1] + b"~%02x" % ord(n[-1:]) | ||
|
r17589 | return path | ||
|
r7229 | |||
|
r43346 | |||
|
r14288 | _maxstorepathlen = 120 | ||
_dirprefixlen = 8 | ||||
_maxshortdirslen = 8 * (_dirprefixlen + 1) - 4 | ||||
|
r17610 | |||
|
r43346 | |||
|
r17610 | def _hashencode(path, dotencode): | ||
|
r44517 | digest = node.hex(hashutil.sha1(path).digest()) | ||
|
r43347 | le = lowerencode(path[5:]).split(b'/') # skips prefix 'data/' or 'meta/' | ||
|
r17610 | parts = _auxencode(le, dotencode) | ||
basename = parts[-1] | ||||
_root, ext = os.path.splitext(basename) | ||||
sdirs = [] | ||||
sdirslen = 0 | ||||
for p in parts[:-1]: | ||||
d = p[:_dirprefixlen] | ||||
|
r43347 | if d[-1] in b'. ': | ||
|
r17610 | # Windows can't access dirs ending in period or space | ||
|
r43347 | d = d[:-1] + b'_' | ||
|
r17610 | if sdirslen == 0: | ||
t = len(d) | ||||
else: | ||||
t = sdirslen + 1 + len(d) | ||||
if t > _maxshortdirslen: | ||||
break | ||||
sdirs.append(d) | ||||
sdirslen = t | ||||
|
r43347 | dirs = b'/'.join(sdirs) | ||
|
r17610 | if len(dirs) > 0: | ||
|
r43347 | dirs += b'/' | ||
res = b'dh/' + dirs + digest + ext | ||||
|
r17610 | spaceleft = _maxstorepathlen - len(res) | ||
if spaceleft > 0: | ||||
filler = basename[:spaceleft] | ||||
|
r43347 | res = b'dh/' + dirs + filler + digest + ext | ||
|
r17610 | return res | ||
|
r43346 | |||
|
r17590 | def _hybridencode(path, dotencode): | ||
|
r7229 | '''encodes path with a length limit | ||
Encodes all paths that begin with 'data/', according to the following. | ||||
Default encoding (reversible): | ||||
Encodes all uppercase letters 'X' as '_x'. All reserved or illegal | ||||
characters are encoded as '~xx', where xx is the two digit hex code | ||||
of the character (see encodefilename). | ||||
Relevant path components consisting of Windows reserved filenames are | ||||
|
r17738 | masked by encoding the third character ('aux' -> 'au~78', see _auxencode). | ||
|
r7229 | |||
Hashed encoding (not reversible): | ||||
|
r14288 | If the default-encoded path is longer than _maxstorepathlen, a | ||
|
r7229 | non-reversible hybrid hashing of the path is done instead. | ||
|
r14288 | This encoding uses up to _dirprefixlen characters of all directory | ||
|
r7229 | levels of the lowerencoded path, but not more levels than can fit into | ||
|
r14288 | _maxshortdirslen. | ||
|
r7229 | Then follows the filler followed by the sha digest of the full path. | ||
The filler is the beginning of the basename of the lowerencoded path | ||||
(the basename is everything after the last path separator). The filler | ||||
is as long as possible, filling in characters from the basename until | ||||
|
r14288 | the encoded path has _maxstorepathlen characters (or all chars of the | ||
basename have been taken). | ||||
|
r7229 | The extension (e.g. '.i' or '.d') is preserved. | ||
The string 'data/' at the beginning is replaced with 'dh/', if the hashed | ||||
encoding was used. | ||||
''' | ||||
|
r17609 | path = encodedir(path) | ||
|
r43347 | ef = _encodefname(path).split(b'/') | ||
res = b'/'.join(_auxencode(ef, dotencode)) | ||||
|
r14288 | if len(res) > _maxstorepathlen: | ||
|
r17610 | res = _hashencode(path, dotencode) | ||
|
r7229 | return res | ||
|
r43346 | |||
|
r17624 | def _pathencode(path): | ||
|
r18435 | de = encodedir(path) | ||
|
r17693 | if len(path) > _maxstorepathlen: | ||
|
r18435 | return _hashencode(de, True) | ||
|
r43347 | ef = _encodefname(de).split(b'/') | ||
res = b'/'.join(_auxencode(ef, True)) | ||||
|
r17624 | if len(res) > _maxstorepathlen: | ||
|
r18435 | return _hashencode(de, True) | ||
|
r17624 | return res | ||
|
r43346 | |||
|
r17624 | _pathencode = getattr(parsers, 'pathencode', _pathencode) | ||
|
r43346 | |||
|
r17623 | def _plainhybridencode(f): | ||
return _hybridencode(f, False) | ||||
|
r43346 | |||
|
r17726 | def _calcmode(vfs): | ||
|
r6898 | try: | ||
# files in .hg/ will be created using this mode | ||||
|
r17726 | mode = vfs.stat().st_mode | ||
|
r43346 | # avoid some useless chmods | ||
|
r25658 | if (0o777 & ~util.umask) == (0o777 & mode): | ||
|
r6898 | mode = None | ||
except OSError: | ||||
mode = None | ||||
return mode | ||||
|
r43346 | |||
_data = ( | ||||
|
r43347 | b'bookmarks narrowspec data meta 00manifest.d 00manifest.i' | ||
b' 00changelog.d 00changelog.i phaseroots obsstore' | ||||
|
r43346 | ) | ||
|
r6903 | |||
|
r37427 | def isrevlog(f, kind, st): | ||
|
r43347 | return kind == stat.S_IFREG and f[-2:] in (b'.i', b'.d') | ||
|
r37427 | |||
|
r43346 | |||
|
r8778 | class basicstore(object): | ||
|
r6840 | '''base class for local repository stores''' | ||
|
r43346 | |||
|
r17651 | def __init__(self, path, vfstype): | ||
|
r17724 | vfs = vfstype(path) | ||
self.path = vfs.base | ||||
|
r17726 | self.createmode = _calcmode(vfs) | ||
|
r17652 | vfs.createmode = self.createmode | ||
|
r17728 | self.rawvfs = vfs | ||
|
r31234 | self.vfs = vfsmod.filtervfs(vfs, encodedir) | ||
|
r17653 | self.opener = self.vfs | ||
|
r6840 | |||
def join(self, f): | ||||
|
r43347 | return self.path + b'/' + encodedir(f) | ||
|
r6840 | |||
|
r37427 | def _walk(self, relpath, recurse, filefilter=isrevlog): | ||
|
r6900 | '''yields (unencoded, encoded, size)''' | ||
|
r13426 | path = self.path | ||
if relpath: | ||||
|
r43347 | path += b'/' + relpath | ||
|
r13426 | striplen = len(self.path) + 1 | ||
|
r6899 | l = [] | ||
|
r17728 | if self.rawvfs.isdir(path): | ||
|
r6899 | visit = [path] | ||
|
r17747 | readdir = self.rawvfs.readdir | ||
|
r6899 | while visit: | ||
p = visit.pop() | ||||
|
r17747 | for f, kind, st in readdir(p, stat=True): | ||
|
r43347 | fp = p + b'/' + f | ||
|
r37427 | if filefilter(f, kind, st): | ||
|
r6900 | n = util.pconvert(fp[striplen:]) | ||
|
r8531 | l.append((decodedir(n), n, st.st_size)) | ||
|
r6899 | elif kind == stat.S_IFDIR and recurse: | ||
visit.append(fp) | ||||
|
r17054 | l.sort() | ||
return l | ||||
|
r6840 | |||
|
r43175 | def changelog(self, trypending): | ||
return changelog.changelog(self.vfs, trypending=trypending) | ||||
def manifestlog(self, repo, storenarrowmatch): | ||||
rootstore = manifest.manifestrevlog(self.vfs) | ||||
|
r43346 | return manifest.manifestlog(self.vfs, repo, rootstore, storenarrowmatch) | ||
|
r43175 | |||
|
r40376 | def datafiles(self, matcher=None): | ||
|
r43347 | return self._walk(b'data', True) + self._walk(b'meta', True) | ||
|
r6840 | |||
|
r19177 | def topfiles(self): | ||
# yield manifest before changelog | ||||
|
r43347 | return reversed(self._walk(b'', False)) | ||
|
r19177 | |||
|
r40376 | def walk(self, matcher=None): | ||
'''yields (unencoded, encoded, size) | ||||
if a matcher is passed, storage files of only those tracked paths | ||||
are passed with matches the matcher | ||||
''' | ||||
|
r6840 | # yield data files first | ||
|
r40376 | for x in self.datafiles(matcher): | ||
|
r6840 | yield x | ||
|
r19177 | for x in self.topfiles(): | ||
|
r6840 | yield x | ||
|
r6903 | def copylist(self): | ||
|
r43347 | return [b'requires'] + _data.split() | ||
|
r6903 | |||
|
r20883 | def write(self, tr): | ||
|
r13391 | pass | ||
|
r20884 | def invalidatecaches(self): | ||
pass | ||||
|
r20885 | def markremoved(self, fn): | ||
pass | ||||
|
r17744 | def __contains__(self, path): | ||
'''Checks if the store contains path''' | ||||
|
r43347 | path = b"/".join((b"data", path)) | ||
|
r17744 | # file? | ||
|
r43347 | if self.vfs.exists(path + b".i"): | ||
|
r17744 | return True | ||
# dir? | ||||
|
r43347 | if not path.endswith(b"/"): | ||
path = path + b"/" | ||||
|
r19903 | return self.vfs.exists(path) | ||
|
r17744 | |||
|
r43346 | |||
|
r6898 | class encodedstore(basicstore): | ||
|
r17651 | def __init__(self, path, vfstype): | ||
|
r43347 | vfs = vfstype(path + b'/store') | ||
|
r17724 | self.path = vfs.base | ||
|
r17726 | self.createmode = _calcmode(vfs) | ||
|
r17652 | vfs.createmode = self.createmode | ||
|
r17728 | self.rawvfs = vfs | ||
|
r31234 | self.vfs = vfsmod.filtervfs(vfs, encodefilename) | ||
|
r17653 | self.opener = self.vfs | ||
|
r6840 | |||
|
r40376 | def datafiles(self, matcher=None): | ||
|
r28007 | for a, b, size in super(encodedstore, self).datafiles(): | ||
|
r6892 | try: | ||
|
r6900 | a = decodefilename(a) | ||
|
r6892 | except KeyError: | ||
|
r6900 | a = None | ||
|
r40620 | if a is not None and not _matchtrackedpath(a, matcher): | ||
continue | ||||
|
r6900 | yield a, b, size | ||
|
r6840 | |||
def join(self, f): | ||||
|
r43347 | return self.path + b'/' + encodefilename(f) | ||
|
r6840 | |||
|
r6903 | def copylist(self): | ||
|
r43347 | return [b'requires', b'00changelog.i'] + [ | ||
b'store/' + f for f in _data.split() | ||||
|
r43346 | ] | ||
|
r6903 | |||
|
r8530 | class fncache(object): | ||
|
r8531 | # the filename used to be partially encoded | ||
# hence the encodedir/decodedir dance | ||||
|
r17722 | def __init__(self, vfs): | ||
self.vfs = vfs | ||||
|
r7229 | self.entries = None | ||
|
r13391 | self._dirty = False | ||
|
r40767 | # set of new additions to fncache | ||
self.addls = set() | ||||
|
r7229 | |||
|
r42960 | def ensureloaded(self, warn=None): | ||
'''read the fncache file if not already read. | ||||
If the file on disk is corrupted, raise. If warn is provided, | ||||
warn and keep going instead.''' | ||||
if self.entries is None: | ||||
self._load(warn) | ||||
def _load(self, warn=None): | ||||
|
r8530 | '''fill the entries from the fncache file''' | ||
|
r13391 | self._dirty = False | ||
|
r8530 | try: | ||
|
r43347 | fp = self.vfs(b'fncache', mode=b'rb') | ||
|
r8530 | except IOError: | ||
# skip nonexistent file | ||||
|
r16404 | self.entries = set() | ||
|
r8530 | return | ||
|
r42144 | |||
self.entries = set() | ||||
chunk = b'' | ||||
for c in iter(functools.partial(fp.read, fncache_chunksize), b''): | ||||
chunk += c | ||||
try: | ||||
p = chunk.rindex(b'\n') | ||||
|
r43346 | self.entries.update(decodedir(chunk[: p + 1]).splitlines()) | ||
chunk = chunk[p + 1 :] | ||||
|
r42144 | except ValueError: | ||
# substring '\n' not found, maybe the entry is bigger than the | ||||
# chunksize, so let's keep iterating | ||||
pass | ||||
|
r42147 | if chunk: | ||
|
r43347 | msg = _(b"fncache does not ends with a newline") | ||
|
r42960 | if warn: | ||
|
r43347 | warn(msg + b'\n') | ||
|
r42960 | else: | ||
|
r43346 | raise error.Abort( | ||
msg, | ||||
hint=_( | ||||
|
r43347 | b"use 'hg debugrebuildfncache' to " | ||
b"rebuild the fncache" | ||||
|
r43346 | ), | ||
) | ||||
|
r42960 | self._checkentries(fp, warn) | ||
|
r42139 | fp.close() | ||
|
r42960 | def _checkentries(self, fp, warn): | ||
|
r42139 | """ make sure there is no empty string in entries """ | ||
|
r43347 | if b'' in self.entries: | ||
|
r16404 | fp.seek(0) | ||
|
r30398 | for n, line in enumerate(util.iterfile(fp)): | ||
|
r43347 | if not line.rstrip(b'\n'): | ||
t = _(b'invalid entry in fncache, line %d') % (n + 1) | ||||
|
r42960 | if warn: | ||
|
r43347 | warn(t + b'\n') | ||
|
r42960 | else: | ||
raise error.Abort(t) | ||||
|
r7229 | |||
|
r20883 | def write(self, tr): | ||
|
r16404 | if self._dirty: | ||
|
r38718 | assert self.entries is not None | ||
|
r40779 | self.entries = self.entries | self.addls | ||
self.addls = set() | ||||
|
r43347 | tr.addbackup(b'fncache') | ||
fp = self.vfs(b'fncache', mode=b'wb', atomictemp=True) | ||||
|
r20879 | if self.entries: | ||
|
r43347 | fp.write(encodedir(b'\n'.join(self.entries) + b'\n')) | ||
|
r20879 | fp.close() | ||
self._dirty = False | ||||
|
r40767 | if self.addls: | ||
# if we have just new entries, let's append them to the fncache | ||||
|
r43347 | tr.addbackup(b'fncache') | ||
fp = self.vfs(b'fncache', mode=b'ab', atomictemp=True) | ||||
|
r40767 | if self.addls: | ||
|
r43347 | fp.write(encodedir(b'\n'.join(self.addls) + b'\n')) | ||
|
r40767 | fp.close() | ||
self.entries = None | ||||
self.addls = set() | ||||
|
r8530 | |||
def add(self, fn): | ||||
if self.entries is None: | ||||
self._load() | ||||
|
r10577 | if fn not in self.entries: | ||
|
r40767 | self.addls.add(fn) | ||
|
r8530 | |||
|
r20885 | def remove(self, fn): | ||
if self.entries is None: | ||||
self._load() | ||||
|
r40767 | if fn in self.addls: | ||
self.addls.remove(fn) | ||||
return | ||||
|
r20885 | try: | ||
self.entries.remove(fn) | ||||
self._dirty = True | ||||
except KeyError: | ||||
pass | ||||
|
r17782 | def __contains__(self, fn): | ||
|
r40767 | if fn in self.addls: | ||
return True | ||||
|
r8530 | if self.entries is None: | ||
self._load() | ||||
|
r17782 | return fn in self.entries | ||
|
r8530 | |||
def __iter__(self): | ||||
if self.entries is None: | ||||
self._load() | ||||
|
r40767 | return iter(self.entries | self.addls) | ||
|
r7229 | |||
|
r43346 | |||
|
r41125 | class _fncachevfs(vfsmod.proxyvfs): | ||
|
r17721 | def __init__(self, vfs, fnc, encode): | ||
|
r33412 | vfsmod.proxyvfs.__init__(self, vfs) | ||
|
r14194 | self.fncache = fnc | ||
self.encode = encode | ||||
|
r43347 | def __call__(self, path, mode=b'r', *args, **kw): | ||
|
r38683 | encoded = self.encode(path) | ||
|
r43347 | if mode not in (b'r', b'rb') and ( | ||
path.startswith(b'data/') or path.startswith(b'meta/') | ||||
|
r43346 | ): | ||
|
r38683 | # do not trigger a fncache load when adding a file that already is | ||
# known to exist. | ||||
notload = self.fncache.entries is None and self.vfs.exists(encoded) | ||||
|
r43347 | if notload and b'a' in mode and not self.vfs.stat(encoded).st_size: | ||
|
r38683 | # when appending to an existing file, if the file has size zero, | ||
# it should be considered as missing. Such zero-size files are | ||||
# the result of truncation when a transaction is aborted. | ||||
notload = False | ||||
if not notload: | ||||
self.fncache.add(path) | ||||
return self.vfs(encoded, mode, *args, **kw) | ||||
|
r14194 | |||
|
r17725 | def join(self, path): | ||
if path: | ||||
return self.vfs.join(self.encode(path)) | ||||
else: | ||||
return self.vfs.join(path) | ||||
|
r43346 | |||
|
r7229 | class fncachestore(basicstore): | ||
|
r17651 | def __init__(self, path, vfstype, dotencode): | ||
|
r17591 | if dotencode: | ||
|
r18435 | encode = _pathencode | ||
|
r17591 | else: | ||
encode = _plainhybridencode | ||||
|
r12687 | self.encode = encode | ||
|
r43347 | vfs = vfstype(path + b'/store') | ||
|
r17724 | self.path = vfs.base | ||
|
r43347 | self.pathsep = self.path + b'/' | ||
|
r17726 | self.createmode = _calcmode(vfs) | ||
|
r17652 | vfs.createmode = self.createmode | ||
|
r17727 | self.rawvfs = vfs | ||
|
r17652 | fnc = fncache(vfs) | ||
|
r9133 | self.fncache = fnc | ||
|
r17653 | self.vfs = _fncachevfs(vfs, fnc, encode) | ||
self.opener = self.vfs | ||||
|
r7229 | |||
def join(self, f): | ||||
|
r17562 | return self.pathsep + self.encode(f) | ||
|
r7229 | |||
|
r17731 | def getsize(self, path): | ||
return self.rawvfs.stat(path).st_size | ||||
|
r40376 | def datafiles(self, matcher=None): | ||
|
r17373 | for f in sorted(self.fncache): | ||
|
r40529 | if not _matchtrackedpath(f, matcher): | ||
continue | ||||
|
r12687 | ef = self.encode(f) | ||
|
r7229 | try: | ||
|
r17731 | yield f, ef, self.getsize(ef) | ||
|
r25660 | except OSError as err: | ||
|
r17374 | if err.errno != errno.ENOENT: | ||
raise | ||||
|
r7229 | |||
def copylist(self): | ||||
|
r43346 | d = ( | ||
|
r43347 | b'bookmarks narrowspec data meta dh fncache phaseroots obsstore' | ||
b' 00manifest.d 00manifest.i 00changelog.d 00changelog.i' | ||||
|
r43346 | ) | ||
|
r43347 | return [b'requires', b'00changelog.i'] + [ | ||
b'store/' + f for f in d.split() | ||||
] | ||||
|
r7229 | |||
|
r20883 | def write(self, tr): | ||
self.fncache.write(tr) | ||||
|
r13391 | |||
|
r20884 | def invalidatecaches(self): | ||
self.fncache.entries = None | ||||
|
r40767 | self.fncache.addls = set() | ||
|
r20884 | |||
|
r20885 | def markremoved(self, fn): | ||
self.fncache.remove(fn) | ||||
|
r17783 | def _exists(self, f): | ||
ef = self.encode(f) | ||||
try: | ||||
self.getsize(ef) | ||||
return True | ||||
|
r25660 | except OSError as err: | ||
|
r17783 | if err.errno != errno.ENOENT: | ||
raise | ||||
# nonexistent entry | ||||
return False | ||||
|
r17745 | def __contains__(self, path): | ||
'''Checks if the store contains path''' | ||||
|
r43347 | path = b"/".join((b"data", path)) | ||
|
r17782 | # check for files (exact match) | ||
|
r43347 | e = path + b'.i' | ||
|
r17784 | if e in self.fncache and self._exists(e): | ||
|
r17782 | return True | ||
# now check for directories (prefix match) | ||||
|
r43347 | if not path.endswith(b'/'): | ||
path += b'/' | ||||
|
r17782 | for e in self.fncache: | ||
|
r17784 | if e.startswith(path) and self._exists(e): | ||
|
r17782 | return True | ||
return False | ||||