branchmap.py
519 lines
| 19.8 KiB
| text/x-python
|
PythonLexer
/ mercurial / branchmap.py
|
r18116 | # branchmap.py - logic to computes, maintain and stores branchmap for local repo | ||
# | ||||
# Copyright 2005-2007 Matt Mackall <mpm@selenic.com> | ||||
# | ||||
# This software may be used and distributed according to the terms of the | ||||
# GNU General Public License version 2 or any later version. | ||||
|
r18117 | |||
|
r25918 | from __future__ import absolute_import | ||
import struct | ||||
from .node import ( | ||||
bin, | ||||
hex, | ||||
nullid, | ||||
nullrev, | ||||
) | ||||
from . import ( | ||||
encoding, | ||||
|
r26587 | error, | ||
|
r25918 | scmutil, | ||
|
r30975 | util, | ||
|
r25918 | ) | ||
calcsize = struct.calcsize | ||||
|
r31370 | pack_into = struct.pack_into | ||
unpack_from = struct.unpack_from | ||||
|
r18117 | |||
|
r18185 | def _filename(repo): | ||
|
r18187 | """name of a branchcache file for a given repo or repoview""" | ||
|
r20185 | filename = "cache/branch2" | ||
|
r18187 | if repo.filtername: | ||
filename = '%s-%s' % (filename, repo.filtername) | ||||
return filename | ||||
|
r18185 | |||
|
r18118 | def read(repo): | ||
try: | ||||
|
r23877 | f = repo.vfs(_filename(repo)) | ||
|
r18118 | lines = f.read().split('\n') | ||
f.close() | ||||
except (IOError, OSError): | ||||
|
r18212 | return None | ||
|
r18118 | |||
try: | ||||
|
r18184 | cachekey = lines.pop(0).split(" ", 2) | ||
last, lrev = cachekey[:2] | ||||
|
r18118 | last, lrev = bin(last), int(lrev) | ||
|
r18184 | filteredhash = None | ||
if len(cachekey) > 2: | ||||
filteredhash = bin(cachekey[2]) | ||||
partial = branchcache(tipnode=last, tiprev=lrev, | ||||
filteredhash=filteredhash) | ||||
|
r18132 | if not partial.validfor(repo): | ||
|
r18118 | # invalidate the cache | ||
|
r18166 | raise ValueError('tip differs') | ||
|
r28364 | cl = repo.changelog | ||
|
r18118 | for l in lines: | ||
if not l: | ||||
continue | ||||
|
r20185 | node, state, label = l.split(" ", 2) | ||
if state not in 'oc': | ||||
raise ValueError('invalid branch state') | ||||
|
r18118 | label = encoding.tolocal(label.strip()) | ||
|
r20185 | node = bin(node) | ||
|
r28364 | if not cl.hasnode(node): | ||
raise ValueError('node %s does not exist' % hex(node)) | ||||
|
r20185 | partial.setdefault(label, []).append(node) | ||
if state == 'c': | ||||
partial._closednodes.add(node) | ||||
|
r18118 | except KeyboardInterrupt: | ||
raise | ||||
|
r25660 | except Exception as inst: | ||
|
r18118 | if repo.ui.debugflag: | ||
|
r18188 | msg = 'invalid branchheads cache' | ||
if repo.filtername is not None: | ||||
msg += ' (%s)' % repo.filtername | ||||
msg += ': %s\n' | ||||
|
r21789 | repo.ui.debug(msg % inst) | ||
|
r18212 | partial = None | ||
|
r18126 | return partial | ||
|
r18118 | |||
|
r20032 | ### Nearest subset relation | ||
# Nearest subset of filter X is a filter Y so that: | ||||
# * Y is included in X, | ||||
# * X - Y is as small as possible. | ||||
# This create and ordering used for branchmap purpose. | ||||
# the ordering may be partial | ||||
subsettable = {None: 'visible', | ||||
'visible': 'served', | ||||
'served': 'immutable', | ||||
'immutable': 'base'} | ||||
|
r18121 | def updatecache(repo): | ||
cl = repo.changelog | ||||
|
r18189 | filtername = repo.filtername | ||
partial = repo._branchcaches.get(filtername) | ||||
|
r18121 | |||
|
r18234 | revs = [] | ||
|
r18132 | if partial is None or not partial.validfor(repo): | ||
|
r18126 | partial = read(repo) | ||
|
r18212 | if partial is None: | ||
|
r20032 | subsetname = subsettable.get(filtername) | ||
|
r18234 | if subsetname is None: | ||
partial = branchcache() | ||||
else: | ||||
subset = repo.filtered(subsetname) | ||||
partial = subset.branchmap().copy() | ||||
extrarevs = subset.changelog.filteredrevs - cl.filteredrevs | ||||
revs.extend(r for r in extrarevs if r <= partial.tiprev) | ||||
revs.extend(cl.revs(start=partial.tiprev + 1)) | ||||
|
r18218 | if revs: | ||
|
r18305 | partial.update(repo, revs) | ||
|
r18128 | partial.write(repo) | ||
|
r24373 | |||
|
r18451 | assert partial.validfor(repo), filtername | ||
|
r18189 | repo._branchcaches[repo.filtername] = partial | ||
|
r18124 | |||
|
r26460 | def replacecache(repo, bm): | ||
"""Replace the branchmap cache for a repo with a branch mapping. | ||||
This is likely only called during clone with a branch map from a remote. | ||||
""" | ||||
rbheads = [] | ||||
closed = [] | ||||
for bheads in bm.itervalues(): | ||||
rbheads.extend(bheads) | ||||
for h in bheads: | ||||
r = repo.changelog.rev(h) | ||||
b, c = repo.changelog.branchinfo(r) | ||||
if c: | ||||
closed.append(h) | ||||
if rbheads: | ||||
rtiprev = max((int(repo.changelog.rev(node)) | ||||
for node in rbheads)) | ||||
cache = branchcache(bm, | ||||
repo[rtiprev].node(), | ||||
rtiprev, | ||||
closednodes=closed) | ||||
# Try to stick it as low as possible | ||||
# filter above served are unlikely to be fetch from a clone | ||||
for candidate in ('base', 'immutable', 'served'): | ||||
rview = repo.filtered(candidate) | ||||
if cache.validfor(rview): | ||||
repo._branchcaches[candidate] = cache | ||||
cache.write(rview) | ||||
break | ||||
|
r18124 | class branchcache(dict): | ||
|
r20181 | """A dict like object that hold branches heads cache. | ||
This cache is used to avoid costly computations to determine all the | ||||
branch heads of a repo. | ||||
The cache is serialized on disk in the following format: | ||||
<tip hex node> <tip rev number> [optional filtered repo hex hash] | ||||
|
r20185 | <branch head hex node> <open/closed state> <branch name> | ||
<branch head hex node> <open/closed state> <branch name> | ||||
|
r20181 | ... | ||
The first line is used to check if the cache is still valid. If the | ||||
branch cache is for a filtered repo view, an optional third hash is | ||||
included that hashes the hashes of all filtered revisions. | ||||
|
r20185 | |||
The open/closed state is represented by a single letter 'o' or 'c'. | ||||
This field can be used to avoid changelog reads when determining if a | ||||
branch head closes a branch or not. | ||||
|
r20181 | """ | ||
|
r18124 | |||
|
r18168 | def __init__(self, entries=(), tipnode=nullid, tiprev=nullrev, | ||
|
r20185 | filteredhash=None, closednodes=None): | ||
|
r18125 | super(branchcache, self).__init__(entries) | ||
self.tipnode = tipnode | ||||
|
r18126 | self.tiprev = tiprev | ||
|
r18168 | self.filteredhash = filteredhash | ||
|
r20185 | # closednodes is a set of nodes that close their branch. If the branch | ||
# cache has been updated, it may contain nodes that are no longer | ||||
# heads. | ||||
if closednodes is None: | ||||
self._closednodes = set() | ||||
else: | ||||
self._closednodes = closednodes | ||||
|
r18168 | |||
|
r18132 | def validfor(self, repo): | ||
|
r18644 | """Is the cache content valid regarding a repo | ||
|
r18132 | |||
|
r18644 | - False when cached tipnode is unknown or if we detect a strip. | ||
|
r18132 | - True when cache is up to date or a subset of current repo.""" | ||
try: | ||||
|
r18168 | return ((self.tipnode == repo.changelog.node(self.tiprev)) | ||
|
r24723 | and (self.filteredhash == \ | ||
scmutil.filteredhash(repo, self.tiprev))) | ||||
|
r18132 | except IndexError: | ||
return False | ||||
|
r20186 | def _branchtip(self, heads): | ||
|
r20245 | '''Return tuple with last open head in heads and false, | ||
otherwise return last closed head and true.''' | ||||
|
r20186 | tip = heads[-1] | ||
closed = True | ||||
for h in reversed(heads): | ||||
if h not in self._closednodes: | ||||
tip = h | ||||
closed = False | ||||
break | ||||
return tip, closed | ||||
def branchtip(self, branch): | ||||
|
r20245 | '''Return the tipmost open head on branch head, otherwise return the | ||
tipmost closed head on branch. | ||||
Raise KeyError for unknown branch.''' | ||||
|
r20186 | return self._branchtip(self[branch])[0] | ||
|
r20188 | def branchheads(self, branch, closed=False): | ||
heads = self[branch] | ||||
if not closed: | ||||
heads = [h for h in heads if h not in self._closednodes] | ||||
return heads | ||||
|
r20190 | def iterbranches(self): | ||
for bn, heads in self.iteritems(): | ||||
yield (bn, heads) + self._branchtip(heads) | ||||
|
r18232 | def copy(self): | ||
"""return an deep copy of the branchcache object""" | ||||
|
r20185 | return branchcache(self, self.tipnode, self.tiprev, self.filteredhash, | ||
self._closednodes) | ||||
|
r18132 | |||
|
r18128 | def write(self, repo): | ||
try: | ||||
|
r23877 | f = repo.vfs(_filename(repo), "w", atomictemp=True) | ||
|
r31348 | cachekey = [hex(self.tipnode), '%d' % self.tiprev] | ||
|
r18184 | if self.filteredhash is not None: | ||
cachekey.append(hex(self.filteredhash)) | ||||
f.write(" ".join(cachekey) + '\n') | ||||
|
r21031 | nodecount = 0 | ||
|
r18357 | for label, nodes in sorted(self.iteritems()): | ||
|
r18128 | for node in nodes: | ||
|
r21031 | nodecount += 1 | ||
|
r20185 | if node in self._closednodes: | ||
state = 'c' | ||||
else: | ||||
state = 'o' | ||||
f.write("%s %s %s\n" % (hex(node), state, | ||||
encoding.fromlocal(label))) | ||||
|
r18128 | f.close() | ||
|
r21031 | repo.ui.log('branchcache', | ||
'wrote %s branch cache with %d labels and %d nodes\n', | ||||
repo.filtername, len(self), nodecount) | ||||
|
r26587 | except (IOError, OSError, error.Abort) as inst: | ||
|
r21788 | repo.ui.debug("couldn't write branch cache: %s\n" % inst) | ||
|
r18214 | # Abort may be raise by read only opener | ||
|
r18128 | pass | ||
|
r18131 | |||
|
r18305 | def update(self, repo, revgen): | ||
|
r18131 | """Given a branchhead cache, self, that may have extra nodes or be | ||
|
r20263 | missing heads, and a generator of nodes that are strictly a superset of | ||
|
r18131 | heads missing, this function updates self to be correct. | ||
""" | ||||
|
r30975 | starttime = util.timer() | ||
|
r18131 | cl = repo.changelog | ||
# collect new branch entries | ||||
newbranches = {} | ||||
|
r24373 | getbranchinfo = repo.revbranchcache().branchinfo | ||
|
r18307 | for r in revgen: | ||
|
r24374 | branch, closesbranch = getbranchinfo(r) | ||
|
r20262 | newbranches.setdefault(branch, []).append(r) | ||
|
r20185 | if closesbranch: | ||
|
r20262 | self._closednodes.add(cl.node(r)) | ||
|
r22357 | |||
# fetch current topological heads to speed up filtering | ||||
topoheads = set(cl.headrevs()) | ||||
|
r18131 | # if older branchheads are reachable from new ones, they aren't | ||
# really branchheads. Note checking parents is insufficient: | ||||
# 1 (branch a) -> 2 (branch b) -> 3 (branch a) | ||||
|
r20262 | for branch, newheadrevs in newbranches.iteritems(): | ||
|
r18131 | bheads = self.setdefault(branch, []) | ||
|
r20264 | bheadset = set(cl.rev(node) for node in bheads) | ||
|
r18131 | |||
|
r20263 | # This have been tested True on all internal usage of this function. | ||
# run it again in case of doubt | ||||
# assert not (set(bheadrevs) & set(newheadrevs)) | ||||
newheadrevs.sort() | ||||
|
r20264 | bheadset.update(newheadrevs) | ||
|
r18131 | |||
|
r22356 | # This prunes out two kinds of heads - heads that are superseded by | ||
# a head in newheadrevs, and newheadrevs that are not heads because | ||||
# an existing head is their descendant. | ||||
|
r22357 | uncertain = bheadset - topoheads | ||
if uncertain: | ||||
floorrev = min(uncertain) | ||||
ancestors = set(cl.ancestors(newheadrevs, floorrev)) | ||||
bheadset -= ancestors | ||||
|
r20264 | bheadrevs = sorted(bheadset) | ||
|
r18131 | self[branch] = [cl.node(rev) for rev in bheadrevs] | ||
|
r20263 | tiprev = bheadrevs[-1] | ||
|
r18131 | if tiprev > self.tiprev: | ||
self.tipnode = cl.node(tiprev) | ||||
self.tiprev = tiprev | ||||
|
r19838 | if not self.validfor(repo): | ||
|
r18131 | # cache key are not valid anymore | ||
self.tipnode = nullid | ||||
self.tiprev = nullrev | ||||
for heads in self.values(): | ||||
tiprev = max(cl.rev(node) for node in heads) | ||||
if tiprev > self.tiprev: | ||||
self.tipnode = cl.node(tiprev) | ||||
self.tiprev = tiprev | ||||
|
r24723 | self.filteredhash = scmutil.filteredhash(repo, self.tiprev) | ||
|
r21031 | |||
|
r30975 | duration = util.timer() - starttime | ||
|
r21031 | repo.ui.log('branchcache', 'updated %s branch cache in %.4f seconds\n', | ||
repo.filtername, duration) | ||||
|
r23785 | |||
# Revision branch info cache | ||||
_rbcversion = '-v1' | ||||
_rbcnames = 'cache/rbc-names' + _rbcversion | ||||
_rbcrevs = 'cache/rbc-revs' + _rbcversion | ||||
# [4 byte hash prefix][4 byte branch name number with sign bit indicating open] | ||||
_rbcrecfmt = '>4sI' | ||||
_rbcrecsize = calcsize(_rbcrecfmt) | ||||
_rbcnodelen = 4 | ||||
_rbcbranchidxmask = 0x7fffffff | ||||
_rbccloseflag = 0x80000000 | ||||
class revbranchcache(object): | ||||
"""Persistent cache, mapping from revision number to branch name and close. | ||||
This is a low level cache, independent of filtering. | ||||
Branch names are stored in rbc-names in internal encoding separated by 0. | ||||
rbc-names is append-only, and each branch name is only stored once and will | ||||
thus have a unique index. | ||||
The branch info for each revision is stored in rbc-revs as constant size | ||||
records. The whole file is read into memory, but it is only 'parsed' on | ||||
demand. The file is usually append-only but will be truncated if repo | ||||
modification is detected. | ||||
The record for each revision contains the first 4 bytes of the | ||||
corresponding node hash, and the record is only used if it still matches. | ||||
Even a completely trashed rbc-revs fill thus still give the right result | ||||
while converging towards full recovery ... assuming no incorrectly matching | ||||
node hashes. | ||||
The record also contains 4 bytes where 31 bits contains the index of the | ||||
branch and the last bit indicate that it is a branch close commit. | ||||
The usage pattern for rbc-revs is thus somewhat similar to 00changelog.i | ||||
and will grow with it but be 1/8th of its size. | ||||
""" | ||||
|
r24159 | def __init__(self, repo, readonly=True): | ||
|
r23785 | assert repo.filtername is None | ||
|
r24374 | self._repo = repo | ||
|
r23785 | self._names = [] # branch names in local encoding with static index | ||
|
r31346 | self._rbcrevs = bytearray() | ||
|
r29615 | self._rbcsnameslen = 0 # length of names read at _rbcsnameslen | ||
|
r23785 | try: | ||
bndata = repo.vfs.read(_rbcnames) | ||||
self._rbcsnameslen = len(bndata) # for verification before writing | ||||
|
r31371 | if bndata: | ||
self._names = [encoding.tolocal(bn) | ||||
for bn in bndata.split('\0')] | ||||
|
r29423 | except (IOError, OSError): | ||
|
r24159 | if readonly: | ||
# don't try to use cache - fall back to the slow path | ||||
self.branchinfo = self._branchinfo | ||||
|
r23785 | if self._names: | ||
try: | ||||
data = repo.vfs.read(_rbcrevs) | ||||
|
r31346 | self._rbcrevs[:] = data | ||
|
r25660 | except (IOError, OSError) as inst: | ||
|
r23785 | repo.ui.debug("couldn't read revision branch cache: %s\n" % | ||
inst) | ||||
# remember number of good records on disk | ||||
self._rbcrevslen = min(len(self._rbcrevs) // _rbcrecsize, | ||||
len(repo.changelog)) | ||||
if self._rbcrevslen == 0: | ||||
self._names = [] | ||||
|
r29615 | self._rbcnamescount = len(self._names) # number of names read at | ||
# _rbcsnameslen | ||||
|
r23785 | self._namesreverse = dict((b, r) for r, b in enumerate(self._names)) | ||
|
r28558 | def _clear(self): | ||
self._rbcsnameslen = 0 | ||||
del self._names[:] | ||||
self._rbcnamescount = 0 | ||||
self._namesreverse.clear() | ||||
self._rbcrevslen = len(self._repo.changelog) | ||||
|
r31346 | self._rbcrevs = bytearray(self._rbcrevslen * _rbcrecsize) | ||
|
r28558 | |||
|
r24374 | def branchinfo(self, rev): | ||
|
r23785 | """Return branch name and close flag for rev, using and updating | ||
persistent cache.""" | ||||
|
r24374 | changelog = self._repo.changelog | ||
|
r23785 | rbcrevidx = rev * _rbcrecsize | ||
|
r25266 | # avoid negative index, changelog.read(nullrev) is fast without cache | ||
if rev == nullrev: | ||||
return changelog.branchinfo(rev) | ||||
|
r29604 | # if requested rev isn't allocated, grow and cache the rev info | ||
|
r23785 | if len(self._rbcrevs) < rbcrevidx + _rbcrecsize: | ||
|
r29604 | return self._branchinfo(rev) | ||
|
r23785 | |||
# fast path: extract data from cache, use it if node is matching | ||||
reponode = changelog.node(rev)[:_rbcnodelen] | ||||
|
r31381 | cachenode, branchidx = unpack_from( | ||
_rbcrecfmt, util.buffer(self._rbcrevs), rbcrevidx) | ||||
|
r23785 | close = bool(branchidx & _rbccloseflag) | ||
if close: | ||||
branchidx &= _rbcbranchidxmask | ||||
|
r24376 | if cachenode == '\0\0\0\0': | ||
pass | ||||
elif cachenode == reponode: | ||||
|
r29615 | try: | ||
|
r28558 | return self._names[branchidx], close | ||
|
r29615 | except IndexError: | ||
# recover from invalid reference to unknown branch | ||||
self._repo.ui.debug("referenced branch names not found" | ||||
" - rebuilding revision branch cache from scratch\n") | ||||
self._clear() | ||||
|
r24376 | else: | ||
# rev/node map has changed, invalidate the cache from here up | ||||
|
r29615 | self._repo.ui.debug("history modification detected - truncating " | ||
"revision branch cache to revision %s\n" % rev) | ||||
|
r24376 | truncate = rbcrevidx + _rbcrecsize | ||
del self._rbcrevs[truncate:] | ||||
self._rbcrevslen = min(self._rbcrevslen, truncate) | ||||
|
r23785 | # fall back to slow path and make sure it will be written to disk | ||
|
r24374 | return self._branchinfo(rev) | ||
|
r23785 | |||
|
r24374 | def _branchinfo(self, rev): | ||
|
r23785 | """Retrieve branch info from changelog and update _rbcrevs""" | ||
|
r24374 | changelog = self._repo.changelog | ||
|
r23785 | b, close = changelog.branchinfo(rev) | ||
if b in self._namesreverse: | ||||
branchidx = self._namesreverse[b] | ||||
else: | ||||
branchidx = len(self._names) | ||||
self._names.append(b) | ||||
self._namesreverse[b] = branchidx | ||||
reponode = changelog.node(rev) | ||||
if close: | ||||
branchidx |= _rbccloseflag | ||||
|
r24375 | self._setcachedata(rev, reponode, branchidx) | ||
return b, close | ||||
def _setcachedata(self, rev, node, branchidx): | ||||
"""Writes the node's branch data to the in-memory cache data.""" | ||||
|
r23785 | rbcrevidx = rev * _rbcrecsize | ||
|
r29604 | if len(self._rbcrevs) < rbcrevidx + _rbcrecsize: | ||
self._rbcrevs.extend('\0' * | ||||
(len(self._repo.changelog) * _rbcrecsize - | ||||
len(self._rbcrevs))) | ||||
|
r31370 | pack_into(_rbcrecfmt, self._rbcrevs, rbcrevidx, node, branchidx) | ||
|
r24376 | self._rbcrevslen = min(self._rbcrevslen, rev) | ||
|
r23785 | |||
|
r24377 | tr = self._repo.currenttransaction() | ||
if tr: | ||||
tr.addfinalize('write-revbranchcache', self.write) | ||||
def write(self, tr=None): | ||||
|
r23785 | """Save branch cache if it is dirty.""" | ||
|
r24374 | repo = self._repo | ||
|
r29744 | wlock = None | ||
|
r29745 | step = '' | ||
|
r29744 | try: | ||
|
r29743 | if self._rbcnamescount < len(self._names): | ||
|
r29745 | step = ' names' | ||
|
r29744 | wlock = repo.wlock(wait=False) | ||
|
r29746 | if self._rbcnamescount != 0: | ||
f = repo.vfs.open(_rbcnames, 'ab') | ||||
if f.tell() == self._rbcsnameslen: | ||||
f.write('\0') | ||||
else: | ||||
f.close() | ||||
repo.ui.debug("%s changed - rewriting it\n" % _rbcnames) | ||||
self._rbcnamescount = 0 | ||||
self._rbcrevslen = 0 | ||||
if self._rbcnamescount == 0: | ||||
# before rewriting names, make sure references are removed | ||||
repo.vfs.unlinkpath(_rbcrevs, ignoremissing=True) | ||||
f = repo.vfs.open(_rbcnames, 'wb') | ||||
f.write('\0'.join(encoding.fromlocal(b) | ||||
for b in self._names[self._rbcnamescount:])) | ||||
self._rbcsnameslen = f.tell() | ||||
f.close() | ||||
|
r29743 | self._rbcnamescount = len(self._names) | ||
|
r23785 | |||
|
r29743 | start = self._rbcrevslen * _rbcrecsize | ||
if start != len(self._rbcrevs): | ||||
|
r29745 | step = '' | ||
|
r29744 | if wlock is None: | ||
wlock = repo.wlock(wait=False) | ||||
|
r29743 | revs = min(len(repo.changelog), | ||
len(self._rbcrevs) // _rbcrecsize) | ||||
|
r29746 | f = repo.vfs.open(_rbcrevs, 'ab') | ||
if f.tell() != start: | ||||
repo.ui.debug("truncating %s to %s\n" % (_rbcrevs, start)) | ||||
f.seek(start) | ||||
|
r28557 | if f.tell() != start: | ||
|
r29746 | start = 0 | ||
|
r28557 | f.seek(start) | ||
|
r29746 | f.truncate() | ||
end = revs * _rbcrecsize | ||||
f.write(self._rbcrevs[start:end]) | ||||
f.close() | ||||
|
r29743 | self._rbcrevslen = revs | ||
|
r29745 | except (IOError, OSError, error.Abort, error.LockError) as inst: | ||
repo.ui.debug("couldn't write revision branch cache%s: %s\n" | ||||
% (step, inst)) | ||||
|
r29744 | finally: | ||
if wlock is not None: | ||||
wlock.release() | ||||