branchmap.py
873 lines
| 31.1 KiB
| text/x-python
|
PythonLexer
/ mercurial / branchmap.py
|
r18116 | # branchmap.py - logic to computes, maintain and stores branchmap for local repo | ||
# | ||||
|
r47575 | # Copyright 2005-2007 Olivia Mackall <olivia@selenic.com> | ||
|
r18116 | # | ||
# This software may be used and distributed according to the terms of the | ||||
# GNU General Public License version 2 or any later version. | ||||
|
r18117 | |||
|
r25918 | from __future__ import absolute_import | ||
import struct | ||||
from .node import ( | ||||
bin, | ||||
hex, | ||||
nullrev, | ||||
) | ||||
from . import ( | ||||
encoding, | ||||
|
r26587 | error, | ||
r49536 | obsolete, | |||
|
r35849 | pycompat, | ||
|
r25918 | scmutil, | ||
|
r30975 | util, | ||
|
r25918 | ) | ||
|
r37102 | from .utils import ( | ||
r42314 | repoviewutil, | |||
|
r37102 | stringutil, | ||
) | ||||
|
r25918 | |||
|
r44212 | if pycompat.TYPE_CHECKING: | ||
|
r44035 | from typing import ( | ||
Any, | ||||
Callable, | ||||
Dict, | ||||
Iterable, | ||||
List, | ||||
Optional, | ||||
Set, | ||||
Tuple, | ||||
Union, | ||||
) | ||||
|
r47552 | from . import localrepo | ||
|
r44035 | |||
assert any( | ||||
|
r46554 | ( | ||
Any, | ||||
Callable, | ||||
Dict, | ||||
Iterable, | ||||
List, | ||||
Optional, | ||||
Set, | ||||
Tuple, | ||||
Union, | ||||
|
r47552 | localrepo, | ||
|
r46554 | ) | ||
|
r44035 | ) | ||
|
r43346 | subsettable = repoviewutil.subsettable | ||
r42314 | ||||
|
r25918 | calcsize = struct.calcsize | ||
|
r31370 | pack_into = struct.pack_into | ||
unpack_from = struct.unpack_from | ||||
|
r18117 | |||
|
r18118 | |||
|
r41764 | class BranchMapCache(object): | ||
|
r41867 | """mapping of filtered views of repo with their branchcache""" | ||
|
r43346 | |||
|
r41764 | def __init__(self): | ||
self._per_filter = {} | ||||
|
r41708 | |||
|
r41764 | def __getitem__(self, repo): | ||
self.updatecache(repo) | ||||
return self._per_filter[repo.filtername] | ||||
def updatecache(self, repo): | ||||
"""Update the cache for the given filtered view on a repository""" | ||||
# This can trigger updates for the caches for subsets of the filtered | ||||
# view, e.g. when there is no cache for this filtered view or the cache | ||||
# is stale. | ||||
|
r18121 | |||
|
r41764 | cl = repo.changelog | ||
filtername = repo.filtername | ||||
bcache = self._per_filter.get(filtername) | ||||
if bcache is None or not bcache.validfor(repo): | ||||
# cache object missing or cache object stale? Read from disk | ||||
bcache = branchcache.fromfile(repo) | ||||
|
r41708 | |||
|
r41764 | revs = [] | ||
if bcache is None: | ||||
# no (fresh) cache available anymore, perhaps we can re-use | ||||
# the cache for a subset, then extend that to add info on missing | ||||
# revisions. | ||||
subsetname = subsettable.get(filtername) | ||||
if subsetname is not None: | ||||
subset = repo.filtered(subsetname) | ||||
bcache = self[subset].copy() | ||||
extrarevs = subset.changelog.filteredrevs - cl.filteredrevs | ||||
revs.extend(r for r in extrarevs if r <= bcache.tiprev) | ||||
else: | ||||
# nothing to fall back on, start empty. | ||||
|
r47538 | bcache = branchcache(repo) | ||
|
r24373 | |||
|
r41764 | revs.extend(cl.revs(start=bcache.tiprev + 1)) | ||
if revs: | ||||
bcache.update(repo, revs) | ||||
|
r18124 | |||
|
r41764 | assert bcache.validfor(repo), filtername | ||
self._per_filter[repo.filtername] = bcache | ||||
def replace(self, repo, remotebranchmap): | ||||
"""Replace the branchmap cache for a repo with a branch mapping. | ||||
This is likely only called during clone with a branch map from a | ||||
remote. | ||||
|
r26460 | |||
|
r41764 | """ | ||
cl = repo.changelog | ||||
clrev = cl.rev | ||||
clbranchinfo = cl.branchinfo | ||||
rbheads = [] | ||||
|
r44086 | closed = set() | ||
|
r43374 | for bheads in pycompat.itervalues(remotebranchmap): | ||
|
r41764 | rbheads += bheads | ||
for h in bheads: | ||||
r = clrev(h) | ||||
b, c = clbranchinfo(r) | ||||
if c: | ||||
|
r44086 | closed.add(h) | ||
|
r26460 | |||
|
r41764 | if rbheads: | ||
rtiprev = max((int(clrev(node)) for node in rbheads)) | ||||
cache = branchcache( | ||||
|
r47538 | repo, | ||
|
r43346 | remotebranchmap, | ||
repo[rtiprev].node(), | ||||
rtiprev, | ||||
|
r44086 | closednodes=closed, | ||
|
r43346 | ) | ||
|
r26460 | |||
|
r41764 | # Try to stick it as low as possible | ||
# filter above served are unlikely to be fetch from a clone | ||||
|
r43347 | for candidate in (b'base', b'immutable', b'served'): | ||
|
r41764 | rview = repo.filtered(candidate) | ||
if cache.validfor(rview): | ||||
self._per_filter[candidate] = cache | ||||
cache.write(rview) | ||||
return | ||||
def clear(self): | ||||
self._per_filter.clear() | ||||
r49526 | def write_delayed(self, repo): | |||
unfi = repo.unfiltered() | ||||
for filtername, cache in self._per_filter.items(): | ||||
if cache._delayed: | ||||
repo = unfi.filtered(filtername) | ||||
cache.write(repo) | ||||
|
r43346 | |||
|
r42289 | def _unknownnode(node): | ||
|
r46554 | """raises ValueError when branchcache found a node which does not exists""" | ||
|
r43906 | raise ValueError('node %s does not exist' % pycompat.sysstr(hex(node))) | ||
|
r26460 | |||
|
r43346 | |||
|
r42805 | def _branchcachedesc(repo): | ||
if repo.filtername is not None: | ||||
|
r43347 | return b'branch cache (%s)' % repo.filtername | ||
|
r42805 | else: | ||
|
r43347 | return b'branch cache' | ||
|
r42805 | |||
|
r43346 | |||
|
r42168 | class branchcache(object): | ||
|
r20181 | """A dict like object that hold branches heads cache. | ||
This cache is used to avoid costly computations to determine all the | ||||
branch heads of a repo. | ||||
The cache is serialized on disk in the following format: | ||||
<tip hex node> <tip rev number> [optional filtered repo hex hash] | ||||
|
r20185 | <branch head hex node> <open/closed state> <branch name> | ||
<branch head hex node> <open/closed state> <branch name> | ||||
|
r20181 | ... | ||
The first line is used to check if the cache is still valid. If the | ||||
branch cache is for a filtered repo view, an optional third hash is | ||||
r49536 | included that hashes the hashes of all filtered and obsolete revisions. | |||
|
r20185 | |||
The open/closed state is represented by a single letter 'o' or 'c'. | ||||
This field can be used to avoid changelog reads when determining if a | ||||
branch head closes a branch or not. | ||||
|
r20181 | """ | ||
|
r41826 | |||
|
r43346 | def __init__( | ||
self, | ||||
|
r47538 | repo, | ||
|
r43346 | entries=(), | ||
|
r47771 | tipnode=None, | ||
|
r43346 | tiprev=nullrev, | ||
filteredhash=None, | ||||
closednodes=None, | ||||
hasnode=None, | ||||
): | ||||
|
r47552 | # type: (localrepo.localrepository, Union[Dict[bytes, List[bytes]], Iterable[Tuple[bytes, List[bytes]]]], bytes, int, Optional[bytes], Optional[Set[bytes]], Optional[Callable[[bytes], bool]]) -> None | ||
|
r46554 | """hasnode is a function which can be used to verify whether changelog | ||
|
r42174 | has a given node or not. If it's not provided, we assume that every node | ||
|
r46554 | we have exists in changelog""" | ||
|
r47538 | self._repo = repo | ||
r49526 | self._delayed = False | |||
|
r47771 | if tipnode is None: | ||
self.tipnode = repo.nullid | ||||
else: | ||||
self.tipnode = tipnode | ||||
|
r41826 | self.tiprev = tiprev | ||
self.filteredhash = filteredhash | ||||
# closednodes is a set of nodes that close their branch. If the branch | ||||
# cache has been updated, it may contain nodes that are no longer | ||||
# heads. | ||||
if closednodes is None: | ||||
self._closednodes = set() | ||||
else: | ||||
|
r44035 | self._closednodes = closednodes | ||
|
r42172 | self._entries = dict(entries) | ||
|
r42173 | # whether closed nodes are verified or not | ||
self._closedverified = False | ||||
# branches for which nodes are verified | ||||
self._verifiedbranches = set() | ||||
|
r42174 | self._hasnode = hasnode | ||
if self._hasnode is None: | ||||
self._hasnode = lambda x: True | ||||
|
r42168 | |||
|
r42289 | def _verifyclosed(self): | ||
|
r47856 | """verify the closed nodes we have""" | ||
|
r42289 | if self._closedverified: | ||
return | ||||
for node in self._closednodes: | ||||
if not self._hasnode(node): | ||||
_unknownnode(node) | ||||
self._closedverified = True | ||||
def _verifybranch(self, branch): | ||||
|
r47856 | """verify head nodes for the given branch.""" | ||
|
r42289 | if branch not in self._entries or branch in self._verifiedbranches: | ||
return | ||||
for n in self._entries[branch]: | ||||
if not self._hasnode(n): | ||||
_unknownnode(n) | ||||
self._verifiedbranches.add(branch) | ||||
def _verifyall(self): | ||||
|
r47856 | """verifies nodes of all the branches""" | ||
|
r42302 | needverification = set(self._entries.keys()) - self._verifiedbranches | ||
for b in needverification: | ||||
|
r42289 | self._verifybranch(b) | ||
|
r42168 | def __iter__(self): | ||
|
r42172 | return iter(self._entries) | ||
|
r42168 | |||
def __setitem__(self, key, value): | ||||
|
r42172 | self._entries[key] = value | ||
|
r42168 | |||
def __getitem__(self, key): | ||||
|
r42290 | self._verifybranch(key) | ||
|
r42172 | return self._entries[key] | ||
|
r42168 | |||
|
r42282 | def __contains__(self, key): | ||
|
r42290 | self._verifybranch(key) | ||
|
r42282 | return key in self._entries | ||
|
r42168 | def iteritems(self): | ||
|
r43376 | for k, v in pycompat.iteritems(self._entries): | ||
|
r42303 | self._verifybranch(k) | ||
yield k, v | ||||
|
r42168 | |||
|
r42809 | items = iteritems | ||
|
r42171 | def hasbranch(self, label): | ||
|
r47856 | """checks whether a branch of this name exists or not""" | ||
|
r42290 | self._verifybranch(label) | ||
|
r42172 | return label in self._entries | ||
|
r42171 | |||
|
r41706 | @classmethod | ||
def fromfile(cls, repo): | ||||
f = None | ||||
try: | ||||
f = repo.cachevfs(cls._filename(repo)) | ||||
lineiter = iter(f) | ||||
|
r43347 | cachekey = next(lineiter).rstrip(b'\n').split(b" ", 2) | ||
|
r41706 | last, lrev = cachekey[:2] | ||
last, lrev = bin(last), int(lrev) | ||||
filteredhash = None | ||||
|
r42174 | hasnode = repo.changelog.hasnode | ||
|
r41706 | if len(cachekey) > 2: | ||
filteredhash = bin(cachekey[2]) | ||||
|
r43346 | bcache = cls( | ||
|
r47538 | repo, | ||
|
r43346 | tipnode=last, | ||
tiprev=lrev, | ||||
filteredhash=filteredhash, | ||||
hasnode=hasnode, | ||||
) | ||||
|
r41706 | if not bcache.validfor(repo): | ||
# invalidate the cache | ||||
|
r43906 | raise ValueError('tip differs') | ||
|
r41974 | bcache.load(repo, lineiter) | ||
|
r41706 | except (IOError, OSError): | ||
return None | ||||
except Exception as inst: | ||||
if repo.ui.debugflag: | ||||
|
r43347 | msg = b'invalid %s: %s\n' | ||
|
r43346 | repo.ui.debug( | ||
|
r43805 | msg | ||
% ( | ||||
_branchcachedesc(repo), | ||||
|
r47512 | stringutil.forcebytestr(inst), | ||
|
r43805 | ) | ||
|
r43346 | ) | ||
|
r41706 | bcache = None | ||
finally: | ||||
if f: | ||||
f.close() | ||||
return bcache | ||||
|
r41974 | def load(self, repo, lineiter): | ||
|
r46554 | """fully loads the branchcache by reading from the file using the line | ||
|
r41974 | iterator passed""" | ||
|
r41959 | for line in lineiter: | ||
|
r43347 | line = line.rstrip(b'\n') | ||
|
r41959 | if not line: | ||
continue | ||||
|
r43347 | node, state, label = line.split(b" ", 2) | ||
if state not in b'oc': | ||||
|
r43906 | raise ValueError('invalid branch state') | ||
|
r41959 | label = encoding.tolocal(label.strip()) | ||
node = bin(node) | ||||
|
r42172 | self._entries.setdefault(label, []).append(node) | ||
|
r43347 | if state == b'c': | ||
|
r41959 | self._closednodes.add(node) | ||
|
r41706 | @staticmethod | ||
def _filename(repo): | ||||
"""name of a branchcache file for a given repo or repoview""" | ||||
|
r43347 | filename = b"branch2" | ||
|
r41706 | if repo.filtername: | ||
|
r43347 | filename = b'%s-%s' % (filename, repo.filtername) | ||
|
r41706 | return filename | ||
|
r18124 | |||
|
r18132 | def validfor(self, repo): | ||
|
r18644 | """Is the cache content valid regarding a repo | ||
|
r18132 | |||
|
r18644 | - False when cached tipnode is unknown or if we detect a strip. | ||
|
r18132 | - True when cache is up to date or a subset of current repo.""" | ||
try: | ||||
|
r43346 | return (self.tipnode == repo.changelog.node(self.tiprev)) and ( | ||
r49536 | self.filteredhash | |||
== scmutil.filteredhash(repo, self.tiprev, needobsolete=True) | ||||
|
r43346 | ) | ||
|
r18132 | except IndexError: | ||
return False | ||||
|
r20186 | def _branchtip(self, heads): | ||
|
r46554 | """Return tuple with last open head in heads and false, | ||
otherwise return last closed head and true.""" | ||||
|
r20186 | tip = heads[-1] | ||
closed = True | ||||
for h in reversed(heads): | ||||
if h not in self._closednodes: | ||||
tip = h | ||||
closed = False | ||||
break | ||||
return tip, closed | ||||
def branchtip(self, branch): | ||||
|
r46554 | """Return the tipmost open head on branch head, otherwise return the | ||
|
r20245 | tipmost closed head on branch. | ||
|
r46554 | Raise KeyError for unknown branch.""" | ||
|
r20186 | return self._branchtip(self[branch])[0] | ||
|
r34076 | def iteropen(self, nodes): | ||
return (n for n in nodes if n not in self._closednodes) | ||||
|
r20188 | def branchheads(self, branch, closed=False): | ||
|
r42290 | self._verifybranch(branch) | ||
|
r42281 | heads = self._entries[branch] | ||
|
r20188 | if not closed: | ||
|
r34076 | heads = list(self.iteropen(heads)) | ||
|
r20188 | return heads | ||
|
r20190 | def iterbranches(self): | ||
|
r43376 | for bn, heads in pycompat.iteritems(self): | ||
|
r20190 | yield (bn, heads) + self._branchtip(heads) | ||
|
r42169 | def iterheads(self): | ||
|
r47856 | """returns all the heads""" | ||
|
r42290 | self._verifyall() | ||
|
r43374 | return pycompat.itervalues(self._entries) | ||
|
r42169 | |||
|
r18232 | def copy(self): | ||
"""return an deep copy of the branchcache object""" | ||||
|
r42280 | return type(self)( | ||
|
r47538 | self._repo, | ||
|
r43346 | self._entries, | ||
self.tipnode, | ||||
self.tiprev, | ||||
self.filteredhash, | ||||
self._closednodes, | ||||
) | ||||
|
r18132 | |||
|
r18128 | def write(self, repo): | ||
r49526 | tr = repo.currenttransaction() | |||
if not getattr(tr, 'finalized', True): | ||||
# Avoid premature writing. | ||||
# | ||||
# (The cache warming setup by localrepo will update the file later.) | ||||
self._delayed = True | ||||
return | ||||
|
r18128 | try: | ||
|
r43347 | f = repo.cachevfs(self._filename(repo), b"w", atomictemp=True) | ||
cachekey = [hex(self.tipnode), b'%d' % self.tiprev] | ||||
|
r18184 | if self.filteredhash is not None: | ||
cachekey.append(hex(self.filteredhash)) | ||||
|
r43347 | f.write(b" ".join(cachekey) + b'\n') | ||
|
r21031 | nodecount = 0 | ||
|
r43376 | for label, nodes in sorted(pycompat.iteritems(self._entries)): | ||
|
r41827 | label = encoding.fromlocal(label) | ||
|
r18128 | for node in nodes: | ||
|
r21031 | nodecount += 1 | ||
|
r20185 | if node in self._closednodes: | ||
|
r43347 | state = b'c' | ||
|
r20185 | else: | ||
|
r43347 | state = b'o' | ||
f.write(b"%s %s %s\n" % (hex(node), state, label)) | ||||
|
r18128 | f.close() | ||
|
r43346 | repo.ui.log( | ||
|
r43347 | b'branchcache', | ||
b'wrote %s with %d labels and %d nodes\n', | ||||
|
r43346 | _branchcachedesc(repo), | ||
len(self._entries), | ||||
nodecount, | ||||
) | ||||
r49526 | self._delayed = False | |||
|
r26587 | except (IOError, OSError, error.Abort) as inst: | ||
|
r34369 | # Abort may be raised by read only opener, so log and continue | ||
|
r43346 | repo.ui.debug( | ||
|
r43347 | b"couldn't write branch cache: %s\n" | ||
|
r43346 | % stringutil.forcebytestr(inst) | ||
) | ||||
|
r18131 | |||
|
r18305 | def update(self, repo, revgen): | ||
|
r18131 | """Given a branchhead cache, self, that may have extra nodes or be | ||
|
r20263 | missing heads, and a generator of nodes that are strictly a superset of | ||
|
r18131 | heads missing, this function updates self to be correct. | ||
""" | ||||
|
r30975 | starttime = util.timer() | ||
|
r18131 | cl = repo.changelog | ||
# collect new branch entries | ||||
newbranches = {} | ||||
|
r24373 | getbranchinfo = repo.revbranchcache().branchinfo | ||
|
r18307 | for r in revgen: | ||
|
r40455 | branch, closesbranch = getbranchinfo(r) | ||
|
r20262 | newbranches.setdefault(branch, []).append(r) | ||
|
r20185 | if closesbranch: | ||
|
r20262 | self._closednodes.add(cl.node(r)) | ||
|
r22357 | |||
|
r42400 | # new tip revision which we found after iterating items from new | ||
# branches | ||||
ntiprev = self.tiprev | ||||
|
r46880 | # Delay fetching the topological heads until they are needed. | ||
# A repository without non-continous branches can skip this part. | ||||
topoheads = None | ||||
# If a changeset is visible, its parents must be visible too, so | ||||
# use the faster unfiltered parent accessor. | ||||
parentrevs = repo.unfiltered().changelog.parentrevs | ||||
r49536 | # Faster than using ctx.obsolete() | |||
obsrevs = obsolete.getrevs(repo, b'obsolete') | ||||
|
r43376 | for branch, newheadrevs in pycompat.iteritems(newbranches): | ||
|
r46880 | # For every branch, compute the new branchheads. | ||
# A branchhead is a revision such that no descendant is on | ||||
# the same branch. | ||||
# | ||||
# The branchheads are computed iteratively in revision order. | ||||
# This ensures topological order, i.e. parents are processed | ||||
# before their children. Ancestors are inclusive here, i.e. | ||||
# any revision is an ancestor of itself. | ||||
# | ||||
# Core observations: | ||||
# - The current revision is always a branchhead for the | ||||
# repository up to that point. | ||||
# - It is the first revision of the branch if and only if | ||||
# there was no branchhead before. In that case, it is the | ||||
# only branchhead as there are no possible ancestors on | ||||
# the same branch. | ||||
# - If a parent is on the same branch, a branchhead can | ||||
# only be an ancestor of that parent, if it is parent | ||||
# itself. Otherwise it would have been removed as ancestor | ||||
# of that parent before. | ||||
# - Therefore, if all parents are on the same branch, they | ||||
# can just be removed from the branchhead set. | ||||
# - If one parent is on the same branch and the other is not | ||||
# and there was exactly one branchhead known, the existing | ||||
# branchhead can only be an ancestor if it is the parent. | ||||
# Otherwise it would have been removed as ancestor of | ||||
# the parent before. The other parent therefore can't have | ||||
# a branchhead as ancestor. | ||||
# - In all other cases, the parents on different branches | ||||
# could have a branchhead as ancestor. Those parents are | ||||
# kept in the "uncertain" set. If all branchheads are also | ||||
# topological heads, they can't have descendants and further | ||||
# checks can be skipped. Otherwise, the ancestors of the | ||||
# "uncertain" set are removed from branchheads. | ||||
# This computation is heavy and avoided if at all possible. | ||||
|
r42172 | bheads = self._entries.setdefault(branch, []) | ||
|
r44937 | bheadset = {cl.rev(node) for node in bheads} | ||
|
r46880 | uncertain = set() | ||
for newrev in sorted(newheadrevs): | ||||
r49536 | if newrev in obsrevs: | |||
# We ignore obsolete changesets as they shouldn't be | ||||
# considered heads. | ||||
continue | ||||
|
r46880 | if not bheadset: | ||
bheadset.add(newrev) | ||||
continue | ||||
|
r18131 | |||
|
r46880 | parents = [p for p in parentrevs(newrev) if p != nullrev] | ||
samebranch = set() | ||||
otherbranch = set() | ||||
r49536 | obsparents = set() | |||
|
r46880 | for p in parents: | ||
r49536 | if p in obsrevs: | |||
# We ignored this obsolete changeset earlier, but now | ||||
# that it has non-ignored children, we need to make | ||||
# sure their ancestors are not considered heads. To | ||||
# achieve that, we will simply treat this obsolete | ||||
# changeset as a parent from other branch. | ||||
obsparents.add(p) | ||||
elif p in bheadset or getbranchinfo(p)[0] == branch: | ||||
|
r46880 | samebranch.add(p) | ||
else: | ||||
otherbranch.add(p) | ||||
r49536 | if not (len(bheadset) == len(samebranch) == 1): | |||
|
r46880 | uncertain.update(otherbranch) | ||
r49536 | uncertain.update(obsparents) | |||
|
r46880 | bheadset.difference_update(samebranch) | ||
bheadset.add(newrev) | ||||
|
r22357 | if uncertain: | ||
|
r46880 | if topoheads is None: | ||
topoheads = set(cl.headrevs()) | ||||
if bheadset - topoheads: | ||||
floorrev = min(bheadset) | ||||
r49536 | if floorrev <= max(uncertain): | |||
ancestors = set(cl.ancestors(uncertain, floorrev)) | ||||
bheadset -= ancestors | ||||
|
r20264 | bheadrevs = sorted(bheadset) | ||
|
r18131 | self[branch] = [cl.node(rev) for rev in bheadrevs] | ||
r49536 | tiprev = max(newheadrevs) | |||
|
r42400 | if tiprev > ntiprev: | ||
ntiprev = tiprev | ||||
if ntiprev > self.tiprev: | ||||
self.tiprev = ntiprev | ||||
self.tipnode = cl.node(ntiprev) | ||||
|
r18131 | |||
|
r19838 | if not self.validfor(repo): | ||
r49536 | # old cache key is now invalid for the repo, but we've just updated | |||
# the cache and we assume it's valid, so let's make the cache key | ||||
# valid as well by recomputing it from the cached data | ||||
|
r47771 | self.tipnode = repo.nullid | ||
|
r18131 | self.tiprev = nullrev | ||
|
r42169 | for heads in self.iterheads(): | ||
r49536 | if not heads: | |||
# all revisions on a branch are obsolete | ||||
continue | ||||
# note: tiprev is not necessarily the tip revision of repo, | ||||
# because the tip could be obsolete (i.e. not a head) | ||||
|
r18131 | tiprev = max(cl.rev(node) for node in heads) | ||
if tiprev > self.tiprev: | ||||
self.tipnode = cl.node(tiprev) | ||||
self.tiprev = tiprev | ||||
r49536 | self.filteredhash = scmutil.filteredhash( | |||
repo, self.tiprev, needobsolete=True | ||||
) | ||||
|
r21031 | |||
|
r30975 | duration = util.timer() - starttime | ||
|
r43346 | repo.ui.log( | ||
|
r43347 | b'branchcache', | ||
b'updated %s in %.4f seconds\n', | ||||
|
r43346 | _branchcachedesc(repo), | ||
duration, | ||||
) | ||||
|
r23785 | |||
|
r41707 | self.write(repo) | ||
class remotebranchcache(branchcache): | ||||
"""Branchmap info for a remote connection, should not write locally""" | ||||
|
r43346 | |||
|
r41707 | def write(self, repo): | ||
pass | ||||
|
r23785 | # Revision branch info cache | ||
|
r43347 | _rbcversion = b'-v1' | ||
_rbcnames = b'rbc-names' + _rbcversion | ||||
_rbcrevs = b'rbc-revs' + _rbcversion | ||||
|
r23785 | # [4 byte hash prefix][4 byte branch name number with sign bit indicating open] | ||
|
r43347 | _rbcrecfmt = b'>4sI' | ||
|
r23785 | _rbcrecsize = calcsize(_rbcrecfmt) | ||
|
r47069 | _rbcmininc = 64 * _rbcrecsize | ||
|
r23785 | _rbcnodelen = 4 | ||
|
r43346 | _rbcbranchidxmask = 0x7FFFFFFF | ||
|
r23785 | _rbccloseflag = 0x80000000 | ||
|
r43346 | |||
|
r23785 | class revbranchcache(object): | ||
"""Persistent cache, mapping from revision number to branch name and close. | ||||
This is a low level cache, independent of filtering. | ||||
Branch names are stored in rbc-names in internal encoding separated by 0. | ||||
rbc-names is append-only, and each branch name is only stored once and will | ||||
thus have a unique index. | ||||
The branch info for each revision is stored in rbc-revs as constant size | ||||
records. The whole file is read into memory, but it is only 'parsed' on | ||||
demand. The file is usually append-only but will be truncated if repo | ||||
modification is detected. | ||||
The record for each revision contains the first 4 bytes of the | ||||
corresponding node hash, and the record is only used if it still matches. | ||||
Even a completely trashed rbc-revs fill thus still give the right result | ||||
while converging towards full recovery ... assuming no incorrectly matching | ||||
node hashes. | ||||
The record also contains 4 bytes where 31 bits contains the index of the | ||||
branch and the last bit indicate that it is a branch close commit. | ||||
The usage pattern for rbc-revs is thus somewhat similar to 00changelog.i | ||||
and will grow with it but be 1/8th of its size. | ||||
""" | ||||
|
r24159 | def __init__(self, repo, readonly=True): | ||
|
r23785 | assert repo.filtername is None | ||
|
r24374 | self._repo = repo | ||
|
r43346 | self._names = [] # branch names in local encoding with static index | ||
|
r31346 | self._rbcrevs = bytearray() | ||
|
r43346 | self._rbcsnameslen = 0 # length of names read at _rbcsnameslen | ||
|
r23785 | try: | ||
|
r33535 | bndata = repo.cachevfs.read(_rbcnames) | ||
|
r43346 | self._rbcsnameslen = len(bndata) # for verification before writing | ||
|
r31371 | if bndata: | ||
|
r43346 | self._names = [ | ||
|
r43347 | encoding.tolocal(bn) for bn in bndata.split(b'\0') | ||
|
r43346 | ] | ||
|
r29423 | except (IOError, OSError): | ||
|
r24159 | if readonly: | ||
# don't try to use cache - fall back to the slow path | ||||
self.branchinfo = self._branchinfo | ||||
|
r23785 | if self._names: | ||
try: | ||||
|
r33535 | data = repo.cachevfs.read(_rbcrevs) | ||
|
r31346 | self._rbcrevs[:] = data | ||
|
r25660 | except (IOError, OSError) as inst: | ||
|
r43346 | repo.ui.debug( | ||
|
r43347 | b"couldn't read revision branch cache: %s\n" | ||
|
r43346 | % stringutil.forcebytestr(inst) | ||
) | ||||
|
r23785 | # remember number of good records on disk | ||
|
r43346 | self._rbcrevslen = min( | ||
len(self._rbcrevs) // _rbcrecsize, len(repo.changelog) | ||||
) | ||||
|
r23785 | if self._rbcrevslen == 0: | ||
self._names = [] | ||||
|
r43346 | self._rbcnamescount = len(self._names) # number of names read at | ||
# _rbcsnameslen | ||||
|
r23785 | |||
|
r28558 | def _clear(self): | ||
self._rbcsnameslen = 0 | ||||
del self._names[:] | ||||
self._rbcnamescount = 0 | ||||
self._rbcrevslen = len(self._repo.changelog) | ||||
|
r31346 | self._rbcrevs = bytearray(self._rbcrevslen * _rbcrecsize) | ||
|
r43347 | util.clearcachedproperty(self, b'_namesreverse') | ||
|
r40746 | |||
@util.propertycache | ||||
def _namesreverse(self): | ||||
|
r44937 | return {b: r for r, b in enumerate(self._names)} | ||
|
r28558 | |||
|
r40455 | def branchinfo(self, rev): | ||
|
r23785 | """Return branch name and close flag for rev, using and updating | ||
persistent cache.""" | ||||
|
r40455 | changelog = self._repo.changelog | ||
|
r23785 | rbcrevidx = rev * _rbcrecsize | ||
|
r25266 | # avoid negative index, changelog.read(nullrev) is fast without cache | ||
if rev == nullrev: | ||||
return changelog.branchinfo(rev) | ||||
|
r29604 | # if requested rev isn't allocated, grow and cache the rev info | ||
|
r23785 | if len(self._rbcrevs) < rbcrevidx + _rbcrecsize: | ||
|
r40455 | return self._branchinfo(rev) | ||
|
r23785 | |||
# fast path: extract data from cache, use it if node is matching | ||||
reponode = changelog.node(rev)[:_rbcnodelen] | ||||
|
r33737 | cachenode, branchidx = unpack_from( | ||
|
r43346 | _rbcrecfmt, util.buffer(self._rbcrevs), rbcrevidx | ||
) | ||||
|
r23785 | close = bool(branchidx & _rbccloseflag) | ||
if close: | ||||
branchidx &= _rbcbranchidxmask | ||||
|
r43347 | if cachenode == b'\0\0\0\0': | ||
|
r24376 | pass | ||
elif cachenode == reponode: | ||||
|
r29615 | try: | ||
|
r28558 | return self._names[branchidx], close | ||
|
r29615 | except IndexError: | ||
# recover from invalid reference to unknown branch | ||||
|
r43346 | self._repo.ui.debug( | ||
|
r43347 | b"referenced branch names not found" | ||
b" - rebuilding revision branch cache from scratch\n" | ||||
|
r43346 | ) | ||
|
r29615 | self._clear() | ||
|
r24376 | else: | ||
# rev/node map has changed, invalidate the cache from here up | ||||
|
r43346 | self._repo.ui.debug( | ||
|
r43347 | b"history modification detected - truncating " | ||
b"revision branch cache to revision %d\n" % rev | ||||
|
r43346 | ) | ||
|
r24376 | truncate = rbcrevidx + _rbcrecsize | ||
del self._rbcrevs[truncate:] | ||||
self._rbcrevslen = min(self._rbcrevslen, truncate) | ||||
|
r23785 | # fall back to slow path and make sure it will be written to disk | ||
|
r40455 | return self._branchinfo(rev) | ||
|
r23785 | |||
|
r40455 | def _branchinfo(self, rev): | ||
|
r23785 | """Retrieve branch info from changelog and update _rbcrevs""" | ||
|
r40455 | changelog = self._repo.changelog | ||
|
r23785 | b, close = changelog.branchinfo(rev) | ||
if b in self._namesreverse: | ||||
branchidx = self._namesreverse[b] | ||||
else: | ||||
branchidx = len(self._names) | ||||
self._names.append(b) | ||||
self._namesreverse[b] = branchidx | ||||
reponode = changelog.node(rev) | ||||
if close: | ||||
branchidx |= _rbccloseflag | ||||
|
r40455 | self._setcachedata(rev, reponode, branchidx) | ||
|
r24375 | return b, close | ||
|
r47084 | def setdata(self, rev, changelogrevision): | ||
|
r36980 | """add new data information to the cache""" | ||
|
r47084 | branch, close = changelogrevision.branchinfo | ||
|
r36980 | if branch in self._namesreverse: | ||
branchidx = self._namesreverse[branch] | ||||
else: | ||||
branchidx = len(self._names) | ||||
self._names.append(branch) | ||||
self._namesreverse[branch] = branchidx | ||||
if close: | ||||
branchidx |= _rbccloseflag | ||||
|
r47084 | self._setcachedata(rev, self._repo.changelog.node(rev), branchidx) | ||
|
r36980 | # If no cache data were readable (non exists, bad permission, etc) | ||
# the cache was bypassing itself by setting: | ||||
# | ||||
# self.branchinfo = self._branchinfo | ||||
# | ||||
# Since we now have data in the cache, we need to drop this bypassing. | ||||
|
r43906 | if 'branchinfo' in vars(self): | ||
|
r36980 | del self.branchinfo | ||
|
r40455 | def _setcachedata(self, rev, node, branchidx): | ||
|
r24375 | """Writes the node's branch data to the in-memory cache data.""" | ||
|
r31454 | if rev == nullrev: | ||
return | ||||
|
r23785 | rbcrevidx = rev * _rbcrecsize | ||
|
r47069 | requiredsize = rbcrevidx + _rbcrecsize | ||
rbccur = len(self._rbcrevs) | ||||
if rbccur < requiredsize: | ||||
# bytearray doesn't allocate extra space at least in Python 3.7. | ||||
# When multiple changesets are added in a row, precise resize would | ||||
# result in quadratic complexity. Overallocate to compensate by | ||||
# use the classic doubling technique for dynamic arrays instead. | ||||
# If there was a gap in the map before, less space will be reserved. | ||||
self._rbcrevs.extend(b'\0' * max(_rbcmininc, requiredsize)) | ||||
|
r31370 | pack_into(_rbcrecfmt, self._rbcrevs, rbcrevidx, node, branchidx) | ||
|
r24376 | self._rbcrevslen = min(self._rbcrevslen, rev) | ||
|
r23785 | |||
|
r24377 | tr = self._repo.currenttransaction() | ||
if tr: | ||||
|
r43347 | tr.addfinalize(b'write-revbranchcache', self.write) | ||
|
r24377 | |||
def write(self, tr=None): | ||||
|
r23785 | """Save branch cache if it is dirty.""" | ||
|
r24374 | repo = self._repo | ||
|
r29744 | wlock = None | ||
|
r43347 | step = b'' | ||
|
r29744 | try: | ||
|
r42363 | # write the new names | ||
|
r29743 | if self._rbcnamescount < len(self._names): | ||
|
r29744 | wlock = repo.wlock(wait=False) | ||
|
r43347 | step = b' names' | ||
|
r42363 | self._writenames(repo) | ||
|
r23785 | |||
|
r42363 | # write the new revs | ||
|
r29743 | start = self._rbcrevslen * _rbcrecsize | ||
if start != len(self._rbcrevs): | ||||
|
r43347 | step = b'' | ||
|
r29744 | if wlock is None: | ||
wlock = repo.wlock(wait=False) | ||||
|
r42363 | self._writerevs(repo, start) | ||
|
r29745 | except (IOError, OSError, error.Abort, error.LockError) as inst: | ||
|
r43346 | repo.ui.debug( | ||
|
r43347 | b"couldn't write revision branch cache%s: %s\n" | ||
|
r43346 | % (step, stringutil.forcebytestr(inst)) | ||
) | ||||
|
r29744 | finally: | ||
if wlock is not None: | ||||
wlock.release() | ||||
|
r42363 | |||
def _writenames(self, repo): | ||||
|
r47856 | """write the new branch names to revbranchcache""" | ||
|
r42363 | if self._rbcnamescount != 0: | ||
|
r43347 | f = repo.cachevfs.open(_rbcnames, b'ab') | ||
|
r42363 | if f.tell() == self._rbcsnameslen: | ||
|
r43347 | f.write(b'\0') | ||
|
r42363 | else: | ||
f.close() | ||||
|
r43347 | repo.ui.debug(b"%s changed - rewriting it\n" % _rbcnames) | ||
|
r42363 | self._rbcnamescount = 0 | ||
self._rbcrevslen = 0 | ||||
if self._rbcnamescount == 0: | ||||
# before rewriting names, make sure references are removed | ||||
repo.cachevfs.unlinkpath(_rbcrevs, ignoremissing=True) | ||||
|
r43347 | f = repo.cachevfs.open(_rbcnames, b'wb') | ||
|
r43346 | f.write( | ||
|
r43347 | b'\0'.join( | ||
|
r43346 | encoding.fromlocal(b) | ||
for b in self._names[self._rbcnamescount :] | ||||
) | ||||
) | ||||
|
r42363 | self._rbcsnameslen = f.tell() | ||
f.close() | ||||
self._rbcnamescount = len(self._names) | ||||
def _writerevs(self, repo, start): | ||||
|
r47856 | """write the new revs to revbranchcache""" | ||
|
r42364 | revs = min(len(repo.changelog), len(self._rbcrevs) // _rbcrecsize) | ||
|
r43347 | with repo.cachevfs.open(_rbcrevs, b'ab') as f: | ||
|
r42363 | if f.tell() != start: | ||
|
r43347 | repo.ui.debug( | ||
b"truncating cache/%s to %d\n" % (_rbcrevs, start) | ||||
) | ||||
|
r42363 | f.seek(start) | ||
|
r42364 | if f.tell() != start: | ||
start = 0 | ||||
f.seek(start) | ||||
f.truncate() | ||||
end = revs * _rbcrecsize | ||||
f.write(self._rbcrevs[start:end]) | ||||
|
r42363 | self._rbcrevslen = revs | ||