##// END OF EJS Templates
revlog: allow tuning of the chunk cache size (via format.chunkcachesize)...
revlog: allow tuning of the chunk cache size (via format.chunkcachesize) Running perfmoonwalk on the Mercurial repo (with almost 20,000 changesets) on Mac OS X with an SSD, before this change: $ hg --config format.chunkcachesize=1024 perfmoonwalk ! wall 2.022021 comb 2.030000 user 1.970000 sys 0.060000 (best of 5) (16,154 cache hits, 3,840 misses.) $ hg --config format.chunkcachesize=4096 perfmoonwalk ! wall 1.901006 comb 1.900000 user 1.880000 sys 0.020000 (best of 6) (19,003 hits, 991 misses.) $ hg --config format.chunkcachesize=16384 perfmoonwalk ! wall 1.802775 comb 1.800000 user 1.800000 sys 0.000000 (best of 6) (19,746 hits, 248 misses.) $ hg --config format.chunkcachesize=32768 perfmoonwalk ! wall 1.818545 comb 1.810000 user 1.810000 sys 0.000000 (best of 6) (19,870 hits, 124 misses.) $ hg --config format.chunkcachesize=65536 perfmoonwalk ! wall 1.801350 comb 1.810000 user 1.800000 sys 0.010000 (best of 6) (19,932 hits, 62 misses.) $ hg --config format.chunkcachesize=131072 perfmoonwalk ! wall 1.805879 comb 1.820000 user 1.810000 sys 0.010000 (best of 6) (19,963 hits, 31 misses.) We may want to change the default size in the future based on testing and user feedback.

File last commit:

r20032:175c6fd8 default
r20180:969148b4 default
Show More
branchmap.py
221 lines | 8.4 KiB | text/x-python | PythonLexer
Pierre-Yves David
branchmap: create a mercurial.branchmap module...
r18116 # branchmap.py - logic to computes, maintain and stores branchmap for local repo
#
# Copyright 2005-2007 Matt Mackall <mpm@selenic.com>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
Pierre-Yves David
branchmap: extract write logic from localrepo
r18117
Pierre-Yves David
branchmap: extract read logic from repo
r18118 from node import bin, hex, nullid, nullrev
Pierre-Yves David
branchmap: extract write logic from localrepo
r18117 import encoding
Augie Fackler
subsettable: move from repoview to branchmap, the only place it's used...
r20032 import util
Pierre-Yves David
branchmap: extract write logic from localrepo
r18117
Pierre-Yves David
branchmap: move the cache file name into a dedicated function...
r18185 def _filename(repo):
Pierre-Yves David
branchmap: use a different file name for filtered view of repo
r18187 """name of a branchcache file for a given repo or repoview"""
filename = "cache/branchheads"
if repo.filtername:
filename = '%s-%s' % (filename, repo.filtername)
return filename
Pierre-Yves David
branchmap: move the cache file name into a dedicated function...
r18185
Pierre-Yves David
branchmap: extract read logic from repo
r18118 def read(repo):
try:
Pierre-Yves David
branchmap: move the cache file name into a dedicated function...
r18185 f = repo.opener(_filename(repo))
Pierre-Yves David
branchmap: extract read logic from repo
r18118 lines = f.read().split('\n')
f.close()
except (IOError, OSError):
Pierre-Yves David
branchmap: read return None in case of failure...
r18212 return None
Pierre-Yves David
branchmap: extract read logic from repo
r18118
try:
Pierre-Yves David
branchmap: read and write key part related to filtered revision...
r18184 cachekey = lines.pop(0).split(" ", 2)
last, lrev = cachekey[:2]
Pierre-Yves David
branchmap: extract read logic from repo
r18118 last, lrev = bin(last), int(lrev)
Pierre-Yves David
branchmap: read and write key part related to filtered revision...
r18184 filteredhash = None
if len(cachekey) > 2:
filteredhash = bin(cachekey[2])
partial = branchcache(tipnode=last, tiprev=lrev,
filteredhash=filteredhash)
Pierre-Yves David
branchmap: move validity logic in the object itself...
r18132 if not partial.validfor(repo):
Pierre-Yves David
branchmap: extract read logic from repo
r18118 # invalidate the cache
Pierre-Yves David
branchmap: improve invalid cache message when reading...
r18166 raise ValueError('tip differs')
Pierre-Yves David
branchmap: extract read logic from repo
r18118 for l in lines:
if not l:
continue
node, label = l.split(" ", 1)
label = encoding.tolocal(label.strip())
if not node in repo:
Pierre-Yves David
branchmap: improve invalid cache message when reading...
r18166 raise ValueError('node %s does not exist' % node)
Pierre-Yves David
branchmap: extract read logic from repo
r18118 partial.setdefault(label, []).append(bin(node))
except KeyboardInterrupt:
raise
except Exception, inst:
if repo.ui.debugflag:
Pierre-Yves David
branchmap: report filtername when read fails...
r18188 msg = 'invalid branchheads cache'
if repo.filtername is not None:
msg += ' (%s)' % repo.filtername
msg += ': %s\n'
repo.ui.warn(msg % inst)
Pierre-Yves David
branchmap: read return None in case of failure...
r18212 partial = None
Pierre-Yves David
branchmap: add the tiprev (cache key) on the branchmap object...
r18126 return partial
Pierre-Yves David
branchmap: extract read logic from repo
r18118
Pierre-Yves David
branchmap: make update responsible to update the cache key...
r18130
Pierre-Yves David
branchmap: extract _updatebranchcache from repo
r18120
Augie Fackler
subsettable: move from repoview to branchmap, the only place it's used...
r20032 ### Nearest subset relation
# Nearest subset of filter X is a filter Y so that:
# * Y is included in X,
# * X - Y is as small as possible.
# This create and ordering used for branchmap purpose.
# the ordering may be partial
subsettable = {None: 'visible',
'visible': 'served',
'served': 'immutable',
'immutable': 'base'}
Pierre-Yves David
branchmap: extract updatebranchcache from repo
r18121 def updatecache(repo):
cl = repo.changelog
Pierre-Yves David
branchmap: enable caching for filtered version too...
r18189 filtername = repo.filtername
partial = repo._branchcaches.get(filtername)
Pierre-Yves David
branchmap: extract updatebranchcache from repo
r18121
Pierre-Yves David
branchmap: allow to use cache of subset...
r18234 revs = []
Pierre-Yves David
branchmap: move validity logic in the object itself...
r18132 if partial is None or not partial.validfor(repo):
Pierre-Yves David
branchmap: add the tiprev (cache key) on the branchmap object...
r18126 partial = read(repo)
Pierre-Yves David
branchmap: read return None in case of failure...
r18212 if partial is None:
Augie Fackler
subsettable: move from repoview to branchmap, the only place it's used...
r20032 subsetname = subsettable.get(filtername)
Pierre-Yves David
branchmap: allow to use cache of subset...
r18234 if subsetname is None:
partial = branchcache()
else:
subset = repo.filtered(subsetname)
partial = subset.branchmap().copy()
extrarevs = subset.changelog.filteredrevs - cl.filteredrevs
revs.extend(r for r in extrarevs if r <= partial.tiprev)
revs.extend(cl.revs(start=partial.tiprev + 1))
Pierre-Yves David
branchmap: drop `_cacheabletip` usage in `updatecache`...
r18218 if revs:
Pierre-Yves David
branchmap: pass revision insteads of changectx to the update function...
r18305 partial.update(repo, revs)
Pierre-Yves David
branchmap: make write a method on the branchmap object
r18128 partial.write(repo)
Pierre-Yves David
branchmap: display filtername when `updatebranch` fails to do its jobs...
r18451 assert partial.validfor(repo), filtername
Pierre-Yves David
branchmap: enable caching for filtered version too...
r18189 repo._branchcaches[repo.filtername] = partial
Pierre-Yves David
branchmap: store branchcache in a dedicated object...
r18124
class branchcache(dict):
"""A dict like object that hold branches heads cache"""
Pierre-Yves David
branchmap: takes filtered revision in account for cache calculation...
r18168 def __init__(self, entries=(), tipnode=nullid, tiprev=nullrev,
filteredhash=None):
Pierre-Yves David
branchmap: add the tipnode (cache key) on the branchcache object...
r18125 super(branchcache, self).__init__(entries)
self.tipnode = tipnode
Pierre-Yves David
branchmap: add the tiprev (cache key) on the branchmap object...
r18126 self.tiprev = tiprev
Pierre-Yves David
branchmap: takes filtered revision in account for cache calculation...
r18168 self.filteredhash = filteredhash
def _hashfiltered(self, repo):
"""build hash of revision filtered in the current cache
Mads Kiilerich
spelling: fix some minor issues found by spell checker
r18644 Tracking tipnode and tiprev is not enough to ensure validity of the
Pierre-Yves David
branchmap: takes filtered revision in account for cache calculation...
r18168 cache as they do not help to distinct cache that ignored various
revision bellow tiprev.
To detect such difference, we build a cache of all ignored revisions.
"""
cl = repo.changelog
if not cl.filteredrevs:
return None
key = None
revs = sorted(r for r in cl.filteredrevs if r <= self.tiprev)
if revs:
s = util.sha1()
for rev in revs:
s.update('%s;' % rev)
key = s.digest()
return key
Pierre-Yves David
branchmap: make write a method on the branchmap object
r18128
Pierre-Yves David
branchmap: move validity logic in the object itself...
r18132 def validfor(self, repo):
Mads Kiilerich
spelling: fix some minor issues found by spell checker
r18644 """Is the cache content valid regarding a repo
Pierre-Yves David
branchmap: move validity logic in the object itself...
r18132
Mads Kiilerich
spelling: fix some minor issues found by spell checker
r18644 - False when cached tipnode is unknown or if we detect a strip.
Pierre-Yves David
branchmap: move validity logic in the object itself...
r18132 - True when cache is up to date or a subset of current repo."""
try:
Pierre-Yves David
branchmap: takes filtered revision in account for cache calculation...
r18168 return ((self.tipnode == repo.changelog.node(self.tiprev))
and (self.filteredhash == self._hashfiltered(repo)))
Pierre-Yves David
branchmap: move validity logic in the object itself...
r18132 except IndexError:
return False
Pierre-Yves David
branchmap: add a copy method...
r18232 def copy(self):
"""return an deep copy of the branchcache object"""
return branchcache(self, self.tipnode, self.tiprev, self.filteredhash)
Pierre-Yves David
branchmap: move validity logic in the object itself...
r18132
Pierre-Yves David
branchmap: make write a method on the branchmap object
r18128 def write(self, repo):
try:
Pierre-Yves David
branchmap: move the cache file name into a dedicated function...
r18185 f = repo.opener(_filename(repo), "w", atomictemp=True)
Pierre-Yves David
branchmap: read and write key part related to filtered revision...
r18184 cachekey = [hex(self.tipnode), str(self.tiprev)]
if self.filteredhash is not None:
cachekey.append(hex(self.filteredhash))
f.write(" ".join(cachekey) + '\n')
Mads Kiilerich
localrepo: store branchheads sorted
r18357 for label, nodes in sorted(self.iteritems()):
Pierre-Yves David
branchmap: make write a method on the branchmap object
r18128 for node in nodes:
f.write("%s %s\n" % (hex(node), encoding.fromlocal(label)))
f.close()
Pierre-Yves David
branchmap: ignore Abort error while writing cache...
r18214 except (IOError, OSError, util.Abort):
# Abort may be raise by read only opener
Pierre-Yves David
branchmap: make write a method on the branchmap object
r18128 pass
Pierre-Yves David
branchmap: make update a method
r18131
Pierre-Yves David
branchmap: pass revision insteads of changectx to the update function...
r18305 def update(self, repo, revgen):
Pierre-Yves David
branchmap: make update a method
r18131 """Given a branchhead cache, self, that may have extra nodes or be
missing heads, and a generator of nodes that are at least a superset of
heads missing, this function updates self to be correct.
"""
cl = repo.changelog
# collect new branch entries
newbranches = {}
Pierre-Yves David
branchmap: Save changectx creation during update...
r18307 getbranch = cl.branch
for r in revgen:
newbranches.setdefault(getbranch(r), []).append(cl.node(r))
Pierre-Yves David
branchmap: make update a method
r18131 # if older branchheads are reachable from new ones, they aren't
# really branchheads. Note checking parents is insufficient:
# 1 (branch a) -> 2 (branch b) -> 3 (branch a)
for branch, newnodes in newbranches.iteritems():
bheads = self.setdefault(branch, [])
# Remove candidate heads that no longer are in the repo (e.g., as
# the result of a strip that just happened). Avoid using 'node in
# self' here because that dives down into branchcache code somewhat
# recursively.
bheadrevs = [cl.rev(node) for node in bheads
if cl.hasnode(node)]
newheadrevs = [cl.rev(node) for node in newnodes
if cl.hasnode(node)]
ctxisnew = bheadrevs and min(newheadrevs) > max(bheadrevs)
# Remove duplicates - nodes that are in newheadrevs and are already
# in bheadrevs. This can happen if you strip a node whose parent
# was already a head (because they're on different branches).
bheadrevs = sorted(set(bheadrevs).union(newheadrevs))
# Starting from tip means fewer passes over reachable. If we know
# the new candidates are not ancestors of existing heads, we don't
# have to examine ancestors of existing heads
if ctxisnew:
iterrevs = sorted(newheadrevs)
else:
iterrevs = list(bheadrevs)
# This loop prunes out two kinds of heads - heads that are
# superseded by a head in newheadrevs, and newheadrevs that are not
# heads because an existing head is their descendant.
while iterrevs:
latest = iterrevs.pop()
if latest not in bheadrevs:
continue
ancestors = set(cl.ancestors([latest],
bheadrevs[0]))
if ancestors:
bheadrevs = [b for b in bheadrevs if b not in ancestors]
self[branch] = [cl.node(rev) for rev in bheadrevs]
tiprev = max(bheadrevs)
if tiprev > self.tiprev:
self.tipnode = cl.node(tiprev)
self.tiprev = tiprev
Pierre-Yves David
branchmap: remove the droppednodes logic...
r19838 if not self.validfor(repo):
Pierre-Yves David
branchmap: make update a method
r18131 # cache key are not valid anymore
self.tipnode = nullid
self.tiprev = nullrev
for heads in self.values():
tiprev = max(cl.rev(node) for node in heads)
if tiprev > self.tiprev:
self.tipnode = cl.node(tiprev)
self.tiprev = tiprev
Pierre-Yves David
branchmap: takes filtered revision in account for cache calculation...
r18168 self.filteredhash = self._hashfiltered(repo)