##// END OF EJS Templates
util: lower water mark when removing nodes after cost limit reached...
util: lower water mark when removing nodes after cost limit reached See the inline comment for the reasoning here. This is a pretty common strategy for garbage collectors, other cache-like primtives. The performance impact is substantial: $ hg perflrucachedict --size 4 --gets 1000000 --sets 1000000 --mixed 1000000 --costlimit 100 ! inserts w/ cost limit ! wall 1.659181 comb 1.650000 user 1.650000 sys 0.000000 (best of 7) ! wall 1.722122 comb 1.720000 user 1.720000 sys 0.000000 (best of 6) ! mixed w/ cost limit ! wall 1.139955 comb 1.140000 user 1.140000 sys 0.000000 (best of 9) ! wall 1.182513 comb 1.180000 user 1.180000 sys 0.000000 (best of 9) $ hg perflrucachedict --size 1000 --gets 1000000 --sets 1000000 --mixed 1000000 --costlimit 10000 ! inserts ! wall 0.679546 comb 0.680000 user 0.680000 sys 0.000000 (best of 15) ! sets ! wall 0.825147 comb 0.830000 user 0.830000 sys 0.000000 (best of 13) ! inserts w/ cost limit ! wall 25.105273 comb 25.080000 user 25.080000 sys 0.000000 (best of 3) ! wall 1.724397 comb 1.720000 user 1.720000 sys 0.000000 (best of 6) ! mixed ! wall 0.807096 comb 0.810000 user 0.810000 sys 0.000000 (best of 13) ! mixed w/ cost limit ! wall 12.104470 comb 12.070000 user 12.070000 sys 0.000000 (best of 3) ! wall 1.190563 comb 1.190000 user 1.190000 sys 0.000000 (best of 9) $ hg perflrucachedict --size 1000 --gets 1000000 --sets 1000000 --mixed 1000000 --costlimit 10000 --mixedgetfreq 90 ! inserts ! wall 0.711177 comb 0.710000 user 0.710000 sys 0.000000 (best of 14) ! sets ! wall 0.846992 comb 0.850000 user 0.850000 sys 0.000000 (best of 12) ! inserts w/ cost limit ! wall 25.963028 comb 25.960000 user 25.960000 sys 0.000000 (best of 3) ! wall 2.184311 comb 2.180000 user 2.180000 sys 0.000000 (best of 5) ! mixed ! wall 0.728256 comb 0.730000 user 0.730000 sys 0.000000 (best of 14) ! mixed w/ cost limit ! wall 3.174256 comb 3.170000 user 3.170000 sys 0.000000 (best of 4) ! wall 0.773186 comb 0.770000 user 0.770000 sys 0.000000 (best of 13) $ hg perflrucachedict --size 100000 --gets 1000000 --sets 1000000 --mixed 1000000 --mixedgetfreq 90 --costlimit 5000000 ! gets ! wall 1.191368 comb 1.190000 user 1.190000 sys 0.000000 (best of 9) ! wall 1.195304 comb 1.190000 user 1.190000 sys 0.000000 (best of 9) ! inserts ! wall 0.950995 comb 0.950000 user 0.950000 sys 0.000000 (best of 11) ! inserts w/ cost limit ! wall 1.589732 comb 1.590000 user 1.590000 sys 0.000000 (best of 7) ! sets ! wall 1.094941 comb 1.100000 user 1.090000 sys 0.010000 (best of 9) ! mixed ! wall 0.936420 comb 0.940000 user 0.930000 sys 0.010000 (best of 10) ! mixed w/ cost limit ! wall 0.882780 comb 0.870000 user 0.870000 sys 0.000000 (best of 11) This puts us ~2x slower than caches without cost accounting. And for read-heavy workloads (the prime use cases for caches), performance is nearly identical. In the worst case (pure write workloads with cost accounting enabled), we're looking at ~1.5us per insert on large caches. That seems "fast enough." Differential Revision: https://phab.mercurial-scm.org/D4505

File last commit:

r38414:59c9d3cc default
r39606:f296c0b3 default
Show More
similar.py
121 lines | 4.0 KiB | text/x-python | PythonLexer
# similar.py - mechanisms for finding similar files
#
# Copyright 2005-2007 Matt Mackall <mpm@selenic.com>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
from __future__ import absolute_import
from .i18n import _
from . import (
mdiff,
)
def _findexactmatches(repo, added, removed):
'''find renamed files that have no changes
Takes a list of new filectxs and a list of removed filectxs, and yields
(before, after) tuples of exact matches.
'''
# Build table of removed files: {hash(fctx.data()): [fctx, ...]}.
# We use hash() to discard fctx.data() from memory.
hashes = {}
progress = repo.ui.makeprogress(_('searching for exact renames'),
total=(len(added) + len(removed)),
unit=_('files'))
for fctx in removed:
progress.increment()
h = hash(fctx.data())
if h not in hashes:
hashes[h] = [fctx]
else:
hashes[h].append(fctx)
# For each added file, see if it corresponds to a removed file.
for fctx in added:
progress.increment()
adata = fctx.data()
h = hash(adata)
for rfctx in hashes.get(h, []):
# compare between actual file contents for exact identity
if adata == rfctx.data():
yield (rfctx, fctx)
break
# Done
progress.complete()
def _ctxdata(fctx):
# lazily load text
orig = fctx.data()
return orig, mdiff.splitnewlines(orig)
def _score(fctx, otherdata):
orig, lines = otherdata
text = fctx.data()
# mdiff.blocks() returns blocks of matching lines
# count the number of bytes in each
equal = 0
matches = mdiff.blocks(text, orig)
for x1, x2, y1, y2 in matches:
for line in lines[y1:y2]:
equal += len(line)
lengths = len(text) + len(orig)
return equal * 2.0 / lengths
def score(fctx1, fctx2):
return _score(fctx1, _ctxdata(fctx2))
def _findsimilarmatches(repo, added, removed, threshold):
'''find potentially renamed files based on similar file content
Takes a list of new filectxs and a list of removed filectxs, and yields
(before, after, score) tuples of partial matches.
'''
copies = {}
progress = repo.ui.makeprogress(_('searching for similar files'),
unit=_('files'), total=len(removed))
for r in removed:
progress.increment()
data = None
for a in added:
bestscore = copies.get(a, (None, threshold))[1]
if data is None:
data = _ctxdata(r)
myscore = _score(a, data)
if myscore > bestscore:
copies[a] = (r, myscore)
progress.complete()
for dest, v in copies.iteritems():
source, bscore = v
yield source, dest, bscore
def _dropempty(fctxs):
return [x for x in fctxs if x.size() > 0]
def findrenames(repo, added, removed, threshold):
'''find renamed files -- yields (before, after, score) tuples'''
wctx = repo[None]
pctx = wctx.p1()
# Zero length files will be frequently unrelated to each other, and
# tracking the deletion/addition of such a file will probably cause more
# harm than good. We strip them out here to avoid matching them later on.
addedfiles = _dropempty(wctx[fp] for fp in sorted(added))
removedfiles = _dropempty(pctx[fp] for fp in sorted(removed) if fp in pctx)
# Find exact matches.
matchedfiles = set()
for (a, b) in _findexactmatches(repo, addedfiles, removedfiles):
matchedfiles.add(b)
yield (a.path(), b.path(), 1.0)
# If the user requested similar files to be matched, search for them also.
if threshold < 1.0:
addedfiles = [x for x in addedfiles if x not in matchedfiles]
for (a, b, score) in _findsimilarmatches(repo, addedfiles,
removedfiles, threshold):
yield (a.path(), b.path(), score)