##// END OF EJS Templates
findrenames: Optimise "addremove -s100" by matching files by their SHA1 hashes....
findrenames: Optimise "addremove -s100" by matching files by their SHA1 hashes. We speed up 'findrenames' for the usecase when a user specifies they want a similarity of 100% by matching files by their exact SHA1 hash value. This reduces the number of comparisons required to find exact matches from O(n^2) to O(n). While it would be nice if we could just use mercurial's pre-calculated SHA1 hash for existing files, this hash includes the file's ancestor information making it unsuitable for our purposes. Instead, we calculate the hash of old content from scratch. The following benchmarks were taken on the current head of crew: addremove 100% similarity: rm -rf *; hg up -C; mv tests tests.new hg --time addremove -s100 --dry-run before: real 176.350 secs (user 128.890+0.000 sys 47.430+0.000) after: real 2.130 secs (user 1.890+0.000 sys 0.240+0.000) addremove 75% similarity: rm -rf *; hg up -C; mv tests tests.new; \ for i in tests.new/*; do echo x >> $i; done hg --time addremove -s75 --dry-run before: real 264.560 secs (user 215.130+0.000 sys 49.410+0.000) after: real 218.710 secs (user 172.790+0.000 sys 45.870+0.000)

File last commit:

r10339:23e608f4 default
r11060:e6df0177 default
Show More
lsprof.py
113 lines | 3.7 KiB | text/x-python | PythonLexer
Dirkjan Ochtman
updating lsprof.py from remote repository
r5992 #! /usr/bin/env python
Vadim Gelfer
add --lsprof option. 3x faster than --profile, more useful output....
r2422
import sys
Joel Rosdahl
Remove unused imports
r6212 from _lsprof import Profiler, profiler_entry
Vadim Gelfer
add --lsprof option. 3x faster than --profile, more useful output....
r2422
__all__ = ['profile', 'Stats']
def profile(f, *args, **kwds):
"""XXX docstring"""
p = Profiler()
Dirkjan Ochtman
updating lsprof.py from remote repository
r5992 p.enable(subcalls=True, builtins=True)
Vadim Gelfer
add --lsprof option. 3x faster than --profile, more useful output....
r2422 try:
Dirkjan Ochtman
updating lsprof.py from remote repository
r5992 f(*args, **kwds)
Vadim Gelfer
add --lsprof option. 3x faster than --profile, more useful output....
r2422 finally:
p.disable()
Dirkjan Ochtman
updating lsprof.py from remote repository
r5992 return Stats(p.getstats())
Vadim Gelfer
add --lsprof option. 3x faster than --profile, more useful output....
r2422
class Stats(object):
"""XXX docstring"""
def __init__(self, data):
self.data = data
def sort(self, crit="inlinetime"):
"""XXX docstring"""
if crit not in profiler_entry.__dict__:
Peter Ruibal
use Exception(args)-style raising consistently (py3k compatibility)
r7008 raise ValueError("Can't sort by %s" % crit)
Alejandro Santos
compat: use 'key' argument instead of 'cmp' when sorting a list
r9032 self.data.sort(key=lambda x: getattr(x, crit), reverse=True)
Vadim Gelfer
add --lsprof option. 3x faster than --profile, more useful output....
r2422 for e in self.data:
if e.calls:
Alejandro Santos
compat: use 'key' argument instead of 'cmp' when sorting a list
r9032 e.calls.sort(key=lambda x: getattr(x, crit), reverse=True)
Vadim Gelfer
add --lsprof option. 3x faster than --profile, more useful output....
r2422
def pprint(self, top=None, file=None, limit=None, climit=None):
"""XXX docstring"""
if file is None:
file = sys.stdout
d = self.data
if top is not None:
d = d[:top]
Dirkjan Ochtman
updating lsprof.py from remote repository
r5992 cols = "% 12s %12s %11.4f %11.4f %s\n"
hcols = "% 12s %12s %12s %12s %s\n"
file.write(hcols % ("CallCount", "Recursive", "Total(ms)",
"Inline(ms)", "module:lineno(function)"))
Vadim Gelfer
add --lsprof option. 3x faster than --profile, more useful output....
r2422 count = 0
for e in d:
Dirkjan Ochtman
updating lsprof.py from remote repository
r5992 file.write(cols % (e.callcount, e.reccallcount, e.totaltime,
Vadim Gelfer
add --lsprof option. 3x faster than --profile, more useful output....
r2422 e.inlinetime, label(e.code)))
count += 1
if limit is not None and count == limit:
return
ccount = 0
if e.calls:
for se in e.calls:
Dirkjan Ochtman
updating lsprof.py from remote repository
r5992 file.write(cols % ("+%s" % se.callcount, se.reccallcount,
Vadim Gelfer
add --lsprof option. 3x faster than --profile, more useful output....
r2422 se.totaltime, se.inlinetime,
"+%s" % label(se.code)))
count += 1
ccount += 1
if limit is not None and count == limit:
return
if climit is not None and ccount == climit:
break
def freeze(self):
"""Replace all references to code objects with string
descriptions; this makes it possible to pickle the instance."""
# this code is probably rather ickier than it needs to be!
for i in range(len(self.data)):
e = self.data[i]
if not isinstance(e.code, str):
self.data[i] = type(e)((label(e.code),) + e[1:])
Dirkjan Ochtman
updating lsprof.py from remote repository
r5992 if e.calls:
for j in range(len(e.calls)):
se = e.calls[j]
if not isinstance(se.code, str):
e.calls[j] = type(se)((label(se.code),) + se[1:])
Vadim Gelfer
add --lsprof option. 3x faster than --profile, more useful output....
r2422
_fn2mod = {}
def label(code):
if isinstance(code, str):
return code
try:
mname = _fn2mod[code.co_filename]
except KeyError:
Dirkjan Ochtman
lsprof: make profile not die when imported modules changes (issue1774)
r9314 for k, v in list(sys.modules.iteritems()):
Vadim Gelfer
add --lsprof option. 3x faster than --profile, more useful output....
r2422 if v is None:
continue
if not hasattr(v, '__file__'):
continue
if not isinstance(v.__file__, str):
continue
if v.__file__.startswith(code.co_filename):
mname = _fn2mod[code.co_filename] = k
break
else:
Benoit Boissinot
fix spaces/identation issues
r10339 mname = _fn2mod[code.co_filename] = '<%s>' % code.co_filename
Vadim Gelfer
add --lsprof option. 3x faster than --profile, more useful output....
r2422
return '%s:%d(%s)' % (mname, code.co_firstlineno, code.co_name)
if __name__ == '__main__':
import os
sys.argv = sys.argv[1:]
if not sys.argv:
print >> sys.stderr, "usage: lsprof.py <script> <arguments...>"
sys.exit(2)
sys.path.insert(0, os.path.abspath(os.path.dirname(sys.argv[0])))
stats = profile(execfile, sys.argv[0], globals(), locals())
stats.sort()
stats.pprint()