revsetbenchmarks.py
317 lines
| 9.6 KiB
| text/x-python
|
PythonLexer
/ contrib / revsetbenchmarks.py
|
r20848 | #!/usr/bin/env python | ||
# Measure the performance of a list of revsets against multiple revisions | ||||
# defined by parameter. Checkout one by one and run perfrevset with every | ||||
# revset in the list to benchmark its performance. | ||||
# | ||||
|
r25535 | # You should run this from the root of your mercurial repository. | ||
|
r20848 | # | ||
|
r25535 | # call with --help for details | ||
|
r20848 | |||
import sys | ||||
|
r21548 | import os | ||
|
r25530 | import re | ||
|
r25539 | import math | ||
|
r20893 | from subprocess import check_call, Popen, CalledProcessError, STDOUT, PIPE | ||
|
r21287 | # cannot use argparse, python 2.7 only | ||
from optparse import OptionParser | ||||
|
r25544 | DEFAULTVARIANTS = ['plain', 'min', 'max', 'first', 'last', | ||
'reverse', 'reverse+first', 'reverse+last', | ||||
'sort', 'sort+first', 'sort+last'] | ||||
|
r25540 | |||
|
r20893 | def check_output(*args, **kwargs): | ||
kwargs.setdefault('stderr', PIPE) | ||||
kwargs.setdefault('stdout', PIPE) | ||||
proc = Popen(*args, **kwargs) | ||||
output, error = proc.communicate() | ||||
if proc.returncode != 0: | ||||
|
r21202 | raise CalledProcessError(proc.returncode, ' '.join(args[0])) | ||
|
r20893 | return output | ||
|
r20848 | |||
|
r20850 | def update(rev): | ||
"""update the repo to a revision""" | ||||
try: | ||||
check_call(['hg', 'update', '--quiet', '--check', str(rev)]) | ||||
|
r25660 | except CalledProcessError as exc: | ||
|
r20850 | print >> sys.stderr, 'update to revision %s failed, aborting' % rev | ||
sys.exit(exc.returncode) | ||||
|
r25528 | |||
def hg(cmd, repo=None): | ||||
"""run a mercurial command | ||||
<cmd> is the list of command + argument, | ||||
<repo> is an optional repository path to run this command in.""" | ||||
fullcmd = ['./hg'] | ||||
if repo is not None: | ||||
fullcmd += ['-R', repo] | ||||
fullcmd += ['--config', | ||||
'extensions.perf=' + os.path.join(contribdir, 'perf.py')] | ||||
fullcmd += cmd | ||||
return check_output(fullcmd, stderr=STDOUT) | ||||
|
r21549 | def perf(revset, target=None): | ||
|
r20851 | """run benchmark for this very revset""" | ||
try: | ||||
|
r25528 | output = hg(['perfrevset', revset], repo=target) | ||
|
r25530 | return parseoutput(output) | ||
|
r25660 | except CalledProcessError as exc: | ||
|
r25529 | print >> sys.stderr, 'abort: cannot run revset benchmark: %s' % exc.cmd | ||
if exc.output is None: | ||||
print >> sys.stderr, '(no ouput)' | ||||
else: | ||||
print >> sys.stderr, exc.output | ||||
|
r25646 | return None | ||
|
r20851 | |||
|
r25530 | outputre = re.compile(r'! wall (\d+.\d+) comb (\d+.\d+) user (\d+.\d+) ' | ||
'sys (\d+.\d+) \(best of (\d+)\)') | ||||
def parseoutput(output): | ||||
"""parse a textual output into a dict | ||||
We cannot just use json because we want to compare with old | ||||
versions of Mercurial that may not support json output. | ||||
""" | ||||
match = outputre.search(output) | ||||
if not match: | ||||
print >> sys.stderr, 'abort: invalid output:' | ||||
print >> sys.stderr, output | ||||
sys.exit(1) | ||||
return {'comb': float(match.group(2)), | ||||
'count': int(match.group(5)), | ||||
'sys': float(match.group(3)), | ||||
'user': float(match.group(4)), | ||||
'wall': float(match.group(1)), | ||||
} | ||||
|
r20852 | def printrevision(rev): | ||
"""print data about a revision""" | ||||
|
r25538 | sys.stdout.write("Revision ") | ||
|
r20852 | sys.stdout.flush() | ||
check_call(['hg', 'log', '--rev', str(rev), '--template', | ||||
|
r25546 | '{if(tags, " ({tags})")} ' | ||
|
r25538 | '{rev}:{node|short}: {desc|firstline}\n']) | ||
|
r20852 | |||
|
r25532 | def idxwidth(nbidx): | ||
"""return the max width of number used for index | ||||
|
r25533 | This is similar to log10(nbidx), but we use custom code here | ||
because we start with zero and we'd rather not deal with all the | ||||
extra rounding business that log10 would imply. | ||||
""" | ||||
|
r25532 | nbidx -= 1 # starts at 0 | ||
idxwidth = 0 | ||||
while nbidx: | ||||
idxwidth += 1 | ||||
nbidx //= 10 | ||||
if not idxwidth: | ||||
idxwidth = 1 | ||||
return idxwidth | ||||
|
r25539 | def getfactor(main, other, field, sensitivity=0.05): | ||
"""return the relative factor between values for 'field' in main and other | ||||
Return None if the factor is insignicant (less than <sensitivity> | ||||
variation).""" | ||||
factor = 1 | ||||
if main is not None: | ||||
factor = other[field] / main[field] | ||||
low, high = 1 - sensitivity, 1 + sensitivity | ||||
if (low < factor < high): | ||||
return None | ||||
return factor | ||||
def formatfactor(factor): | ||||
"""format a factor into a 4 char string | ||||
22% | ||||
156% | ||||
x2.4 | ||||
x23 | ||||
x789 | ||||
x1e4 | ||||
x5x7 | ||||
""" | ||||
if factor is None: | ||||
return ' ' | ||||
elif factor < 2: | ||||
return '%3i%%' % (factor * 100) | ||||
elif factor < 10: | ||||
return 'x%3.1f' % factor | ||||
elif factor < 1000: | ||||
return '%4s' % ('x%i' % factor) | ||||
else: | ||||
order = int(math.log(factor)) + 1 | ||||
while 1 < math.log(factor): | ||||
factor //= 0 | ||||
return 'x%ix%i' % (factor, order) | ||||
|
r25541 | def formattiming(value): | ||
"""format a value to strictly 8 char, dropping some precision if needed""" | ||||
if value < 10**7: | ||||
return ('%.6f' % value)[:8] | ||||
else: | ||||
# value is HUGE very unlikely to happen (4+ month run) | ||||
return '%i' % value | ||||
|
r25539 | _marker = object() | ||
|
r25540 | def printresult(variants, idx, data, maxidx, verbose=False, reference=_marker): | ||
|
r25531 | """print a line of result to stdout""" | ||
|
r25532 | mask = '%%0%ii) %%s' % idxwidth(maxidx) | ||
|
r25646 | |||
|
r25540 | out = [] | ||
for var in variants: | ||||
|
r25646 | if data[var] is None: | ||
out.append('error ') | ||||
out.append(' ' * 4) | ||||
continue | ||||
|
r25541 | out.append(formattiming(data[var]['wall'])) | ||
|
r25540 | if reference is not _marker: | ||
factor = None | ||||
if reference is not None: | ||||
factor = getfactor(reference[var], data[var], 'wall') | ||||
out.append(formatfactor(factor)) | ||||
if verbose: | ||||
|
r25541 | out.append(formattiming(data[var]['comb'])) | ||
out.append(formattiming(data[var]['user'])) | ||||
out.append(formattiming(data[var]['sys'])) | ||||
|
r25540 | out.append('%6d' % data[var]['count']) | ||
|
r25534 | print mask % (idx, ' '.join(out)) | ||
|
r25530 | |||
|
r25540 | def printheader(variants, maxidx, verbose=False, relative=False): | ||
header = [' ' * (idxwidth(maxidx) + 1)] | ||||
for var in variants: | ||||
if not var: | ||||
var = 'iter' | ||||
|
r25541 | if 8 < len(var): | ||
var = var[:3] + '..' + var[-3:] | ||||
header.append('%-8s' % var) | ||||
|
r25540 | if relative: | ||
header.append(' ') | ||||
if verbose: | ||||
|
r25541 | header.append('%-8s' % 'comb') | ||
header.append('%-8s' % 'user') | ||||
header.append('%-8s' % 'sys') | ||||
|
r25540 | header.append('%6s' % 'count') | ||
|
r25534 | print ' '.join(header) | ||
|
r25530 | |||
|
r20853 | def getrevs(spec): | ||
"""get the list of rev matched by a revset""" | ||||
try: | ||||
out = check_output(['hg', 'log', '--template={rev}\n', '--rev', spec]) | ||||
|
r25660 | except CalledProcessError as exc: | ||
|
r20853 | print >> sys.stderr, "abort, can't get revision from %s" % spec | ||
sys.exit(exc.returncode) | ||||
return [r for r in out.split() if r] | ||||
|
r25540 | def applyvariants(revset, variant): | ||
if variant == 'plain': | ||||
return revset | ||||
|
r25543 | for var in variant.split('+'): | ||
revset = '%s(%s)' % (var, revset) | ||||
return revset | ||||
|
r25540 | |||
|
r25607 | helptext="""This script will run multiple variants of provided revsets using | ||
different revisions in your mercurial repository. After the benchmark are run | ||||
summary output is provided. Use itto demonstrate speed improvements or pin | ||||
point regressions. Revsets to run are specified in a file (or from stdin), one | ||||
revsets per line. Line starting with '#' will be ignored, allowing insertion of | ||||
comments.""" | ||||
parser = OptionParser(usage="usage: %prog [options] <revs>", | ||||
description=helptext) | ||||
|
r21287 | parser.add_option("-f", "--file", | ||
|
r23139 | help="read revset from FILE (stdin if omitted)", | ||
|
r22555 | metavar="FILE") | ||
|
r21549 | parser.add_option("-R", "--repo", | ||
help="run benchmark on REPO", metavar="REPO") | ||||
|
r21287 | |||
|
r25537 | parser.add_option("-v", "--verbose", | ||
action='store_true', | ||||
help="display all timing data (not just best total time)") | ||||
|
r25540 | parser.add_option("", "--variants", | ||
default=','.join(DEFAULTVARIANTS), | ||||
help="comma separated list of variant to test " | ||||
"(eg: plain,min,sorted) (plain = no modification)") | ||||
|
r21287 | (options, args) = parser.parse_args() | ||
|
r20848 | |||
|
r25535 | if not args: | ||
|
r21287 | parser.print_help() | ||
|
r21286 | sys.exit(255) | ||
|
r21548 | # the directory where both this script and the perf.py extension live. | ||
contribdir = os.path.dirname(__file__) | ||||
|
r21287 | |||
|
r20848 | revsetsfile = sys.stdin | ||
|
r21287 | if options.file: | ||
revsetsfile = open(options.file) | ||||
|
r20848 | |||
|
r22556 | revsets = [l.strip() for l in revsetsfile if not l.startswith('#')] | ||
|
r25642 | revsets = [l for l in revsets if l] | ||
|
r20848 | |||
print "Revsets to benchmark" | ||||
print "----------------------------" | ||||
for idx, rset in enumerate(revsets): | ||||
print "%i) %s" % (idx, rset) | ||||
print "----------------------------" | ||||
|
r25535 | revs = [] | ||
for a in args: | ||||
revs.extend(getrevs(a)) | ||||
|
r20848 | |||
|
r25540 | variants = options.variants.split(',') | ||
|
r20855 | results = [] | ||
|
r20848 | for r in revs: | ||
print "----------------------------" | ||||
|
r20852 | printrevision(r) | ||
|
r20848 | print "----------------------------" | ||
|
r20850 | update(r) | ||
|
r20855 | res = [] | ||
results.append(res) | ||||
|
r25540 | printheader(variants, len(revsets), verbose=options.verbose) | ||
|
r20848 | for idx, rset in enumerate(revsets): | ||
|
r25540 | varres = {} | ||
for var in variants: | ||||
varrset = applyvariants(rset, var) | ||||
data = perf(varrset, target=options.repo) | ||||
varres[var] = data | ||||
res.append(varres) | ||||
printresult(variants, idx, varres, len(revsets), | ||||
verbose=options.verbose) | ||||
|
r20855 | sys.stdout.flush() | ||
|
r20848 | print "----------------------------" | ||
|
r20855 | |||
print """ | ||||
Result by revset | ||||
================ | ||||
""" | ||||
|
r25538 | print 'Revision:' | ||
|
r20855 | for idx, rev in enumerate(revs): | ||
sys.stdout.write('%i) ' % idx) | ||||
sys.stdout.flush() | ||||
printrevision(rev) | ||||
for ridx, rset in enumerate(revsets): | ||||
print "revset #%i: %s" % (ridx, rset) | ||||
|
r25540 | printheader(variants, len(results), verbose=options.verbose, relative=True) | ||
|
r25539 | ref = None | ||
|
r20855 | for idx, data in enumerate(results): | ||
|
r25540 | printresult(variants, idx, data[ridx], len(results), | ||
verbose=options.verbose, reference=ref) | ||||
|
r25539 | ref = data[ridx] | ||
|
r20855 | |||