##// END OF EJS Templates
worker: Use buffered input from the pickle stream...
worker: Use buffered input from the pickle stream On Python 3, "pickle.load" will raise an exception ("_pickle.UnpicklingError: pickle data was truncated") when it gets a short read, i.e. it receives fewer bytes than it requested. On our build machine, Mercurial seems to frequently hit this problem while updating a mozilla-central clone iff it gets scheduled in batch mode. It is easy to trigger with: #wipe the workdir rm -rf * hg update null chrt -b 0 hg update default I've also written the following program, which demonstrates the core problem: from __future__ import print_function import io import os import pickle import time obj = {"a": 1, "b": 2} obj_data = pickle.dumps(obj) assert len(obj_data) > 10 rfd, wfd = os.pipe() pid = os.fork() if pid == 0: os.close(rfd) for _ in range(4): time.sleep(0.5) print("First write") os.write(wfd, obj_data[:10]) time.sleep(0.5) print("Second write") os.write(wfd, obj_data[10:]) os._exit(0) try: os.close(wfd) rfile = os.fdopen(rfd, "rb", 0) print("Reading") while True: try: obj_copy = pickle.load(rfile) assert obj == obj_copy except EOFError: break print("Success") finally: os.kill(pid, 15) The program reliably fails with Python 3.8 and succeeds with Python 2.7. Providing the unpickler with a buffered reader fixes the issue, so let "os.fdopen" create one. https://bugzilla.mozilla.org/show_bug.cgi?id=1604486 Differential Revision: https://phab.mercurial-scm.org/D8051

File last commit:

r43346:2372284d default
r44718:cb52e619 stable
Show More
__init__.py
126 lines | 3.9 KiB | text/x-python | PythonLexer
Philippe Pepiot
perf: add asv benchmarks...
r30406 # __init__.py - asv benchmark suite
#
# Copyright 2016 Logilab SA <contact@logilab.fr>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
Philippe Pepiot
perf: add historical support of ui.load()...
r30588 # "historical portability" policy of contrib/benchmarks:
#
# We have to make this code work correctly with current mercurial stable branch
# and if possible with reasonable cost with early Mercurial versions.
Philippe Pepiot
perf: add asv benchmarks...
r30406 '''ASV (https://asv.readthedocs.io) benchmark suite
Benchmark are parameterized against reference repositories found in the
directory pointed by the REPOS_DIR environment variable.
Invocation example:
$ export REPOS_DIR=~/hgperf/repos
# run suite on given revision
$ asv --config contrib/asv.conf.json run REV
# run suite on new changesets found in stable and default branch
$ asv --config contrib/asv.conf.json run NEW
# display a comparative result table of benchmark results between two given
# revisions
$ asv --config contrib/asv.conf.json compare REV1 REV2
# compute regression detection and generate ASV static website
$ asv --config contrib/asv.conf.json publish
# serve the static website
$ asv --config contrib/asv.conf.json preview
'''
from __future__ import absolute_import
import functools
import os
import re
from mercurial import (
extensions,
hg,
ui as uimod,
Philippe Pepiot
perf: add historical support of ui.load()...
r30588 util,
Philippe Pepiot
perf: add asv benchmarks...
r30406 )
Augie Fackler
formatting: blacken the codebase...
r43346 basedir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir)
)
Philippe Pepiot
perf: add asv benchmarks...
r30406 reposdir = os.environ['REPOS_DIR']
Augie Fackler
formatting: blacken the codebase...
r43346 reposnames = [
name
for name in os.listdir(reposdir)
if os.path.isdir(os.path.join(reposdir, name, ".hg"))
]
Philippe Pepiot
perf: add asv benchmarks...
r30406 if not reposnames:
raise ValueError("No repositories found in $REPO_DIR")
Augie Fackler
formatting: blacken the codebase...
r43346 outputre = re.compile(
(
r'! wall (\d+.\d+) comb \d+.\d+ user \d+.\d+ sys '
r'\d+.\d+ \(best of \d+\)'
)
)
Philippe Pepiot
perf: add asv benchmarks...
r30406
def runperfcommand(reponame, command, *args, **kwargs):
os.environ["HGRCPATH"] = os.environ.get("ASVHGRCPATH", "")
Philippe Pepiot
perf: add historical support of ui.load()...
r30588 # for "historical portability"
# ui.load() has been available since d83ca85
if util.safehasattr(uimod.ui, "load"):
ui = uimod.ui.load()
else:
ui = uimod.ui()
Philippe Pepiot
perf: add asv benchmarks...
r30406 repo = hg.repository(ui, os.path.join(reposdir, reponame))
Augie Fackler
formatting: blacken the codebase...
r43346 perfext = extensions.load(
ui, 'perfext', os.path.join(basedir, 'contrib', 'perf.py')
)
Philippe Pepiot
perf: add asv benchmarks...
r30406 cmd = getattr(perfext, command)
ui.pushbuffer()
cmd(ui, repo, *args, **kwargs)
output = ui.popbuffer()
match = outputre.search(output)
if not match:
raise ValueError("Invalid output {0}".format(output))
return float(match.group(1))
Augie Fackler
formatting: blacken the codebase...
r43346
Philippe Pepiot
perf: add asv benchmarks...
r30406 def perfbench(repos=reposnames, name=None, params=None):
"""decorator to declare ASV benchmark based on contrib/perf.py extension
An ASV benchmark is a python function with the given attributes:
__name__: should start with track_, time_ or mem_ to be collected by ASV
params and param_name: parameter matrix to display multiple graphs on the
same page.
pretty_name: If defined it's displayed in web-ui instead of __name__
(useful for revsets)
the module name is prepended to the benchmark name and displayed as
"category" in webui.
Benchmarks are automatically parameterized with repositories found in the
REPOS_DIR environment variable.
`params` is the param matrix in the form of a list of tuple
(param_name, [value0, value1])
For example [(x, [a, b]), (y, [c, d])] declare benchmarks for
(a, c), (a, d), (b, c) and (b, d).
"""
params = list(params or [])
params.insert(0, ("repo", repos))
def decorator(func):
@functools.wraps(func)
def wrapped(repo, *args):
def perf(command, *a, **kw):
return runperfcommand(repo, command, *a, **kw)
Augie Fackler
formatting: blacken the codebase...
r43346
Philippe Pepiot
perf: add asv benchmarks...
r30406 return func(perf, *args)
wrapped.params = [p[1] for p in params]
wrapped.param_names = [p[0] for p in params]
wrapped.pretty_name = name
return wrapped
Augie Fackler
formatting: blacken the codebase...
r43346
Philippe Pepiot
perf: add asv benchmarks...
r30406 return decorator