##// END OF EJS Templates
discovery: slowly increase sampling size...
discovery: slowly increase sampling size Some pathological discovery runs can requires many roundtrip. When this happens things can get very slow. To make the algorithm more resilience again such pathological case. We slowly increase the sample size with each roundtrip (+5%). This will have a negligible impact on "normal" discovery with few roundtrips, but a large positive impact of case with many roundtrips. Asking more question per roundtrip helps to reduce the undecided set faster. Instead of reducing the undecided set a linear speed (in the worst case), we reduce it as a guaranteed (small) exponential rate. The data below show this slow ramp up in sample size: round trip | 1 | 5 | 10 | 20 | 50 | 100 | 130 | sample size | 200 | 254 | 321 | 517 | 2 199 | 25 123 | 108 549 | covered nodes | 200 | 1 357 | 2 821 | 7 031 | 42 658 | 524 530 | 2 276 755 | To be a bit more concrete, lets take a very pathological case as an example. We are doing discovery from a copy of Mozilla-try to a more recent version of mozilla-unified. Mozilla-unified heads are unknown to the mozilla-try repo and there are over 1 million "missing" changesets. (the discovery is "local" to avoid network interference) Without this change, the discovery: - last 1858 seconds (31 minutes), - does 1700 round trip, - asking about 340 000 nodes. With this change, the discovery: - last 218 seconds (3 minutes, 38 seconds a -88% improvement), - does 94 round trip (-94%), - asking about 344 211 nodes (+1%). Of course, this is an extreme case (and 3 minutes is still slow). However this give a good example of how this sample size increase act as a safety net catching any bad situations. We could image a steeper increase than 5%. For example 10% would give the following number: round trip | 1 | 5 | 10 | 20 | 50 | 75 | 100 | sample size | 200 | 321 | 514 | 1 326 | 23 060 | 249 812 | 2 706 594 | covered nodes | 200 | 1 541 | 3 690 | 12 671 | 251 871 | 2 746 254 | 29 770 966 | In parallel, it is useful to understand these pathological cases and improve them. However the current change provides a general purpose safety net to smooth the impact of pathological cases. To avoid issue with older http server, the increase in sample size only occurs if the protocol has not limit on command argument size.

File last commit:

r38505:4c068365 @58 default
r42546:dbd0fcca default
Show More
namespaces.py
214 lines | 8.0 KiB | text/x-python | PythonLexer
from __future__ import absolute_import
from .i18n import _
from . import (
registrar,
templatekw,
util,
)
def tolist(val):
"""
a convenience method to return an empty list instead of None
"""
if val is None:
return []
else:
return [val]
class namespaces(object):
"""provides an interface to register and operate on multiple namespaces. See
the namespace class below for details on the namespace object.
"""
_names_version = 0
def __init__(self):
self._names = util.sortdict()
columns = templatekw.getlogcolumns()
# we need current mercurial named objects (bookmarks, tags, and
# branches) to be initialized somewhere, so that place is here
bmknames = lambda repo: repo._bookmarks.keys()
bmknamemap = lambda repo, name: tolist(repo._bookmarks.get(name))
bmknodemap = lambda repo, node: repo.nodebookmarks(node)
n = namespace("bookmarks", templatename="bookmark",
logfmt=columns['bookmark'],
listnames=bmknames,
namemap=bmknamemap, nodemap=bmknodemap,
builtin=True)
self.addnamespace(n)
tagnames = lambda repo: [t for t, n in repo.tagslist()]
tagnamemap = lambda repo, name: tolist(repo._tagscache.tags.get(name))
tagnodemap = lambda repo, node: repo.nodetags(node)
n = namespace("tags", templatename="tag",
logfmt=columns['tag'],
listnames=tagnames,
namemap=tagnamemap, nodemap=tagnodemap,
deprecated={'tip'},
builtin=True)
self.addnamespace(n)
bnames = lambda repo: repo.branchmap().keys()
bnamemap = lambda repo, name: tolist(repo.branchtip(name, True))
bnodemap = lambda repo, node: [repo[node].branch()]
n = namespace("branches", templatename="branch",
logfmt=columns['branch'],
listnames=bnames,
namemap=bnamemap, nodemap=bnodemap,
builtin=True)
self.addnamespace(n)
def __getitem__(self, namespace):
"""returns the namespace object"""
return self._names[namespace]
def __iter__(self):
return self._names.__iter__()
def items(self):
return self._names.iteritems()
iteritems = items
def addnamespace(self, namespace, order=None):
"""register a namespace
namespace: the name to be registered (in plural form)
order: optional argument to specify the order of namespaces
(e.g. 'branches' should be listed before 'bookmarks')
"""
if order is not None:
self._names.insert(order, namespace.name, namespace)
else:
self._names[namespace.name] = namespace
# we only generate a template keyword if one does not already exist
if namespace.name not in templatekw.keywords:
templatekeyword = registrar.templatekeyword(templatekw.keywords)
@templatekeyword(namespace.name, requires={'repo', 'ctx'})
def generatekw(context, mapping):
return templatekw.shownames(context, mapping, namespace.name)
def singlenode(self, repo, name):
"""
Return the 'best' node for the given name. What's best is defined
by the namespace's singlenode() function. The first match returned by
a namespace in the defined precedence order is used.
Raises a KeyError if there is no such node.
"""
for ns, v in self._names.iteritems():
n = v.singlenode(repo, name)
if n:
return n
raise KeyError(_('no such name: %s') % name)
class namespace(object):
"""provides an interface to a namespace
Namespaces are basically generic many-to-many mapping between some
(namespaced) names and nodes. The goal here is to control the pollution of
jamming things into tags or bookmarks (in extension-land) and to simplify
internal bits of mercurial: log output, tab completion, etc.
More precisely, we define a mapping of names to nodes, and a mapping from
nodes to names. Each mapping returns a list.
Furthermore, each name mapping will be passed a name to lookup which might
not be in its domain. In this case, each method should return an empty list
and not raise an error.
This namespace object will define the properties we need:
'name': the namespace (plural form)
'templatename': name to use for templating (usually the singular form
of the plural namespace name)
'listnames': list of all names in the namespace (usually the keys of a
dictionary)
'namemap': function that takes a name and returns a list of nodes
'nodemap': function that takes a node and returns a list of names
'deprecated': set of names to be masked for ordinary use
'builtin': bool indicating if this namespace is supported by core
Mercurial.
"""
def __init__(self, name, templatename=None, logname=None, colorname=None,
logfmt=None, listnames=None, namemap=None, nodemap=None,
deprecated=None, builtin=False, singlenode=None):
"""create a namespace
name: the namespace to be registered (in plural form)
templatename: the name to use for templating
logname: the name to use for log output; if not specified templatename
is used
colorname: the name to use for colored log output; if not specified
logname is used
logfmt: the format to use for (i18n-ed) log output; if not specified
it is composed from logname
listnames: function to list all names
namemap: function that inputs a name, output node(s)
nodemap: function that inputs a node, output name(s)
deprecated: set of names to be masked for ordinary use
builtin: whether namespace is implemented by core Mercurial
singlenode: function that inputs a name, output best node (or None)
"""
self.name = name
self.templatename = templatename
self.logname = logname
self.colorname = colorname
self.logfmt = logfmt
self.listnames = listnames
self.namemap = namemap
self.nodemap = nodemap
if singlenode:
self.singlenode = singlenode
# if logname is not specified, use the template name as backup
if self.logname is None:
self.logname = self.templatename
# if colorname is not specified, just use the logname as a backup
if self.colorname is None:
self.colorname = self.logname
# if logfmt is not specified, compose it from logname as backup
if self.logfmt is None:
# i18n: column positioning for "hg log"
self.logfmt = ("%s:" % self.logname).ljust(13) + "%s\n"
if deprecated is None:
self.deprecated = set()
else:
self.deprecated = deprecated
self.builtin = builtin
def names(self, repo, node):
"""method that returns a (sorted) list of names in a namespace that
match a given node"""
return sorted(self.nodemap(repo, node))
def nodes(self, repo, name):
"""method that returns a list of nodes in a namespace that
match a given name.
"""
return sorted(self.namemap(repo, name))
def singlenode(self, repo, name):
"""returns the best node for the given name
By default, the best node is the node from nodes() with the highest
revision number. It can be overriden by the namespace."""
n = self.namemap(repo, name)
if n:
# return max revision number
if len(n) > 1:
cl = repo.changelog
maxrev = max(cl.rev(node) for node in n)
return cl.node(maxrev)
return n[0]
return None