##// END OF EJS Templates
branchmap: make branchcache responsible for reading...
Martijn Pieters -
r41706:bf7fb97a default
parent child Browse files
Show More
@@ -1,2761 +1,2772 b''
1 1 # perf.py - performance test routines
2 2 '''helper extension to measure performance'''
3 3
4 4 # "historical portability" policy of perf.py:
5 5 #
6 6 # We have to do:
7 7 # - make perf.py "loadable" with as wide Mercurial version as possible
8 8 # This doesn't mean that perf commands work correctly with that Mercurial.
9 9 # BTW, perf.py itself has been available since 1.1 (or eb240755386d).
10 10 # - make historical perf command work correctly with as wide Mercurial
11 11 # version as possible
12 12 #
13 13 # We have to do, if possible with reasonable cost:
14 14 # - make recent perf command for historical feature work correctly
15 15 # with early Mercurial
16 16 #
17 17 # We don't have to do:
18 18 # - make perf command for recent feature work correctly with early
19 19 # Mercurial
20 20
21 21 from __future__ import absolute_import
22 22 import contextlib
23 23 import functools
24 24 import gc
25 25 import os
26 26 import random
27 27 import shutil
28 28 import struct
29 29 import sys
30 30 import tempfile
31 31 import threading
32 32 import time
33 33 from mercurial import (
34 34 changegroup,
35 35 cmdutil,
36 36 commands,
37 37 copies,
38 38 error,
39 39 extensions,
40 40 hg,
41 41 mdiff,
42 42 merge,
43 43 revlog,
44 44 util,
45 45 )
46 46
47 47 # for "historical portability":
48 48 # try to import modules separately (in dict order), and ignore
49 49 # failure, because these aren't available with early Mercurial
50 50 try:
51 51 from mercurial import branchmap # since 2.5 (or bcee63733aad)
52 52 except ImportError:
53 53 pass
54 54 try:
55 55 from mercurial import obsolete # since 2.3 (or ad0d6c2b3279)
56 56 except ImportError:
57 57 pass
58 58 try:
59 59 from mercurial import registrar # since 3.7 (or 37d50250b696)
60 60 dir(registrar) # forcibly load it
61 61 except ImportError:
62 62 registrar = None
63 63 try:
64 64 from mercurial import repoview # since 2.5 (or 3a6ddacb7198)
65 65 except ImportError:
66 66 pass
67 67 try:
68 68 from mercurial import scmutil # since 1.9 (or 8b252e826c68)
69 69 except ImportError:
70 70 pass
71 71 try:
72 72 from mercurial import setdiscovery # since 1.9 (or cb98fed52495)
73 73 except ImportError:
74 74 pass
75 75
76 76
77 77 def identity(a):
78 78 return a
79 79
80 80 try:
81 81 from mercurial import pycompat
82 82 getargspec = pycompat.getargspec # added to module after 4.5
83 83 _byteskwargs = pycompat.byteskwargs # since 4.1 (or fbc3f73dc802)
84 84 _sysstr = pycompat.sysstr # since 4.0 (or 2219f4f82ede)
85 85 _xrange = pycompat.xrange # since 4.8 (or 7eba8f83129b)
86 86 fsencode = pycompat.fsencode # since 3.9 (or f4a5e0e86a7e)
87 87 if pycompat.ispy3:
88 88 _maxint = sys.maxsize # per py3 docs for replacing maxint
89 89 else:
90 90 _maxint = sys.maxint
91 91 except (ImportError, AttributeError):
92 92 import inspect
93 93 getargspec = inspect.getargspec
94 94 _byteskwargs = identity
95 95 fsencode = identity # no py3 support
96 96 _maxint = sys.maxint # no py3 support
97 97 _sysstr = lambda x: x # no py3 support
98 98 _xrange = xrange
99 99
100 100 try:
101 101 # 4.7+
102 102 queue = pycompat.queue.Queue
103 103 except (AttributeError, ImportError):
104 104 # <4.7.
105 105 try:
106 106 queue = pycompat.queue
107 107 except (AttributeError, ImportError):
108 108 queue = util.queue
109 109
110 110 try:
111 111 from mercurial import logcmdutil
112 112 makelogtemplater = logcmdutil.maketemplater
113 113 except (AttributeError, ImportError):
114 114 try:
115 115 makelogtemplater = cmdutil.makelogtemplater
116 116 except (AttributeError, ImportError):
117 117 makelogtemplater = None
118 118
119 119 # for "historical portability":
120 120 # define util.safehasattr forcibly, because util.safehasattr has been
121 121 # available since 1.9.3 (or 94b200a11cf7)
122 122 _undefined = object()
123 123 def safehasattr(thing, attr):
124 124 return getattr(thing, _sysstr(attr), _undefined) is not _undefined
125 125 setattr(util, 'safehasattr', safehasattr)
126 126
127 127 # for "historical portability":
128 128 # define util.timer forcibly, because util.timer has been available
129 129 # since ae5d60bb70c9
130 130 if safehasattr(time, 'perf_counter'):
131 131 util.timer = time.perf_counter
132 132 elif os.name == b'nt':
133 133 util.timer = time.clock
134 134 else:
135 135 util.timer = time.time
136 136
137 137 # for "historical portability":
138 138 # use locally defined empty option list, if formatteropts isn't
139 139 # available, because commands.formatteropts has been available since
140 140 # 3.2 (or 7a7eed5176a4), even though formatting itself has been
141 141 # available since 2.2 (or ae5f92e154d3)
142 142 formatteropts = getattr(cmdutil, "formatteropts",
143 143 getattr(commands, "formatteropts", []))
144 144
145 145 # for "historical portability":
146 146 # use locally defined option list, if debugrevlogopts isn't available,
147 147 # because commands.debugrevlogopts has been available since 3.7 (or
148 148 # 5606f7d0d063), even though cmdutil.openrevlog() has been available
149 149 # since 1.9 (or a79fea6b3e77).
150 150 revlogopts = getattr(cmdutil, "debugrevlogopts",
151 151 getattr(commands, "debugrevlogopts", [
152 152 (b'c', b'changelog', False, (b'open changelog')),
153 153 (b'm', b'manifest', False, (b'open manifest')),
154 154 (b'', b'dir', False, (b'open directory manifest')),
155 155 ]))
156 156
157 157 cmdtable = {}
158 158
159 159 # for "historical portability":
160 160 # define parsealiases locally, because cmdutil.parsealiases has been
161 161 # available since 1.5 (or 6252852b4332)
162 162 def parsealiases(cmd):
163 163 return cmd.split(b"|")
164 164
165 165 if safehasattr(registrar, 'command'):
166 166 command = registrar.command(cmdtable)
167 167 elif safehasattr(cmdutil, 'command'):
168 168 command = cmdutil.command(cmdtable)
169 169 if b'norepo' not in getargspec(command).args:
170 170 # for "historical portability":
171 171 # wrap original cmdutil.command, because "norepo" option has
172 172 # been available since 3.1 (or 75a96326cecb)
173 173 _command = command
174 174 def command(name, options=(), synopsis=None, norepo=False):
175 175 if norepo:
176 176 commands.norepo += b' %s' % b' '.join(parsealiases(name))
177 177 return _command(name, list(options), synopsis)
178 178 else:
179 179 # for "historical portability":
180 180 # define "@command" annotation locally, because cmdutil.command
181 181 # has been available since 1.9 (or 2daa5179e73f)
182 182 def command(name, options=(), synopsis=None, norepo=False):
183 183 def decorator(func):
184 184 if synopsis:
185 185 cmdtable[name] = func, list(options), synopsis
186 186 else:
187 187 cmdtable[name] = func, list(options)
188 188 if norepo:
189 189 commands.norepo += b' %s' % b' '.join(parsealiases(name))
190 190 return func
191 191 return decorator
192 192
193 193 try:
194 194 import mercurial.registrar
195 195 import mercurial.configitems
196 196 configtable = {}
197 197 configitem = mercurial.registrar.configitem(configtable)
198 198 configitem(b'perf', b'presleep',
199 199 default=mercurial.configitems.dynamicdefault,
200 200 )
201 201 configitem(b'perf', b'stub',
202 202 default=mercurial.configitems.dynamicdefault,
203 203 )
204 204 configitem(b'perf', b'parentscount',
205 205 default=mercurial.configitems.dynamicdefault,
206 206 )
207 207 configitem(b'perf', b'all-timing',
208 208 default=mercurial.configitems.dynamicdefault,
209 209 )
210 210 except (ImportError, AttributeError):
211 211 pass
212 212
213 213 def getlen(ui):
214 214 if ui.configbool(b"perf", b"stub", False):
215 215 return lambda x: 1
216 216 return len
217 217
218 218 def gettimer(ui, opts=None):
219 219 """return a timer function and formatter: (timer, formatter)
220 220
221 221 This function exists to gather the creation of formatter in a single
222 222 place instead of duplicating it in all performance commands."""
223 223
224 224 # enforce an idle period before execution to counteract power management
225 225 # experimental config: perf.presleep
226 226 time.sleep(getint(ui, b"perf", b"presleep", 1))
227 227
228 228 if opts is None:
229 229 opts = {}
230 230 # redirect all to stderr unless buffer api is in use
231 231 if not ui._buffers:
232 232 ui = ui.copy()
233 233 uifout = safeattrsetter(ui, b'fout', ignoremissing=True)
234 234 if uifout:
235 235 # for "historical portability":
236 236 # ui.fout/ferr have been available since 1.9 (or 4e1ccd4c2b6d)
237 237 uifout.set(ui.ferr)
238 238
239 239 # get a formatter
240 240 uiformatter = getattr(ui, 'formatter', None)
241 241 if uiformatter:
242 242 fm = uiformatter(b'perf', opts)
243 243 else:
244 244 # for "historical portability":
245 245 # define formatter locally, because ui.formatter has been
246 246 # available since 2.2 (or ae5f92e154d3)
247 247 from mercurial import node
248 248 class defaultformatter(object):
249 249 """Minimized composition of baseformatter and plainformatter
250 250 """
251 251 def __init__(self, ui, topic, opts):
252 252 self._ui = ui
253 253 if ui.debugflag:
254 254 self.hexfunc = node.hex
255 255 else:
256 256 self.hexfunc = node.short
257 257 def __nonzero__(self):
258 258 return False
259 259 __bool__ = __nonzero__
260 260 def startitem(self):
261 261 pass
262 262 def data(self, **data):
263 263 pass
264 264 def write(self, fields, deftext, *fielddata, **opts):
265 265 self._ui.write(deftext % fielddata, **opts)
266 266 def condwrite(self, cond, fields, deftext, *fielddata, **opts):
267 267 if cond:
268 268 self._ui.write(deftext % fielddata, **opts)
269 269 def plain(self, text, **opts):
270 270 self._ui.write(text, **opts)
271 271 def end(self):
272 272 pass
273 273 fm = defaultformatter(ui, b'perf', opts)
274 274
275 275 # stub function, runs code only once instead of in a loop
276 276 # experimental config: perf.stub
277 277 if ui.configbool(b"perf", b"stub", False):
278 278 return functools.partial(stub_timer, fm), fm
279 279
280 280 # experimental config: perf.all-timing
281 281 displayall = ui.configbool(b"perf", b"all-timing", False)
282 282 return functools.partial(_timer, fm, displayall=displayall), fm
283 283
284 284 def stub_timer(fm, func, setup=None, title=None):
285 285 if setup is not None:
286 286 setup()
287 287 func()
288 288
289 289 @contextlib.contextmanager
290 290 def timeone():
291 291 r = []
292 292 ostart = os.times()
293 293 cstart = util.timer()
294 294 yield r
295 295 cstop = util.timer()
296 296 ostop = os.times()
297 297 a, b = ostart, ostop
298 298 r.append((cstop - cstart, b[0] - a[0], b[1]-a[1]))
299 299
300 300 def _timer(fm, func, setup=None, title=None, displayall=False):
301 301 gc.collect()
302 302 results = []
303 303 begin = util.timer()
304 304 count = 0
305 305 while True:
306 306 if setup is not None:
307 307 setup()
308 308 with timeone() as item:
309 309 r = func()
310 310 count += 1
311 311 results.append(item[0])
312 312 cstop = util.timer()
313 313 if cstop - begin > 3 and count >= 100:
314 314 break
315 315 if cstop - begin > 10 and count >= 3:
316 316 break
317 317
318 318 formatone(fm, results, title=title, result=r,
319 319 displayall=displayall)
320 320
321 321 def formatone(fm, timings, title=None, result=None, displayall=False):
322 322
323 323 count = len(timings)
324 324
325 325 fm.startitem()
326 326
327 327 if title:
328 328 fm.write(b'title', b'! %s\n', title)
329 329 if result:
330 330 fm.write(b'result', b'! result: %s\n', result)
331 331 def display(role, entry):
332 332 prefix = b''
333 333 if role != b'best':
334 334 prefix = b'%s.' % role
335 335 fm.plain(b'!')
336 336 fm.write(prefix + b'wall', b' wall %f', entry[0])
337 337 fm.write(prefix + b'comb', b' comb %f', entry[1] + entry[2])
338 338 fm.write(prefix + b'user', b' user %f', entry[1])
339 339 fm.write(prefix + b'sys', b' sys %f', entry[2])
340 340 fm.write(prefix + b'count', b' (%s of %%d)' % role, count)
341 341 fm.plain(b'\n')
342 342 timings.sort()
343 343 min_val = timings[0]
344 344 display(b'best', min_val)
345 345 if displayall:
346 346 max_val = timings[-1]
347 347 display(b'max', max_val)
348 348 avg = tuple([sum(x) / count for x in zip(*timings)])
349 349 display(b'avg', avg)
350 350 median = timings[len(timings) // 2]
351 351 display(b'median', median)
352 352
353 353 # utilities for historical portability
354 354
355 355 def getint(ui, section, name, default):
356 356 # for "historical portability":
357 357 # ui.configint has been available since 1.9 (or fa2b596db182)
358 358 v = ui.config(section, name, None)
359 359 if v is None:
360 360 return default
361 361 try:
362 362 return int(v)
363 363 except ValueError:
364 364 raise error.ConfigError((b"%s.%s is not an integer ('%s')")
365 365 % (section, name, v))
366 366
367 367 def safeattrsetter(obj, name, ignoremissing=False):
368 368 """Ensure that 'obj' has 'name' attribute before subsequent setattr
369 369
370 370 This function is aborted, if 'obj' doesn't have 'name' attribute
371 371 at runtime. This avoids overlooking removal of an attribute, which
372 372 breaks assumption of performance measurement, in the future.
373 373
374 374 This function returns the object to (1) assign a new value, and
375 375 (2) restore an original value to the attribute.
376 376
377 377 If 'ignoremissing' is true, missing 'name' attribute doesn't cause
378 378 abortion, and this function returns None. This is useful to
379 379 examine an attribute, which isn't ensured in all Mercurial
380 380 versions.
381 381 """
382 382 if not util.safehasattr(obj, name):
383 383 if ignoremissing:
384 384 return None
385 385 raise error.Abort((b"missing attribute %s of %s might break assumption"
386 386 b" of performance measurement") % (name, obj))
387 387
388 388 origvalue = getattr(obj, _sysstr(name))
389 389 class attrutil(object):
390 390 def set(self, newvalue):
391 391 setattr(obj, _sysstr(name), newvalue)
392 392 def restore(self):
393 393 setattr(obj, _sysstr(name), origvalue)
394 394
395 395 return attrutil()
396 396
397 397 # utilities to examine each internal API changes
398 398
399 399 def getbranchmapsubsettable():
400 400 # for "historical portability":
401 401 # subsettable is defined in:
402 402 # - branchmap since 2.9 (or 175c6fd8cacc)
403 403 # - repoview since 2.5 (or 59a9f18d4587)
404 404 for mod in (branchmap, repoview):
405 405 subsettable = getattr(mod, 'subsettable', None)
406 406 if subsettable:
407 407 return subsettable
408 408
409 409 # bisecting in bcee63733aad::59a9f18d4587 can reach here (both
410 410 # branchmap and repoview modules exist, but subsettable attribute
411 411 # doesn't)
412 412 raise error.Abort((b"perfbranchmap not available with this Mercurial"),
413 413 hint=b"use 2.5 or later")
414 414
415 415 def getsvfs(repo):
416 416 """Return appropriate object to access files under .hg/store
417 417 """
418 418 # for "historical portability":
419 419 # repo.svfs has been available since 2.3 (or 7034365089bf)
420 420 svfs = getattr(repo, 'svfs', None)
421 421 if svfs:
422 422 return svfs
423 423 else:
424 424 return getattr(repo, 'sopener')
425 425
426 426 def getvfs(repo):
427 427 """Return appropriate object to access files under .hg
428 428 """
429 429 # for "historical portability":
430 430 # repo.vfs has been available since 2.3 (or 7034365089bf)
431 431 vfs = getattr(repo, 'vfs', None)
432 432 if vfs:
433 433 return vfs
434 434 else:
435 435 return getattr(repo, 'opener')
436 436
437 437 def repocleartagscachefunc(repo):
438 438 """Return the function to clear tags cache according to repo internal API
439 439 """
440 440 if util.safehasattr(repo, b'_tagscache'): # since 2.0 (or 9dca7653b525)
441 441 # in this case, setattr(repo, '_tagscache', None) or so isn't
442 442 # correct way to clear tags cache, because existing code paths
443 443 # expect _tagscache to be a structured object.
444 444 def clearcache():
445 445 # _tagscache has been filteredpropertycache since 2.5 (or
446 446 # 98c867ac1330), and delattr() can't work in such case
447 447 if b'_tagscache' in vars(repo):
448 448 del repo.__dict__[b'_tagscache']
449 449 return clearcache
450 450
451 451 repotags = safeattrsetter(repo, b'_tags', ignoremissing=True)
452 452 if repotags: # since 1.4 (or 5614a628d173)
453 453 return lambda : repotags.set(None)
454 454
455 455 repotagscache = safeattrsetter(repo, b'tagscache', ignoremissing=True)
456 456 if repotagscache: # since 0.6 (or d7df759d0e97)
457 457 return lambda : repotagscache.set(None)
458 458
459 459 # Mercurial earlier than 0.6 (or d7df759d0e97) logically reaches
460 460 # this point, but it isn't so problematic, because:
461 461 # - repo.tags of such Mercurial isn't "callable", and repo.tags()
462 462 # in perftags() causes failure soon
463 463 # - perf.py itself has been available since 1.1 (or eb240755386d)
464 464 raise error.Abort((b"tags API of this hg command is unknown"))
465 465
466 466 # utilities to clear cache
467 467
468 468 def clearfilecache(obj, attrname):
469 469 unfiltered = getattr(obj, 'unfiltered', None)
470 470 if unfiltered is not None:
471 471 obj = obj.unfiltered()
472 472 if attrname in vars(obj):
473 473 delattr(obj, attrname)
474 474 obj._filecache.pop(attrname, None)
475 475
476 476 def clearchangelog(repo):
477 477 if repo is not repo.unfiltered():
478 478 object.__setattr__(repo, r'_clcachekey', None)
479 479 object.__setattr__(repo, r'_clcache', None)
480 480 clearfilecache(repo.unfiltered(), 'changelog')
481 481
482 482 # perf commands
483 483
484 484 @command(b'perfwalk', formatteropts)
485 485 def perfwalk(ui, repo, *pats, **opts):
486 486 opts = _byteskwargs(opts)
487 487 timer, fm = gettimer(ui, opts)
488 488 m = scmutil.match(repo[None], pats, {})
489 489 timer(lambda: len(list(repo.dirstate.walk(m, subrepos=[], unknown=True,
490 490 ignored=False))))
491 491 fm.end()
492 492
493 493 @command(b'perfannotate', formatteropts)
494 494 def perfannotate(ui, repo, f, **opts):
495 495 opts = _byteskwargs(opts)
496 496 timer, fm = gettimer(ui, opts)
497 497 fc = repo[b'.'][f]
498 498 timer(lambda: len(fc.annotate(True)))
499 499 fm.end()
500 500
501 501 @command(b'perfstatus',
502 502 [(b'u', b'unknown', False,
503 503 b'ask status to look for unknown files')] + formatteropts)
504 504 def perfstatus(ui, repo, **opts):
505 505 opts = _byteskwargs(opts)
506 506 #m = match.always(repo.root, repo.getcwd())
507 507 #timer(lambda: sum(map(len, repo.dirstate.status(m, [], False, False,
508 508 # False))))
509 509 timer, fm = gettimer(ui, opts)
510 510 timer(lambda: sum(map(len, repo.status(unknown=opts[b'unknown']))))
511 511 fm.end()
512 512
513 513 @command(b'perfaddremove', formatteropts)
514 514 def perfaddremove(ui, repo, **opts):
515 515 opts = _byteskwargs(opts)
516 516 timer, fm = gettimer(ui, opts)
517 517 try:
518 518 oldquiet = repo.ui.quiet
519 519 repo.ui.quiet = True
520 520 matcher = scmutil.match(repo[None])
521 521 opts[b'dry_run'] = True
522 522 timer(lambda: scmutil.addremove(repo, matcher, b"", opts))
523 523 finally:
524 524 repo.ui.quiet = oldquiet
525 525 fm.end()
526 526
527 527 def clearcaches(cl):
528 528 # behave somewhat consistently across internal API changes
529 529 if util.safehasattr(cl, b'clearcaches'):
530 530 cl.clearcaches()
531 531 elif util.safehasattr(cl, b'_nodecache'):
532 532 from mercurial.node import nullid, nullrev
533 533 cl._nodecache = {nullid: nullrev}
534 534 cl._nodepos = None
535 535
536 536 @command(b'perfheads', formatteropts)
537 537 def perfheads(ui, repo, **opts):
538 538 """benchmark the computation of a changelog heads"""
539 539 opts = _byteskwargs(opts)
540 540 timer, fm = gettimer(ui, opts)
541 541 cl = repo.changelog
542 542 def s():
543 543 clearcaches(cl)
544 544 def d():
545 545 len(cl.headrevs())
546 546 timer(d, setup=s)
547 547 fm.end()
548 548
549 549 @command(b'perftags', formatteropts+
550 550 [
551 551 (b'', b'clear-revlogs', False, b'refresh changelog and manifest'),
552 552 ])
553 553 def perftags(ui, repo, **opts):
554 554 opts = _byteskwargs(opts)
555 555 timer, fm = gettimer(ui, opts)
556 556 repocleartagscache = repocleartagscachefunc(repo)
557 557 clearrevlogs = opts[b'clear_revlogs']
558 558 def s():
559 559 if clearrevlogs:
560 560 clearchangelog(repo)
561 561 clearfilecache(repo.unfiltered(), 'manifest')
562 562 repocleartagscache()
563 563 def t():
564 564 return len(repo.tags())
565 565 timer(t, setup=s)
566 566 fm.end()
567 567
568 568 @command(b'perfancestors', formatteropts)
569 569 def perfancestors(ui, repo, **opts):
570 570 opts = _byteskwargs(opts)
571 571 timer, fm = gettimer(ui, opts)
572 572 heads = repo.changelog.headrevs()
573 573 def d():
574 574 for a in repo.changelog.ancestors(heads):
575 575 pass
576 576 timer(d)
577 577 fm.end()
578 578
579 579 @command(b'perfancestorset', formatteropts)
580 580 def perfancestorset(ui, repo, revset, **opts):
581 581 opts = _byteskwargs(opts)
582 582 timer, fm = gettimer(ui, opts)
583 583 revs = repo.revs(revset)
584 584 heads = repo.changelog.headrevs()
585 585 def d():
586 586 s = repo.changelog.ancestors(heads)
587 587 for rev in revs:
588 588 rev in s
589 589 timer(d)
590 590 fm.end()
591 591
592 592 @command(b'perfdiscovery', formatteropts, b'PATH')
593 593 def perfdiscovery(ui, repo, path, **opts):
594 594 """benchmark discovery between local repo and the peer at given path
595 595 """
596 596 repos = [repo, None]
597 597 timer, fm = gettimer(ui, opts)
598 598 path = ui.expandpath(path)
599 599
600 600 def s():
601 601 repos[1] = hg.peer(ui, opts, path)
602 602 def d():
603 603 setdiscovery.findcommonheads(ui, *repos)
604 604 timer(d, setup=s)
605 605 fm.end()
606 606
607 607 @command(b'perfbookmarks', formatteropts +
608 608 [
609 609 (b'', b'clear-revlogs', False, b'refresh changelog and manifest'),
610 610 ])
611 611 def perfbookmarks(ui, repo, **opts):
612 612 """benchmark parsing bookmarks from disk to memory"""
613 613 opts = _byteskwargs(opts)
614 614 timer, fm = gettimer(ui, opts)
615 615
616 616 clearrevlogs = opts[b'clear_revlogs']
617 617 def s():
618 618 if clearrevlogs:
619 619 clearchangelog(repo)
620 620 clearfilecache(repo, b'_bookmarks')
621 621 def d():
622 622 repo._bookmarks
623 623 timer(d, setup=s)
624 624 fm.end()
625 625
626 626 @command(b'perfbundleread', formatteropts, b'BUNDLE')
627 627 def perfbundleread(ui, repo, bundlepath, **opts):
628 628 """Benchmark reading of bundle files.
629 629
630 630 This command is meant to isolate the I/O part of bundle reading as
631 631 much as possible.
632 632 """
633 633 from mercurial import (
634 634 bundle2,
635 635 exchange,
636 636 streamclone,
637 637 )
638 638
639 639 opts = _byteskwargs(opts)
640 640
641 641 def makebench(fn):
642 642 def run():
643 643 with open(bundlepath, b'rb') as fh:
644 644 bundle = exchange.readbundle(ui, fh, bundlepath)
645 645 fn(bundle)
646 646
647 647 return run
648 648
649 649 def makereadnbytes(size):
650 650 def run():
651 651 with open(bundlepath, b'rb') as fh:
652 652 bundle = exchange.readbundle(ui, fh, bundlepath)
653 653 while bundle.read(size):
654 654 pass
655 655
656 656 return run
657 657
658 658 def makestdioread(size):
659 659 def run():
660 660 with open(bundlepath, b'rb') as fh:
661 661 while fh.read(size):
662 662 pass
663 663
664 664 return run
665 665
666 666 # bundle1
667 667
668 668 def deltaiter(bundle):
669 669 for delta in bundle.deltaiter():
670 670 pass
671 671
672 672 def iterchunks(bundle):
673 673 for chunk in bundle.getchunks():
674 674 pass
675 675
676 676 # bundle2
677 677
678 678 def forwardchunks(bundle):
679 679 for chunk in bundle._forwardchunks():
680 680 pass
681 681
682 682 def iterparts(bundle):
683 683 for part in bundle.iterparts():
684 684 pass
685 685
686 686 def iterpartsseekable(bundle):
687 687 for part in bundle.iterparts(seekable=True):
688 688 pass
689 689
690 690 def seek(bundle):
691 691 for part in bundle.iterparts(seekable=True):
692 692 part.seek(0, os.SEEK_END)
693 693
694 694 def makepartreadnbytes(size):
695 695 def run():
696 696 with open(bundlepath, b'rb') as fh:
697 697 bundle = exchange.readbundle(ui, fh, bundlepath)
698 698 for part in bundle.iterparts():
699 699 while part.read(size):
700 700 pass
701 701
702 702 return run
703 703
704 704 benches = [
705 705 (makestdioread(8192), b'read(8k)'),
706 706 (makestdioread(16384), b'read(16k)'),
707 707 (makestdioread(32768), b'read(32k)'),
708 708 (makestdioread(131072), b'read(128k)'),
709 709 ]
710 710
711 711 with open(bundlepath, b'rb') as fh:
712 712 bundle = exchange.readbundle(ui, fh, bundlepath)
713 713
714 714 if isinstance(bundle, changegroup.cg1unpacker):
715 715 benches.extend([
716 716 (makebench(deltaiter), b'cg1 deltaiter()'),
717 717 (makebench(iterchunks), b'cg1 getchunks()'),
718 718 (makereadnbytes(8192), b'cg1 read(8k)'),
719 719 (makereadnbytes(16384), b'cg1 read(16k)'),
720 720 (makereadnbytes(32768), b'cg1 read(32k)'),
721 721 (makereadnbytes(131072), b'cg1 read(128k)'),
722 722 ])
723 723 elif isinstance(bundle, bundle2.unbundle20):
724 724 benches.extend([
725 725 (makebench(forwardchunks), b'bundle2 forwardchunks()'),
726 726 (makebench(iterparts), b'bundle2 iterparts()'),
727 727 (makebench(iterpartsseekable), b'bundle2 iterparts() seekable'),
728 728 (makebench(seek), b'bundle2 part seek()'),
729 729 (makepartreadnbytes(8192), b'bundle2 part read(8k)'),
730 730 (makepartreadnbytes(16384), b'bundle2 part read(16k)'),
731 731 (makepartreadnbytes(32768), b'bundle2 part read(32k)'),
732 732 (makepartreadnbytes(131072), b'bundle2 part read(128k)'),
733 733 ])
734 734 elif isinstance(bundle, streamclone.streamcloneapplier):
735 735 raise error.Abort(b'stream clone bundles not supported')
736 736 else:
737 737 raise error.Abort(b'unhandled bundle type: %s' % type(bundle))
738 738
739 739 for fn, title in benches:
740 740 timer, fm = gettimer(ui, opts)
741 741 timer(fn, title=title)
742 742 fm.end()
743 743
744 744 @command(b'perfchangegroupchangelog', formatteropts +
745 745 [(b'', b'cgversion', b'02', b'changegroup version'),
746 746 (b'r', b'rev', b'', b'revisions to add to changegroup')])
747 747 def perfchangegroupchangelog(ui, repo, cgversion=b'02', rev=None, **opts):
748 748 """Benchmark producing a changelog group for a changegroup.
749 749
750 750 This measures the time spent processing the changelog during a
751 751 bundle operation. This occurs during `hg bundle` and on a server
752 752 processing a `getbundle` wire protocol request (handles clones
753 753 and pull requests).
754 754
755 755 By default, all revisions are added to the changegroup.
756 756 """
757 757 opts = _byteskwargs(opts)
758 758 cl = repo.changelog
759 759 nodes = [cl.lookup(r) for r in repo.revs(rev or b'all()')]
760 760 bundler = changegroup.getbundler(cgversion, repo)
761 761
762 762 def d():
763 763 state, chunks = bundler._generatechangelog(cl, nodes)
764 764 for chunk in chunks:
765 765 pass
766 766
767 767 timer, fm = gettimer(ui, opts)
768 768
769 769 # Terminal printing can interfere with timing. So disable it.
770 770 with ui.configoverride({(b'progress', b'disable'): True}):
771 771 timer(d)
772 772
773 773 fm.end()
774 774
775 775 @command(b'perfdirs', formatteropts)
776 776 def perfdirs(ui, repo, **opts):
777 777 opts = _byteskwargs(opts)
778 778 timer, fm = gettimer(ui, opts)
779 779 dirstate = repo.dirstate
780 780 b'a' in dirstate
781 781 def d():
782 782 dirstate.hasdir(b'a')
783 783 del dirstate._map._dirs
784 784 timer(d)
785 785 fm.end()
786 786
787 787 @command(b'perfdirstate', formatteropts)
788 788 def perfdirstate(ui, repo, **opts):
789 789 opts = _byteskwargs(opts)
790 790 timer, fm = gettimer(ui, opts)
791 791 b"a" in repo.dirstate
792 792 def d():
793 793 repo.dirstate.invalidate()
794 794 b"a" in repo.dirstate
795 795 timer(d)
796 796 fm.end()
797 797
798 798 @command(b'perfdirstatedirs', formatteropts)
799 799 def perfdirstatedirs(ui, repo, **opts):
800 800 opts = _byteskwargs(opts)
801 801 timer, fm = gettimer(ui, opts)
802 802 b"a" in repo.dirstate
803 803 def d():
804 804 repo.dirstate.hasdir(b"a")
805 805 del repo.dirstate._map._dirs
806 806 timer(d)
807 807 fm.end()
808 808
809 809 @command(b'perfdirstatefoldmap', formatteropts)
810 810 def perfdirstatefoldmap(ui, repo, **opts):
811 811 opts = _byteskwargs(opts)
812 812 timer, fm = gettimer(ui, opts)
813 813 dirstate = repo.dirstate
814 814 b'a' in dirstate
815 815 def d():
816 816 dirstate._map.filefoldmap.get(b'a')
817 817 del dirstate._map.filefoldmap
818 818 timer(d)
819 819 fm.end()
820 820
821 821 @command(b'perfdirfoldmap', formatteropts)
822 822 def perfdirfoldmap(ui, repo, **opts):
823 823 opts = _byteskwargs(opts)
824 824 timer, fm = gettimer(ui, opts)
825 825 dirstate = repo.dirstate
826 826 b'a' in dirstate
827 827 def d():
828 828 dirstate._map.dirfoldmap.get(b'a')
829 829 del dirstate._map.dirfoldmap
830 830 del dirstate._map._dirs
831 831 timer(d)
832 832 fm.end()
833 833
834 834 @command(b'perfdirstatewrite', formatteropts)
835 835 def perfdirstatewrite(ui, repo, **opts):
836 836 opts = _byteskwargs(opts)
837 837 timer, fm = gettimer(ui, opts)
838 838 ds = repo.dirstate
839 839 b"a" in ds
840 840 def d():
841 841 ds._dirty = True
842 842 ds.write(repo.currenttransaction())
843 843 timer(d)
844 844 fm.end()
845 845
846 846 @command(b'perfmergecalculate',
847 847 [(b'r', b'rev', b'.', b'rev to merge against')] + formatteropts)
848 848 def perfmergecalculate(ui, repo, rev, **opts):
849 849 opts = _byteskwargs(opts)
850 850 timer, fm = gettimer(ui, opts)
851 851 wctx = repo[None]
852 852 rctx = scmutil.revsingle(repo, rev, rev)
853 853 ancestor = wctx.ancestor(rctx)
854 854 # we don't want working dir files to be stat'd in the benchmark, so prime
855 855 # that cache
856 856 wctx.dirty()
857 857 def d():
858 858 # acceptremote is True because we don't want prompts in the middle of
859 859 # our benchmark
860 860 merge.calculateupdates(repo, wctx, rctx, [ancestor], False, False,
861 861 acceptremote=True, followcopies=True)
862 862 timer(d)
863 863 fm.end()
864 864
865 865 @command(b'perfpathcopies', [], b"REV REV")
866 866 def perfpathcopies(ui, repo, rev1, rev2, **opts):
867 867 """benchmark the copy tracing logic"""
868 868 opts = _byteskwargs(opts)
869 869 timer, fm = gettimer(ui, opts)
870 870 ctx1 = scmutil.revsingle(repo, rev1, rev1)
871 871 ctx2 = scmutil.revsingle(repo, rev2, rev2)
872 872 def d():
873 873 copies.pathcopies(ctx1, ctx2)
874 874 timer(d)
875 875 fm.end()
876 876
877 877 @command(b'perfphases',
878 878 [(b'', b'full', False, b'include file reading time too'),
879 879 ], b"")
880 880 def perfphases(ui, repo, **opts):
881 881 """benchmark phasesets computation"""
882 882 opts = _byteskwargs(opts)
883 883 timer, fm = gettimer(ui, opts)
884 884 _phases = repo._phasecache
885 885 full = opts.get(b'full')
886 886 def d():
887 887 phases = _phases
888 888 if full:
889 889 clearfilecache(repo, b'_phasecache')
890 890 phases = repo._phasecache
891 891 phases.invalidate()
892 892 phases.loadphaserevs(repo)
893 893 timer(d)
894 894 fm.end()
895 895
896 896 @command(b'perfphasesremote',
897 897 [], b"[DEST]")
898 898 def perfphasesremote(ui, repo, dest=None, **opts):
899 899 """benchmark time needed to analyse phases of the remote server"""
900 900 from mercurial.node import (
901 901 bin,
902 902 )
903 903 from mercurial import (
904 904 exchange,
905 905 hg,
906 906 phases,
907 907 )
908 908 opts = _byteskwargs(opts)
909 909 timer, fm = gettimer(ui, opts)
910 910
911 911 path = ui.paths.getpath(dest, default=(b'default-push', b'default'))
912 912 if not path:
913 913 raise error.Abort((b'default repository not configured!'),
914 914 hint=(b"see 'hg help config.paths'"))
915 915 dest = path.pushloc or path.loc
916 916 ui.status((b'analysing phase of %s\n') % util.hidepassword(dest))
917 917 other = hg.peer(repo, opts, dest)
918 918
919 919 # easier to perform discovery through the operation
920 920 op = exchange.pushoperation(repo, other)
921 921 exchange._pushdiscoverychangeset(op)
922 922
923 923 remotesubset = op.fallbackheads
924 924
925 925 with other.commandexecutor() as e:
926 926 remotephases = e.callcommand(b'listkeys',
927 927 {b'namespace': b'phases'}).result()
928 928 del other
929 929 publishing = remotephases.get(b'publishing', False)
930 930 if publishing:
931 931 ui.status((b'publishing: yes\n'))
932 932 else:
933 933 ui.status((b'publishing: no\n'))
934 934
935 935 nodemap = repo.changelog.nodemap
936 936 nonpublishroots = 0
937 937 for nhex, phase in remotephases.iteritems():
938 938 if nhex == b'publishing': # ignore data related to publish option
939 939 continue
940 940 node = bin(nhex)
941 941 if node in nodemap and int(phase):
942 942 nonpublishroots += 1
943 943 ui.status((b'number of roots: %d\n') % len(remotephases))
944 944 ui.status((b'number of known non public roots: %d\n') % nonpublishroots)
945 945 def d():
946 946 phases.remotephasessummary(repo,
947 947 remotesubset,
948 948 remotephases)
949 949 timer(d)
950 950 fm.end()
951 951
952 952 @command(b'perfmanifest',[
953 953 (b'm', b'manifest-rev', False, b'Look up a manifest node revision'),
954 954 (b'', b'clear-disk', False, b'clear on-disk caches too'),
955 955 ] + formatteropts, b'REV|NODE')
956 956 def perfmanifest(ui, repo, rev, manifest_rev=False, clear_disk=False, **opts):
957 957 """benchmark the time to read a manifest from disk and return a usable
958 958 dict-like object
959 959
960 960 Manifest caches are cleared before retrieval."""
961 961 opts = _byteskwargs(opts)
962 962 timer, fm = gettimer(ui, opts)
963 963 if not manifest_rev:
964 964 ctx = scmutil.revsingle(repo, rev, rev)
965 965 t = ctx.manifestnode()
966 966 else:
967 967 from mercurial.node import bin
968 968
969 969 if len(rev) == 40:
970 970 t = bin(rev)
971 971 else:
972 972 try:
973 973 rev = int(rev)
974 974
975 975 if util.safehasattr(repo.manifestlog, b'getstorage'):
976 976 t = repo.manifestlog.getstorage(b'').node(rev)
977 977 else:
978 978 t = repo.manifestlog._revlog.lookup(rev)
979 979 except ValueError:
980 980 raise error.Abort(b'manifest revision must be integer or full '
981 981 b'node')
982 982 def d():
983 983 repo.manifestlog.clearcaches(clear_persisted_data=clear_disk)
984 984 repo.manifestlog[t].read()
985 985 timer(d)
986 986 fm.end()
987 987
988 988 @command(b'perfchangeset', formatteropts)
989 989 def perfchangeset(ui, repo, rev, **opts):
990 990 opts = _byteskwargs(opts)
991 991 timer, fm = gettimer(ui, opts)
992 992 n = scmutil.revsingle(repo, rev).node()
993 993 def d():
994 994 repo.changelog.read(n)
995 995 #repo.changelog._cache = None
996 996 timer(d)
997 997 fm.end()
998 998
999 999 @command(b'perfignore', formatteropts)
1000 1000 def perfignore(ui, repo, **opts):
1001 1001 """benchmark operation related to computing ignore"""
1002 1002 opts = _byteskwargs(opts)
1003 1003 timer, fm = gettimer(ui, opts)
1004 1004 dirstate = repo.dirstate
1005 1005
1006 1006 def setupone():
1007 1007 dirstate.invalidate()
1008 1008 clearfilecache(dirstate, b'_ignore')
1009 1009
1010 1010 def runone():
1011 1011 dirstate._ignore
1012 1012
1013 1013 timer(runone, setup=setupone, title=b"load")
1014 1014 fm.end()
1015 1015
1016 1016 @command(b'perfindex', [
1017 1017 (b'', b'rev', [], b'revision to be looked up (default tip)'),
1018 1018 (b'', b'no-lookup', None, b'do not revision lookup post creation'),
1019 1019 ] + formatteropts)
1020 1020 def perfindex(ui, repo, **opts):
1021 1021 """benchmark index creation time followed by a lookup
1022 1022
1023 1023 The default is to look `tip` up. Depending on the index implementation,
1024 1024 the revision looked up can matters. For example, an implementation
1025 1025 scanning the index will have a faster lookup time for `--rev tip` than for
1026 1026 `--rev 0`. The number of looked up revisions and their order can also
1027 1027 matters.
1028 1028
1029 1029 Example of useful set to test:
1030 1030 * tip
1031 1031 * 0
1032 1032 * -10:
1033 1033 * :10
1034 1034 * -10: + :10
1035 1035 * :10: + -10:
1036 1036 * -10000:
1037 1037 * -10000: + 0
1038 1038
1039 1039 It is not currently possible to check for lookup of a missing node. For
1040 1040 deeper lookup benchmarking, checkout the `perfnodemap` command."""
1041 1041 import mercurial.revlog
1042 1042 opts = _byteskwargs(opts)
1043 1043 timer, fm = gettimer(ui, opts)
1044 1044 mercurial.revlog._prereadsize = 2**24 # disable lazy parser in old hg
1045 1045 if opts[b'no_lookup']:
1046 1046 if opts['rev']:
1047 1047 raise error.Abort('--no-lookup and --rev are mutually exclusive')
1048 1048 nodes = []
1049 1049 elif not opts[b'rev']:
1050 1050 nodes = [repo[b"tip"].node()]
1051 1051 else:
1052 1052 revs = scmutil.revrange(repo, opts[b'rev'])
1053 1053 cl = repo.changelog
1054 1054 nodes = [cl.node(r) for r in revs]
1055 1055
1056 1056 unfi = repo.unfiltered()
1057 1057 # find the filecache func directly
1058 1058 # This avoid polluting the benchmark with the filecache logic
1059 1059 makecl = unfi.__class__.changelog.func
1060 1060 def setup():
1061 1061 # probably not necessary, but for good measure
1062 1062 clearchangelog(unfi)
1063 1063 def d():
1064 1064 cl = makecl(unfi)
1065 1065 for n in nodes:
1066 1066 cl.rev(n)
1067 1067 timer(d, setup=setup)
1068 1068 fm.end()
1069 1069
1070 1070 @command(b'perfnodemap', [
1071 1071 (b'', b'rev', [], b'revision to be looked up (default tip)'),
1072 1072 (b'', b'clear-caches', True, b'clear revlog cache between calls'),
1073 1073 ] + formatteropts)
1074 1074 def perfnodemap(ui, repo, **opts):
1075 1075 """benchmark the time necessary to look up revision from a cold nodemap
1076 1076
1077 1077 Depending on the implementation, the amount and order of revision we look
1078 1078 up can varies. Example of useful set to test:
1079 1079 * tip
1080 1080 * 0
1081 1081 * -10:
1082 1082 * :10
1083 1083 * -10: + :10
1084 1084 * :10: + -10:
1085 1085 * -10000:
1086 1086 * -10000: + 0
1087 1087
1088 1088 The command currently focus on valid binary lookup. Benchmarking for
1089 1089 hexlookup, prefix lookup and missing lookup would also be valuable.
1090 1090 """
1091 1091 import mercurial.revlog
1092 1092 opts = _byteskwargs(opts)
1093 1093 timer, fm = gettimer(ui, opts)
1094 1094 mercurial.revlog._prereadsize = 2**24 # disable lazy parser in old hg
1095 1095
1096 1096 unfi = repo.unfiltered()
1097 1097 clearcaches = opts['clear_caches']
1098 1098 # find the filecache func directly
1099 1099 # This avoid polluting the benchmark with the filecache logic
1100 1100 makecl = unfi.__class__.changelog.func
1101 1101 if not opts[b'rev']:
1102 1102 raise error.Abort('use --rev to specify revisions to look up')
1103 1103 revs = scmutil.revrange(repo, opts[b'rev'])
1104 1104 cl = repo.changelog
1105 1105 nodes = [cl.node(r) for r in revs]
1106 1106
1107 1107 # use a list to pass reference to a nodemap from one closure to the next
1108 1108 nodeget = [None]
1109 1109 def setnodeget():
1110 1110 # probably not necessary, but for good measure
1111 1111 clearchangelog(unfi)
1112 1112 nodeget[0] = makecl(unfi).nodemap.get
1113 1113
1114 1114 def d():
1115 1115 get = nodeget[0]
1116 1116 for n in nodes:
1117 1117 get(n)
1118 1118
1119 1119 setup = None
1120 1120 if clearcaches:
1121 1121 def setup():
1122 1122 setnodeget()
1123 1123 else:
1124 1124 setnodeget()
1125 1125 d() # prewarm the data structure
1126 1126 timer(d, setup=setup)
1127 1127 fm.end()
1128 1128
1129 1129 @command(b'perfstartup', formatteropts)
1130 1130 def perfstartup(ui, repo, **opts):
1131 1131 opts = _byteskwargs(opts)
1132 1132 timer, fm = gettimer(ui, opts)
1133 1133 def d():
1134 1134 if os.name != r'nt':
1135 1135 os.system(b"HGRCPATH= %s version -q > /dev/null" %
1136 1136 fsencode(sys.argv[0]))
1137 1137 else:
1138 1138 os.environ[r'HGRCPATH'] = r' '
1139 1139 os.system(r"%s version -q > NUL" % sys.argv[0])
1140 1140 timer(d)
1141 1141 fm.end()
1142 1142
1143 1143 @command(b'perfparents', formatteropts)
1144 1144 def perfparents(ui, repo, **opts):
1145 1145 opts = _byteskwargs(opts)
1146 1146 timer, fm = gettimer(ui, opts)
1147 1147 # control the number of commits perfparents iterates over
1148 1148 # experimental config: perf.parentscount
1149 1149 count = getint(ui, b"perf", b"parentscount", 1000)
1150 1150 if len(repo.changelog) < count:
1151 1151 raise error.Abort(b"repo needs %d commits for this test" % count)
1152 1152 repo = repo.unfiltered()
1153 1153 nl = [repo.changelog.node(i) for i in _xrange(count)]
1154 1154 def d():
1155 1155 for n in nl:
1156 1156 repo.changelog.parents(n)
1157 1157 timer(d)
1158 1158 fm.end()
1159 1159
1160 1160 @command(b'perfctxfiles', formatteropts)
1161 1161 def perfctxfiles(ui, repo, x, **opts):
1162 1162 opts = _byteskwargs(opts)
1163 1163 x = int(x)
1164 1164 timer, fm = gettimer(ui, opts)
1165 1165 def d():
1166 1166 len(repo[x].files())
1167 1167 timer(d)
1168 1168 fm.end()
1169 1169
1170 1170 @command(b'perfrawfiles', formatteropts)
1171 1171 def perfrawfiles(ui, repo, x, **opts):
1172 1172 opts = _byteskwargs(opts)
1173 1173 x = int(x)
1174 1174 timer, fm = gettimer(ui, opts)
1175 1175 cl = repo.changelog
1176 1176 def d():
1177 1177 len(cl.read(x)[3])
1178 1178 timer(d)
1179 1179 fm.end()
1180 1180
1181 1181 @command(b'perflookup', formatteropts)
1182 1182 def perflookup(ui, repo, rev, **opts):
1183 1183 opts = _byteskwargs(opts)
1184 1184 timer, fm = gettimer(ui, opts)
1185 1185 timer(lambda: len(repo.lookup(rev)))
1186 1186 fm.end()
1187 1187
1188 1188 @command(b'perflinelogedits',
1189 1189 [(b'n', b'edits', 10000, b'number of edits'),
1190 1190 (b'', b'max-hunk-lines', 10, b'max lines in a hunk'),
1191 1191 ], norepo=True)
1192 1192 def perflinelogedits(ui, **opts):
1193 1193 from mercurial import linelog
1194 1194
1195 1195 opts = _byteskwargs(opts)
1196 1196
1197 1197 edits = opts[b'edits']
1198 1198 maxhunklines = opts[b'max_hunk_lines']
1199 1199
1200 1200 maxb1 = 100000
1201 1201 random.seed(0)
1202 1202 randint = random.randint
1203 1203 currentlines = 0
1204 1204 arglist = []
1205 1205 for rev in _xrange(edits):
1206 1206 a1 = randint(0, currentlines)
1207 1207 a2 = randint(a1, min(currentlines, a1 + maxhunklines))
1208 1208 b1 = randint(0, maxb1)
1209 1209 b2 = randint(b1, b1 + maxhunklines)
1210 1210 currentlines += (b2 - b1) - (a2 - a1)
1211 1211 arglist.append((rev, a1, a2, b1, b2))
1212 1212
1213 1213 def d():
1214 1214 ll = linelog.linelog()
1215 1215 for args in arglist:
1216 1216 ll.replacelines(*args)
1217 1217
1218 1218 timer, fm = gettimer(ui, opts)
1219 1219 timer(d)
1220 1220 fm.end()
1221 1221
1222 1222 @command(b'perfrevrange', formatteropts)
1223 1223 def perfrevrange(ui, repo, *specs, **opts):
1224 1224 opts = _byteskwargs(opts)
1225 1225 timer, fm = gettimer(ui, opts)
1226 1226 revrange = scmutil.revrange
1227 1227 timer(lambda: len(revrange(repo, specs)))
1228 1228 fm.end()
1229 1229
1230 1230 @command(b'perfnodelookup', formatteropts)
1231 1231 def perfnodelookup(ui, repo, rev, **opts):
1232 1232 opts = _byteskwargs(opts)
1233 1233 timer, fm = gettimer(ui, opts)
1234 1234 import mercurial.revlog
1235 1235 mercurial.revlog._prereadsize = 2**24 # disable lazy parser in old hg
1236 1236 n = scmutil.revsingle(repo, rev).node()
1237 1237 cl = mercurial.revlog.revlog(getsvfs(repo), b"00changelog.i")
1238 1238 def d():
1239 1239 cl.rev(n)
1240 1240 clearcaches(cl)
1241 1241 timer(d)
1242 1242 fm.end()
1243 1243
1244 1244 @command(b'perflog',
1245 1245 [(b'', b'rename', False, b'ask log to follow renames')
1246 1246 ] + formatteropts)
1247 1247 def perflog(ui, repo, rev=None, **opts):
1248 1248 opts = _byteskwargs(opts)
1249 1249 if rev is None:
1250 1250 rev=[]
1251 1251 timer, fm = gettimer(ui, opts)
1252 1252 ui.pushbuffer()
1253 1253 timer(lambda: commands.log(ui, repo, rev=rev, date=b'', user=b'',
1254 1254 copies=opts.get(b'rename')))
1255 1255 ui.popbuffer()
1256 1256 fm.end()
1257 1257
1258 1258 @command(b'perfmoonwalk', formatteropts)
1259 1259 def perfmoonwalk(ui, repo, **opts):
1260 1260 """benchmark walking the changelog backwards
1261 1261
1262 1262 This also loads the changelog data for each revision in the changelog.
1263 1263 """
1264 1264 opts = _byteskwargs(opts)
1265 1265 timer, fm = gettimer(ui, opts)
1266 1266 def moonwalk():
1267 1267 for i in repo.changelog.revs(start=(len(repo) - 1), stop=-1):
1268 1268 ctx = repo[i]
1269 1269 ctx.branch() # read changelog data (in addition to the index)
1270 1270 timer(moonwalk)
1271 1271 fm.end()
1272 1272
1273 1273 @command(b'perftemplating',
1274 1274 [(b'r', b'rev', [], b'revisions to run the template on'),
1275 1275 ] + formatteropts)
1276 1276 def perftemplating(ui, repo, testedtemplate=None, **opts):
1277 1277 """test the rendering time of a given template"""
1278 1278 if makelogtemplater is None:
1279 1279 raise error.Abort((b"perftemplating not available with this Mercurial"),
1280 1280 hint=b"use 4.3 or later")
1281 1281
1282 1282 opts = _byteskwargs(opts)
1283 1283
1284 1284 nullui = ui.copy()
1285 1285 nullui.fout = open(os.devnull, r'wb')
1286 1286 nullui.disablepager()
1287 1287 revs = opts.get(b'rev')
1288 1288 if not revs:
1289 1289 revs = [b'all()']
1290 1290 revs = list(scmutil.revrange(repo, revs))
1291 1291
1292 1292 defaulttemplate = (b'{date|shortdate} [{rev}:{node|short}]'
1293 1293 b' {author|person}: {desc|firstline}\n')
1294 1294 if testedtemplate is None:
1295 1295 testedtemplate = defaulttemplate
1296 1296 displayer = makelogtemplater(nullui, repo, testedtemplate)
1297 1297 def format():
1298 1298 for r in revs:
1299 1299 ctx = repo[r]
1300 1300 displayer.show(ctx)
1301 1301 displayer.flush(ctx)
1302 1302
1303 1303 timer, fm = gettimer(ui, opts)
1304 1304 timer(format)
1305 1305 fm.end()
1306 1306
1307 1307 @command(b'perfhelper-pathcopies', formatteropts +
1308 1308 [
1309 1309 (b'r', b'revs', [], b'restrict search to these revisions'),
1310 1310 (b'', b'timing', False, b'provides extra data (costly)'),
1311 1311 ])
1312 1312 def perfhelperpathcopies(ui, repo, revs=[], **opts):
1313 1313 """find statistic about potential parameters for the `perftracecopies`
1314 1314
1315 1315 This command find source-destination pair relevant for copytracing testing.
1316 1316 It report value for some of the parameters that impact copy tracing time.
1317 1317
1318 1318 If `--timing` is set, rename detection is run and the associated timing
1319 1319 will be reported. The extra details comes at the cost of a slower command
1320 1320 execution.
1321 1321
1322 1322 Since the rename detection is only run once, other factors might easily
1323 1323 affect the precision of the timing. However it should give a good
1324 1324 approximation of which revision pairs are very costly.
1325 1325 """
1326 1326 opts = _byteskwargs(opts)
1327 1327 fm = ui.formatter(b'perf', opts)
1328 1328 dotiming = opts[b'timing']
1329 1329
1330 1330 if dotiming:
1331 1331 header = '%12s %12s %12s %12s %12s %12s\n'
1332 1332 output = ("%(source)12s %(destination)12s "
1333 1333 "%(nbrevs)12d %(nbmissingfiles)12d "
1334 1334 "%(nbrenamedfiles)12d %(time)18.5f\n")
1335 1335 header_names = ("source", "destination", "nb-revs", "nb-files",
1336 1336 "nb-renames", "time")
1337 1337 fm.plain(header % header_names)
1338 1338 else:
1339 1339 header = '%12s %12s %12s %12s\n'
1340 1340 output = ("%(source)12s %(destination)12s "
1341 1341 "%(nbrevs)12d %(nbmissingfiles)12d\n")
1342 1342 fm.plain(header % ("source", "destination", "nb-revs", "nb-files"))
1343 1343
1344 1344 if not revs:
1345 1345 revs = ['all()']
1346 1346 revs = scmutil.revrange(repo, revs)
1347 1347
1348 1348 roi = repo.revs('merge() and %ld', revs)
1349 1349 for r in roi:
1350 1350 ctx = repo[r]
1351 1351 p1 = ctx.p1().rev()
1352 1352 p2 = ctx.p2().rev()
1353 1353 bases = repo.changelog._commonancestorsheads(p1, p2)
1354 1354 for p in (p1, p2):
1355 1355 for b in bases:
1356 1356 base = repo[b]
1357 1357 parent = repo[p]
1358 1358 missing = copies._computeforwardmissing(base, parent)
1359 1359 if not missing:
1360 1360 continue
1361 1361 data = {
1362 1362 b'source': base.hex(),
1363 1363 b'destination': parent.hex(),
1364 1364 b'nbrevs': len(repo.revs('%d::%d', b, p)),
1365 1365 b'nbmissingfiles': len(missing),
1366 1366 }
1367 1367 if dotiming:
1368 1368 begin = util.timer()
1369 1369 renames = copies.pathcopies(base, parent)
1370 1370 end = util.timer()
1371 1371 # not very stable timing since we did only one run
1372 1372 data['time'] = end - begin
1373 1373 data['nbrenamedfiles'] = len(renames)
1374 1374 fm.startitem()
1375 1375 fm.data(**data)
1376 1376 out = data.copy()
1377 1377 out['source'] = fm.hexfunc(base.node())
1378 1378 out['destination'] = fm.hexfunc(parent.node())
1379 1379 fm.plain(output % out)
1380 1380
1381 1381 fm.end()
1382 1382
1383 1383 @command(b'perfcca', formatteropts)
1384 1384 def perfcca(ui, repo, **opts):
1385 1385 opts = _byteskwargs(opts)
1386 1386 timer, fm = gettimer(ui, opts)
1387 1387 timer(lambda: scmutil.casecollisionauditor(ui, False, repo.dirstate))
1388 1388 fm.end()
1389 1389
1390 1390 @command(b'perffncacheload', formatteropts)
1391 1391 def perffncacheload(ui, repo, **opts):
1392 1392 opts = _byteskwargs(opts)
1393 1393 timer, fm = gettimer(ui, opts)
1394 1394 s = repo.store
1395 1395 def d():
1396 1396 s.fncache._load()
1397 1397 timer(d)
1398 1398 fm.end()
1399 1399
1400 1400 @command(b'perffncachewrite', formatteropts)
1401 1401 def perffncachewrite(ui, repo, **opts):
1402 1402 opts = _byteskwargs(opts)
1403 1403 timer, fm = gettimer(ui, opts)
1404 1404 s = repo.store
1405 1405 lock = repo.lock()
1406 1406 s.fncache._load()
1407 1407 tr = repo.transaction(b'perffncachewrite')
1408 1408 tr.addbackup(b'fncache')
1409 1409 def d():
1410 1410 s.fncache._dirty = True
1411 1411 s.fncache.write(tr)
1412 1412 timer(d)
1413 1413 tr.close()
1414 1414 lock.release()
1415 1415 fm.end()
1416 1416
1417 1417 @command(b'perffncacheencode', formatteropts)
1418 1418 def perffncacheencode(ui, repo, **opts):
1419 1419 opts = _byteskwargs(opts)
1420 1420 timer, fm = gettimer(ui, opts)
1421 1421 s = repo.store
1422 1422 s.fncache._load()
1423 1423 def d():
1424 1424 for p in s.fncache.entries:
1425 1425 s.encode(p)
1426 1426 timer(d)
1427 1427 fm.end()
1428 1428
1429 1429 def _bdiffworker(q, blocks, xdiff, ready, done):
1430 1430 while not done.is_set():
1431 1431 pair = q.get()
1432 1432 while pair is not None:
1433 1433 if xdiff:
1434 1434 mdiff.bdiff.xdiffblocks(*pair)
1435 1435 elif blocks:
1436 1436 mdiff.bdiff.blocks(*pair)
1437 1437 else:
1438 1438 mdiff.textdiff(*pair)
1439 1439 q.task_done()
1440 1440 pair = q.get()
1441 1441 q.task_done() # for the None one
1442 1442 with ready:
1443 1443 ready.wait()
1444 1444
1445 1445 def _manifestrevision(repo, mnode):
1446 1446 ml = repo.manifestlog
1447 1447
1448 1448 if util.safehasattr(ml, b'getstorage'):
1449 1449 store = ml.getstorage(b'')
1450 1450 else:
1451 1451 store = ml._revlog
1452 1452
1453 1453 return store.revision(mnode)
1454 1454
1455 1455 @command(b'perfbdiff', revlogopts + formatteropts + [
1456 1456 (b'', b'count', 1, b'number of revisions to test (when using --startrev)'),
1457 1457 (b'', b'alldata', False, b'test bdiffs for all associated revisions'),
1458 1458 (b'', b'threads', 0, b'number of thread to use (disable with 0)'),
1459 1459 (b'', b'blocks', False, b'test computing diffs into blocks'),
1460 1460 (b'', b'xdiff', False, b'use xdiff algorithm'),
1461 1461 ],
1462 1462
1463 1463 b'-c|-m|FILE REV')
1464 1464 def perfbdiff(ui, repo, file_, rev=None, count=None, threads=0, **opts):
1465 1465 """benchmark a bdiff between revisions
1466 1466
1467 1467 By default, benchmark a bdiff between its delta parent and itself.
1468 1468
1469 1469 With ``--count``, benchmark bdiffs between delta parents and self for N
1470 1470 revisions starting at the specified revision.
1471 1471
1472 1472 With ``--alldata``, assume the requested revision is a changeset and
1473 1473 measure bdiffs for all changes related to that changeset (manifest
1474 1474 and filelogs).
1475 1475 """
1476 1476 opts = _byteskwargs(opts)
1477 1477
1478 1478 if opts[b'xdiff'] and not opts[b'blocks']:
1479 1479 raise error.CommandError(b'perfbdiff', b'--xdiff requires --blocks')
1480 1480
1481 1481 if opts[b'alldata']:
1482 1482 opts[b'changelog'] = True
1483 1483
1484 1484 if opts.get(b'changelog') or opts.get(b'manifest'):
1485 1485 file_, rev = None, file_
1486 1486 elif rev is None:
1487 1487 raise error.CommandError(b'perfbdiff', b'invalid arguments')
1488 1488
1489 1489 blocks = opts[b'blocks']
1490 1490 xdiff = opts[b'xdiff']
1491 1491 textpairs = []
1492 1492
1493 1493 r = cmdutil.openrevlog(repo, b'perfbdiff', file_, opts)
1494 1494
1495 1495 startrev = r.rev(r.lookup(rev))
1496 1496 for rev in range(startrev, min(startrev + count, len(r) - 1)):
1497 1497 if opts[b'alldata']:
1498 1498 # Load revisions associated with changeset.
1499 1499 ctx = repo[rev]
1500 1500 mtext = _manifestrevision(repo, ctx.manifestnode())
1501 1501 for pctx in ctx.parents():
1502 1502 pman = _manifestrevision(repo, pctx.manifestnode())
1503 1503 textpairs.append((pman, mtext))
1504 1504
1505 1505 # Load filelog revisions by iterating manifest delta.
1506 1506 man = ctx.manifest()
1507 1507 pman = ctx.p1().manifest()
1508 1508 for filename, change in pman.diff(man).items():
1509 1509 fctx = repo.file(filename)
1510 1510 f1 = fctx.revision(change[0][0] or -1)
1511 1511 f2 = fctx.revision(change[1][0] or -1)
1512 1512 textpairs.append((f1, f2))
1513 1513 else:
1514 1514 dp = r.deltaparent(rev)
1515 1515 textpairs.append((r.revision(dp), r.revision(rev)))
1516 1516
1517 1517 withthreads = threads > 0
1518 1518 if not withthreads:
1519 1519 def d():
1520 1520 for pair in textpairs:
1521 1521 if xdiff:
1522 1522 mdiff.bdiff.xdiffblocks(*pair)
1523 1523 elif blocks:
1524 1524 mdiff.bdiff.blocks(*pair)
1525 1525 else:
1526 1526 mdiff.textdiff(*pair)
1527 1527 else:
1528 1528 q = queue()
1529 1529 for i in _xrange(threads):
1530 1530 q.put(None)
1531 1531 ready = threading.Condition()
1532 1532 done = threading.Event()
1533 1533 for i in _xrange(threads):
1534 1534 threading.Thread(target=_bdiffworker,
1535 1535 args=(q, blocks, xdiff, ready, done)).start()
1536 1536 q.join()
1537 1537 def d():
1538 1538 for pair in textpairs:
1539 1539 q.put(pair)
1540 1540 for i in _xrange(threads):
1541 1541 q.put(None)
1542 1542 with ready:
1543 1543 ready.notify_all()
1544 1544 q.join()
1545 1545 timer, fm = gettimer(ui, opts)
1546 1546 timer(d)
1547 1547 fm.end()
1548 1548
1549 1549 if withthreads:
1550 1550 done.set()
1551 1551 for i in _xrange(threads):
1552 1552 q.put(None)
1553 1553 with ready:
1554 1554 ready.notify_all()
1555 1555
1556 1556 @command(b'perfunidiff', revlogopts + formatteropts + [
1557 1557 (b'', b'count', 1, b'number of revisions to test (when using --startrev)'),
1558 1558 (b'', b'alldata', False, b'test unidiffs for all associated revisions'),
1559 1559 ], b'-c|-m|FILE REV')
1560 1560 def perfunidiff(ui, repo, file_, rev=None, count=None, **opts):
1561 1561 """benchmark a unified diff between revisions
1562 1562
1563 1563 This doesn't include any copy tracing - it's just a unified diff
1564 1564 of the texts.
1565 1565
1566 1566 By default, benchmark a diff between its delta parent and itself.
1567 1567
1568 1568 With ``--count``, benchmark diffs between delta parents and self for N
1569 1569 revisions starting at the specified revision.
1570 1570
1571 1571 With ``--alldata``, assume the requested revision is a changeset and
1572 1572 measure diffs for all changes related to that changeset (manifest
1573 1573 and filelogs).
1574 1574 """
1575 1575 opts = _byteskwargs(opts)
1576 1576 if opts[b'alldata']:
1577 1577 opts[b'changelog'] = True
1578 1578
1579 1579 if opts.get(b'changelog') or opts.get(b'manifest'):
1580 1580 file_, rev = None, file_
1581 1581 elif rev is None:
1582 1582 raise error.CommandError(b'perfunidiff', b'invalid arguments')
1583 1583
1584 1584 textpairs = []
1585 1585
1586 1586 r = cmdutil.openrevlog(repo, b'perfunidiff', file_, opts)
1587 1587
1588 1588 startrev = r.rev(r.lookup(rev))
1589 1589 for rev in range(startrev, min(startrev + count, len(r) - 1)):
1590 1590 if opts[b'alldata']:
1591 1591 # Load revisions associated with changeset.
1592 1592 ctx = repo[rev]
1593 1593 mtext = _manifestrevision(repo, ctx.manifestnode())
1594 1594 for pctx in ctx.parents():
1595 1595 pman = _manifestrevision(repo, pctx.manifestnode())
1596 1596 textpairs.append((pman, mtext))
1597 1597
1598 1598 # Load filelog revisions by iterating manifest delta.
1599 1599 man = ctx.manifest()
1600 1600 pman = ctx.p1().manifest()
1601 1601 for filename, change in pman.diff(man).items():
1602 1602 fctx = repo.file(filename)
1603 1603 f1 = fctx.revision(change[0][0] or -1)
1604 1604 f2 = fctx.revision(change[1][0] or -1)
1605 1605 textpairs.append((f1, f2))
1606 1606 else:
1607 1607 dp = r.deltaparent(rev)
1608 1608 textpairs.append((r.revision(dp), r.revision(rev)))
1609 1609
1610 1610 def d():
1611 1611 for left, right in textpairs:
1612 1612 # The date strings don't matter, so we pass empty strings.
1613 1613 headerlines, hunks = mdiff.unidiff(
1614 1614 left, b'', right, b'', b'left', b'right', binary=False)
1615 1615 # consume iterators in roughly the way patch.py does
1616 1616 b'\n'.join(headerlines)
1617 1617 b''.join(sum((list(hlines) for hrange, hlines in hunks), []))
1618 1618 timer, fm = gettimer(ui, opts)
1619 1619 timer(d)
1620 1620 fm.end()
1621 1621
1622 1622 @command(b'perfdiffwd', formatteropts)
1623 1623 def perfdiffwd(ui, repo, **opts):
1624 1624 """Profile diff of working directory changes"""
1625 1625 opts = _byteskwargs(opts)
1626 1626 timer, fm = gettimer(ui, opts)
1627 1627 options = {
1628 1628 'w': 'ignore_all_space',
1629 1629 'b': 'ignore_space_change',
1630 1630 'B': 'ignore_blank_lines',
1631 1631 }
1632 1632
1633 1633 for diffopt in ('', 'w', 'b', 'B', 'wB'):
1634 1634 opts = dict((options[c], b'1') for c in diffopt)
1635 1635 def d():
1636 1636 ui.pushbuffer()
1637 1637 commands.diff(ui, repo, **opts)
1638 1638 ui.popbuffer()
1639 1639 diffopt = diffopt.encode('ascii')
1640 1640 title = b'diffopts: %s' % (diffopt and (b'-' + diffopt) or b'none')
1641 1641 timer(d, title=title)
1642 1642 fm.end()
1643 1643
1644 1644 @command(b'perfrevlogindex', revlogopts + formatteropts,
1645 1645 b'-c|-m|FILE')
1646 1646 def perfrevlogindex(ui, repo, file_=None, **opts):
1647 1647 """Benchmark operations against a revlog index.
1648 1648
1649 1649 This tests constructing a revlog instance, reading index data,
1650 1650 parsing index data, and performing various operations related to
1651 1651 index data.
1652 1652 """
1653 1653
1654 1654 opts = _byteskwargs(opts)
1655 1655
1656 1656 rl = cmdutil.openrevlog(repo, b'perfrevlogindex', file_, opts)
1657 1657
1658 1658 opener = getattr(rl, 'opener') # trick linter
1659 1659 indexfile = rl.indexfile
1660 1660 data = opener.read(indexfile)
1661 1661
1662 1662 header = struct.unpack(b'>I', data[0:4])[0]
1663 1663 version = header & 0xFFFF
1664 1664 if version == 1:
1665 1665 revlogio = revlog.revlogio()
1666 1666 inline = header & (1 << 16)
1667 1667 else:
1668 1668 raise error.Abort((b'unsupported revlog version: %d') % version)
1669 1669
1670 1670 rllen = len(rl)
1671 1671
1672 1672 node0 = rl.node(0)
1673 1673 node25 = rl.node(rllen // 4)
1674 1674 node50 = rl.node(rllen // 2)
1675 1675 node75 = rl.node(rllen // 4 * 3)
1676 1676 node100 = rl.node(rllen - 1)
1677 1677
1678 1678 allrevs = range(rllen)
1679 1679 allrevsrev = list(reversed(allrevs))
1680 1680 allnodes = [rl.node(rev) for rev in range(rllen)]
1681 1681 allnodesrev = list(reversed(allnodes))
1682 1682
1683 1683 def constructor():
1684 1684 revlog.revlog(opener, indexfile)
1685 1685
1686 1686 def read():
1687 1687 with opener(indexfile) as fh:
1688 1688 fh.read()
1689 1689
1690 1690 def parseindex():
1691 1691 revlogio.parseindex(data, inline)
1692 1692
1693 1693 def getentry(revornode):
1694 1694 index = revlogio.parseindex(data, inline)[0]
1695 1695 index[revornode]
1696 1696
1697 1697 def getentries(revs, count=1):
1698 1698 index = revlogio.parseindex(data, inline)[0]
1699 1699
1700 1700 for i in range(count):
1701 1701 for rev in revs:
1702 1702 index[rev]
1703 1703
1704 1704 def resolvenode(node):
1705 1705 nodemap = revlogio.parseindex(data, inline)[1]
1706 1706 # This only works for the C code.
1707 1707 if nodemap is None:
1708 1708 return
1709 1709
1710 1710 try:
1711 1711 nodemap[node]
1712 1712 except error.RevlogError:
1713 1713 pass
1714 1714
1715 1715 def resolvenodes(nodes, count=1):
1716 1716 nodemap = revlogio.parseindex(data, inline)[1]
1717 1717 if nodemap is None:
1718 1718 return
1719 1719
1720 1720 for i in range(count):
1721 1721 for node in nodes:
1722 1722 try:
1723 1723 nodemap[node]
1724 1724 except error.RevlogError:
1725 1725 pass
1726 1726
1727 1727 benches = [
1728 1728 (constructor, b'revlog constructor'),
1729 1729 (read, b'read'),
1730 1730 (parseindex, b'create index object'),
1731 1731 (lambda: getentry(0), b'retrieve index entry for rev 0'),
1732 1732 (lambda: resolvenode(b'a' * 20), b'look up missing node'),
1733 1733 (lambda: resolvenode(node0), b'look up node at rev 0'),
1734 1734 (lambda: resolvenode(node25), b'look up node at 1/4 len'),
1735 1735 (lambda: resolvenode(node50), b'look up node at 1/2 len'),
1736 1736 (lambda: resolvenode(node75), b'look up node at 3/4 len'),
1737 1737 (lambda: resolvenode(node100), b'look up node at tip'),
1738 1738 # 2x variation is to measure caching impact.
1739 1739 (lambda: resolvenodes(allnodes),
1740 1740 b'look up all nodes (forward)'),
1741 1741 (lambda: resolvenodes(allnodes, 2),
1742 1742 b'look up all nodes 2x (forward)'),
1743 1743 (lambda: resolvenodes(allnodesrev),
1744 1744 b'look up all nodes (reverse)'),
1745 1745 (lambda: resolvenodes(allnodesrev, 2),
1746 1746 b'look up all nodes 2x (reverse)'),
1747 1747 (lambda: getentries(allrevs),
1748 1748 b'retrieve all index entries (forward)'),
1749 1749 (lambda: getentries(allrevs, 2),
1750 1750 b'retrieve all index entries 2x (forward)'),
1751 1751 (lambda: getentries(allrevsrev),
1752 1752 b'retrieve all index entries (reverse)'),
1753 1753 (lambda: getentries(allrevsrev, 2),
1754 1754 b'retrieve all index entries 2x (reverse)'),
1755 1755 ]
1756 1756
1757 1757 for fn, title in benches:
1758 1758 timer, fm = gettimer(ui, opts)
1759 1759 timer(fn, title=title)
1760 1760 fm.end()
1761 1761
1762 1762 @command(b'perfrevlogrevisions', revlogopts + formatteropts +
1763 1763 [(b'd', b'dist', 100, b'distance between the revisions'),
1764 1764 (b's', b'startrev', 0, b'revision to start reading at'),
1765 1765 (b'', b'reverse', False, b'read in reverse')],
1766 1766 b'-c|-m|FILE')
1767 1767 def perfrevlogrevisions(ui, repo, file_=None, startrev=0, reverse=False,
1768 1768 **opts):
1769 1769 """Benchmark reading a series of revisions from a revlog.
1770 1770
1771 1771 By default, we read every ``-d/--dist`` revision from 0 to tip of
1772 1772 the specified revlog.
1773 1773
1774 1774 The start revision can be defined via ``-s/--startrev``.
1775 1775 """
1776 1776 opts = _byteskwargs(opts)
1777 1777
1778 1778 rl = cmdutil.openrevlog(repo, b'perfrevlogrevisions', file_, opts)
1779 1779 rllen = getlen(ui)(rl)
1780 1780
1781 1781 if startrev < 0:
1782 1782 startrev = rllen + startrev
1783 1783
1784 1784 def d():
1785 1785 rl.clearcaches()
1786 1786
1787 1787 beginrev = startrev
1788 1788 endrev = rllen
1789 1789 dist = opts[b'dist']
1790 1790
1791 1791 if reverse:
1792 1792 beginrev, endrev = endrev - 1, beginrev - 1
1793 1793 dist = -1 * dist
1794 1794
1795 1795 for x in _xrange(beginrev, endrev, dist):
1796 1796 # Old revisions don't support passing int.
1797 1797 n = rl.node(x)
1798 1798 rl.revision(n)
1799 1799
1800 1800 timer, fm = gettimer(ui, opts)
1801 1801 timer(d)
1802 1802 fm.end()
1803 1803
1804 1804 @command(b'perfrevlogwrite', revlogopts + formatteropts +
1805 1805 [(b's', b'startrev', 1000, b'revision to start writing at'),
1806 1806 (b'', b'stoprev', -1, b'last revision to write'),
1807 1807 (b'', b'count', 3, b'last revision to write'),
1808 1808 (b'', b'details', False, b'print timing for every revisions tested'),
1809 1809 (b'', b'source', b'full', b'the kind of data feed in the revlog'),
1810 1810 (b'', b'lazydeltabase', True, b'try the provided delta first'),
1811 1811 (b'', b'clear-caches', True, b'clear revlog cache between calls'),
1812 1812 ],
1813 1813 b'-c|-m|FILE')
1814 1814 def perfrevlogwrite(ui, repo, file_=None, startrev=1000, stoprev=-1, **opts):
1815 1815 """Benchmark writing a series of revisions to a revlog.
1816 1816
1817 1817 Possible source values are:
1818 1818 * `full`: add from a full text (default).
1819 1819 * `parent-1`: add from a delta to the first parent
1820 1820 * `parent-2`: add from a delta to the second parent if it exists
1821 1821 (use a delta from the first parent otherwise)
1822 1822 * `parent-smallest`: add from the smallest delta (either p1 or p2)
1823 1823 * `storage`: add from the existing precomputed deltas
1824 1824 """
1825 1825 opts = _byteskwargs(opts)
1826 1826
1827 1827 rl = cmdutil.openrevlog(repo, b'perfrevlogwrite', file_, opts)
1828 1828 rllen = getlen(ui)(rl)
1829 1829 if startrev < 0:
1830 1830 startrev = rllen + startrev
1831 1831 if stoprev < 0:
1832 1832 stoprev = rllen + stoprev
1833 1833
1834 1834 lazydeltabase = opts['lazydeltabase']
1835 1835 source = opts['source']
1836 1836 clearcaches = opts['clear_caches']
1837 1837 validsource = (b'full', b'parent-1', b'parent-2', b'parent-smallest',
1838 1838 b'storage')
1839 1839 if source not in validsource:
1840 1840 raise error.Abort('invalid source type: %s' % source)
1841 1841
1842 1842 ### actually gather results
1843 1843 count = opts['count']
1844 1844 if count <= 0:
1845 1845 raise error.Abort('invalide run count: %d' % count)
1846 1846 allresults = []
1847 1847 for c in range(count):
1848 1848 timing = _timeonewrite(ui, rl, source, startrev, stoprev, c + 1,
1849 1849 lazydeltabase=lazydeltabase,
1850 1850 clearcaches=clearcaches)
1851 1851 allresults.append(timing)
1852 1852
1853 1853 ### consolidate the results in a single list
1854 1854 results = []
1855 1855 for idx, (rev, t) in enumerate(allresults[0]):
1856 1856 ts = [t]
1857 1857 for other in allresults[1:]:
1858 1858 orev, ot = other[idx]
1859 1859 assert orev == rev
1860 1860 ts.append(ot)
1861 1861 results.append((rev, ts))
1862 1862 resultcount = len(results)
1863 1863
1864 1864 ### Compute and display relevant statistics
1865 1865
1866 1866 # get a formatter
1867 1867 fm = ui.formatter(b'perf', opts)
1868 1868 displayall = ui.configbool(b"perf", b"all-timing", False)
1869 1869
1870 1870 # print individual details if requested
1871 1871 if opts['details']:
1872 1872 for idx, item in enumerate(results, 1):
1873 1873 rev, data = item
1874 1874 title = 'revisions #%d of %d, rev %d' % (idx, resultcount, rev)
1875 1875 formatone(fm, data, title=title, displayall=displayall)
1876 1876
1877 1877 # sorts results by median time
1878 1878 results.sort(key=lambda x: sorted(x[1])[len(x[1]) // 2])
1879 1879 # list of (name, index) to display)
1880 1880 relevants = [
1881 1881 ("min", 0),
1882 1882 ("10%", resultcount * 10 // 100),
1883 1883 ("25%", resultcount * 25 // 100),
1884 1884 ("50%", resultcount * 70 // 100),
1885 1885 ("75%", resultcount * 75 // 100),
1886 1886 ("90%", resultcount * 90 // 100),
1887 1887 ("95%", resultcount * 95 // 100),
1888 1888 ("99%", resultcount * 99 // 100),
1889 1889 ("99.9%", resultcount * 999 // 1000),
1890 1890 ("99.99%", resultcount * 9999 // 10000),
1891 1891 ("99.999%", resultcount * 99999 // 100000),
1892 1892 ("max", -1),
1893 1893 ]
1894 1894 if not ui.quiet:
1895 1895 for name, idx in relevants:
1896 1896 data = results[idx]
1897 1897 title = '%s of %d, rev %d' % (name, resultcount, data[0])
1898 1898 formatone(fm, data[1], title=title, displayall=displayall)
1899 1899
1900 1900 # XXX summing that many float will not be very precise, we ignore this fact
1901 1901 # for now
1902 1902 totaltime = []
1903 1903 for item in allresults:
1904 1904 totaltime.append((sum(x[1][0] for x in item),
1905 1905 sum(x[1][1] for x in item),
1906 1906 sum(x[1][2] for x in item),)
1907 1907 )
1908 1908 formatone(fm, totaltime, title="total time (%d revs)" % resultcount,
1909 1909 displayall=displayall)
1910 1910 fm.end()
1911 1911
1912 1912 class _faketr(object):
1913 1913 def add(s, x, y, z=None):
1914 1914 return None
1915 1915
1916 1916 def _timeonewrite(ui, orig, source, startrev, stoprev, runidx=None,
1917 1917 lazydeltabase=True, clearcaches=True):
1918 1918 timings = []
1919 1919 tr = _faketr()
1920 1920 with _temprevlog(ui, orig, startrev) as dest:
1921 1921 dest._lazydeltabase = lazydeltabase
1922 1922 revs = list(orig.revs(startrev, stoprev))
1923 1923 total = len(revs)
1924 1924 topic = 'adding'
1925 1925 if runidx is not None:
1926 1926 topic += ' (run #%d)' % runidx
1927 1927 # Support both old and new progress API
1928 1928 if util.safehasattr(ui, 'makeprogress'):
1929 1929 progress = ui.makeprogress(topic, unit='revs', total=total)
1930 1930 def updateprogress(pos):
1931 1931 progress.update(pos)
1932 1932 def completeprogress():
1933 1933 progress.complete()
1934 1934 else:
1935 1935 def updateprogress(pos):
1936 1936 ui.progress(topic, pos, unit='revs', total=total)
1937 1937 def completeprogress():
1938 1938 ui.progress(topic, None, unit='revs', total=total)
1939 1939
1940 1940 for idx, rev in enumerate(revs):
1941 1941 updateprogress(idx)
1942 1942 addargs, addkwargs = _getrevisionseed(orig, rev, tr, source)
1943 1943 if clearcaches:
1944 1944 dest.index.clearcaches()
1945 1945 dest.clearcaches()
1946 1946 with timeone() as r:
1947 1947 dest.addrawrevision(*addargs, **addkwargs)
1948 1948 timings.append((rev, r[0]))
1949 1949 updateprogress(total)
1950 1950 completeprogress()
1951 1951 return timings
1952 1952
1953 1953 def _getrevisionseed(orig, rev, tr, source):
1954 1954 from mercurial.node import nullid
1955 1955
1956 1956 linkrev = orig.linkrev(rev)
1957 1957 node = orig.node(rev)
1958 1958 p1, p2 = orig.parents(node)
1959 1959 flags = orig.flags(rev)
1960 1960 cachedelta = None
1961 1961 text = None
1962 1962
1963 1963 if source == b'full':
1964 1964 text = orig.revision(rev)
1965 1965 elif source == b'parent-1':
1966 1966 baserev = orig.rev(p1)
1967 1967 cachedelta = (baserev, orig.revdiff(p1, rev))
1968 1968 elif source == b'parent-2':
1969 1969 parent = p2
1970 1970 if p2 == nullid:
1971 1971 parent = p1
1972 1972 baserev = orig.rev(parent)
1973 1973 cachedelta = (baserev, orig.revdiff(parent, rev))
1974 1974 elif source == b'parent-smallest':
1975 1975 p1diff = orig.revdiff(p1, rev)
1976 1976 parent = p1
1977 1977 diff = p1diff
1978 1978 if p2 != nullid:
1979 1979 p2diff = orig.revdiff(p2, rev)
1980 1980 if len(p1diff) > len(p2diff):
1981 1981 parent = p2
1982 1982 diff = p2diff
1983 1983 baserev = orig.rev(parent)
1984 1984 cachedelta = (baserev, diff)
1985 1985 elif source == b'storage':
1986 1986 baserev = orig.deltaparent(rev)
1987 1987 cachedelta = (baserev, orig.revdiff(orig.node(baserev), rev))
1988 1988
1989 1989 return ((text, tr, linkrev, p1, p2),
1990 1990 {'node': node, 'flags': flags, 'cachedelta': cachedelta})
1991 1991
1992 1992 @contextlib.contextmanager
1993 1993 def _temprevlog(ui, orig, truncaterev):
1994 1994 from mercurial import vfs as vfsmod
1995 1995
1996 1996 if orig._inline:
1997 1997 raise error.Abort('not supporting inline revlog (yet)')
1998 1998
1999 1999 origindexpath = orig.opener.join(orig.indexfile)
2000 2000 origdatapath = orig.opener.join(orig.datafile)
2001 2001 indexname = 'revlog.i'
2002 2002 dataname = 'revlog.d'
2003 2003
2004 2004 tmpdir = tempfile.mkdtemp(prefix='tmp-hgperf-')
2005 2005 try:
2006 2006 # copy the data file in a temporary directory
2007 2007 ui.debug('copying data in %s\n' % tmpdir)
2008 2008 destindexpath = os.path.join(tmpdir, 'revlog.i')
2009 2009 destdatapath = os.path.join(tmpdir, 'revlog.d')
2010 2010 shutil.copyfile(origindexpath, destindexpath)
2011 2011 shutil.copyfile(origdatapath, destdatapath)
2012 2012
2013 2013 # remove the data we want to add again
2014 2014 ui.debug('truncating data to be rewritten\n')
2015 2015 with open(destindexpath, 'ab') as index:
2016 2016 index.seek(0)
2017 2017 index.truncate(truncaterev * orig._io.size)
2018 2018 with open(destdatapath, 'ab') as data:
2019 2019 data.seek(0)
2020 2020 data.truncate(orig.start(truncaterev))
2021 2021
2022 2022 # instantiate a new revlog from the temporary copy
2023 2023 ui.debug('truncating adding to be rewritten\n')
2024 2024 vfs = vfsmod.vfs(tmpdir)
2025 2025 vfs.options = getattr(orig.opener, 'options', None)
2026 2026
2027 2027 dest = revlog.revlog(vfs,
2028 2028 indexfile=indexname,
2029 2029 datafile=dataname)
2030 2030 if dest._inline:
2031 2031 raise error.Abort('not supporting inline revlog (yet)')
2032 2032 # make sure internals are initialized
2033 2033 dest.revision(len(dest) - 1)
2034 2034 yield dest
2035 2035 del dest, vfs
2036 2036 finally:
2037 2037 shutil.rmtree(tmpdir, True)
2038 2038
2039 2039 @command(b'perfrevlogchunks', revlogopts + formatteropts +
2040 2040 [(b'e', b'engines', b'', b'compression engines to use'),
2041 2041 (b's', b'startrev', 0, b'revision to start at')],
2042 2042 b'-c|-m|FILE')
2043 2043 def perfrevlogchunks(ui, repo, file_=None, engines=None, startrev=0, **opts):
2044 2044 """Benchmark operations on revlog chunks.
2045 2045
2046 2046 Logically, each revlog is a collection of fulltext revisions. However,
2047 2047 stored within each revlog are "chunks" of possibly compressed data. This
2048 2048 data needs to be read and decompressed or compressed and written.
2049 2049
2050 2050 This command measures the time it takes to read+decompress and recompress
2051 2051 chunks in a revlog. It effectively isolates I/O and compression performance.
2052 2052 For measurements of higher-level operations like resolving revisions,
2053 2053 see ``perfrevlogrevisions`` and ``perfrevlogrevision``.
2054 2054 """
2055 2055 opts = _byteskwargs(opts)
2056 2056
2057 2057 rl = cmdutil.openrevlog(repo, b'perfrevlogchunks', file_, opts)
2058 2058
2059 2059 # _chunkraw was renamed to _getsegmentforrevs.
2060 2060 try:
2061 2061 segmentforrevs = rl._getsegmentforrevs
2062 2062 except AttributeError:
2063 2063 segmentforrevs = rl._chunkraw
2064 2064
2065 2065 # Verify engines argument.
2066 2066 if engines:
2067 2067 engines = set(e.strip() for e in engines.split(b','))
2068 2068 for engine in engines:
2069 2069 try:
2070 2070 util.compressionengines[engine]
2071 2071 except KeyError:
2072 2072 raise error.Abort(b'unknown compression engine: %s' % engine)
2073 2073 else:
2074 2074 engines = []
2075 2075 for e in util.compengines:
2076 2076 engine = util.compengines[e]
2077 2077 try:
2078 2078 if engine.available():
2079 2079 engine.revlogcompressor().compress(b'dummy')
2080 2080 engines.append(e)
2081 2081 except NotImplementedError:
2082 2082 pass
2083 2083
2084 2084 revs = list(rl.revs(startrev, len(rl) - 1))
2085 2085
2086 2086 def rlfh(rl):
2087 2087 if rl._inline:
2088 2088 return getsvfs(repo)(rl.indexfile)
2089 2089 else:
2090 2090 return getsvfs(repo)(rl.datafile)
2091 2091
2092 2092 def doread():
2093 2093 rl.clearcaches()
2094 2094 for rev in revs:
2095 2095 segmentforrevs(rev, rev)
2096 2096
2097 2097 def doreadcachedfh():
2098 2098 rl.clearcaches()
2099 2099 fh = rlfh(rl)
2100 2100 for rev in revs:
2101 2101 segmentforrevs(rev, rev, df=fh)
2102 2102
2103 2103 def doreadbatch():
2104 2104 rl.clearcaches()
2105 2105 segmentforrevs(revs[0], revs[-1])
2106 2106
2107 2107 def doreadbatchcachedfh():
2108 2108 rl.clearcaches()
2109 2109 fh = rlfh(rl)
2110 2110 segmentforrevs(revs[0], revs[-1], df=fh)
2111 2111
2112 2112 def dochunk():
2113 2113 rl.clearcaches()
2114 2114 fh = rlfh(rl)
2115 2115 for rev in revs:
2116 2116 rl._chunk(rev, df=fh)
2117 2117
2118 2118 chunks = [None]
2119 2119
2120 2120 def dochunkbatch():
2121 2121 rl.clearcaches()
2122 2122 fh = rlfh(rl)
2123 2123 # Save chunks as a side-effect.
2124 2124 chunks[0] = rl._chunks(revs, df=fh)
2125 2125
2126 2126 def docompress(compressor):
2127 2127 rl.clearcaches()
2128 2128
2129 2129 try:
2130 2130 # Swap in the requested compression engine.
2131 2131 oldcompressor = rl._compressor
2132 2132 rl._compressor = compressor
2133 2133 for chunk in chunks[0]:
2134 2134 rl.compress(chunk)
2135 2135 finally:
2136 2136 rl._compressor = oldcompressor
2137 2137
2138 2138 benches = [
2139 2139 (lambda: doread(), b'read'),
2140 2140 (lambda: doreadcachedfh(), b'read w/ reused fd'),
2141 2141 (lambda: doreadbatch(), b'read batch'),
2142 2142 (lambda: doreadbatchcachedfh(), b'read batch w/ reused fd'),
2143 2143 (lambda: dochunk(), b'chunk'),
2144 2144 (lambda: dochunkbatch(), b'chunk batch'),
2145 2145 ]
2146 2146
2147 2147 for engine in sorted(engines):
2148 2148 compressor = util.compengines[engine].revlogcompressor()
2149 2149 benches.append((functools.partial(docompress, compressor),
2150 2150 b'compress w/ %s' % engine))
2151 2151
2152 2152 for fn, title in benches:
2153 2153 timer, fm = gettimer(ui, opts)
2154 2154 timer(fn, title=title)
2155 2155 fm.end()
2156 2156
2157 2157 @command(b'perfrevlogrevision', revlogopts + formatteropts +
2158 2158 [(b'', b'cache', False, b'use caches instead of clearing')],
2159 2159 b'-c|-m|FILE REV')
2160 2160 def perfrevlogrevision(ui, repo, file_, rev=None, cache=None, **opts):
2161 2161 """Benchmark obtaining a revlog revision.
2162 2162
2163 2163 Obtaining a revlog revision consists of roughly the following steps:
2164 2164
2165 2165 1. Compute the delta chain
2166 2166 2. Slice the delta chain if applicable
2167 2167 3. Obtain the raw chunks for that delta chain
2168 2168 4. Decompress each raw chunk
2169 2169 5. Apply binary patches to obtain fulltext
2170 2170 6. Verify hash of fulltext
2171 2171
2172 2172 This command measures the time spent in each of these phases.
2173 2173 """
2174 2174 opts = _byteskwargs(opts)
2175 2175
2176 2176 if opts.get(b'changelog') or opts.get(b'manifest'):
2177 2177 file_, rev = None, file_
2178 2178 elif rev is None:
2179 2179 raise error.CommandError(b'perfrevlogrevision', b'invalid arguments')
2180 2180
2181 2181 r = cmdutil.openrevlog(repo, b'perfrevlogrevision', file_, opts)
2182 2182
2183 2183 # _chunkraw was renamed to _getsegmentforrevs.
2184 2184 try:
2185 2185 segmentforrevs = r._getsegmentforrevs
2186 2186 except AttributeError:
2187 2187 segmentforrevs = r._chunkraw
2188 2188
2189 2189 node = r.lookup(rev)
2190 2190 rev = r.rev(node)
2191 2191
2192 2192 def getrawchunks(data, chain):
2193 2193 start = r.start
2194 2194 length = r.length
2195 2195 inline = r._inline
2196 2196 iosize = r._io.size
2197 2197 buffer = util.buffer
2198 2198
2199 2199 chunks = []
2200 2200 ladd = chunks.append
2201 2201 for idx, item in enumerate(chain):
2202 2202 offset = start(item[0])
2203 2203 bits = data[idx]
2204 2204 for rev in item:
2205 2205 chunkstart = start(rev)
2206 2206 if inline:
2207 2207 chunkstart += (rev + 1) * iosize
2208 2208 chunklength = length(rev)
2209 2209 ladd(buffer(bits, chunkstart - offset, chunklength))
2210 2210
2211 2211 return chunks
2212 2212
2213 2213 def dodeltachain(rev):
2214 2214 if not cache:
2215 2215 r.clearcaches()
2216 2216 r._deltachain(rev)
2217 2217
2218 2218 def doread(chain):
2219 2219 if not cache:
2220 2220 r.clearcaches()
2221 2221 for item in slicedchain:
2222 2222 segmentforrevs(item[0], item[-1])
2223 2223
2224 2224 def doslice(r, chain, size):
2225 2225 for s in slicechunk(r, chain, targetsize=size):
2226 2226 pass
2227 2227
2228 2228 def dorawchunks(data, chain):
2229 2229 if not cache:
2230 2230 r.clearcaches()
2231 2231 getrawchunks(data, chain)
2232 2232
2233 2233 def dodecompress(chunks):
2234 2234 decomp = r.decompress
2235 2235 for chunk in chunks:
2236 2236 decomp(chunk)
2237 2237
2238 2238 def dopatch(text, bins):
2239 2239 if not cache:
2240 2240 r.clearcaches()
2241 2241 mdiff.patches(text, bins)
2242 2242
2243 2243 def dohash(text):
2244 2244 if not cache:
2245 2245 r.clearcaches()
2246 2246 r.checkhash(text, node, rev=rev)
2247 2247
2248 2248 def dorevision():
2249 2249 if not cache:
2250 2250 r.clearcaches()
2251 2251 r.revision(node)
2252 2252
2253 2253 try:
2254 2254 from mercurial.revlogutils.deltas import slicechunk
2255 2255 except ImportError:
2256 2256 slicechunk = getattr(revlog, '_slicechunk', None)
2257 2257
2258 2258 size = r.length(rev)
2259 2259 chain = r._deltachain(rev)[0]
2260 2260 if not getattr(r, '_withsparseread', False):
2261 2261 slicedchain = (chain,)
2262 2262 else:
2263 2263 slicedchain = tuple(slicechunk(r, chain, targetsize=size))
2264 2264 data = [segmentforrevs(seg[0], seg[-1])[1] for seg in slicedchain]
2265 2265 rawchunks = getrawchunks(data, slicedchain)
2266 2266 bins = r._chunks(chain)
2267 2267 text = bytes(bins[0])
2268 2268 bins = bins[1:]
2269 2269 text = mdiff.patches(text, bins)
2270 2270
2271 2271 benches = [
2272 2272 (lambda: dorevision(), b'full'),
2273 2273 (lambda: dodeltachain(rev), b'deltachain'),
2274 2274 (lambda: doread(chain), b'read'),
2275 2275 ]
2276 2276
2277 2277 if getattr(r, '_withsparseread', False):
2278 2278 slicing = (lambda: doslice(r, chain, size), b'slice-sparse-chain')
2279 2279 benches.append(slicing)
2280 2280
2281 2281 benches.extend([
2282 2282 (lambda: dorawchunks(data, slicedchain), b'rawchunks'),
2283 2283 (lambda: dodecompress(rawchunks), b'decompress'),
2284 2284 (lambda: dopatch(text, bins), b'patch'),
2285 2285 (lambda: dohash(text), b'hash'),
2286 2286 ])
2287 2287
2288 2288 timer, fm = gettimer(ui, opts)
2289 2289 for fn, title in benches:
2290 2290 timer(fn, title=title)
2291 2291 fm.end()
2292 2292
2293 2293 @command(b'perfrevset',
2294 2294 [(b'C', b'clear', False, b'clear volatile cache between each call.'),
2295 2295 (b'', b'contexts', False, b'obtain changectx for each revision')]
2296 2296 + formatteropts, b"REVSET")
2297 2297 def perfrevset(ui, repo, expr, clear=False, contexts=False, **opts):
2298 2298 """benchmark the execution time of a revset
2299 2299
2300 2300 Use the --clean option if need to evaluate the impact of build volatile
2301 2301 revisions set cache on the revset execution. Volatile cache hold filtered
2302 2302 and obsolete related cache."""
2303 2303 opts = _byteskwargs(opts)
2304 2304
2305 2305 timer, fm = gettimer(ui, opts)
2306 2306 def d():
2307 2307 if clear:
2308 2308 repo.invalidatevolatilesets()
2309 2309 if contexts:
2310 2310 for ctx in repo.set(expr): pass
2311 2311 else:
2312 2312 for r in repo.revs(expr): pass
2313 2313 timer(d)
2314 2314 fm.end()
2315 2315
2316 2316 @command(b'perfvolatilesets',
2317 2317 [(b'', b'clear-obsstore', False, b'drop obsstore between each call.'),
2318 2318 ] + formatteropts)
2319 2319 def perfvolatilesets(ui, repo, *names, **opts):
2320 2320 """benchmark the computation of various volatile set
2321 2321
2322 2322 Volatile set computes element related to filtering and obsolescence."""
2323 2323 opts = _byteskwargs(opts)
2324 2324 timer, fm = gettimer(ui, opts)
2325 2325 repo = repo.unfiltered()
2326 2326
2327 2327 def getobs(name):
2328 2328 def d():
2329 2329 repo.invalidatevolatilesets()
2330 2330 if opts[b'clear_obsstore']:
2331 2331 clearfilecache(repo, b'obsstore')
2332 2332 obsolete.getrevs(repo, name)
2333 2333 return d
2334 2334
2335 2335 allobs = sorted(obsolete.cachefuncs)
2336 2336 if names:
2337 2337 allobs = [n for n in allobs if n in names]
2338 2338
2339 2339 for name in allobs:
2340 2340 timer(getobs(name), title=name)
2341 2341
2342 2342 def getfiltered(name):
2343 2343 def d():
2344 2344 repo.invalidatevolatilesets()
2345 2345 if opts[b'clear_obsstore']:
2346 2346 clearfilecache(repo, b'obsstore')
2347 2347 repoview.filterrevs(repo, name)
2348 2348 return d
2349 2349
2350 2350 allfilter = sorted(repoview.filtertable)
2351 2351 if names:
2352 2352 allfilter = [n for n in allfilter if n in names]
2353 2353
2354 2354 for name in allfilter:
2355 2355 timer(getfiltered(name), title=name)
2356 2356 fm.end()
2357 2357
2358 2358 @command(b'perfbranchmap',
2359 2359 [(b'f', b'full', False,
2360 2360 b'Includes build time of subset'),
2361 2361 (b'', b'clear-revbranch', False,
2362 2362 b'purge the revbranch cache between computation'),
2363 2363 ] + formatteropts)
2364 2364 def perfbranchmap(ui, repo, *filternames, **opts):
2365 2365 """benchmark the update of a branchmap
2366 2366
2367 2367 This benchmarks the full repo.branchmap() call with read and write disabled
2368 2368 """
2369 2369 opts = _byteskwargs(opts)
2370 2370 full = opts.get(b"full", False)
2371 2371 clear_revbranch = opts.get(b"clear_revbranch", False)
2372 2372 timer, fm = gettimer(ui, opts)
2373 2373 def getbranchmap(filtername):
2374 2374 """generate a benchmark function for the filtername"""
2375 2375 if filtername is None:
2376 2376 view = repo
2377 2377 else:
2378 2378 view = repo.filtered(filtername)
2379 2379 def d():
2380 2380 if clear_revbranch:
2381 2381 repo.revbranchcache()._clear()
2382 2382 if full:
2383 2383 view._branchcaches.clear()
2384 2384 else:
2385 2385 view._branchcaches.pop(filtername, None)
2386 2386 view.branchmap()
2387 2387 return d
2388 2388 # add filter in smaller subset to bigger subset
2389 2389 possiblefilters = set(repoview.filtertable)
2390 2390 if filternames:
2391 2391 possiblefilters &= set(filternames)
2392 2392 subsettable = getbranchmapsubsettable()
2393 2393 allfilters = []
2394 2394 while possiblefilters:
2395 2395 for name in possiblefilters:
2396 2396 subset = subsettable.get(name)
2397 2397 if subset not in possiblefilters:
2398 2398 break
2399 2399 else:
2400 2400 assert False, b'subset cycle %s!' % possiblefilters
2401 2401 allfilters.append(name)
2402 2402 possiblefilters.remove(name)
2403 2403
2404 2404 # warm the cache
2405 2405 if not full:
2406 2406 for name in allfilters:
2407 2407 repo.filtered(name).branchmap()
2408 2408 if not filternames or b'unfiltered' in filternames:
2409 2409 # add unfiltered
2410 2410 allfilters.append(None)
2411 2411
2412 branchcacheread = safeattrsetter(branchmap, b'read')
2412 if util.safehasattr(branchmap.branchcache, 'fromfile'):
2413 branchcacheread = safeattrsetter(branchmap.branchcache, b'fromfile')
2414 branchcacheread.set(classmethod(lambda *args: None))
2415 else:
2416 # older versions
2417 branchcacheread = safeattrsetter(branchmap, b'read')
2418 branchcacheread.set(lambda *args: None)
2413 2419 branchcachewrite = safeattrsetter(branchmap.branchcache, b'write')
2414 branchcacheread.set(lambda repo: None)
2415 branchcachewrite.set(lambda bc, repo: None)
2420 branchcachewrite.set(lambda *args: None)
2416 2421 try:
2417 2422 for name in allfilters:
2418 2423 printname = name
2419 2424 if name is None:
2420 2425 printname = b'unfiltered'
2421 2426 timer(getbranchmap(name), title=str(printname))
2422 2427 finally:
2423 2428 branchcacheread.restore()
2424 2429 branchcachewrite.restore()
2425 2430 fm.end()
2426 2431
2427 2432 @command(b'perfbranchmapupdate', [
2428 2433 (b'', b'base', [], b'subset of revision to start from'),
2429 2434 (b'', b'target', [], b'subset of revision to end with'),
2430 2435 (b'', b'clear-caches', False, b'clear cache between each runs')
2431 2436 ] + formatteropts)
2432 2437 def perfbranchmapupdate(ui, repo, base=(), target=(), **opts):
2433 2438 """benchmark branchmap update from for <base> revs to <target> revs
2434 2439
2435 2440 If `--clear-caches` is passed, the following items will be reset before
2436 2441 each update:
2437 2442 * the changelog instance and associated indexes
2438 2443 * the rev-branch-cache instance
2439 2444
2440 2445 Examples:
2441 2446
2442 2447 # update for the one last revision
2443 2448 $ hg perfbranchmapupdate --base 'not tip' --target 'tip'
2444 2449
2445 2450 $ update for change coming with a new branch
2446 2451 $ hg perfbranchmapupdate --base 'stable' --target 'default'
2447 2452 """
2448 2453 from mercurial import branchmap
2449 2454 from mercurial import repoview
2450 2455 opts = _byteskwargs(opts)
2451 2456 timer, fm = gettimer(ui, opts)
2452 2457 clearcaches = opts[b'clear_caches']
2453 2458 unfi = repo.unfiltered()
2454 2459 x = [None] # used to pass data between closure
2455 2460
2456 2461 # we use a `list` here to avoid possible side effect from smartset
2457 2462 baserevs = list(scmutil.revrange(repo, base))
2458 2463 targetrevs = list(scmutil.revrange(repo, target))
2459 2464 if not baserevs:
2460 2465 raise error.Abort(b'no revisions selected for --base')
2461 2466 if not targetrevs:
2462 2467 raise error.Abort(b'no revisions selected for --target')
2463 2468
2464 2469 # make sure the target branchmap also contains the one in the base
2465 2470 targetrevs = list(set(baserevs) | set(targetrevs))
2466 2471 targetrevs.sort()
2467 2472
2468 2473 cl = repo.changelog
2469 2474 allbaserevs = list(cl.ancestors(baserevs, inclusive=True))
2470 2475 allbaserevs.sort()
2471 2476 alltargetrevs = frozenset(cl.ancestors(targetrevs, inclusive=True))
2472 2477
2473 2478 newrevs = list(alltargetrevs.difference(allbaserevs))
2474 2479 newrevs.sort()
2475 2480
2476 2481 allrevs = frozenset(unfi.changelog.revs())
2477 2482 basefilterrevs = frozenset(allrevs.difference(allbaserevs))
2478 2483 targetfilterrevs = frozenset(allrevs.difference(alltargetrevs))
2479 2484
2480 2485 def basefilter(repo, visibilityexceptions=None):
2481 2486 return basefilterrevs
2482 2487
2483 2488 def targetfilter(repo, visibilityexceptions=None):
2484 2489 return targetfilterrevs
2485 2490
2486 2491 msg = b'benchmark of branchmap with %d revisions with %d new ones\n'
2487 2492 ui.status(msg % (len(allbaserevs), len(newrevs)))
2488 2493 if targetfilterrevs:
2489 2494 msg = b'(%d revisions still filtered)\n'
2490 2495 ui.status(msg % len(targetfilterrevs))
2491 2496
2492 2497 try:
2493 2498 repoview.filtertable[b'__perf_branchmap_update_base'] = basefilter
2494 2499 repoview.filtertable[b'__perf_branchmap_update_target'] = targetfilter
2495 2500
2496 2501 baserepo = repo.filtered(b'__perf_branchmap_update_base')
2497 2502 targetrepo = repo.filtered(b'__perf_branchmap_update_target')
2498 2503
2499 2504 # try to find an existing branchmap to reuse
2500 2505 subsettable = getbranchmapsubsettable()
2501 2506 candidatefilter = subsettable.get(None)
2502 2507 while candidatefilter is not None:
2503 2508 candidatebm = repo.filtered(candidatefilter).branchmap()
2504 2509 if candidatebm.validfor(baserepo):
2505 2510 filtered = repoview.filterrevs(repo, candidatefilter)
2506 2511 missing = [r for r in allbaserevs if r in filtered]
2507 2512 base = candidatebm.copy()
2508 2513 base.update(baserepo, missing)
2509 2514 break
2510 2515 candidatefilter = subsettable.get(candidatefilter)
2511 2516 else:
2512 2517 # no suitable subset where found
2513 2518 base = branchmap.branchcache()
2514 2519 base.update(baserepo, allbaserevs)
2515 2520
2516 2521 def setup():
2517 2522 x[0] = base.copy()
2518 2523 if clearcaches:
2519 2524 unfi._revbranchcache = None
2520 2525 clearchangelog(repo)
2521 2526
2522 2527 def bench():
2523 2528 x[0].update(targetrepo, newrevs)
2524 2529
2525 2530 timer(bench, setup=setup)
2526 2531 fm.end()
2527 2532 finally:
2528 2533 repoview.filtertable.pop(b'__perf_branchmap_update_base', None)
2529 2534 repoview.filtertable.pop(b'__perf_branchmap_update_target', None)
2530 2535
2531 2536 @command(b'perfbranchmapload', [
2532 2537 (b'f', b'filter', b'', b'Specify repoview filter'),
2533 2538 (b'', b'list', False, b'List brachmap filter caches'),
2534 2539 (b'', b'clear-revlogs', False, b'refresh changelog and manifest'),
2535 2540
2536 2541 ] + formatteropts)
2537 2542 def perfbranchmapload(ui, repo, filter=b'', list=False, **opts):
2538 2543 """benchmark reading the branchmap"""
2539 2544 opts = _byteskwargs(opts)
2540 2545 clearrevlogs = opts[b'clear_revlogs']
2541 2546
2542 2547 if list:
2543 2548 for name, kind, st in repo.cachevfs.readdir(stat=True):
2544 2549 if name.startswith(b'branch2'):
2545 2550 filtername = name.partition(b'-')[2] or b'unfiltered'
2546 2551 ui.status(b'%s - %s\n'
2547 2552 % (filtername, util.bytecount(st.st_size)))
2548 2553 return
2549 2554 if not filter:
2550 2555 filter = None
2551 2556 subsettable = getbranchmapsubsettable()
2552 2557 if filter is None:
2553 2558 repo = repo.unfiltered()
2554 2559 else:
2555 2560 repo = repoview.repoview(repo, filter)
2556 2561
2557 2562 repo.branchmap() # make sure we have a relevant, up to date branchmap
2558 2563
2564 try:
2565 fromfile = branchmap.branchcache.fromfile
2566 except AttributeError:
2567 # older versions
2568 fromfile = branchmap.read
2569
2559 2570 currentfilter = filter
2560 2571 # try once without timer, the filter may not be cached
2561 while branchmap.read(repo) is None:
2572 while fromfile(repo) is None:
2562 2573 currentfilter = subsettable.get(currentfilter)
2563 2574 if currentfilter is None:
2564 2575 raise error.Abort(b'No branchmap cached for %s repo'
2565 2576 % (filter or b'unfiltered'))
2566 2577 repo = repo.filtered(currentfilter)
2567 2578 timer, fm = gettimer(ui, opts)
2568 2579 def setup():
2569 2580 if clearrevlogs:
2570 2581 clearchangelog(repo)
2571 2582 def bench():
2572 branchmap.read(repo)
2583 fromfile(repo)
2573 2584 timer(bench, setup=setup)
2574 2585 fm.end()
2575 2586
2576 2587 @command(b'perfloadmarkers')
2577 2588 def perfloadmarkers(ui, repo):
2578 2589 """benchmark the time to parse the on-disk markers for a repo
2579 2590
2580 2591 Result is the number of markers in the repo."""
2581 2592 timer, fm = gettimer(ui)
2582 2593 svfs = getsvfs(repo)
2583 2594 timer(lambda: len(obsolete.obsstore(svfs)))
2584 2595 fm.end()
2585 2596
2586 2597 @command(b'perflrucachedict', formatteropts +
2587 2598 [(b'', b'costlimit', 0, b'maximum total cost of items in cache'),
2588 2599 (b'', b'mincost', 0, b'smallest cost of items in cache'),
2589 2600 (b'', b'maxcost', 100, b'maximum cost of items in cache'),
2590 2601 (b'', b'size', 4, b'size of cache'),
2591 2602 (b'', b'gets', 10000, b'number of key lookups'),
2592 2603 (b'', b'sets', 10000, b'number of key sets'),
2593 2604 (b'', b'mixed', 10000, b'number of mixed mode operations'),
2594 2605 (b'', b'mixedgetfreq', 50, b'frequency of get vs set ops in mixed mode')],
2595 2606 norepo=True)
2596 2607 def perflrucache(ui, mincost=0, maxcost=100, costlimit=0, size=4,
2597 2608 gets=10000, sets=10000, mixed=10000, mixedgetfreq=50, **opts):
2598 2609 opts = _byteskwargs(opts)
2599 2610
2600 2611 def doinit():
2601 2612 for i in _xrange(10000):
2602 2613 util.lrucachedict(size)
2603 2614
2604 2615 costrange = list(range(mincost, maxcost + 1))
2605 2616
2606 2617 values = []
2607 2618 for i in _xrange(size):
2608 2619 values.append(random.randint(0, _maxint))
2609 2620
2610 2621 # Get mode fills the cache and tests raw lookup performance with no
2611 2622 # eviction.
2612 2623 getseq = []
2613 2624 for i in _xrange(gets):
2614 2625 getseq.append(random.choice(values))
2615 2626
2616 2627 def dogets():
2617 2628 d = util.lrucachedict(size)
2618 2629 for v in values:
2619 2630 d[v] = v
2620 2631 for key in getseq:
2621 2632 value = d[key]
2622 2633 value # silence pyflakes warning
2623 2634
2624 2635 def dogetscost():
2625 2636 d = util.lrucachedict(size, maxcost=costlimit)
2626 2637 for i, v in enumerate(values):
2627 2638 d.insert(v, v, cost=costs[i])
2628 2639 for key in getseq:
2629 2640 try:
2630 2641 value = d[key]
2631 2642 value # silence pyflakes warning
2632 2643 except KeyError:
2633 2644 pass
2634 2645
2635 2646 # Set mode tests insertion speed with cache eviction.
2636 2647 setseq = []
2637 2648 costs = []
2638 2649 for i in _xrange(sets):
2639 2650 setseq.append(random.randint(0, _maxint))
2640 2651 costs.append(random.choice(costrange))
2641 2652
2642 2653 def doinserts():
2643 2654 d = util.lrucachedict(size)
2644 2655 for v in setseq:
2645 2656 d.insert(v, v)
2646 2657
2647 2658 def doinsertscost():
2648 2659 d = util.lrucachedict(size, maxcost=costlimit)
2649 2660 for i, v in enumerate(setseq):
2650 2661 d.insert(v, v, cost=costs[i])
2651 2662
2652 2663 def dosets():
2653 2664 d = util.lrucachedict(size)
2654 2665 for v in setseq:
2655 2666 d[v] = v
2656 2667
2657 2668 # Mixed mode randomly performs gets and sets with eviction.
2658 2669 mixedops = []
2659 2670 for i in _xrange(mixed):
2660 2671 r = random.randint(0, 100)
2661 2672 if r < mixedgetfreq:
2662 2673 op = 0
2663 2674 else:
2664 2675 op = 1
2665 2676
2666 2677 mixedops.append((op,
2667 2678 random.randint(0, size * 2),
2668 2679 random.choice(costrange)))
2669 2680
2670 2681 def domixed():
2671 2682 d = util.lrucachedict(size)
2672 2683
2673 2684 for op, v, cost in mixedops:
2674 2685 if op == 0:
2675 2686 try:
2676 2687 d[v]
2677 2688 except KeyError:
2678 2689 pass
2679 2690 else:
2680 2691 d[v] = v
2681 2692
2682 2693 def domixedcost():
2683 2694 d = util.lrucachedict(size, maxcost=costlimit)
2684 2695
2685 2696 for op, v, cost in mixedops:
2686 2697 if op == 0:
2687 2698 try:
2688 2699 d[v]
2689 2700 except KeyError:
2690 2701 pass
2691 2702 else:
2692 2703 d.insert(v, v, cost=cost)
2693 2704
2694 2705 benches = [
2695 2706 (doinit, b'init'),
2696 2707 ]
2697 2708
2698 2709 if costlimit:
2699 2710 benches.extend([
2700 2711 (dogetscost, b'gets w/ cost limit'),
2701 2712 (doinsertscost, b'inserts w/ cost limit'),
2702 2713 (domixedcost, b'mixed w/ cost limit'),
2703 2714 ])
2704 2715 else:
2705 2716 benches.extend([
2706 2717 (dogets, b'gets'),
2707 2718 (doinserts, b'inserts'),
2708 2719 (dosets, b'sets'),
2709 2720 (domixed, b'mixed')
2710 2721 ])
2711 2722
2712 2723 for fn, title in benches:
2713 2724 timer, fm = gettimer(ui, opts)
2714 2725 timer(fn, title=title)
2715 2726 fm.end()
2716 2727
2717 2728 @command(b'perfwrite', formatteropts)
2718 2729 def perfwrite(ui, repo, **opts):
2719 2730 """microbenchmark ui.write
2720 2731 """
2721 2732 opts = _byteskwargs(opts)
2722 2733
2723 2734 timer, fm = gettimer(ui, opts)
2724 2735 def write():
2725 2736 for i in range(100000):
2726 2737 ui.write((b'Testing write performance\n'))
2727 2738 timer(write)
2728 2739 fm.end()
2729 2740
2730 2741 def uisetup(ui):
2731 2742 if (util.safehasattr(cmdutil, b'openrevlog') and
2732 2743 not util.safehasattr(commands, b'debugrevlogopts')):
2733 2744 # for "historical portability":
2734 2745 # In this case, Mercurial should be 1.9 (or a79fea6b3e77) -
2735 2746 # 3.7 (or 5606f7d0d063). Therefore, '--dir' option for
2736 2747 # openrevlog() should cause failure, because it has been
2737 2748 # available since 3.5 (or 49c583ca48c4).
2738 2749 def openrevlog(orig, repo, cmd, file_, opts):
2739 2750 if opts.get(b'dir') and not util.safehasattr(repo, b'dirlog'):
2740 2751 raise error.Abort(b"This version doesn't support --dir option",
2741 2752 hint=b"use 3.5 or later")
2742 2753 return orig(repo, cmd, file_, opts)
2743 2754 extensions.wrapfunction(cmdutil, b'openrevlog', openrevlog)
2744 2755
2745 2756 @command(b'perfprogress', formatteropts + [
2746 2757 (b'', b'topic', b'topic', b'topic for progress messages'),
2747 2758 (b'c', b'total', 1000000, b'total value we are progressing to'),
2748 2759 ], norepo=True)
2749 2760 def perfprogress(ui, topic=None, total=None, **opts):
2750 2761 """printing of progress bars"""
2751 2762 opts = _byteskwargs(opts)
2752 2763
2753 2764 timer, fm = gettimer(ui, opts)
2754 2765
2755 2766 def doprogress():
2756 2767 with ui.makeprogress(topic, total=total) as progress:
2757 2768 for i in pycompat.xrange(total):
2758 2769 progress.increment()
2759 2770
2760 2771 timer(doprogress)
2761 2772 fm.end()
@@ -1,560 +1,561 b''
1 1 # branchmap.py - logic to computes, maintain and stores branchmap for local repo
2 2 #
3 3 # Copyright 2005-2007 Matt Mackall <mpm@selenic.com>
4 4 #
5 5 # This software may be used and distributed according to the terms of the
6 6 # GNU General Public License version 2 or any later version.
7 7
8 8 from __future__ import absolute_import
9 9
10 10 import struct
11 11
12 12 from .node import (
13 13 bin,
14 14 hex,
15 15 nullid,
16 16 nullrev,
17 17 )
18 18 from . import (
19 19 encoding,
20 20 error,
21 21 pycompat,
22 22 scmutil,
23 23 util,
24 24 )
25 25 from .utils import (
26 26 stringutil,
27 27 )
28 28
29 29 calcsize = struct.calcsize
30 30 pack_into = struct.pack_into
31 31 unpack_from = struct.unpack_from
32 32
33 def _filename(repo):
34 """name of a branchcache file for a given repo or repoview"""
35 filename = "branch2"
36 if repo.filtername:
37 filename = '%s-%s' % (filename, repo.filtername)
38 return filename
39
40 def read(repo):
41 f = None
42 try:
43 f = repo.cachevfs(_filename(repo))
44 lineiter = iter(f)
45 cachekey = next(lineiter).rstrip('\n').split(" ", 2)
46 last, lrev = cachekey[:2]
47 last, lrev = bin(last), int(lrev)
48 filteredhash = None
49 if len(cachekey) > 2:
50 filteredhash = bin(cachekey[2])
51 bcache = branchcache(tipnode=last, tiprev=lrev,
52 filteredhash=filteredhash)
53 if not bcache.validfor(repo):
54 # invalidate the cache
55 raise ValueError(r'tip differs')
56 cl = repo.changelog
57 for l in lineiter:
58 l = l.rstrip('\n')
59 if not l:
60 continue
61 node, state, label = l.split(" ", 2)
62 if state not in 'oc':
63 raise ValueError(r'invalid branch state')
64 label = encoding.tolocal(label.strip())
65 node = bin(node)
66 if not cl.hasnode(node):
67 raise ValueError(
68 r'node %s does not exist' % pycompat.sysstr(hex(node)))
69 bcache.setdefault(label, []).append(node)
70 if state == 'c':
71 bcache._closednodes.add(node)
72
73 except (IOError, OSError):
74 return None
75
76 except Exception as inst:
77 if repo.ui.debugflag:
78 msg = 'invalid branchheads cache'
79 if repo.filtername is not None:
80 msg += ' (%s)' % repo.filtername
81 msg += ': %s\n'
82 repo.ui.debug(msg % pycompat.bytestr(inst))
83 bcache = None
84
85 finally:
86 if f:
87 f.close()
88
89 return bcache
90 33
91 34 ### Nearest subset relation
92 35 # Nearest subset of filter X is a filter Y so that:
93 36 # * Y is included in X,
94 37 # * X - Y is as small as possible.
95 38 # This create and ordering used for branchmap purpose.
96 39 # the ordering may be partial
97 40 subsettable = {None: 'visible',
98 41 'visible-hidden': 'visible',
99 42 'visible': 'served',
100 43 'served': 'immutable',
101 44 'immutable': 'base'}
102 45
103 46 def updatecache(repo):
104 47 cl = repo.changelog
105 48 filtername = repo.filtername
106 49 bcache = repo._branchcaches.get(filtername)
107 50
108 51 revs = []
109 52 if bcache is None or not bcache.validfor(repo):
110 bcache = read(repo)
53 bcache = branchcache.fromfile(repo)
111 54 if bcache is None:
112 55 subsetname = subsettable.get(filtername)
113 56 if subsetname is None:
114 57 bcache = branchcache()
115 58 else:
116 59 subset = repo.filtered(subsetname)
117 60 bcache = subset.branchmap().copy()
118 61 extrarevs = subset.changelog.filteredrevs - cl.filteredrevs
119 62 revs.extend(r for r in extrarevs if r <= bcache.tiprev)
120 63 revs.extend(cl.revs(start=bcache.tiprev + 1))
121 64 if revs:
122 65 bcache.update(repo, revs)
123 66 bcache.write(repo)
124 67
125 68 assert bcache.validfor(repo), filtername
126 69 repo._branchcaches[repo.filtername] = bcache
127 70
128 71 def replacecache(repo, bm):
129 72 """Replace the branchmap cache for a repo with a branch mapping.
130 73
131 74 This is likely only called during clone with a branch map from a remote.
132 75 """
133 76 cl = repo.changelog
134 77 clrev = cl.rev
135 78 clbranchinfo = cl.branchinfo
136 79 rbheads = []
137 80 closed = []
138 81 for bheads in bm.itervalues():
139 82 rbheads.extend(bheads)
140 83 for h in bheads:
141 84 r = clrev(h)
142 85 b, c = clbranchinfo(r)
143 86 if c:
144 87 closed.append(h)
145 88
146 89 if rbheads:
147 90 rtiprev = max((int(clrev(node))
148 91 for node in rbheads))
149 92 cache = branchcache(bm,
150 93 repo[rtiprev].node(),
151 94 rtiprev,
152 95 closednodes=closed)
153 96
154 97 # Try to stick it as low as possible
155 98 # filter above served are unlikely to be fetch from a clone
156 99 for candidate in ('base', 'immutable', 'served'):
157 100 rview = repo.filtered(candidate)
158 101 if cache.validfor(rview):
159 102 repo._branchcaches[candidate] = cache
160 103 cache.write(rview)
161 104 break
162 105
163 106 class branchcache(dict):
164 107 """A dict like object that hold branches heads cache.
165 108
166 109 This cache is used to avoid costly computations to determine all the
167 110 branch heads of a repo.
168 111
169 112 The cache is serialized on disk in the following format:
170 113
171 114 <tip hex node> <tip rev number> [optional filtered repo hex hash]
172 115 <branch head hex node> <open/closed state> <branch name>
173 116 <branch head hex node> <open/closed state> <branch name>
174 117 ...
175 118
176 119 The first line is used to check if the cache is still valid. If the
177 120 branch cache is for a filtered repo view, an optional third hash is
178 121 included that hashes the hashes of all filtered revisions.
179 122
180 123 The open/closed state is represented by a single letter 'o' or 'c'.
181 124 This field can be used to avoid changelog reads when determining if a
182 125 branch head closes a branch or not.
183 126 """
127 @classmethod
128 def fromfile(cls, repo):
129 f = None
130 try:
131 f = repo.cachevfs(cls._filename(repo))
132 lineiter = iter(f)
133 cachekey = next(lineiter).rstrip('\n').split(" ", 2)
134 last, lrev = cachekey[:2]
135 last, lrev = bin(last), int(lrev)
136 filteredhash = None
137 if len(cachekey) > 2:
138 filteredhash = bin(cachekey[2])
139 bcache = cls(tipnode=last, tiprev=lrev, filteredhash=filteredhash)
140 if not bcache.validfor(repo):
141 # invalidate the cache
142 raise ValueError(r'tip differs')
143 cl = repo.changelog
144 for line in lineiter:
145 line = line.rstrip('\n')
146 if not line:
147 continue
148 node, state, label = line.split(" ", 2)
149 if state not in 'oc':
150 raise ValueError(r'invalid branch state')
151 label = encoding.tolocal(label.strip())
152 node = bin(node)
153 if not cl.hasnode(node):
154 raise ValueError(
155 r'node %s does not exist' % pycompat.sysstr(hex(node)))
156 bcache.setdefault(label, []).append(node)
157 if state == 'c':
158 bcache._closednodes.add(node)
159
160 except (IOError, OSError):
161 return None
162
163 except Exception as inst:
164 if repo.ui.debugflag:
165 msg = 'invalid branchheads cache'
166 if repo.filtername is not None:
167 msg += ' (%s)' % repo.filtername
168 msg += ': %s\n'
169 repo.ui.debug(msg % pycompat.bytestr(inst))
170 bcache = None
171
172 finally:
173 if f:
174 f.close()
175
176 return bcache
177
178 @staticmethod
179 def _filename(repo):
180 """name of a branchcache file for a given repo or repoview"""
181 filename = "branch2"
182 if repo.filtername:
183 filename = '%s-%s' % (filename, repo.filtername)
184 return filename
184 185
185 186 def __init__(self, entries=(), tipnode=nullid, tiprev=nullrev,
186 187 filteredhash=None, closednodes=None):
187 188 super(branchcache, self).__init__(entries)
188 189 self.tipnode = tipnode
189 190 self.tiprev = tiprev
190 191 self.filteredhash = filteredhash
191 192 # closednodes is a set of nodes that close their branch. If the branch
192 193 # cache has been updated, it may contain nodes that are no longer
193 194 # heads.
194 195 if closednodes is None:
195 196 self._closednodes = set()
196 197 else:
197 198 self._closednodes = closednodes
198 199
199 200 def validfor(self, repo):
200 201 """Is the cache content valid regarding a repo
201 202
202 203 - False when cached tipnode is unknown or if we detect a strip.
203 204 - True when cache is up to date or a subset of current repo."""
204 205 try:
205 206 return ((self.tipnode == repo.changelog.node(self.tiprev))
206 207 and (self.filteredhash == \
207 208 scmutil.filteredhash(repo, self.tiprev)))
208 209 except IndexError:
209 210 return False
210 211
211 212 def _branchtip(self, heads):
212 213 '''Return tuple with last open head in heads and false,
213 214 otherwise return last closed head and true.'''
214 215 tip = heads[-1]
215 216 closed = True
216 217 for h in reversed(heads):
217 218 if h not in self._closednodes:
218 219 tip = h
219 220 closed = False
220 221 break
221 222 return tip, closed
222 223
223 224 def branchtip(self, branch):
224 225 '''Return the tipmost open head on branch head, otherwise return the
225 226 tipmost closed head on branch.
226 227 Raise KeyError for unknown branch.'''
227 228 return self._branchtip(self[branch])[0]
228 229
229 230 def iteropen(self, nodes):
230 231 return (n for n in nodes if n not in self._closednodes)
231 232
232 233 def branchheads(self, branch, closed=False):
233 234 heads = self[branch]
234 235 if not closed:
235 236 heads = list(self.iteropen(heads))
236 237 return heads
237 238
238 239 def iterbranches(self):
239 240 for bn, heads in self.iteritems():
240 241 yield (bn, heads) + self._branchtip(heads)
241 242
242 243 def copy(self):
243 244 """return an deep copy of the branchcache object"""
244 245 return branchcache(self, self.tipnode, self.tiprev, self.filteredhash,
245 246 self._closednodes)
246 247
247 248 def write(self, repo):
248 249 try:
249 f = repo.cachevfs(_filename(repo), "w", atomictemp=True)
250 f = repo.cachevfs(self._filename(repo), "w", atomictemp=True)
250 251 cachekey = [hex(self.tipnode), '%d' % self.tiprev]
251 252 if self.filteredhash is not None:
252 253 cachekey.append(hex(self.filteredhash))
253 254 f.write(" ".join(cachekey) + '\n')
254 255 nodecount = 0
255 256 for label, nodes in sorted(self.iteritems()):
256 257 for node in nodes:
257 258 nodecount += 1
258 259 if node in self._closednodes:
259 260 state = 'c'
260 261 else:
261 262 state = 'o'
262 263 f.write("%s %s %s\n" % (hex(node), state,
263 264 encoding.fromlocal(label)))
264 265 f.close()
265 266 repo.ui.log('branchcache',
266 267 'wrote %s branch cache with %d labels and %d nodes\n',
267 268 repo.filtername, len(self), nodecount)
268 269 except (IOError, OSError, error.Abort) as inst:
269 270 # Abort may be raised by read only opener, so log and continue
270 271 repo.ui.debug("couldn't write branch cache: %s\n" %
271 272 stringutil.forcebytestr(inst))
272 273
273 274 def update(self, repo, revgen):
274 275 """Given a branchhead cache, self, that may have extra nodes or be
275 276 missing heads, and a generator of nodes that are strictly a superset of
276 277 heads missing, this function updates self to be correct.
277 278 """
278 279 starttime = util.timer()
279 280 cl = repo.changelog
280 281 # collect new branch entries
281 282 newbranches = {}
282 283 getbranchinfo = repo.revbranchcache().branchinfo
283 284 for r in revgen:
284 285 branch, closesbranch = getbranchinfo(r)
285 286 newbranches.setdefault(branch, []).append(r)
286 287 if closesbranch:
287 288 self._closednodes.add(cl.node(r))
288 289
289 290 # fetch current topological heads to speed up filtering
290 291 topoheads = set(cl.headrevs())
291 292
292 293 # if older branchheads are reachable from new ones, they aren't
293 294 # really branchheads. Note checking parents is insufficient:
294 295 # 1 (branch a) -> 2 (branch b) -> 3 (branch a)
295 296 for branch, newheadrevs in newbranches.iteritems():
296 297 bheads = self.setdefault(branch, [])
297 298 bheadset = set(cl.rev(node) for node in bheads)
298 299
299 300 # This have been tested True on all internal usage of this function.
300 301 # run it again in case of doubt
301 302 # assert not (set(bheadrevs) & set(newheadrevs))
302 303 bheadset.update(newheadrevs)
303 304
304 305 # This prunes out two kinds of heads - heads that are superseded by
305 306 # a head in newheadrevs, and newheadrevs that are not heads because
306 307 # an existing head is their descendant.
307 308 uncertain = bheadset - topoheads
308 309 if uncertain:
309 310 floorrev = min(uncertain)
310 311 ancestors = set(cl.ancestors(newheadrevs, floorrev))
311 312 bheadset -= ancestors
312 313 bheadrevs = sorted(bheadset)
313 314 self[branch] = [cl.node(rev) for rev in bheadrevs]
314 315 tiprev = bheadrevs[-1]
315 316 if tiprev > self.tiprev:
316 317 self.tipnode = cl.node(tiprev)
317 318 self.tiprev = tiprev
318 319
319 320 if not self.validfor(repo):
320 321 # cache key are not valid anymore
321 322 self.tipnode = nullid
322 323 self.tiprev = nullrev
323 324 for heads in self.values():
324 325 tiprev = max(cl.rev(node) for node in heads)
325 326 if tiprev > self.tiprev:
326 327 self.tipnode = cl.node(tiprev)
327 328 self.tiprev = tiprev
328 329 self.filteredhash = scmutil.filteredhash(repo, self.tiprev)
329 330
330 331 duration = util.timer() - starttime
331 332 repo.ui.log('branchcache', 'updated %s branch cache in %.4f seconds\n',
332 333 repo.filtername, duration)
333 334
334 335 # Revision branch info cache
335 336
336 337 _rbcversion = '-v1'
337 338 _rbcnames = 'rbc-names' + _rbcversion
338 339 _rbcrevs = 'rbc-revs' + _rbcversion
339 340 # [4 byte hash prefix][4 byte branch name number with sign bit indicating open]
340 341 _rbcrecfmt = '>4sI'
341 342 _rbcrecsize = calcsize(_rbcrecfmt)
342 343 _rbcnodelen = 4
343 344 _rbcbranchidxmask = 0x7fffffff
344 345 _rbccloseflag = 0x80000000
345 346
346 347 class revbranchcache(object):
347 348 """Persistent cache, mapping from revision number to branch name and close.
348 349 This is a low level cache, independent of filtering.
349 350
350 351 Branch names are stored in rbc-names in internal encoding separated by 0.
351 352 rbc-names is append-only, and each branch name is only stored once and will
352 353 thus have a unique index.
353 354
354 355 The branch info for each revision is stored in rbc-revs as constant size
355 356 records. The whole file is read into memory, but it is only 'parsed' on
356 357 demand. The file is usually append-only but will be truncated if repo
357 358 modification is detected.
358 359 The record for each revision contains the first 4 bytes of the
359 360 corresponding node hash, and the record is only used if it still matches.
360 361 Even a completely trashed rbc-revs fill thus still give the right result
361 362 while converging towards full recovery ... assuming no incorrectly matching
362 363 node hashes.
363 364 The record also contains 4 bytes where 31 bits contains the index of the
364 365 branch and the last bit indicate that it is a branch close commit.
365 366 The usage pattern for rbc-revs is thus somewhat similar to 00changelog.i
366 367 and will grow with it but be 1/8th of its size.
367 368 """
368 369
369 370 def __init__(self, repo, readonly=True):
370 371 assert repo.filtername is None
371 372 self._repo = repo
372 373 self._names = [] # branch names in local encoding with static index
373 374 self._rbcrevs = bytearray()
374 375 self._rbcsnameslen = 0 # length of names read at _rbcsnameslen
375 376 try:
376 377 bndata = repo.cachevfs.read(_rbcnames)
377 378 self._rbcsnameslen = len(bndata) # for verification before writing
378 379 if bndata:
379 380 self._names = [encoding.tolocal(bn)
380 381 for bn in bndata.split('\0')]
381 382 except (IOError, OSError):
382 383 if readonly:
383 384 # don't try to use cache - fall back to the slow path
384 385 self.branchinfo = self._branchinfo
385 386
386 387 if self._names:
387 388 try:
388 389 data = repo.cachevfs.read(_rbcrevs)
389 390 self._rbcrevs[:] = data
390 391 except (IOError, OSError) as inst:
391 392 repo.ui.debug("couldn't read revision branch cache: %s\n" %
392 393 stringutil.forcebytestr(inst))
393 394 # remember number of good records on disk
394 395 self._rbcrevslen = min(len(self._rbcrevs) // _rbcrecsize,
395 396 len(repo.changelog))
396 397 if self._rbcrevslen == 0:
397 398 self._names = []
398 399 self._rbcnamescount = len(self._names) # number of names read at
399 400 # _rbcsnameslen
400 401
401 402 def _clear(self):
402 403 self._rbcsnameslen = 0
403 404 del self._names[:]
404 405 self._rbcnamescount = 0
405 406 self._rbcrevslen = len(self._repo.changelog)
406 407 self._rbcrevs = bytearray(self._rbcrevslen * _rbcrecsize)
407 408 util.clearcachedproperty(self, '_namesreverse')
408 409
409 410 @util.propertycache
410 411 def _namesreverse(self):
411 412 return dict((b, r) for r, b in enumerate(self._names))
412 413
413 414 def branchinfo(self, rev):
414 415 """Return branch name and close flag for rev, using and updating
415 416 persistent cache."""
416 417 changelog = self._repo.changelog
417 418 rbcrevidx = rev * _rbcrecsize
418 419
419 420 # avoid negative index, changelog.read(nullrev) is fast without cache
420 421 if rev == nullrev:
421 422 return changelog.branchinfo(rev)
422 423
423 424 # if requested rev isn't allocated, grow and cache the rev info
424 425 if len(self._rbcrevs) < rbcrevidx + _rbcrecsize:
425 426 return self._branchinfo(rev)
426 427
427 428 # fast path: extract data from cache, use it if node is matching
428 429 reponode = changelog.node(rev)[:_rbcnodelen]
429 430 cachenode, branchidx = unpack_from(
430 431 _rbcrecfmt, util.buffer(self._rbcrevs), rbcrevidx)
431 432 close = bool(branchidx & _rbccloseflag)
432 433 if close:
433 434 branchidx &= _rbcbranchidxmask
434 435 if cachenode == '\0\0\0\0':
435 436 pass
436 437 elif cachenode == reponode:
437 438 try:
438 439 return self._names[branchidx], close
439 440 except IndexError:
440 441 # recover from invalid reference to unknown branch
441 442 self._repo.ui.debug("referenced branch names not found"
442 443 " - rebuilding revision branch cache from scratch\n")
443 444 self._clear()
444 445 else:
445 446 # rev/node map has changed, invalidate the cache from here up
446 447 self._repo.ui.debug("history modification detected - truncating "
447 448 "revision branch cache to revision %d\n" % rev)
448 449 truncate = rbcrevidx + _rbcrecsize
449 450 del self._rbcrevs[truncate:]
450 451 self._rbcrevslen = min(self._rbcrevslen, truncate)
451 452
452 453 # fall back to slow path and make sure it will be written to disk
453 454 return self._branchinfo(rev)
454 455
455 456 def _branchinfo(self, rev):
456 457 """Retrieve branch info from changelog and update _rbcrevs"""
457 458 changelog = self._repo.changelog
458 459 b, close = changelog.branchinfo(rev)
459 460 if b in self._namesreverse:
460 461 branchidx = self._namesreverse[b]
461 462 else:
462 463 branchidx = len(self._names)
463 464 self._names.append(b)
464 465 self._namesreverse[b] = branchidx
465 466 reponode = changelog.node(rev)
466 467 if close:
467 468 branchidx |= _rbccloseflag
468 469 self._setcachedata(rev, reponode, branchidx)
469 470 return b, close
470 471
471 472 def setdata(self, branch, rev, node, close):
472 473 """add new data information to the cache"""
473 474 if branch in self._namesreverse:
474 475 branchidx = self._namesreverse[branch]
475 476 else:
476 477 branchidx = len(self._names)
477 478 self._names.append(branch)
478 479 self._namesreverse[branch] = branchidx
479 480 if close:
480 481 branchidx |= _rbccloseflag
481 482 self._setcachedata(rev, node, branchidx)
482 483 # If no cache data were readable (non exists, bad permission, etc)
483 484 # the cache was bypassing itself by setting:
484 485 #
485 486 # self.branchinfo = self._branchinfo
486 487 #
487 488 # Since we now have data in the cache, we need to drop this bypassing.
488 489 if r'branchinfo' in vars(self):
489 490 del self.branchinfo
490 491
491 492 def _setcachedata(self, rev, node, branchidx):
492 493 """Writes the node's branch data to the in-memory cache data."""
493 494 if rev == nullrev:
494 495 return
495 496 rbcrevidx = rev * _rbcrecsize
496 497 if len(self._rbcrevs) < rbcrevidx + _rbcrecsize:
497 498 self._rbcrevs.extend('\0' *
498 499 (len(self._repo.changelog) * _rbcrecsize -
499 500 len(self._rbcrevs)))
500 501 pack_into(_rbcrecfmt, self._rbcrevs, rbcrevidx, node, branchidx)
501 502 self._rbcrevslen = min(self._rbcrevslen, rev)
502 503
503 504 tr = self._repo.currenttransaction()
504 505 if tr:
505 506 tr.addfinalize('write-revbranchcache', self.write)
506 507
507 508 def write(self, tr=None):
508 509 """Save branch cache if it is dirty."""
509 510 repo = self._repo
510 511 wlock = None
511 512 step = ''
512 513 try:
513 514 if self._rbcnamescount < len(self._names):
514 515 step = ' names'
515 516 wlock = repo.wlock(wait=False)
516 517 if self._rbcnamescount != 0:
517 518 f = repo.cachevfs.open(_rbcnames, 'ab')
518 519 if f.tell() == self._rbcsnameslen:
519 520 f.write('\0')
520 521 else:
521 522 f.close()
522 523 repo.ui.debug("%s changed - rewriting it\n" % _rbcnames)
523 524 self._rbcnamescount = 0
524 525 self._rbcrevslen = 0
525 526 if self._rbcnamescount == 0:
526 527 # before rewriting names, make sure references are removed
527 528 repo.cachevfs.unlinkpath(_rbcrevs, ignoremissing=True)
528 529 f = repo.cachevfs.open(_rbcnames, 'wb')
529 530 f.write('\0'.join(encoding.fromlocal(b)
530 531 for b in self._names[self._rbcnamescount:]))
531 532 self._rbcsnameslen = f.tell()
532 533 f.close()
533 534 self._rbcnamescount = len(self._names)
534 535
535 536 start = self._rbcrevslen * _rbcrecsize
536 537 if start != len(self._rbcrevs):
537 538 step = ''
538 539 if wlock is None:
539 540 wlock = repo.wlock(wait=False)
540 541 revs = min(len(repo.changelog),
541 542 len(self._rbcrevs) // _rbcrecsize)
542 543 f = repo.cachevfs.open(_rbcrevs, 'ab')
543 544 if f.tell() != start:
544 545 repo.ui.debug("truncating cache/%s to %d\n"
545 546 % (_rbcrevs, start))
546 547 f.seek(start)
547 548 if f.tell() != start:
548 549 start = 0
549 550 f.seek(start)
550 551 f.truncate()
551 552 end = revs * _rbcrecsize
552 553 f.write(self._rbcrevs[start:end])
553 554 f.close()
554 555 self._rbcrevslen = revs
555 556 except (IOError, OSError, error.Abort, error.LockError) as inst:
556 557 repo.ui.debug("couldn't write revision branch cache%s: %s\n"
557 558 % (step, stringutil.forcebytestr(inst)))
558 559 finally:
559 560 if wlock is not None:
560 561 wlock.release()
General Comments 0
You need to be logged in to leave comments. Login now