##// END OF EJS Templates
delta-chain: move the debugdeltachain command in revlogutils...
delta-chain: move the debugdeltachain command in revlogutils There is a dedicated `mercurial.revlogutils.debug` module were this code fits well.

File last commit:

r51960:26dcdbe1 default
r51963:d7f975e4 default
Show More
deltas.py
1630 lines | 56.5 KiB | text/x-python | PythonLexer
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 # revlogdeltas.py - Logic around delta computation for revlog
#
Raphaël Gomès
contributor: change mentions of mpm to olivia...
r47575 # Copyright 2005-2007 Olivia Mackall <olivia@selenic.com>
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 # Copyright 2018 Octobus <contact@octobus.net>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
"""Helper class to compute deltas stored inside revlogs"""
Boris Feld
snapshot: search for unrelated but reusable full-snapshot...
r39529 import collections
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 import struct
# import stuff from node for others to import from revlog
Augie Fackler
formatting: blacken the codebase...
r43346 from ..node import nullrev
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 from ..i18n import _
from .constants import (
revlog: factor the logic to determine the delta compression out...
r48245 COMP_MODE_DEFAULT,
COMP_MODE_INLINE,
COMP_MODE_PLAIN,
delta-find: add a delta-reuse policy that blindly accepts incoming deltas...
r50662 DELTA_BASE_REUSE_FORCE,
find-delta: pass the cache-delta usage policy alongside the cache-delta...
r50572 DELTA_BASE_REUSE_NO,
deltas: add code to display information about the result of `finddeltainfo`...
r50121 KIND_CHANGELOG,
KIND_FILELOG,
KIND_MANIFESTLOG,
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 REVIDX_ISCENSORED,
REVIDX_RAWTEXT_CHANGING_FLAGS,
)
Augie Fackler
formatting: blacken the codebase...
r43346 from ..thirdparty import attr
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366
from .. import (
error,
mdiff,
Boris Feld
delta: have a native implementation of _findsnapshot...
r41141 util,
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 )
Augie Fackler
formatting: blacken the codebase...
r43346 from . import flagutil
flagprocessors: make `processflagsraw` a module level function...
r43262
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 # maximum <delta-chain-data>/<revision-text-length> ratio
LIMIT_DELTA2TEXT = 2
Augie Fackler
formatting: blacken the codebase...
r43346
Gregory Szorc
py3: use class X: instead of class X(object):...
r49801 class _testrevlog:
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 """minimalist fake revlog to use in doctests"""
Boris Feld
doctest: add a `issnapshot` method to _testrevlog...
r40677 def __init__(self, data, density=0.5, mingap=0, snapshot=()):
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 """data is an list of revision payload boundaries"""
revlog: use the new Config classes in _testrevlog...
r51938 from .. import revlog
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 self._data = data
revlog: use the new Config classes in _testrevlog...
r51938 self.data_config = revlog.DataConfig()
self.data_config.sr_density_threshold = density
self.data_config.sr_min_gap_size = mingap
self.delta_config = revlog.DeltaConfig()
self.feature_config = revlog.FeatureConfig()
Boris Feld
doctest: add a `issnapshot` method to _testrevlog...
r40677 self._snapshot = set(snapshot)
Boris Feld
sparse-revlog: put the native implementation of slicechunktodensity to use...
r40745 self.index = None
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366
def start(self, rev):
Boris Feld
revlog: fix pure python slicing test when chain contains nullrev...
r41110 if rev == nullrev:
return 0
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 if rev == 0:
return 0
return self._data[rev - 1]
def end(self, rev):
Boris Feld
revlog: fix pure python slicing test when chain contains nullrev...
r41110 if rev == nullrev:
return 0
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 return self._data[rev]
def length(self, rev):
return self.end(rev) - self.start(rev)
def __len__(self):
return len(self._data)
Boris Feld
doctest: add a `issnapshot` method to _testrevlog...
r40677 def issnapshot(self, rev):
Boris Feld
revlog: fix pure python slicing test when chain contains nullrev...
r41110 if rev == nullrev:
return True
Boris Feld
doctest: add a `issnapshot` method to _testrevlog...
r40677 return rev in self._snapshot
Augie Fackler
formatting: blacken the codebase...
r43346
Boris Feld
sparse-revlog: drop unused deltainfo parameter from _slicechunktodensity...
r40640 def slicechunk(revlog, revs, targetsize=None):
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 """slice revs to reduce the amount of unrelated data to be read from disk.
``revs`` is sliced into groups that should be read in one time.
Assume that revs are sorted.
The initial chunk is sliced until the overall density (payload/chunks-span
revlog: remove legacy usage of `_srdensitythreshold`...
r51955 ratio) is above `revlog.data_config.sr_density_threshold`. No gap smaller
revlog: remove legacy usage of `_srmingapsize`...
r51956 than `revlog.data_config.sr_min_gap_size` is skipped.
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366
If `targetsize` is set, no chunk larger than `targetsize` will be yield.
For consistency with other slicing choice, this limit won't go lower than
revlog: remove legacy usage of `_srmingapsize`...
r51956 `revlog.data_config.sr_min_gap_size`.
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366
If individual revisions chunk are larger than this limit, they will still
be raised individually.
Boris Feld
sparse-revlog: rework the way we enforce chunk size limit...
r40678 >>> data = [
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 ... 5, #00 (5)
... 10, #01 (5)
... 12, #02 (2)
... 12, #03 (empty)
... 27, #04 (15)
... 31, #05 (4)
... 31, #06 (empty)
... 42, #07 (11)
... 47, #08 (5)
... 47, #09 (empty)
... 48, #10 (1)
... 51, #11 (3)
... 74, #12 (23)
... 85, #13 (11)
... 86, #14 (1)
... 91, #15 (5)
Boris Feld
sparse-revlog: rework the way we enforce chunk size limit...
r40678 ... ]
>>> revlog = _testrevlog(data, snapshot=range(16))
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366
>>> list(slicechunk(revlog, list(range(16))))
[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]]
>>> list(slicechunk(revlog, [0, 15]))
[[0], [15]]
>>> list(slicechunk(revlog, [0, 11, 15]))
[[0], [11], [15]]
>>> list(slicechunk(revlog, [0, 11, 13, 15]))
[[0], [11, 13, 15]]
>>> list(slicechunk(revlog, [1, 2, 3, 5, 8, 10, 11, 14]))
[[1, 2], [5, 8, 10, 11], [14]]
Slicing with a maximum chunk size
>>> list(slicechunk(revlog, [0, 11, 13, 15], targetsize=15))
[[0], [11], [13], [15]]
>>> list(slicechunk(revlog, [0, 11, 13, 15], targetsize=20))
[[0], [11], [13, 15]]
Boris Feld
revlog: fix pure python slicing test when chain contains nullrev...
r41110
Slicing involving nullrev
>>> list(slicechunk(revlog, [-1, 0, 11, 13, 15], targetsize=20))
[[-1, 0], [11], [13, 15]]
>>> list(slicechunk(revlog, [-1, 13, 15], targetsize=5))
[[-1], [13], [15]]
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 """
if targetsize is not None:
revlog: remove legacy usage of `_srmingapsize`...
r51956 targetsize = max(targetsize, revlog.data_config.sr_min_gap_size)
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 # targetsize should not be specified when evaluating delta candidates:
# * targetsize is used to ensure we stay within specification when reading,
Boris Feld
sparse-revlog: put the native implementation of slicechunktodensity to use...
r40745 densityslicing = getattr(revlog.index, 'slicechunktodensity', None)
if densityslicing is None:
densityslicing = lambda x, y, z: _slicechunktodensity(revlog, x, y, z)
Augie Fackler
formatting: blacken the codebase...
r43346 for chunk in densityslicing(
revlog: remove legacy usage of `_srmingapsize`...
r51956 revs,
revlog.data_config.sr_density_threshold,
revlog.data_config.sr_min_gap_size,
Augie Fackler
formatting: blacken the codebase...
r43346 ):
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 for subchunk in _slicechunktosize(revlog, chunk, targetsize):
yield subchunk
Augie Fackler
formatting: blacken the codebase...
r43346
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 def _slicechunktosize(revlog, revs, targetsize=None):
"""slice revs to match the target size
This is intended to be used on chunk that density slicing selected by that
are still too large compared to the read garantee of revlog. This might
happens when "minimal gap size" interrupted the slicing or when chain are
built in a way that create large blocks next to each other.
Boris Feld
sparse-revlog: rework the way we enforce chunk size limit...
r40678 >>> data = [
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 ... 3, #0 (3)
... 5, #1 (2)
... 6, #2 (1)
... 8, #3 (2)
... 8, #4 (empty)
... 11, #5 (3)
... 12, #6 (1)
... 13, #7 (1)
... 14, #8 (1)
Boris Feld
sparse-revlog: rework the way we enforce chunk size limit...
r40678 ... ]
== All snapshots cases ==
>>> revlog = _testrevlog(data, snapshot=range(9))
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366
Cases where chunk is already small enough
>>> list(_slicechunktosize(revlog, [0], 3))
[[0]]
>>> list(_slicechunktosize(revlog, [6, 7], 3))
[[6, 7]]
>>> list(_slicechunktosize(revlog, [0], None))
[[0]]
>>> list(_slicechunktosize(revlog, [6, 7], None))
[[6, 7]]
cases where we need actual slicing
>>> list(_slicechunktosize(revlog, [0, 1], 3))
[[0], [1]]
>>> list(_slicechunktosize(revlog, [1, 3], 3))
[[1], [3]]
>>> list(_slicechunktosize(revlog, [1, 2, 3], 3))
[[1, 2], [3]]
>>> list(_slicechunktosize(revlog, [3, 5], 3))
[[3], [5]]
>>> list(_slicechunktosize(revlog, [3, 4, 5], 3))
[[3], [5]]
>>> list(_slicechunktosize(revlog, [5, 6, 7, 8], 3))
[[5], [6, 7, 8]]
>>> list(_slicechunktosize(revlog, [0, 1, 2, 3, 4, 5, 6, 7, 8], 3))
[[0], [1, 2], [3], [5], [6, 7, 8]]
Case with too large individual chunk (must return valid chunk)
>>> list(_slicechunktosize(revlog, [0, 1], 2))
[[0], [1]]
>>> list(_slicechunktosize(revlog, [1, 3], 1))
[[1], [3]]
>>> list(_slicechunktosize(revlog, [3, 4, 5], 2))
[[3], [5]]
Boris Feld
sparse-revlog: rework the way we enforce chunk size limit...
r40678
== No Snapshot cases ==
>>> revlog = _testrevlog(data)
Cases where chunk is already small enough
>>> list(_slicechunktosize(revlog, [0], 3))
[[0]]
>>> list(_slicechunktosize(revlog, [6, 7], 3))
[[6, 7]]
>>> list(_slicechunktosize(revlog, [0], None))
[[0]]
>>> list(_slicechunktosize(revlog, [6, 7], None))
[[6, 7]]
cases where we need actual slicing
>>> list(_slicechunktosize(revlog, [0, 1], 3))
[[0], [1]]
>>> list(_slicechunktosize(revlog, [1, 3], 3))
[[1], [3]]
>>> list(_slicechunktosize(revlog, [1, 2, 3], 3))
[[1], [2, 3]]
>>> list(_slicechunktosize(revlog, [3, 5], 3))
[[3], [5]]
>>> list(_slicechunktosize(revlog, [3, 4, 5], 3))
[[3], [4, 5]]
>>> list(_slicechunktosize(revlog, [5, 6, 7, 8], 3))
[[5], [6, 7, 8]]
>>> list(_slicechunktosize(revlog, [0, 1, 2, 3, 4, 5, 6, 7, 8], 3))
[[0], [1, 2], [3], [5], [6, 7, 8]]
Case with too large individual chunk (must return valid chunk)
>>> list(_slicechunktosize(revlog, [0, 1], 2))
[[0], [1]]
>>> list(_slicechunktosize(revlog, [1, 3], 1))
[[1], [3]]
>>> list(_slicechunktosize(revlog, [3, 4, 5], 2))
[[3], [5]]
== mixed case ==
>>> revlog = _testrevlog(data, snapshot=[0, 1, 2])
>>> list(_slicechunktosize(revlog, list(range(9)), 5))
[[0, 1], [2], [3, 4, 5], [6, 7, 8]]
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 """
assert targetsize is None or 0 <= targetsize
Boris Feld
sparse-revlog: rework the way we enforce chunk size limit...
r40678 startdata = revlog.start(revs[0])
enddata = revlog.end(revs[-1])
fullspan = enddata - startdata
if targetsize is None or fullspan <= targetsize:
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 yield revs
return
startrevidx = 0
Boris Feld
sparse-revlog: align endrevidx usages in the _slicechunktosize...
r40693 endrevidx = 1
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 iterrevs = enumerate(revs)
Augie Fackler
formatting: blacken the codebase...
r43346 next(iterrevs) # skip first rev.
Boris Feld
sparse-revlog: rework the way we enforce chunk size limit...
r40678 # first step: get snapshots out of the way
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 for idx, r in iterrevs:
span = revlog.end(r) - startdata
Boris Feld
sparse-revlog: rework the way we enforce chunk size limit...
r40678 snapshot = revlog.issnapshot(r)
if span <= targetsize and snapshot:
Boris Feld
sparse-revlog: align endrevidx usages in the _slicechunktosize...
r40693 endrevidx = idx + 1
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 else:
Boris Feld
sparse-revlog: align endrevidx usages in the _slicechunktosize...
r40693 chunk = _trimchunk(revlog, revs, startrevidx, endrevidx)
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 if chunk:
yield chunk
startrevidx = idx
startdata = revlog.start(r)
Boris Feld
sparse-revlog: align endrevidx usages in the _slicechunktosize...
r40693 endrevidx = idx + 1
Boris Feld
sparse-revlog: rework the way we enforce chunk size limit...
r40678 if not snapshot:
break
# for the others, we use binary slicing to quickly converge toward valid
# chunks (otherwise, we might end up looking for start/end of many
# revisions). This logic is not looking for the perfect slicing point, it
# focuses on quickly converging toward valid chunks.
nbitem = len(revs)
while (enddata - startdata) > targetsize:
endrevidx = nbitem
if nbitem - startrevidx <= 1:
Augie Fackler
formatting: blacken the codebase...
r43346 break # protect against individual chunk larger than limit
Boris Feld
sparse-revlog: rework the way we enforce chunk size limit...
r40678 localenddata = revlog.end(revs[endrevidx - 1])
span = localenddata - startdata
Boris Feld
sparse-revlog: use `span` variable as intended...
r40690 while span > targetsize:
Boris Feld
sparse-revlog: rework the way we enforce chunk size limit...
r40678 if endrevidx - startrevidx <= 1:
Augie Fackler
formatting: blacken the codebase...
r43346 break # protect against individual chunk larger than limit
Boris Feld
sparse-revlog: rework the way we enforce chunk size limit...
r40678 endrevidx -= (endrevidx - startrevidx) // 2
localenddata = revlog.end(revs[endrevidx - 1])
span = localenddata - startdata
chunk = _trimchunk(revlog, revs, startrevidx, endrevidx)
if chunk:
yield chunk
startrevidx = endrevidx
startdata = revlog.start(revs[startrevidx])
chunk = _trimchunk(revlog, revs, startrevidx)
if chunk:
yield chunk
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366
Augie Fackler
formatting: blacken the codebase...
r43346
def _slicechunktodensity(revlog, revs, targetdensity=0.5, mingapsize=0):
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 """slice revs to reduce the amount of unrelated data to be read from disk.
``revs`` is sliced into groups that should be read in one time.
Assume that revs are sorted.
The initial chunk is sliced until the overall density (payload/chunks-span
ratio) is above `targetdensity`. No gap smaller than `mingapsize` is
skipped.
>>> revlog = _testrevlog([
... 5, #00 (5)
... 10, #01 (5)
... 12, #02 (2)
... 12, #03 (empty)
... 27, #04 (15)
... 31, #05 (4)
... 31, #06 (empty)
... 42, #07 (11)
... 47, #08 (5)
... 47, #09 (empty)
... 48, #10 (1)
... 51, #11 (3)
... 74, #12 (23)
... 85, #13 (11)
... 86, #14 (1)
... 91, #15 (5)
... ])
>>> list(_slicechunktodensity(revlog, list(range(16))))
[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]]
>>> list(_slicechunktodensity(revlog, [0, 15]))
[[0], [15]]
>>> list(_slicechunktodensity(revlog, [0, 11, 15]))
[[0], [11], [15]]
>>> list(_slicechunktodensity(revlog, [0, 11, 13, 15]))
[[0], [11, 13, 15]]
>>> list(_slicechunktodensity(revlog, [1, 2, 3, 5, 8, 10, 11, 14]))
[[1, 2], [5, 8, 10, 11], [14]]
>>> list(_slicechunktodensity(revlog, [1, 2, 3, 5, 8, 10, 11, 14],
... mingapsize=20))
[[1, 2, 3, 5, 8, 10, 11], [14]]
>>> list(_slicechunktodensity(revlog, [1, 2, 3, 5, 8, 10, 11, 14],
... targetdensity=0.95))
[[1, 2], [5], [8, 10, 11], [14]]
>>> list(_slicechunktodensity(revlog, [1, 2, 3, 5, 8, 10, 11, 14],
... targetdensity=0.95, mingapsize=12))
[[1, 2], [5, 8, 10, 11], [14]]
"""
start = revlog.start
length = revlog.length
if len(revs) <= 1:
yield revs
return
Boris Feld
sparse-revlog: drop unused deltainfo parameter from _slicechunktodensity...
r40640 deltachainspan = segmentspan(revlog, revs)
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366
if deltachainspan < mingapsize:
yield revs
return
readdata = deltachainspan
Boris Feld
sparse-revlog: fast-path before computing payload size...
r40642 chainpayload = sum(length(r) for r in revs)
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366
if deltachainspan:
density = chainpayload / float(deltachainspan)
else:
density = 1.0
if density >= targetdensity:
yield revs
return
# Store the gaps in a heap to have them sorted by decreasing size
Boris Feld
sparse-revlog: stop using a heap to track gaps...
r40643 gaps = []
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 prevend = None
for i, rev in enumerate(revs):
Boris Feld
sparse-revlog: drop unused deltainfo parameter from _slicechunktodensity...
r40640 revstart = start(rev)
revlen = length(rev)
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366
# Skip empty revisions to form larger holes
if revlen == 0:
continue
if prevend is not None:
gapsize = revstart - prevend
# only consider holes that are large enough
if gapsize > mingapsize:
Boris Feld
sparse-revlog: stop using a heap to track gaps...
r40643 gaps.append((gapsize, i))
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366
prevend = revstart + revlen
Boris Feld
sparse-revlog: stop using a heap to track gaps...
r40643 # sort the gaps to pop them from largest to small
gaps.sort()
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366
# Collect the indices of the largest holes until the density is acceptable
Boris Feld
sparse-revlog: stop using a heap to track selected gap...
r40644 selected = []
Boris Feld
sparse-revlog: stop using a heap to track gaps...
r40643 while gaps and density < targetdensity:
gapsize, gapidx = gaps.pop()
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366
Boris Feld
sparse-revlog: stop using a heap to track selected gap...
r40644 selected.append(gapidx)
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366
# the gap sizes are stored as negatives to be sorted decreasingly
# by the heap
Boris Feld
sparse-revlog: stop using a heap to track gaps...
r40643 readdata -= gapsize
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 if readdata > 0:
density = chainpayload / float(readdata)
else:
density = 1.0
Boris Feld
sparse-revlog: stop using a heap to track selected gap...
r40644 selected.sort()
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366
# Cut the revs at collected indices
previdx = 0
Boris Feld
sparse-revlog: stop using a heap to track selected gap...
r40644 for idx in selected:
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366
chunk = _trimchunk(revlog, revs, previdx, idx)
if chunk:
yield chunk
previdx = idx
chunk = _trimchunk(revlog, revs, previdx)
if chunk:
yield chunk
Augie Fackler
formatting: blacken the codebase...
r43346
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 def _trimchunk(revlog, revs, startidx, endidx=None):
"""returns revs[startidx:endidx] without empty trailing revs
Doctest Setup
>>> revlog = _testrevlog([
... 5, #0
... 10, #1
... 12, #2
... 12, #3 (empty)
... 17, #4
... 21, #5
... 21, #6 (empty)
... ])
Contiguous cases:
>>> _trimchunk(revlog, [0, 1, 2, 3, 4, 5, 6], 0)
[0, 1, 2, 3, 4, 5]
>>> _trimchunk(revlog, [0, 1, 2, 3, 4, 5, 6], 0, 5)
[0, 1, 2, 3, 4]
>>> _trimchunk(revlog, [0, 1, 2, 3, 4, 5, 6], 0, 4)
[0, 1, 2]
>>> _trimchunk(revlog, [0, 1, 2, 3, 4, 5, 6], 2, 4)
[2]
>>> _trimchunk(revlog, [0, 1, 2, 3, 4, 5, 6], 3)
[3, 4, 5]
>>> _trimchunk(revlog, [0, 1, 2, 3, 4, 5, 6], 3, 5)
[3, 4]
Discontiguous cases:
>>> _trimchunk(revlog, [1, 3, 5, 6], 0)
[1, 3, 5]
>>> _trimchunk(revlog, [1, 3, 5, 6], 0, 2)
[1]
>>> _trimchunk(revlog, [1, 3, 5, 6], 1, 3)
[3, 5]
>>> _trimchunk(revlog, [1, 3, 5, 6], 1)
[3, 5]
"""
length = revlog.length
if endidx is None:
endidx = len(revs)
# If we have a non-emtpy delta candidate, there are nothing to trim
if revs[endidx - 1] < len(revlog):
# Trim empty revs at the end, except the very first revision of a chain
Augie Fackler
formatting: blacken the codebase...
r43346 while (
endidx > 1 and endidx > startidx and length(revs[endidx - 1]) == 0
):
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 endidx -= 1
return revs[startidx:endidx]
Augie Fackler
formatting: blacken the codebase...
r43346
Boris Feld
sparse-revlog: drop unused deltainfo parameter from segmentspan...
r40641 def segmentspan(revlog, revs):
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 """Get the byte span of a segment of revisions
revs is a sorted array of revision numbers
>>> revlog = _testrevlog([
... 5, #0
... 10, #1
... 12, #2
... 12, #3 (empty)
... 17, #4
... ])
>>> segmentspan(revlog, [0, 1, 2, 3, 4])
17
>>> segmentspan(revlog, [0, 4])
17
>>> segmentspan(revlog, [3, 4])
5
>>> segmentspan(revlog, [1, 2, 3,])
7
>>> segmentspan(revlog, [1, 3])
7
"""
if not revs:
return 0
Boris Feld
sparse-revlog: drop unused deltainfo parameter from segmentspan...
r40641 end = revlog.end(revs[-1])
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 return end - revlog.start(revs[0])
Augie Fackler
formatting: blacken the codebase...
r43346
delta-computer: stop explicitly taking file handle...
r51913 def _textfromdelta(revlog, baserev, delta, p1, p2, flags, expectednode):
Boris Feld
revlogdeltas: extra fulltext building in its own function...
r39367 """build full text from a (base, delta) pair and other metadata"""
# special case deltas which replace entire base; no need to decode
# base revision. this neatly avoids censored bases, which throw when
# they're decoded.
Augie Fackler
formatting: byteify all mercurial/ and hgext/ string literals...
r43347 hlen = struct.calcsize(b">lll")
Augie Fackler
formatting: blacken the codebase...
r43346 if delta[:hlen] == mdiff.replacediffheader(
revlog.rawsize(baserev), len(delta) - hlen
):
Boris Feld
revlogdeltas: extra fulltext building in its own function...
r39367 fulltext = delta[hlen:]
else:
# deltabase is rawtext before changed by flag processors, which is
# equivalent to non-raw text
delta-computer: stop explicitly taking file handle...
r51913 basetext = revlog.revision(baserev)
Boris Feld
revlogdeltas: extra fulltext building in its own function...
r39367 fulltext = mdiff.patch(basetext, delta)
try:
flagprocessors: make `processflagsraw` a module level function...
r43262 validatehash = flagutil.processflagsraw(revlog, fulltext, flags)
Boris Feld
revlogdeltas: extra fulltext building in its own function...
r39367 if validatehash:
revlog.checkhash(fulltext, expectednode, p1=p1, p2=p2)
if flags & REVIDX_ISCENSORED:
Augie Fackler
formatting: blacken the codebase...
r43346 raise error.StorageError(
Augie Fackler
formatting: byteify all mercurial/ and hgext/ string literals...
r43347 _(b'node %s is not censored') % expectednode
Augie Fackler
formatting: blacken the codebase...
r43346 )
Gregory Szorc
revlog: drop some more error aliases (API)...
r39810 except error.CensoredNodeError:
Boris Feld
revlogdeltas: extra fulltext building in its own function...
r39367 # must pass the censored index flag to add censored revisions
if not flags & REVIDX_ISCENSORED:
raise
return fulltext
Augie Fackler
formatting: blacken the codebase...
r43346
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 @attr.s(slots=True, frozen=True)
Gregory Szorc
py3: use class X: instead of class X(object):...
r49801 class _deltainfo:
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 distance = attr.ib()
deltalen = attr.ib()
data = attr.ib()
base = attr.ib()
chainbase = attr.ib()
chainlen = attr.ib()
compresseddeltalen = attr.ib()
snapshotdepth = attr.ib()
Augie Fackler
formatting: blacken the codebase...
r43346
revlog: introduce a plain compression mode...
r48027 def drop_u_compression(delta):
"""turn into a "u" (no-compression) into no-compression without header
This is useful for revlog format that has better compression method.
"""
assert delta.data[0] == b'u', delta.data[0]
return _deltainfo(
delta.distance,
delta.deltalen - 1,
(b'', delta.data[1]),
delta.base,
delta.chainbase,
delta.chainlen,
delta.compresseddeltalen,
delta.snapshotdepth,
)
find-delta: rename _isgooddeltainfo...
r50569 def is_good_delta_info(revlog, deltainfo, revinfo):
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 """Returns True if the given delta is good. Good means that it is within
the disk span, disk size, and chain length bounds that we know to be
performant."""
if deltainfo is None:
return False
delta-find: remove dead code intended to deal with forced delta reuse...
r51591 # the DELTA_BASE_REUSE_FORCE case should have been taken care of sooner so
# we should never end up asking such question. Adding the assert as a
# safe-guard to detect anything that would be fishy in this regard.
assert (
revinfo.cachedelta is None
or revinfo.cachedelta[2] != DELTA_BASE_REUSE_FORCE
revlog: remove legacy usage of `_generaldelta`...
r51939 or not revlog.delta_config.general_delta
delta-find: remove dead code intended to deal with forced delta reuse...
r51591 )
delta-find: add a delta-reuse policy that blindly accepts incoming deltas...
r50662
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 # - 'deltainfo.distance' is the distance from the base revision --
# bounding it limits the amount of I/O we need to do.
# - 'deltainfo.compresseddeltalen' is the sum of the total size of
# deltas we need to apply -- bounding it limits the amount of CPU
# we consume.
textlen = revinfo.textlen
defaultmax = textlen * 4
revlog: remove legacy usage of `_maxdeltachainspan`...
r51951 maxdist = revlog.delta_config.max_deltachain_span
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 if not maxdist:
Augie Fackler
formatting: blacken the codebase...
r43346 maxdist = deltainfo.distance # ensure the conditional pass
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 maxdist = max(maxdist, defaultmax)
# Bad delta from read span:
#
# If the span of data read is larger than the maximum allowed.
Boris Feld
sparse-revlog: skip the span check in the sparse-revlog case...
r40639 #
# In the sparse-revlog case, we rely on the associated "sparse reading"
# to avoid issue related to the span of data. In theory, it would be
# possible to build pathological revlog where delta pattern would lead
# to too many reads. However, they do not happen in practice at all. So
# we skip the span check entirely.
revlog: remove legacy usage of `_sparserevlog`...
r51953 if not revlog.delta_config.sparse_revlog and maxdist < deltainfo.distance:
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 return False
# Bad delta from new delta size:
#
# If the delta size is larger than the target text, storing the
# delta will be inefficient.
if textlen < deltainfo.deltalen:
return False
# Bad delta from cumulated payload size:
#
# If the sum of delta get larger than K * target text length.
if textlen * LIMIT_DELTA2TEXT < deltainfo.compresseddeltalen:
return False
# Bad delta from chain length:
#
# If the number of delta in the chain gets too high.
revlog: remove legacy usage of `_maxchainlen`...
r51945 if (
revlog.delta_config.max_chain_len
and revlog.delta_config.max_chain_len < deltainfo.chainlen
):
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 return False
# bad delta from intermediate snapshot size limit
#
# If an intermediate snapshot size is higher than the limit. The
# limit exist to prevent endless chain of intermediate delta to be
# created.
Augie Fackler
formatting: blacken the codebase...
r43346 if (
deltainfo.snapshotdepth is not None
and (textlen >> deltainfo.snapshotdepth) < deltainfo.deltalen
):
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 return False
# bad delta if new intermediate snapshot is larger than the previous
# snapshot
Augie Fackler
formatting: blacken the codebase...
r43346 if (
deltainfo.snapshotdepth
and revlog.length(deltainfo.base) < deltainfo.deltalen
):
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 return False
return True
Augie Fackler
formatting: blacken the codebase...
r43346
Boris Feld
delta: exclude base candidate much smaller than the target...
r41014 # If a revision's full text is that much bigger than a base candidate full
# text's, it is very unlikely that it will produce a valid delta. We no longer
# consider these candidates.
Boris Feld
revlog: limit base to rev size ratio to 500 instead of 50...
r41062 LIMIT_BASE2TEXT = 500
Boris Feld
delta: exclude base candidate much smaller than the target...
r41014
Augie Fackler
formatting: blacken the codebase...
r43346
delta-find: expand a function definition and call before extendin it...
r50507 def _candidategroups(
revlog,
textlen,
p1,
p2,
cachedelta,
delta-find: move pre-filtering with other pre-filtering logic...
r50508 excluded_bases=None,
target_rev=None,
delta-find: use a single snapshot cache when applying a group to an object...
r50576 snapshot_cache=None,
delta-find: expand a function definition and call before extendin it...
r50507 ):
Boris Feld
revlogdeltas: split candidate groups selection from the filtering logic...
r39372 """Provides group of revision to be tested as delta base
This top level function focus on emitting groups with unique and worthwhile
content. See _raw_candidate_groups for details about the group order.
Boris Feld
revlogdeltas: extract _getcandidaterevs in a function...
r39370 """
Boris Feld
revlogdeltas: split candidate groups selection from the filtering logic...
r39372 # should we try to build a delta?
if not (len(revlog) and revlog._storedeltachains):
Boris Feld
snapshot: use None as a stop value when looking for a good delta...
r39533 yield None
Boris Feld
revlogdeltas: split candidate groups selection from the filtering logic...
r39372 return
delta-find: assume the target-rev if not specified...
r51330 if target_rev is None:
target_rev = len(revlog)
revlog: remove legacy usage of `_generaldelta`...
r51939 if not revlog.delta_config.general_delta:
delta-find: never do anything fancy when general delta is off...
r51331 # before general delta, there is only one possible delta base
yield (target_rev - 1,)
yield None
return
delta-find: remove dead code intended to deal with forced delta reuse...
r51591 # the DELTA_BASE_REUSE_FORCE case should have been taken care of sooner so
# we should never end up asking such question. Adding the assert as a
# safe-guard to detect anything that would be fishy in this regard.
assert (
cachedelta is None
or cachedelta[2] != DELTA_BASE_REUSE_FORCE
revlog: remove legacy usage of `_generaldelta`...
r51939 or not revlog.delta_config.general_delta
delta-find: remove dead code intended to deal with forced delta reuse...
r51591 )
bundle: when forcing acceptance of incoming delta also accept snapshot...
r50677
Boris Feld
revlogdeltas: move finddeltainfo filtering inside _candidategroups...
r39373 deltalength = revlog.length
deltaparent = revlog.deltaparent
revlog: remove legacy usage of `_sparserevlog`...
r51953 sparse = revlog.delta_config.sparse_revlog
Boris Feld
snapshot: add refining logic at the findeltainfo level...
r39534 good = None
Boris Feld
revlogdeltas: move finddeltainfo filtering inside _candidategroups...
r39373
deltas_limit = textlen * LIMIT_DELTA2TEXT
revlog: remove legacy usage of `_candidate_group_chunk_size`...
r51948 group_chunk_size = revlog.delta_config.candidate_group_chunk_size
Boris Feld
revlogdeltas: move finddeltainfo filtering inside _candidategroups...
r39373
Martin von Zweigbergk
cleanup: use set literals where possible...
r42224 tested = {nullrev}
debug: add an option to display statistic about a unbundling operation...
r50506 candidates = _refinedgroups(
revlog,
p1,
p2,
cachedelta,
delta-find: use a single snapshot cache when applying a group to an object...
r50576 snapshot_cache=snapshot_cache,
debug: add an option to display statistic about a unbundling operation...
r50506 )
Boris Feld
snapshot: also use None as a stop value for `_refinegroup`...
r39535 while True:
Boris Feld
snapshot: turn _refinedgroups into a coroutine...
r39536 temptative = candidates.send(good)
Boris Feld
snapshot: also use None as a stop value for `_refinegroup`...
r39535 if temptative is None:
break
Boris Feld
revlogdeltas: move finddeltainfo filtering inside _candidategroups...
r39373 group = []
for rev in temptative:
# skip over empty delta (no need to include them in a chain)
delta-find: simply code that is now never invoqued without general delta...
r51333 while not (rev == nullrev or rev in tested or deltalength(rev)):
Boris Feld
snapshot: fix line order when skipping over empty deltas...
r39630 tested.add(rev)
Boris Feld
revlogdeltas: move finddeltainfo filtering inside _candidategroups...
r39373 rev = deltaparent(rev)
Boris Feld
delta: filter nullrev out first...
r40993 # no need to try a delta against nullrev, this will be done as a
# last resort.
if rev == nullrev:
continue
Boris Feld
revlogdeltas: move finddeltainfo filtering inside _candidategroups...
r39373 # filter out revision we tested already
if rev in tested:
continue
delta-find: add a delta-reuse policy that blindly accepts incoming deltas...
r50662
delta-find: move pre-filtering with other pre-filtering logic...
r50508 # an higher authority deamed the base unworthy (e.g. censored)
if excluded_bases is not None and rev in excluded_bases:
delta-find: adjust the moment when we mark something as "tested"...
r50511 tested.add(rev)
delta-find: move pre-filtering with other pre-filtering logic...
r50508 continue
# We are in some recomputation cases and that rev is too high in
# the revlog
if target_rev is not None and rev >= target_rev:
delta-find: adjust the moment when we mark something as "tested"...
r50511 tested.add(rev)
delta-find: move pre-filtering with other pre-filtering logic...
r50508 continue
Boris Feld
revlogdeltas: move finddeltainfo filtering inside _candidategroups...
r39373 # filter out delta base that will never produce good delta
if deltas_limit < revlog.length(rev):
delta-find: adjust the moment when we mark something as "tested"...
r50511 tested.add(rev)
Boris Feld
revlogdeltas: move finddeltainfo filtering inside _candidategroups...
r39373 continue
Boris Feld
delta: exclude base candidate much smaller than the target...
r41014 if sparse and revlog.rawsize(rev) < (textlen // LIMIT_BASE2TEXT):
delta-find: adjust the moment when we mark something as "tested"...
r50511 tested.add(rev)
Boris Feld
delta: exclude base candidate much smaller than the target...
r41014 continue
Boris Feld
revlogdeltas: move finddeltainfo filtering inside _candidategroups...
r39373 # no delta for rawtext-changing revs (see "candelta" for why)
if revlog.flags(rev) & REVIDX_RAWTEXT_CHANGING_FLAGS:
delta-find: adjust the moment when we mark something as "tested"...
r50511 tested.add(rev)
Boris Feld
revlogdeltas: move finddeltainfo filtering inside _candidategroups...
r39373 continue
debug: add an option to display statistic about a unbundling operation...
r50506
Boris Feld
delta: ignore base whose chains already don't match expectations...
r41015 # If we reach here, we are about to build and test a delta.
# The delta building process will compute the chaininfo in all
# case, since that computation is cached, it is fine to access it
# here too.
chainlen, chainsize = revlog._chaininfo(rev)
# if chain will be too long, skip base
revlog: remove legacy usage of `_maxchainlen`...
r51945 if (
revlog.delta_config.max_chain_len
and chainlen >= revlog.delta_config.max_chain_len
):
delta-find: adjust the moment when we mark something as "tested"...
r50511 tested.add(rev)
Boris Feld
delta: ignore base whose chains already don't match expectations...
r41015 continue
# if chain already have too much data, skip base
if deltas_limit < chainsize:
delta-find: adjust the moment when we mark something as "tested"...
r50511 tested.add(rev)
Boris Feld
delta: ignore base whose chains already don't match expectations...
r41015 continue
Valentin Gatien-Baron , Pierre-Yves David pierre-yves.david@octobus.net
deltas: skip if projected delta size does not match text size constraint...
r42663 if sparse and revlog.upperboundcomp is not None:
maxcomp = revlog.upperboundcomp
basenotsnap = (p1, p2, nullrev)
if rev not in basenotsnap and revlog.issnapshot(rev):
snapshotdepth = revlog.snapshotdepth(rev)
# If text is significantly larger than the base, we can
# expect the resulting delta to be proportional to the size
# difference
revsize = revlog.rawsize(rev)
rawsizedistance = max(textlen - revsize, 0)
# use an estimate of the compression upper bound.
lowestrealisticdeltalen = rawsizedistance // maxcomp
# check the absolute constraint on the delta size
snapshotlimit = textlen >> snapshotdepth
if snapshotlimit < lowestrealisticdeltalen:
# delta lower bound is larger than accepted upper bound
delta-find: adjust the moment when we mark something as "tested"...
r50511 tested.add(rev)
Valentin Gatien-Baron , Pierre-Yves David pierre-yves.david@octobus.net
deltas: skip if projected delta size does not match text size constraint...
r42663 continue
Valentin Gatien-Baron
deltas: skip if projected delta size is bigger than previous snapshot...
r42664 # check the relative constraint on the delta size
revlength = revlog.length(rev)
if revlength < lowestrealisticdeltalen:
# delta probable lower bound is larger than target base
delta-find: adjust the moment when we mark something as "tested"...
r50511 tested.add(rev)
Valentin Gatien-Baron
deltas: skip if projected delta size is bigger than previous snapshot...
r42664 continue
Boris Feld
revlogdeltas: move finddeltainfo filtering inside _candidategroups...
r39373 group.append(rev)
Boris Feld
revlogdeltas: split candidate groups selection from the filtering logic...
r39372 if group:
delta-find: add a way to control the number of bases tested at the same time...
r50552 # When the size of the candidate group is big, it can result in a
# quite significant performance impact. To reduce this, we can send
# them in smaller batches until the new batch does not provide any
# improvements.
#
# This might reduce the overall efficiency of the compression in
# some corner cases, but that should also prevent very pathological
# cases from being an issue. (eg. 20 000 candidates).
#
# XXX note that the ordering of the group becomes important as it
# now impacts the final result. The current order is unprocessed
# and can be improved.
if group_chunk_size == 0:
tested.update(group)
good = yield tuple(group)
else:
prev_good = good
for start in range(0, len(group), group_chunk_size):
sub_group = group[start : start + group_chunk_size]
tested.update(sub_group)
good = yield tuple(sub_group)
if prev_good == good:
break
Boris Feld
snapshot: use None as a stop value when looking for a good delta...
r39533 yield None
Boris Feld
revlogdeltas: split candidate groups selection from the filtering logic...
r39372
Augie Fackler
formatting: blacken the codebase...
r43346
delta-find: use a single snapshot cache when applying a group to an object...
r50576 def _refinedgroups(revlog, p1, p2, cachedelta, snapshot_cache=None):
Boris Feld
snapshot: introduce an intermediate `_refinedgroups` generator...
r39532 good = None
Boris Feld
snapshot: make sure we'll never refine delta base from a reused source...
r39537 # First we try to reuse a the delta contained in the bundle.
# (or from the source revlog)
#
# This logic only applies to general delta repositories and can be disabled
# through configuration. Disabling reuse source delta is useful when
# we want to make sure we recomputed "optimal" deltas.
debug: add an option to display statistic about a unbundling operation...
r50506 debug_info = None
find-delta: pass the cache-delta usage policy alongside the cache-delta...
r50572 if cachedelta is not None and cachedelta[2] > DELTA_BASE_REUSE_NO:
Boris Feld
snapshot: make sure we'll never refine delta base from a reused source...
r39537 # Assume what we received from the server is a good choice
# build delta will reuse the cache
debug: add an option to display statistic about a unbundling operation...
r50506 if debug_info is not None:
debug_info['cached-delta.tested'] += 1
Boris Feld
snapshot: make sure we'll never refine delta base from a reused source...
r39537 good = yield (cachedelta[0],)
if good is not None:
debug: add an option to display statistic about a unbundling operation...
r50506 if debug_info is not None:
debug_info['cached-delta.accepted'] += 1
Boris Feld
snapshot: make sure we'll never refine delta base from a reused source...
r39537 yield None
return
delta-find: use a single snapshot cache when applying a group to an object...
r50576 if snapshot_cache is None:
snapshot_cache = SnapshotCache()
debug: add an option to display statistic about a unbundling operation...
r50506 groups = _rawgroups(
revlog,
p1,
p2,
cachedelta,
delta-find: use a smarter object for snapshot caching...
r50573 snapshot_cache,
debug: add an option to display statistic about a unbundling operation...
r50506 )
for candidates in groups:
Boris Feld
snapshot: introduce an intermediate `_refinedgroups` generator...
r39532 good = yield candidates
if good is not None:
break
Boris Feld
snapshot: try to refine new snapshot base down the chain...
r39538
Boris Feld
sparse-revlog: only refine delta candidates in the sparse case (issue6006)...
r40479 # If sparse revlog is enabled, we can try to refine the available deltas
revlog: remove legacy usage of `_sparserevlog`...
r51953 if not revlog.delta_config.sparse_revlog:
Boris Feld
sparse-revlog: only refine delta candidates in the sparse case (issue6006)...
r40479 yield None
return
Boris Feld
snapshot: try to refine new snapshot base down the chain...
r39538 # if we have a refinable value, try to refine it
if good is not None and good not in (p1, p2) and revlog.issnapshot(good):
# refine snapshot down
previous = None
while previous != good:
previous = good
base = revlog.deltaparent(good)
if base == nullrev:
break
good = yield (base,)
Boris Feld
snapshot: refine candidate snapshot base upward...
r39539 # refine snapshot up
delta-find: use a smarter object for snapshot caching...
r50573 if not snapshot_cache.snapshots:
snapshot_cache.update(revlog, good + 1)
Boris Feld
snapshot: refine candidate snapshot base upward...
r39539 previous = None
while good != previous:
previous = good
delta-find: use a smarter object for snapshot caching...
r50573 children = tuple(sorted(c for c in snapshot_cache.snapshots[good]))
Boris Feld
snapshot: refine candidate snapshot base upward...
r39539 good = yield children
debug: add an option to display statistic about a unbundling operation...
r50506 if debug_info is not None:
if good is None:
debug_info['no-solution'] += 1
Boris Feld
snapshot: also use None as a stop value for `_refinegroup`...
r39535 yield None
Boris Feld
snapshot: introduce an intermediate `_refinedgroups` generator...
r39532
Augie Fackler
formatting: blacken the codebase...
r43346
delta-find: use a smarter object for snapshot caching...
r50573 def _rawgroups(revlog, p1, p2, cachedelta, snapshot_cache=None):
Boris Feld
revlogdeltas: split candidate groups selection from the filtering logic...
r39372 """Provides group of revision to be tested as delta base
This lower level function focus on emitting delta theorically interresting
without looking it any practical details.
The group order aims at providing fast or small candidates first.
Boris Feld
revlogdeltas: extract _getcandidaterevs in a function...
r39370 """
delta-find: simply code that is now never invoqued without general delta...
r51333 # Why search for delta base if we cannot use a delta base ?
revlog: remove legacy usage of `_generaldelta`...
r51939 assert revlog.delta_config.general_delta
delta-find: simply code that is now never invoqued without general delta...
r51333 # also see issue6056
revlog: remove legacy usage of `_sparserevlog`...
r51953 sparse = revlog.delta_config.sparse_revlog
Boris Feld
revlogdeltas: extract _getcandidaterevs in a function...
r39370 curr = len(revlog)
prev = curr - 1
Boris Feld
snapshot: try intermediate snapshot against parents' base...
r39528 deltachain = lambda rev: revlog._deltachain(rev)[0]
Boris Feld
revlogdeltas: extract _getcandidaterevs in a function...
r39370
delta-find: simply code that is now never invoqued without general delta...
r51333 # exclude already lazy tested base if any
parents = [p for p in (p1, p2) if p != nullrev]
Boris Feld
revlogdeltas: extract _getcandidaterevs in a function...
r39370
revlog: remove legacy usage of `_deltabothparents`...
r51946 if not revlog.delta_config.delta_both_parents and len(parents) == 2:
delta-find: simply code that is now never invoqued without general delta...
r51333 parents.sort()
# To minimize the chance of having to build a fulltext,
# pick first whichever parent is closest to us (max rev)
yield (parents[1],)
# then the other one (min rev) if the first did not fit
yield (parents[0],)
elif len(parents) > 0:
# Test all parents (1 or 2), and keep the best candidate
yield parents
Boris Feld
revlogdeltas: extract _getcandidaterevs in a function...
r39370
Boris Feld
snapshot: try intermediate snapshot against parents' base...
r39528 if sparse and parents:
delta-find: use a smarter object for snapshot caching...
r50573 if snapshot_cache is None:
delta-find: small documentation update...
r50509 # map: base-rev: [snapshot-revs]
delta-find: use a smarter object for snapshot caching...
r50573 snapshot_cache = SnapshotCache()
Boris Feld
snapshot: try intermediate snapshot against parents' base...
r39528 # See if we can use an existing snapshot in the parent chains to use as
# a base for a new intermediate-snapshot
Boris Feld
snapshot: consider all snapshots in the parents' chains...
r39530 #
# search for snapshot in parents delta chain
# map: snapshot-level: snapshot-rev
parents_snaps = collections.defaultdict(set)
Boris Feld
snapshot: extract parent chain computation...
r39540 candidate_chains = [deltachain(p) for p in parents]
for chain in candidate_chains:
for idx, s in enumerate(chain):
Boris Feld
snapshot: consider all snapshots in the parents' chains...
r39530 if not revlog.issnapshot(s):
break
parents_snaps[idx].add(s)
Boris Feld
snapshot: consider unrelated snapshots at a similar level first...
r39531 snapfloor = min(parents_snaps[0]) + 1
delta-find: use a smarter object for snapshot caching...
r50573 snapshot_cache.update(revlog, snapfloor)
Boris Feld
snapshot: also consider the snapshot chain of one unrelated revision...
r39541 # search for the highest "unrelated" revision
#
# Adding snapshots used by "unrelated" revision increase the odd we
# reuse an independant, yet better snapshot chain.
#
# XXX instead of building a set of revisions, we could lazily enumerate
# over the chains. That would be more efficient, however we stick to
# simple code for now.
all_revs = set()
for chain in candidate_chains:
all_revs.update(chain)
other = None
for r in revlog.revs(prev, snapfloor):
if r not in all_revs:
other = r
break
if other is not None:
# To avoid unfair competition, we won't use unrelated intermediate
# snapshot that are deeper than the ones from the parent delta
# chain.
max_depth = max(parents_snaps.keys())
chain = deltachain(other)
delta-find: rename a variable for clarity...
r50510 for depth, s in enumerate(chain):
Boris Feld
snapshot: also consider the snapshot chain of one unrelated revision...
r39541 if s < snapfloor:
continue
delta-find: rename a variable for clarity...
r50510 if max_depth < depth:
Boris Feld
snapshot: also consider the snapshot chain of one unrelated revision...
r39541 break
if not revlog.issnapshot(s):
break
delta-find: rename a variable for clarity...
r50510 parents_snaps[depth].add(s)
Boris Feld
snapshot: consider all snapshots in the parents' chains...
r39530 # Test them as possible intermediate snapshot base
# We test them from highest to lowest level. High level one are more
# likely to result in small delta
Boris Feld
snapshot: consider unrelated snapshots at a similar level first...
r39531 floor = None
Boris Feld
snapshot: consider all snapshots in the parents' chains...
r39530 for idx, snaps in sorted(parents_snaps.items(), reverse=True):
Boris Feld
snapshot: consider unrelated snapshots at a similar level first...
r39531 siblings = set()
for s in snaps:
delta-find: use a smarter object for snapshot caching...
r50573 siblings.update(snapshot_cache.snapshots[s])
Boris Feld
snapshot: consider unrelated snapshots at a similar level first...
r39531 # Before considering making a new intermediate snapshot, we check
# if an existing snapshot, children of base we consider, would be
# suitable.
#
# It give a change to reuse a delta chain "unrelated" to the
# current revision instead of starting our own. Without such
# re-use, topological branches would keep reopening new chains.
# Creating more and more snapshot as the repository grow.
if floor is not None:
# We only do this for siblings created after the one in our
# parent's delta chain. Those created before has less chances
# to be valid base since our ancestors had to create a new
# snapshot.
siblings = [r for r in siblings if floor < r]
yield tuple(sorted(siblings))
# then test the base from our parent's delta chain.
Boris Feld
snapshot: consider all snapshots in the parents' chains...
r39530 yield tuple(sorted(snaps))
Boris Feld
snapshot: consider unrelated snapshots at a similar level first...
r39531 floor = min(snaps)
Boris Feld
snapshot: search for unrelated but reusable full-snapshot...
r39529 # No suitable base found in the parent chain, search if any full
# snapshots emitted since parent's base would be a suitable base for an
# intermediate snapshot.
#
# It give a chance to reuse a delta chain unrelated to the current
# revisions instead of starting our own. Without such re-use,
# topological branches would keep reopening new full chains. Creating
# more and more snapshot as the repository grow.
delta-find: make sure we only use newer full snapshot as candidate...
r50575 full = [r for r in snapshot_cache.snapshots[nullrev] if snapfloor <= r]
yield tuple(sorted(full))
Boris Feld
snapshot: try intermediate snapshot against parents' base...
r39528
Boris Feld
snapshot: also consider the snapshot chain of one unrelated revision...
r39541 if not sparse:
# other approach failed try against prev to hopefully save us a
# fulltext.
yield (prev,)
Boris Feld
revlogdeltas: extract _getcandidaterevs in a function...
r39370
Augie Fackler
formatting: blacken the codebase...
r43346
delta-find: use a smarter object for snapshot caching...
r50573 class SnapshotCache:
__slots__ = ('snapshots', '_start_rev', '_end_rev')
def __init__(self):
delta-find: use sets instead of list in the snapshot cache...
r50574 self.snapshots = collections.defaultdict(set)
delta-find: use a smarter object for snapshot caching...
r50573 self._start_rev = None
self._end_rev = None
def update(self, revlog, start_rev=0):
"""find snapshots from start_rev to tip"""
nb_revs = len(revlog)
end_rev = nb_revs - 1
if start_rev > end_rev:
return # range is empty
if self._start_rev is None:
assert self._end_rev is None
self._update(revlog, start_rev, end_rev)
elif not (self._start_rev <= start_rev and end_rev <= self._end_rev):
if start_rev < self._start_rev:
self._update(revlog, start_rev, self._start_rev - 1)
if self._end_rev < end_rev:
self._update(revlog, self._end_rev + 1, end_rev)
if self._start_rev is None:
assert self._end_rev is None
self._end_rev = end_rev
self._start_rev = start_rev
else:
self._start_rev = min(self._start_rev, start_rev)
self._end_rev = max(self._end_rev, end_rev)
assert self._start_rev <= self._end_rev, (
self._start_rev,
self._end_rev,
)
def _update(self, revlog, start_rev, end_rev):
"""internal method that actually do update content"""
assert self._start_rev is None or (
start_rev < self._start_rev or start_rev > self._end_rev
), (self._start_rev, self._end_rev, start_rev, end_rev)
assert self._start_rev is None or (
end_rev < self._start_rev or end_rev > self._end_rev
), (self._start_rev, self._end_rev, start_rev, end_rev)
cache = self.snapshots
safehasattr: drop usage in favor of hasattr...
r51821 if hasattr(revlog.index, 'findsnapshots'):
delta-find: use a smarter object for snapshot caching...
r50573 revlog.index.findsnapshots(cache, start_rev, end_rev)
else:
deltaparent = revlog.deltaparent
issnapshot = revlog.issnapshot
for rev in revlog.revs(start_rev, end_rev):
if issnapshot(rev):
delta-find: use sets instead of list in the snapshot cache...
r50574 cache[deltaparent(rev)].add(rev)
delta-find: use a smarter object for snapshot caching...
r50573
Gregory Szorc
py3: use class X: instead of class X(object):...
r49801 class deltacomputer:
debug: add an option to display statistic about a unbundling operation...
r50506 def __init__(
self,
revlog,
write_debug=None,
debug_search=False,
debug_info=None,
):
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 self.revlog = revlog
deltas: add code to display information about the result of `finddeltainfo`...
r50121 self._write_debug = write_debug
delta-fine: use the `_debug_search` attribute directly...
r51542 if write_debug is None:
self._debug_search = False
else:
self._debug_search = debug_search
debug: add an option to display statistic about a unbundling operation...
r50506 self._debug_info = debug_info
delta-find: use a single snapshot cache when applying a group to an object...
r50576 self._snapshot_cache = SnapshotCache()
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366
delta-find: move the `gather_debug` logic in a property...
r51541 @property
def _gather_debug(self):
return self._write_debug is not None or self._debug_info is not None
delta-computer: stop explicitly taking file handle...
r51913 def buildtext(self, revinfo):
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 """Builds a fulltext version of a revision
revlog: move `revisioninfo` in `revlogutils`...
r48191 revinfo: revisioninfo instance that contains all needed info
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 """
btext = revinfo.btext
if btext[0] is not None:
return btext[0]
revlog = self.revlog
cachedelta = revinfo.cachedelta
baserev = cachedelta[0]
delta = cachedelta[1]
Augie Fackler
formatting: blacken the codebase...
r43346 fulltext = btext[0] = _textfromdelta(
revlog,
baserev,
delta,
revinfo.p1,
revinfo.p2,
revinfo.flags,
revinfo.node,
)
Boris Feld
revlogdeltas: extra fulltext building in its own function...
r39367 return fulltext
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366
delta-computer: stop explicitly taking file handle...
r51913 def _builddeltadiff(self, base, revinfo):
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 revlog = self.revlog
delta-computer: stop explicitly taking file handle...
r51913 t = self.buildtext(revinfo)
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 if revlog.iscensored(base):
# deltas based on a censored revision must replace the
# full content in one patch, so delta works everywhere
header = mdiff.replacediffheader(revlog.rawsize(base), len(t))
delta = header + t
else:
delta-computer: stop explicitly taking file handle...
r51913 ptext = revlog.rawdata(base)
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 delta = mdiff.textdiff(ptext, t)
return delta
delta-computer: stop explicitly taking file handle...
r51913 def _builddeltainfo(self, revinfo, base, target_rev=None):
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 # can we use the cached delta?
Valentin Gatien-Baron
delta: move some delta chain related computation earlier in deltainfo...
r42665 revlog = self.revlog
chainbase = revlog.chainbase(base)
revlog: remove legacy usage of `_generaldelta`...
r51939 if revlog.delta_config.general_delta:
Valentin Gatien-Baron
delta: move some delta chain related computation earlier in deltainfo...
r42665 deltabase = base
else:
delta-find: add a simple safeguard to prevent bad non-general-delta...
r51332 if target_rev is not None and base != target_rev - 1:
msg = (
b'general delta cannot use delta for something else '
b'than `prev`: %d<-%d'
)
msg %= (base, target_rev)
raise error.ProgrammingError(msg)
Valentin Gatien-Baron
delta: move some delta chain related computation earlier in deltainfo...
r42665 deltabase = chainbase
snapshotdepth = None
revlog: remove legacy usage of `_sparserevlog`...
r51953 if revlog.delta_config.sparse_revlog and deltabase == nullrev:
Valentin Gatien-Baron
delta: move some delta chain related computation earlier in deltainfo...
r42665 snapshotdepth = 0
revlog: remove legacy usage of `_sparserevlog`...
r51953 elif revlog.delta_config.sparse_revlog and revlog.issnapshot(deltabase):
Valentin Gatien-Baron
delta: move some delta chain related computation earlier in deltainfo...
r42665 # A delta chain should always be one full snapshot,
# zero or more semi-snapshots, and zero or more deltas
p1, p2 = revlog.rev(revinfo.p1), revlog.rev(revinfo.p2)
if deltabase not in (p1, p2) and revlog.issnapshot(deltabase):
snapshotdepth = len(revlog._deltachain(deltabase)[0])
Boris Feld
revlog: reuse cached delta for identical base revision (issue5975)...
r39631 delta = None
if revinfo.cachedelta:
find-delta: minor preparatory change...
r50570 cachebase = revinfo.cachedelta[0]
Augie Fackler
formatting: blacken the codebase...
r43346 # check if the diff still apply
Boris Feld
revlog: reuse cached delta for identical base revision (issue5975)...
r39631 currentbase = cachebase
Augie Fackler
formatting: blacken the codebase...
r43346 while (
currentbase != nullrev
and currentbase != base
and self.revlog.length(currentbase) == 0
):
Boris Feld
revlog: reuse cached delta for identical base revision (issue5975)...
r39631 currentbase = self.revlog.deltaparent(currentbase)
revlog: remove legacy usage of `_lazydeltabase`...
r51960 if self.revlog.delta_config.lazy_delta and currentbase == base:
Boris Feld
revlog: reuse cached delta for identical base revision (issue5975)...
r39631 delta = revinfo.cachedelta[1]
if delta is None:
delta-computer: stop explicitly taking file handle...
r51913 delta = self._builddeltadiff(base, revinfo)
delta-fine: use the `_debug_search` attribute directly...
r51542 if self._debug_search:
deltas: add a debug-delta-find command to analyse delta search...
r50123 msg = b"DBG-DELTAS-SEARCH: uncompressed-delta-size=%d\n"
msg %= len(delta)
self._write_debug(msg)
Valentin Gatien-Baron
deltas: skip if projected compressed size does not match text size constraint...
r42667 # snapshotdept need to be neither None nor 0 level snapshot
if revlog.upperboundcomp is not None and snapshotdepth:
lowestrealisticdeltalen = len(delta) // revlog.upperboundcomp
snapshotlimit = revinfo.textlen >> snapshotdepth
delta-fine: use the `_debug_search` attribute directly...
r51542 if self._debug_search:
deltas: add a debug-delta-find command to analyse delta search...
r50123 msg = b"DBG-DELTAS-SEARCH: projected-lower-size=%d\n"
msg %= lowestrealisticdeltalen
self._write_debug(msg)
Valentin Gatien-Baron
deltas: skip if projected compressed size does not match text size constraint...
r42667 if snapshotlimit < lowestrealisticdeltalen:
delta-fine: use the `_debug_search` attribute directly...
r51542 if self._debug_search:
deltas: add a debug-delta-find command to analyse delta search...
r50123 msg = b"DBG-DELTAS-SEARCH: DISCARDED (snapshot limit)\n"
self._write_debug(msg)
Valentin Gatien-Baron
deltas: skip if projected compressed size does not match text size constraint...
r42667 return None
Valentin Gatien-Baron
deltas: skip if projected compressed size is bigger than previous snapshot...
r42668 if revlog.length(base) < lowestrealisticdeltalen:
delta-fine: use the `_debug_search` attribute directly...
r51542 if self._debug_search:
deltas: add a debug-delta-find command to analyse delta search...
r50123 msg = b"DBG-DELTAS-SEARCH: DISCARDED (prev size)\n"
self._write_debug(msg)
Valentin Gatien-Baron
deltas: skip if projected compressed size is bigger than previous snapshot...
r42668 return None
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 header, data = revlog.compress(delta)
deltalen = len(header) + len(data)
offset = revlog.end(len(revlog) - 1)
dist = deltalen + offset - revlog.start(chainbase)
chainlen, compresseddeltalen = revlog._chaininfo(base)
chainlen += 1
compresseddeltalen += deltalen
Augie Fackler
formatting: blacken the codebase...
r43346 return _deltainfo(
dist,
deltalen,
(header, data),
deltabase,
chainbase,
chainlen,
compresseddeltalen,
snapshotdepth,
)
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366
delta-computer: stop explicitly taking file handle...
r51913 def _fullsnapshotinfo(self, revinfo, curr):
rawtext = self.buildtext(revinfo)
Boris Feld
revlogdeltas: always return a delta info object in finddeltainfo...
r39369 data = self.revlog.compress(rawtext)
compresseddeltalen = deltalen = dist = len(data[1]) + len(data[0])
deltabase = chainbase = curr
snapshotdepth = 0
chainlen = 1
Augie Fackler
formatting: blacken the codebase...
r43346 return _deltainfo(
dist,
deltalen,
data,
deltabase,
chainbase,
chainlen,
compresseddeltalen,
snapshotdepth,
)
Boris Feld
revlogdeltas: always return a delta info object in finddeltainfo...
r39369
delta-computer: stop explicitly taking file handle...
r51913 def finddeltainfo(self, revinfo, excluded_bases=None, target_rev=None):
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 """Find an acceptable delta against a candidate revision
revinfo: information about the revision (instance of _revisioninfo)
Returns the first acceptable candidate revision, as ordered by
Boris Feld
revlogdeltas: extract _getcandidaterevs in a function...
r39370 _candidategroups
Boris Feld
revlogdeltas: always return a delta info object in finddeltainfo...
r39369
If no suitable deltabase is found, we return delta info for a full
snapshot.
revlog: add a ways to blacklist some revision when searching for a delta...
r48193
`excluded_bases` is an optional set of revision that cannot be used as
a delta base. Use this to recompute delta suitable in censor or strip
context.
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 """
deltas: at a `target_rev` parameter to finddeltainfo...
r48249 if target_rev is None:
censor: implement censoring for revlogv2...
r48250 target_rev = len(self.revlog)
deltas: at a `target_rev` parameter to finddeltainfo...
r48249
delta-find: initialize the debug information much sooner (when possible)...
r51546 gather_debug = self._gather_debug
cachedelta = revinfo.cachedelta
revlog = self.revlog
delta-find: fix pulled-delta-reuse-policy=forced behavior...
r51550 p1r = p2r = None
delta-find: initialize the debug information much sooner (when possible)...
r51546
delta-find: fix pulled-delta-reuse-policy=forced behavior...
r51550 if excluded_bases is None:
excluded_bases = set()
delta-find: initialize the debug information much sooner (when possible)...
r51546
if gather_debug:
start = util.timer()
dbg = self._one_dbg_data()
dbg['revision'] = target_rev
target_revlog = b"UNKNOWN"
target_type = self.revlog.target[0]
target_key = self.revlog.target[1]
if target_type == KIND_CHANGELOG:
target_revlog = b'CHANGELOG:'
elif target_type == KIND_MANIFESTLOG:
target_revlog = b'MANIFESTLOG:'
if target_key:
target_revlog += b'%s:' % target_key
elif target_type == KIND_FILELOG:
target_revlog = b'FILELOG:'
if target_key:
target_revlog += b'%s:' % target_key
dbg['target-revlog'] = target_revlog
p1r = revlog.rev(revinfo.p1)
p2r = revlog.rev(revinfo.p2)
if p1r != nullrev:
p1_chain_len = revlog._chaininfo(p1r)[0]
else:
p1_chain_len = -1
if p2r != nullrev:
p2_chain_len = revlog._chaininfo(p2r)[0]
else:
p2_chain_len = -1
dbg['p1-chain-len'] = p1_chain_len
dbg['p2-chain-len'] = p2_chain_len
delta-find: gather the condition to blindly use a full snapshot together...
r51547 # 1) if the revision is empty, no amount of delta can beat it
#
# 2) no delta for flag processor revision (see "candelta" for why)
# not calling candelta since only one revision needs test, also to
# avoid overhead fetching flags again.
if not revinfo.textlen or revinfo.flags & REVIDX_RAWTEXT_CHANGING_FLAGS:
delta-computer: stop explicitly taking file handle...
r51913 deltainfo = self._fullsnapshotinfo(revinfo, target_rev)
deltafind: issue debug information when we fast-path rivial case too...
r51548 if gather_debug:
end = util.timer()
dbg['duration'] = end - start
dbg[
'delta-base'
] = deltainfo.base # pytype: disable=attribute-error
dbg['search_round_count'] = 0
delta-find: fix pulled-delta-reuse-policy=forced behavior...
r51550 dbg['using-cached-base'] = False
deltafind: issue debug information when we fast-path rivial case too...
r51548 dbg['delta_try_count'] = 0
dbg['type'] = b"full"
dbg['snapshot-depth'] = 0
self._dbg_process_data(dbg)
return deltainfo
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366
delta-find: fix pulled-delta-reuse-policy=forced behavior...
r51550 deltainfo = None
# If this source delta are to be forcibly reuse, let us comply early.
if (
revlog: remove legacy usage of `_generaldelta`...
r51939 revlog.delta_config.general_delta
delta-find: fix pulled-delta-reuse-policy=forced behavior...
r51550 and revinfo.cachedelta is not None
and revinfo.cachedelta[2] == DELTA_BASE_REUSE_FORCE
):
base = revinfo.cachedelta[0]
if base == nullrev:
dbg_type = b"full"
delta-computer: stop explicitly taking file handle...
r51913 deltainfo = self._fullsnapshotinfo(revinfo, target_rev)
delta-find: fix pulled-delta-reuse-policy=forced behavior...
r51550 if gather_debug:
snapshotdepth = 0
elif base not in excluded_bases:
delta = revinfo.cachedelta[1]
header, data = revlog.compress(delta)
deltalen = len(header) + len(data)
if gather_debug:
offset = revlog.end(len(revlog) - 1)
chainbase = revlog.chainbase(base)
distance = deltalen + offset - revlog.start(chainbase)
chainlen, compresseddeltalen = revlog._chaininfo(base)
chainlen += 1
compresseddeltalen += deltalen
if base == p1r or base == p2r:
dbg_type = b"delta"
snapshotdepth = None
elif not revlog.issnapshot(base):
snapshotdepth = None
else:
dbg_type = b"snapshot"
snapshotdepth = revlog.snapshotdepth(base) + 1
else:
distance = None
chainbase = None
chainlen = None
compresseddeltalen = None
snapshotdepth = None
deltainfo = _deltainfo(
distance=distance,
deltalen=deltalen,
data=(header, data),
base=base,
chainbase=chainbase,
chainlen=chainlen,
compresseddeltalen=compresseddeltalen,
snapshotdepth=snapshotdepth,
)
if deltainfo is not None:
if gather_debug:
end = util.timer()
dbg['duration'] = end - start
dbg[
'delta-base'
] = deltainfo.base # pytype: disable=attribute-error
dbg['search_round_count'] = 0
dbg['using-cached-base'] = True
dbg['delta_try_count'] = 0
dbg['type'] = b"full"
if snapshotdepth is None:
dbg['snapshot-depth'] = 0
else:
dbg['snapshot-depth'] = snapshotdepth
self._dbg_process_data(dbg)
return deltainfo
revlog: add a ways to blacklist some revision when searching for a delta...
r48193
deltas: add code to display information about the result of `finddeltainfo`...
r50121 # count the number of different delta we tried (for debug purpose)
dbg_try_count = 0
# count the number of "search round" we did. (for debug purpose)
dbg_try_rounds = 0
dbg_type = b'unknown'
delta-find: initialize the debug information much sooner (when possible)...
r51546 if p1r is None:
p1r = revlog.rev(revinfo.p1)
p2r = revlog.rev(revinfo.p2)
deltas: add code to display information about the result of `finddeltainfo`...
r50121
delta-fine: use the `_debug_search` attribute directly...
r51542 if self._debug_search:
deltas: add a debug-delta-find command to analyse delta search...
r50123 msg = b"DBG-DELTAS-SEARCH: SEARCH rev=%d\n"
msg %= target_rev
self._write_debug(msg)
deltas: add code to display information about the result of `finddeltainfo`...
r50121
Augie Fackler
formatting: blacken the codebase...
r43346 groups = _candidategroups(
delta-find: expand a function definition and call before extendin it...
r50507 self.revlog,
revinfo.textlen,
p1r,
p2r,
cachedelta,
delta-find: move pre-filtering with other pre-filtering logic...
r50508 excluded_bases,
target_rev,
delta-find: use a single snapshot cache when applying a group to an object...
r50576 snapshot_cache=self._snapshot_cache,
Augie Fackler
formatting: blacken the codebase...
r43346 )
Boris Feld
snapshot: use None as a stop value when looking for a good delta...
r39533 candidaterevs = next(groups)
while candidaterevs is not None:
deltas: add code to display information about the result of `finddeltainfo`...
r50121 dbg_try_rounds += 1
delta-fine: use the `_debug_search` attribute directly...
r51542 if self._debug_search:
deltas: add a debug-delta-find command to analyse delta search...
r50123 prev = None
if deltainfo is not None:
prev = deltainfo.base
delta-find: add debug information about reuse of cached data...
r50504 if (
cachedelta is not None
and len(candidaterevs) == 1
and cachedelta[0] in candidaterevs
):
round_type = b"cached-delta"
delta-find: fix `parents` round detection...
r51545 elif p1r in candidaterevs or p2r in candidaterevs:
deltas: add a debug-delta-find command to analyse delta search...
r50123 round_type = b"parents"
elif prev is not None and all(c < prev for c in candidaterevs):
round_type = b"refine-down"
elif prev is not None and all(c > prev for c in candidaterevs):
round_type = b"refine-up"
else:
round_type = b"search-down"
msg = b"DBG-DELTAS-SEARCH: ROUND #%d - %d candidates - %s\n"
msg %= (dbg_try_rounds, len(candidaterevs), round_type)
self._write_debug(msg)
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 nominateddeltas = []
Boris Feld
snapshot: add refining logic at the findeltainfo level...
r39534 if deltainfo is not None:
delta-fine: use the `_debug_search` attribute directly...
r51542 if self._debug_search:
deltas: add a debug-delta-find command to analyse delta search...
r50123 msg = (
b"DBG-DELTAS-SEARCH: CONTENDER: rev=%d - length=%d\n"
)
msg %= (deltainfo.base, deltainfo.deltalen)
self._write_debug(msg)
Boris Feld
snapshot: add refining logic at the findeltainfo level...
r39534 # if we already found a good delta,
# challenge it against refined candidates
nominateddeltas.append(deltainfo)
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 for candidaterev in candidaterevs:
delta-fine: use the `_debug_search` attribute directly...
r51542 if self._debug_search:
deltas: add a debug-delta-find command to analyse delta search...
r50123 msg = b"DBG-DELTAS-SEARCH: CANDIDATE: rev=%d\n"
msg %= candidaterev
self._write_debug(msg)
candidate_type = None
delta-find: fix `parents` round detection...
r51545 if candidaterev == p1r:
deltas: add a debug-delta-find command to analyse delta search...
r50123 candidate_type = b"p1"
delta-find: fix `parents` round detection...
r51545 elif candidaterev == p2r:
deltas: add a debug-delta-find command to analyse delta search...
r50123 candidate_type = b"p2"
elif self.revlog.issnapshot(candidaterev):
candidate_type = b"snapshot-%d"
candidate_type %= self.revlog.snapshotdepth(
candidaterev
)
if candidate_type is not None:
msg = b"DBG-DELTAS-SEARCH: type=%s\n"
msg %= candidate_type
self._write_debug(msg)
msg = b"DBG-DELTAS-SEARCH: size=%d\n"
msg %= self.revlog.length(candidaterev)
self._write_debug(msg)
msg = b"DBG-DELTAS-SEARCH: base=%d\n"
msg %= self.revlog.deltaparent(candidaterev)
self._write_debug(msg)
delta-find: move pre-filtering with other pre-filtering logic...
r50508
deltas: add code to display information about the result of `finddeltainfo`...
r50121 dbg_try_count += 1
deltas: add a debug-delta-find command to analyse delta search...
r50123
delta-fine: use the `_debug_search` attribute directly...
r51542 if self._debug_search:
deltas: add a debug-delta-find command to analyse delta search...
r50123 delta_start = util.timer()
delta-find: add a simple safeguard to prevent bad non-general-delta...
r51332 candidatedelta = self._builddeltainfo(
revinfo,
candidaterev,
target_rev=target_rev,
)
delta-fine: use the `_debug_search` attribute directly...
r51542 if self._debug_search:
deltas: add a debug-delta-find command to analyse delta search...
r50123 delta_end = util.timer()
msg = b"DBG-DELTAS-SEARCH: delta-search-time=%f\n"
msg %= delta_end - delta_start
self._write_debug(msg)
Valentin Gatien-Baron
deltas: accept and skip None return for delta info...
r42666 if candidatedelta is not None:
find-delta: rename _isgooddeltainfo...
r50569 if is_good_delta_info(self.revlog, candidatedelta, revinfo):
delta-fine: use the `_debug_search` attribute directly...
r51542 if self._debug_search:
deltas: add a debug-delta-find command to analyse delta search...
r50123 msg = b"DBG-DELTAS-SEARCH: DELTA: length=%d (GOOD)\n"
msg %= candidatedelta.deltalen
self._write_debug(msg)
Valentin Gatien-Baron
deltas: accept and skip None return for delta info...
r42666 nominateddeltas.append(candidatedelta)
delta-fine: use the `_debug_search` attribute directly...
r51542 elif self._debug_search:
deltas: add a debug-delta-find command to analyse delta search...
r50123 msg = b"DBG-DELTAS-SEARCH: DELTA: length=%d (BAD)\n"
msg %= candidatedelta.deltalen
self._write_debug(msg)
delta-fine: use the `_debug_search` attribute directly...
r51542 elif self._debug_search:
deltas: add a debug-delta-find command to analyse delta search...
r50123 msg = b"DBG-DELTAS-SEARCH: NO-DELTA\n"
self._write_debug(msg)
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366 if nominateddeltas:
deltainfo = min(nominateddeltas, key=lambda x: x.deltalen)
Boris Feld
snapshot: add refining logic at the findeltainfo level...
r39534 if deltainfo is not None:
candidaterevs = groups.send(deltainfo.base)
else:
candidaterevs = next(groups)
Boris Feld
revlog: split functionality related to deltas computation in a new module...
r39366
Boris Feld
revlogdeltas: always return a delta info object in finddeltainfo...
r39369 if deltainfo is None:
deltas: add code to display information about the result of `finddeltainfo`...
r50121 dbg_type = b"full"
delta-computer: stop explicitly taking file handle...
r51913 deltainfo = self._fullsnapshotinfo(revinfo, target_rev)
deltas: add a debug-delta-find command to analyse delta search...
r50123 elif deltainfo.snapshotdepth: # pytype: disable=attribute-error
deltas: add code to display information about the result of `finddeltainfo`...
r50121 dbg_type = b"snapshot"
else:
dbg_type = b"delta"
debug: add an option to display statistic about a unbundling operation...
r50506 if gather_debug:
deltas: add code to display information about the result of `finddeltainfo`...
r50121 end = util.timer()
delta-find: properly report full snapshot used from cache as such...
r50676 if dbg_type == b'full':
used_cached = (
cachedelta is not None
and dbg_try_rounds == 0
and dbg_try_count == 0
and cachedelta[0] == nullrev
)
else:
used_cached = (
cachedelta is not None
and dbg_try_rounds == 1
and dbg_try_count == 1
and deltainfo.base == cachedelta[0]
)
delta-find: intrduce a `_one_dbg_data` method...
r51544 dbg['duration'] = end - start
dbg[
'delta-base'
] = deltainfo.base # pytype: disable=attribute-error
dbg['search_round_count'] = dbg_try_rounds
dbg['using-cached-base'] = used_cached
dbg['delta_try_count'] = dbg_try_count
dbg['type'] = dbg_type
deltas: add a debug-delta-find command to analyse delta search...
r50123 if (
deltainfo.snapshotdepth # pytype: disable=attribute-error
is not None
):
dbg[
'snapshot-depth'
] = deltainfo.snapshotdepth # pytype: disable=attribute-error
deltas: add code to display information about the result of `finddeltainfo`...
r50121 else:
dbg['snapshot-depth'] = 0
delta-find: move final debug processing in a `_dbg_process_data` method...
r51543 self._dbg_process_data(dbg)
return deltainfo
deltas: add code to display information about the result of `finddeltainfo`...
r50121
delta-find: intrduce a `_one_dbg_data` method...
r51544 def _one_dbg_data(self):
return {
'duration': None,
'revision': None,
'delta-base': None,
'search_round_count': None,
'using-cached-base': None,
'delta_try_count': None,
'type': None,
'p1-chain-len': None,
'p2-chain-len': None,
'snapshot-depth': None,
'target-revlog': None,
}
delta-find: move final debug processing in a `_dbg_process_data` method...
r51543 def _dbg_process_data(self, dbg):
if self._debug_info is not None:
self._debug_info.append(dbg)
debug: add an option to display statistic about a unbundling operation...
r50506
delta-find: move final debug processing in a `_dbg_process_data` method...
r51543 if self._write_debug is not None:
msg = (
b"DBG-DELTAS:"
b" %-12s"
b" rev=%d:"
b" delta-base=%d"
b" is-cached=%d"
b" - search-rounds=%d"
b" try-count=%d"
b" - delta-type=%-6s"
b" snap-depth=%d"
b" - p1-chain-length=%d"
b" p2-chain-length=%d"
b" - duration=%f"
b"\n"
)
msg %= (
dbg["target-revlog"],
dbg["revision"],
dbg["delta-base"],
dbg["using-cached-base"],
dbg["search_round_count"],
dbg["delta_try_count"],
dbg["type"],
dbg["snapshot-depth"],
dbg["p1-chain-len"],
dbg["p2-chain-len"],
dbg["duration"],
)
self._write_debug(msg)
revlog: factor the logic to determine the delta compression out...
r48245
def delta_compression(default_compression_header, deltainfo):
"""return (COMPRESSION_MODE, deltainfo)
used by revlog v2+ format to dispatch between PLAIN and DEFAULT
compression.
"""
h, d = deltainfo.data
compression_mode = COMP_MODE_INLINE
if not h and not d:
# not data to store at all... declare them uncompressed
compression_mode = COMP_MODE_PLAIN
elif not h:
t = d[0:1]
if t == b'\0':
compression_mode = COMP_MODE_PLAIN
elif t == default_compression_header:
compression_mode = COMP_MODE_DEFAULT
elif h == b'u':
# we have a more efficient way to declare uncompressed
h = b''
compression_mode = COMP_MODE_PLAIN
deltainfo = drop_u_compression(deltainfo)
return compression_mode, deltainfo