Show More
@@ -14,7 +14,6 b' import struct' | |||
|
14 | 14 | from .node import nullrev |
|
15 | 15 | from . import ( |
|
16 | 16 | error, |
|
17 | pycompat, | |
|
18 | 17 | requirements as requirementsmod, |
|
19 | 18 | util, |
|
20 | 19 | ) |
@@ -837,14 +836,6 b' def set_sidedata_spec_for_repo(repo):' | |||
|
837 | 836 | ) |
|
838 | 837 | |
|
839 | 838 | |
|
840 | def getsidedataadder(srcrepo, destrepo): | |
|
841 | use_w = srcrepo.ui.configbool(b'experimental', b'worker.repository-upgrade') | |
|
842 | if pycompat.iswindows or not use_w: | |
|
843 | return _get_simple_sidedata_adder(srcrepo, destrepo) | |
|
844 | else: | |
|
845 | return _get_worker_sidedata_adder(srcrepo, destrepo) | |
|
846 | ||
|
847 | ||
|
848 | 839 | def _sidedata_worker(srcrepo, revs_queue, sidedata_queue, tokens): |
|
849 | 840 | """The function used by worker precomputing sidedata |
|
850 | 841 | |
@@ -911,57 +902,21 b' def _get_worker_sidedata_adder(srcrepo, ' | |||
|
911 | 902 | # received, when shelve 43 for later use. |
|
912 | 903 | staging = {} |
|
913 | 904 | |
|
914 | def sidedata_companion(revlog, rev): | |
|
915 | data = {}, False | |
|
916 | if util.safehasattr(revlog, b'filteredrevs'): # this is a changelog | |
|
917 | # Is the data previously shelved ? | |
|
918 | data = staging.pop(rev, None) | |
|
919 | if data is None: | |
|
920 | # look at the queued result until we find the one we are lookig | |
|
921 | # for (shelve the other ones) | |
|
905 | def sidedata_companion(repo, revlog, rev, old_sidedata): | |
|
906 | # Is the data previously shelved ? | |
|
907 | data = staging.pop(rev, None) | |
|
908 | if data is None: | |
|
909 | # look at the queued result until we find the one we are lookig | |
|
910 | # for (shelve the other ones) | |
|
911 | r, data = sidedataq.get() | |
|
912 | while r != rev: | |
|
913 | staging[r] = data | |
|
922 | 914 | r, data = sidedataq.get() |
|
923 | while r != rev: | |
|
924 | staging[r] = data | |
|
925 | r, data = sidedataq.get() | |
|
926 | tokens.release() | |
|
915 | tokens.release() | |
|
927 | 916 | sidedata, has_copies_info = data |
|
928 | 917 | new_flag = 0 |
|
929 | 918 | if has_copies_info: |
|
930 | 919 | new_flag = sidedataflag.REVIDX_HASCOPIESINFO |
|
931 |
return |
|
|
920 | return sidedata, (new_flag, 0) | |
|
932 | 921 | |
|
933 | 922 | return sidedata_companion |
|
934 | ||
|
935 | ||
|
936 | def _get_simple_sidedata_adder(srcrepo, destrepo): | |
|
937 | """The simple version of the sidedata computation | |
|
938 | ||
|
939 | It just compute it in the same thread on request""" | |
|
940 | ||
|
941 | def sidedatacompanion(revlog, rev): | |
|
942 | sidedata, has_copies_info = {}, False | |
|
943 | if util.safehasattr(revlog, 'filteredrevs'): # this is a changelog | |
|
944 | sidedata, has_copies_info = _getsidedata(srcrepo, rev) | |
|
945 | new_flag = 0 | |
|
946 | if has_copies_info: | |
|
947 | new_flag = sidedataflag.REVIDX_HASCOPIESINFO | |
|
948 | ||
|
949 | return False, (), sidedata, new_flag, 0 | |
|
950 | ||
|
951 | return sidedatacompanion | |
|
952 | ||
|
953 | ||
|
954 | def getsidedataremover(srcrepo, destrepo): | |
|
955 | def sidedatacompanion(revlog, rev): | |
|
956 | f = () | |
|
957 | if util.safehasattr(revlog, 'filteredrevs'): # this is a changelog | |
|
958 | if revlog.flags(rev) & sidedataflag.REVIDX_SIDEDATA: | |
|
959 | f = ( | |
|
960 | sidedatamod.SD_P1COPIES, | |
|
961 | sidedatamod.SD_P2COPIES, | |
|
962 | sidedatamod.SD_FILESADDED, | |
|
963 | sidedatamod.SD_FILESREMOVED, | |
|
964 | ) | |
|
965 | return False, f, {}, 0, sidedataflag.REVIDX_HASCOPIESINFO | |
|
966 | ||
|
967 | return sidedatacompanion |
@@ -85,6 +85,7 b' from .utils import (' | |||
|
85 | 85 | |
|
86 | 86 | # blanked usage of all the name to prevent pyflakes constraints |
|
87 | 87 | # We need these name available in the module for extensions. |
|
88 | ||
|
88 | 89 | REVLOGV0 |
|
89 | 90 | REVLOGV1 |
|
90 | 91 | REVLOGV2 |
@@ -1997,7 +1998,7 b' class revlog(object):' | |||
|
1997 | 1998 | |
|
1998 | 1999 | if sidedata is None: |
|
1999 | 2000 | sidedata = {} |
|
2000 | elif not self.hassidedata: | |
|
2001 | elif sidedata and not self.hassidedata: | |
|
2001 | 2002 | raise error.ProgrammingError( |
|
2002 | 2003 | _(b"trying to add sidedata to a revlog who don't support them") |
|
2003 | 2004 | ) |
@@ -2658,7 +2659,7 b' class revlog(object):' | |||
|
2658 | 2659 | addrevisioncb=None, |
|
2659 | 2660 | deltareuse=DELTAREUSESAMEREVS, |
|
2660 | 2661 | forcedeltabothparents=None, |
|
2661 |
sidedata |
|
|
2662 | sidedata_helpers=None, | |
|
2662 | 2663 | ): |
|
2663 | 2664 | """Copy this revlog to another, possibly with format changes. |
|
2664 | 2665 | |
@@ -2701,21 +2702,7 b' class revlog(object):' | |||
|
2701 | 2702 | argument controls whether to force compute deltas against both parents |
|
2702 | 2703 | for merges. By default, the current default is used. |
|
2703 | 2704 | |
|
2704 | If not None, the `sidedatacompanion` is callable that accept two | |
|
2705 | arguments: | |
|
2706 | ||
|
2707 | (srcrevlog, rev) | |
|
2708 | ||
|
2709 | and return a quintet that control changes to sidedata content from the | |
|
2710 | old revision to the new clone result: | |
|
2711 | ||
|
2712 | (dropall, filterout, update, new_flags, dropped_flags) | |
|
2713 | ||
|
2714 | * if `dropall` is True, all sidedata should be dropped | |
|
2715 | * `filterout` is a set of sidedata keys that should be dropped | |
|
2716 | * `update` is a mapping of additionnal/new key -> value | |
|
2717 | * new_flags is a bitfields of new flags that the revision should get | |
|
2718 | * dropped_flags is a bitfields of new flags that the revision shoudl not longer have | |
|
2705 | See `storageutil.emitrevisions` for the doc on `sidedata_helpers`. | |
|
2719 | 2706 | """ |
|
2720 | 2707 | if deltareuse not in self.DELTAREUSEALL: |
|
2721 | 2708 | raise ValueError( |
@@ -2755,7 +2742,7 b' class revlog(object):' | |||
|
2755 | 2742 | addrevisioncb, |
|
2756 | 2743 | deltareuse, |
|
2757 | 2744 | forcedeltabothparents, |
|
2758 |
sidedata |
|
|
2745 | sidedata_helpers, | |
|
2759 | 2746 | ) |
|
2760 | 2747 | |
|
2761 | 2748 | finally: |
@@ -2770,7 +2757,7 b' class revlog(object):' | |||
|
2770 | 2757 | addrevisioncb, |
|
2771 | 2758 | deltareuse, |
|
2772 | 2759 | forcedeltabothparents, |
|
2773 |
sidedata |
|
|
2760 | sidedata_helpers, | |
|
2774 | 2761 | ): |
|
2775 | 2762 | """perform the core duty of `revlog.clone` after parameter processing""" |
|
2776 | 2763 | deltacomputer = deltautil.deltacomputer(destrevlog) |
@@ -2786,31 +2773,18 b' class revlog(object):' | |||
|
2786 | 2773 | p2 = index[entry[6]][7] |
|
2787 | 2774 | node = entry[7] |
|
2788 | 2775 | |
|
2789 | sidedataactions = (False, [], {}, 0, 0) | |
|
2790 | if sidedatacompanion is not None: | |
|
2791 | sidedataactions = sidedatacompanion(self, rev) | |
|
2792 | ||
|
2793 | 2776 | # (Possibly) reuse the delta from the revlog if allowed and |
|
2794 | 2777 | # the revlog chunk is a delta. |
|
2795 | 2778 | cachedelta = None |
|
2796 | 2779 | rawtext = None |
|
2797 |
if |
|
|
2798 | dropall = sidedataactions[0] | |
|
2799 | filterout = sidedataactions[1] | |
|
2800 | update = sidedataactions[2] | |
|
2801 | new_flags = sidedataactions[3] | |
|
2802 | dropped_flags = sidedataactions[4] | |
|
2780 | if deltareuse == self.DELTAREUSEFULLADD: | |
|
2803 | 2781 | text, sidedata = self._revisiondata(rev) |
|
2804 | if dropall: | |
|
2805 |
|
|
|
2806 | for key in filterout: | |
|
2807 |
|
|
|
2808 | sidedata.update(update) | |
|
2809 | if not sidedata: | |
|
2810 | sidedata = None | |
|
2811 | ||
|
2812 | flags |= new_flags | |
|
2813 | flags &= ~dropped_flags | |
|
2782 | ||
|
2783 | if sidedata_helpers is not None: | |
|
2784 | (sidedata, new_flags) = storageutil.run_sidedata_helpers( | |
|
2785 | self, sidedata_helpers, sidedata, rev | |
|
2786 | ) | |
|
2787 | flags = flags | new_flags[0] & ~new_flags[1] | |
|
2814 | 2788 | |
|
2815 | 2789 | destrevlog.addrevision( |
|
2816 | 2790 | text, |
@@ -2830,8 +2804,17 b' class revlog(object):' | |||
|
2830 | 2804 | if dp != nullrev: |
|
2831 | 2805 | cachedelta = (dp, bytes(self._chunk(rev))) |
|
2832 | 2806 | |
|
2807 | sidedata = None | |
|
2833 | 2808 | if not cachedelta: |
|
2834 |
rawtext = self. |
|
|
2809 | rawtext, sidedata = self._revisiondata(rev) | |
|
2810 | if sidedata is None: | |
|
2811 | sidedata = self.sidedata(rev) | |
|
2812 | ||
|
2813 | if sidedata_helpers is not None: | |
|
2814 | (sidedata, new_flags) = storageutil.run_sidedata_helpers( | |
|
2815 | self, sidedata_helpers, sidedata, rev | |
|
2816 | ) | |
|
2817 | flags = flags | new_flags[0] & ~new_flags[1] | |
|
2835 | 2818 | |
|
2836 | 2819 | ifh = destrevlog.opener( |
|
2837 | 2820 | destrevlog.indexfile, b'a+', checkambig=False |
@@ -2852,6 +2835,7 b' class revlog(object):' | |||
|
2852 | 2835 | ifh, |
|
2853 | 2836 | dfh, |
|
2854 | 2837 | deltacomputer=deltacomputer, |
|
2838 | sidedata=sidedata, | |
|
2855 | 2839 | ) |
|
2856 | 2840 | finally: |
|
2857 | 2841 | if dfh: |
@@ -12,6 +12,7 b' import stat' | |||
|
12 | 12 | from ..i18n import _ |
|
13 | 13 | from ..pycompat import getattr |
|
14 | 14 | from .. import ( |
|
15 | changegroup, | |
|
15 | 16 | changelog, |
|
16 | 17 | error, |
|
17 | 18 | filelog, |
@@ -19,13 +20,32 b' from .. import (' | |||
|
19 | 20 | metadata, |
|
20 | 21 | pycompat, |
|
21 | 22 | requirements, |
|
22 | revlog, | |
|
23 | 23 | scmutil, |
|
24 | 24 | store, |
|
25 | 25 | util, |
|
26 | 26 | vfs as vfsmod, |
|
27 | 27 | ) |
|
28 |
from ..revlogutils import |
|
|
28 | from ..revlogutils import ( | |
|
29 | constants as revlogconst, | |
|
30 | flagutil, | |
|
31 | nodemap, | |
|
32 | sidedata as sidedatamod, | |
|
33 | ) | |
|
34 | ||
|
35 | ||
|
36 | def get_sidedata_helpers(srcrepo, dstrepo): | |
|
37 | use_w = srcrepo.ui.configbool(b'experimental', b'worker.repository-upgrade') | |
|
38 | sequential = pycompat.iswindows or not use_w | |
|
39 | if not sequential: | |
|
40 | srcrepo.register_sidedata_computer( | |
|
41 | revlogconst.KIND_CHANGELOG, | |
|
42 | sidedatamod.SD_FILES, | |
|
43 | (sidedatamod.SD_FILES,), | |
|
44 | metadata._get_worker_sidedata_adder(srcrepo, dstrepo), | |
|
45 | flagutil.REVIDX_HASCOPIESINFO, | |
|
46 | replace=True, | |
|
47 | ) | |
|
48 | return changegroup.get_sidedata_helpers(srcrepo, dstrepo._wanted_sidedata) | |
|
29 | 49 | |
|
30 | 50 | |
|
31 | 51 | def _revlogfrompath(repo, rl_type, path): |
@@ -89,25 +109,6 b' UPGRADE_ALL_REVLOGS = frozenset(' | |||
|
89 | 109 | ) |
|
90 | 110 | |
|
91 | 111 | |
|
92 | def getsidedatacompanion(srcrepo, dstrepo): | |
|
93 | sidedatacompanion = None | |
|
94 | removedreqs = srcrepo.requirements - dstrepo.requirements | |
|
95 | addedreqs = dstrepo.requirements - srcrepo.requirements | |
|
96 | if requirements.SIDEDATA_REQUIREMENT in removedreqs: | |
|
97 | ||
|
98 | def sidedatacompanion(rl, rev): | |
|
99 | rl = getattr(rl, '_revlog', rl) | |
|
100 | if rl.flags(rev) & revlog.REVIDX_SIDEDATA: | |
|
101 | return True, (), {}, 0, 0 | |
|
102 | return False, (), {}, 0, 0 | |
|
103 | ||
|
104 | elif requirements.COPIESSDC_REQUIREMENT in addedreqs: | |
|
105 | sidedatacompanion = metadata.getsidedataadder(srcrepo, dstrepo) | |
|
106 | elif requirements.COPIESSDC_REQUIREMENT in removedreqs: | |
|
107 | sidedatacompanion = metadata.getsidedataremover(srcrepo, dstrepo) | |
|
108 | return sidedatacompanion | |
|
109 | ||
|
110 | ||
|
111 | 112 | def matchrevlog(revlogfilter, rl_type): |
|
112 | 113 | """check if a revlog is selected for cloning. |
|
113 | 114 | |
@@ -131,7 +132,7 b' def _perform_clone(' | |||
|
131 | 132 | rl_type, |
|
132 | 133 | unencoded, |
|
133 | 134 | upgrade_op, |
|
134 |
sidedata |
|
|
135 | sidedata_helpers, | |
|
135 | 136 | oncopiedrevision, |
|
136 | 137 | ): |
|
137 | 138 | """ returns the new revlog object created""" |
@@ -147,7 +148,7 b' def _perform_clone(' | |||
|
147 | 148 | addrevisioncb=oncopiedrevision, |
|
148 | 149 | deltareuse=upgrade_op.delta_reuse_mode, |
|
149 | 150 | forcedeltabothparents=upgrade_op.force_re_delta_both_parents, |
|
150 |
sidedata |
|
|
151 | sidedata_helpers=sidedata_helpers, | |
|
151 | 152 | ) |
|
152 | 153 | else: |
|
153 | 154 | msg = _(b'blindly copying %s containing %i revisions\n') |
@@ -257,7 +258,7 b' def _clonerevlogs(' | |||
|
257 | 258 | def oncopiedrevision(rl, rev, node): |
|
258 | 259 | progress.increment() |
|
259 | 260 | |
|
260 |
sidedata |
|
|
261 | sidedata_helpers = get_sidedata_helpers(srcrepo, dstrepo) | |
|
261 | 262 | |
|
262 | 263 | # Migrating filelogs |
|
263 | 264 | ui.status( |
@@ -282,7 +283,7 b' def _clonerevlogs(' | |||
|
282 | 283 | rl_type, |
|
283 | 284 | unencoded, |
|
284 | 285 | upgrade_op, |
|
285 |
sidedata |
|
|
286 | sidedata_helpers, | |
|
286 | 287 | oncopiedrevision, |
|
287 | 288 | ) |
|
288 | 289 | info = newrl.storageinfo(storedsize=True) |
@@ -322,7 +323,7 b' def _clonerevlogs(' | |||
|
322 | 323 | rl_type, |
|
323 | 324 | unencoded, |
|
324 | 325 | upgrade_op, |
|
325 |
sidedata |
|
|
326 | sidedata_helpers, | |
|
326 | 327 | oncopiedrevision, |
|
327 | 328 | ) |
|
328 | 329 | info = newrl.storageinfo(storedsize=True) |
@@ -361,7 +362,7 b' def _clonerevlogs(' | |||
|
361 | 362 | rl_type, |
|
362 | 363 | unencoded, |
|
363 | 364 | upgrade_op, |
|
364 |
sidedata |
|
|
365 | sidedata_helpers, | |
|
365 | 366 | oncopiedrevision, |
|
366 | 367 | ) |
|
367 | 368 | info = newrl.storageinfo(storedsize=True) |
@@ -419,7 +419,7 b' Test committing half a rename' | |||
|
419 | 419 | Test upgrading/downgrading to sidedata storage |
|
420 | 420 | ============================================== |
|
421 | 421 | |
|
422 | downgrading (keeping some sidedata) | |
|
422 | downgrading | |
|
423 | 423 | |
|
424 | 424 |
$ |
|
425 | 425 | format-variant repo config default |
@@ -465,11 +465,7 b' downgrading (keeping some sidedata)' | |||
|
465 | 465 | compression: zstd zstd zstd (zstd !) |
|
466 | 466 | compression-level: default default default |
|
467 | 467 | $ hg debugsidedata -c -- 0 |
|
468 | 1 sidedata entries | |
|
469 | entry-0014 size 14 | |
|
470 | 468 | $ hg debugsidedata -c -- 1 |
|
471 | 1 sidedata entries | |
|
472 | entry-0014 size 14 | |
|
473 | 469 | $ hg debugsidedata -m -- 0 |
|
474 | 470 | |
|
475 | 471 | upgrading |
@@ -12,6 +12,7 b' import struct' | |||
|
12 | 12 | |
|
13 | 13 | from mercurial.node import nullrev |
|
14 | 14 | from mercurial import ( |
|
15 | changegroup, | |
|
15 | 16 | extensions, |
|
16 | 17 | requirements, |
|
17 | 18 | revlog, |
@@ -19,6 +20,7 b' from mercurial import (' | |||
|
19 | 20 | |
|
20 | 21 | from mercurial.upgrade_utils import engine as upgrade_engine |
|
21 | 22 | |
|
23 | from mercurial.revlogutils import constants | |
|
22 | 24 | from mercurial.revlogutils import sidedata |
|
23 | 25 | |
|
24 | 26 | |
@@ -54,13 +56,15 b' def wrap_revisiondata(orig, self, nodeor' | |||
|
54 | 56 | return text, sd |
|
55 | 57 | |
|
56 | 58 | |
|
57 |
def wrapgetsidedata |
|
|
58 |
|
|
|
59 | def wrapget_sidedata_helpers(orig, srcrepo, dstrepo): | |
|
60 | repo, computers, removers = orig(srcrepo, dstrepo) | |
|
61 | assert not computers and not removers # deal with composition later | |
|
59 | 62 | addedreqs = dstrepo.requirements - srcrepo.requirements |
|
63 | ||
|
60 | 64 | if requirements.SIDEDATA_REQUIREMENT in addedreqs: |
|
61 | assert sidedatacompanion is None # deal with composition later | |
|
62 | 65 | |
|
63 |
def |
|
|
66 | def computer(repo, revlog, rev, old_sidedata): | |
|
67 | assert not old_sidedata # not supported yet | |
|
64 | 68 | update = {} |
|
65 | 69 | revlog.sidedatanocheck = True |
|
66 | 70 | try: |
@@ -73,16 +77,25 b' def wrapgetsidedatacompanion(orig, srcre' | |||
|
73 | 77 | # and sha2 hashes |
|
74 | 78 | sha256 = hashlib.sha256(text).digest() |
|
75 | 79 | update[sidedata.SD_TEST2] = struct.pack('>32s', sha256) |
|
76 |
return |
|
|
80 | return update, (0, 0) | |
|
77 | 81 | |
|
78 | return sidedatacompanion | |
|
82 | srcrepo.register_sidedata_computer( | |
|
83 | constants.KIND_CHANGELOG, | |
|
84 | b"whatever", | |
|
85 | (sidedata.SD_TEST1, sidedata.SD_TEST2), | |
|
86 | computer, | |
|
87 | 0, | |
|
88 | ) | |
|
89 | dstrepo.register_wanted_sidedata(b"whatever") | |
|
90 | ||
|
91 | return changegroup.get_sidedata_helpers(srcrepo, dstrepo._wanted_sidedata) | |
|
79 | 92 | |
|
80 | 93 | |
|
81 | 94 | def extsetup(ui): |
|
82 | 95 | extensions.wrapfunction(revlog.revlog, 'addrevision', wrapaddrevision) |
|
83 | 96 | extensions.wrapfunction(revlog.revlog, '_revisiondata', wrap_revisiondata) |
|
84 | 97 | extensions.wrapfunction( |
|
85 |
upgrade_engine, 'getsidedata |
|
|
98 | upgrade_engine, 'get_sidedata_helpers', wrapget_sidedata_helpers | |
|
86 | 99 | ) |
|
87 | 100 | |
|
88 | 101 |
General Comments 0
You need to be logged in to leave comments.
Login now