##// END OF EJS Templates
revlogutils: for issue6528 fix, pre-cache nullrev as metadata-free
Joerg Sonnenberger -
r52806:576876a5 default
parent child Browse files
Show More
@@ -1,886 +1,886
1 # censor code related to censoring revision
1 # censor code related to censoring revision
2 # coding: utf8
2 # coding: utf8
3 #
3 #
4 # Copyright 2021 Pierre-Yves David <pierre-yves.david@octobus.net>
4 # Copyright 2021 Pierre-Yves David <pierre-yves.david@octobus.net>
5 # Copyright 2015 Google, Inc <martinvonz@google.com>
5 # Copyright 2015 Google, Inc <martinvonz@google.com>
6 #
6 #
7 # This software may be used and distributed according to the terms of the
7 # This software may be used and distributed according to the terms of the
8 # GNU General Public License version 2 or any later version.
8 # GNU General Public License version 2 or any later version.
9
9
10 from __future__ import annotations
10 from __future__ import annotations
11
11
12 import binascii
12 import binascii
13 import contextlib
13 import contextlib
14 import os
14 import os
15 import struct
15 import struct
16
16
17 from ..node import (
17 from ..node import (
18 nullrev,
18 nullrev,
19 )
19 )
20 from .constants import (
20 from .constants import (
21 COMP_MODE_PLAIN,
21 COMP_MODE_PLAIN,
22 ENTRY_DATA_COMPRESSED_LENGTH,
22 ENTRY_DATA_COMPRESSED_LENGTH,
23 ENTRY_DATA_COMPRESSION_MODE,
23 ENTRY_DATA_COMPRESSION_MODE,
24 ENTRY_DATA_OFFSET,
24 ENTRY_DATA_OFFSET,
25 ENTRY_DATA_UNCOMPRESSED_LENGTH,
25 ENTRY_DATA_UNCOMPRESSED_LENGTH,
26 ENTRY_DELTA_BASE,
26 ENTRY_DELTA_BASE,
27 ENTRY_LINK_REV,
27 ENTRY_LINK_REV,
28 ENTRY_NODE_ID,
28 ENTRY_NODE_ID,
29 ENTRY_PARENT_1,
29 ENTRY_PARENT_1,
30 ENTRY_PARENT_2,
30 ENTRY_PARENT_2,
31 ENTRY_SIDEDATA_COMPRESSED_LENGTH,
31 ENTRY_SIDEDATA_COMPRESSED_LENGTH,
32 ENTRY_SIDEDATA_COMPRESSION_MODE,
32 ENTRY_SIDEDATA_COMPRESSION_MODE,
33 ENTRY_SIDEDATA_OFFSET,
33 ENTRY_SIDEDATA_OFFSET,
34 REVIDX_ISCENSORED,
34 REVIDX_ISCENSORED,
35 REVLOGV0,
35 REVLOGV0,
36 REVLOGV1,
36 REVLOGV1,
37 )
37 )
38 from ..i18n import _
38 from ..i18n import _
39
39
40 from .. import (
40 from .. import (
41 error,
41 error,
42 mdiff,
42 mdiff,
43 pycompat,
43 pycompat,
44 revlogutils,
44 revlogutils,
45 util,
45 util,
46 )
46 )
47 from ..utils import (
47 from ..utils import (
48 storageutil,
48 storageutil,
49 )
49 )
50 from . import (
50 from . import (
51 constants,
51 constants,
52 deltas,
52 deltas,
53 )
53 )
54
54
55
55
56 def v1_censor(rl, tr, censor_nodes, tombstone=b''):
56 def v1_censor(rl, tr, censor_nodes, tombstone=b''):
57 """censors a revision in a "version 1" revlog"""
57 """censors a revision in a "version 1" revlog"""
58 assert rl._format_version == constants.REVLOGV1, rl._format_version
58 assert rl._format_version == constants.REVLOGV1, rl._format_version
59
59
60 # avoid cycle
60 # avoid cycle
61 from .. import revlog
61 from .. import revlog
62
62
63 censor_revs = set(rl.rev(node) for node in censor_nodes)
63 censor_revs = set(rl.rev(node) for node in censor_nodes)
64 tombstone = storageutil.packmeta({b'censored': tombstone}, b'')
64 tombstone = storageutil.packmeta({b'censored': tombstone}, b'')
65
65
66 # Rewriting the revlog in place is hard. Our strategy for censoring is
66 # Rewriting the revlog in place is hard. Our strategy for censoring is
67 # to create a new revlog, copy all revisions to it, then replace the
67 # to create a new revlog, copy all revisions to it, then replace the
68 # revlogs on transaction close.
68 # revlogs on transaction close.
69 #
69 #
70 # This is a bit dangerous. We could easily have a mismatch of state.
70 # This is a bit dangerous. We could easily have a mismatch of state.
71 newrl = revlog.revlog(
71 newrl = revlog.revlog(
72 rl.opener,
72 rl.opener,
73 target=rl.target,
73 target=rl.target,
74 radix=rl.radix,
74 radix=rl.radix,
75 postfix=b'tmpcensored',
75 postfix=b'tmpcensored',
76 censorable=True,
76 censorable=True,
77 data_config=rl.data_config,
77 data_config=rl.data_config,
78 delta_config=rl.delta_config,
78 delta_config=rl.delta_config,
79 feature_config=rl.feature_config,
79 feature_config=rl.feature_config,
80 may_inline=rl._inline,
80 may_inline=rl._inline,
81 )
81 )
82 # inline splitting will prepare some transaction work that will get
82 # inline splitting will prepare some transaction work that will get
83 # confused by the final file move. So if there is a risk of not being
83 # confused by the final file move. So if there is a risk of not being
84 # inline at the end, we prevent the new revlog to be inline in the first
84 # inline at the end, we prevent the new revlog to be inline in the first
85 # place.
85 # place.
86 assert not (newrl._inline and not rl._inline)
86 assert not (newrl._inline and not rl._inline)
87
87
88 for rev in rl.revs():
88 for rev in rl.revs():
89 node = rl.node(rev)
89 node = rl.node(rev)
90 p1, p2 = rl.parents(node)
90 p1, p2 = rl.parents(node)
91
91
92 if rev in censor_revs:
92 if rev in censor_revs:
93 newrl.addrawrevision(
93 newrl.addrawrevision(
94 tombstone,
94 tombstone,
95 tr,
95 tr,
96 rl.linkrev(rev),
96 rl.linkrev(rev),
97 p1,
97 p1,
98 p2,
98 p2,
99 node,
99 node,
100 constants.REVIDX_ISCENSORED,
100 constants.REVIDX_ISCENSORED,
101 )
101 )
102
102
103 if newrl.deltaparent(rev) != nullrev:
103 if newrl.deltaparent(rev) != nullrev:
104 m = _(b'censored revision stored as delta; cannot censor')
104 m = _(b'censored revision stored as delta; cannot censor')
105 h = _(
105 h = _(
106 b'censoring of revlogs is not fully implemented;'
106 b'censoring of revlogs is not fully implemented;'
107 b' please report this bug'
107 b' please report this bug'
108 )
108 )
109 raise error.Abort(m, hint=h)
109 raise error.Abort(m, hint=h)
110 continue
110 continue
111
111
112 if rl.iscensored(rev):
112 if rl.iscensored(rev):
113 if rl.deltaparent(rev) != nullrev:
113 if rl.deltaparent(rev) != nullrev:
114 m = _(
114 m = _(
115 b'cannot censor due to censored '
115 b'cannot censor due to censored '
116 b'revision having delta stored'
116 b'revision having delta stored'
117 )
117 )
118 raise error.Abort(m)
118 raise error.Abort(m)
119 rawtext = rl._inner._chunk(rev)
119 rawtext = rl._inner._chunk(rev)
120 else:
120 else:
121 rawtext = rl.rawdata(rev)
121 rawtext = rl.rawdata(rev)
122
122
123 newrl.addrawrevision(
123 newrl.addrawrevision(
124 rawtext, tr, rl.linkrev(rev), p1, p2, node, rl.flags(rev)
124 rawtext, tr, rl.linkrev(rev), p1, p2, node, rl.flags(rev)
125 )
125 )
126
126
127 tr.addbackup(rl._indexfile, location=b'store')
127 tr.addbackup(rl._indexfile, location=b'store')
128 if not rl._inline:
128 if not rl._inline:
129 tr.addbackup(rl._datafile, location=b'store')
129 tr.addbackup(rl._datafile, location=b'store')
130
130
131 rl.opener.rename(newrl._indexfile, rl._indexfile)
131 rl.opener.rename(newrl._indexfile, rl._indexfile)
132 if newrl._inline:
132 if newrl._inline:
133 assert rl._inline
133 assert rl._inline
134 else:
134 else:
135 assert not rl._inline
135 assert not rl._inline
136 rl.opener.rename(newrl._datafile, rl._datafile)
136 rl.opener.rename(newrl._datafile, rl._datafile)
137
137
138 rl.clearcaches()
138 rl.clearcaches()
139 chunk_cache = rl._loadindex()
139 chunk_cache = rl._loadindex()
140 rl._load_inner(chunk_cache)
140 rl._load_inner(chunk_cache)
141
141
142
142
143 def v2_censor(revlog, tr, censor_nodes, tombstone=b''):
143 def v2_censor(revlog, tr, censor_nodes, tombstone=b''):
144 """censors a revision in a "version 2" revlog"""
144 """censors a revision in a "version 2" revlog"""
145 assert revlog._format_version != REVLOGV0, revlog._format_version
145 assert revlog._format_version != REVLOGV0, revlog._format_version
146 assert revlog._format_version != REVLOGV1, revlog._format_version
146 assert revlog._format_version != REVLOGV1, revlog._format_version
147
147
148 censor_revs = {revlog.rev(node) for node in censor_nodes}
148 censor_revs = {revlog.rev(node) for node in censor_nodes}
149 _rewrite_v2(revlog, tr, censor_revs, tombstone)
149 _rewrite_v2(revlog, tr, censor_revs, tombstone)
150
150
151
151
152 def _rewrite_v2(revlog, tr, censor_revs, tombstone=b''):
152 def _rewrite_v2(revlog, tr, censor_revs, tombstone=b''):
153 """rewrite a revlog to censor some of its content
153 """rewrite a revlog to censor some of its content
154
154
155 General principle
155 General principle
156
156
157 We create new revlog files (index/data/sidedata) to copy the content of
157 We create new revlog files (index/data/sidedata) to copy the content of
158 the existing data without the censored data.
158 the existing data without the censored data.
159
159
160 We need to recompute new delta for any revision that used the censored
160 We need to recompute new delta for any revision that used the censored
161 revision as delta base. As the cumulative size of the new delta may be
161 revision as delta base. As the cumulative size of the new delta may be
162 large, we store them in a temporary file until they are stored in their
162 large, we store them in a temporary file until they are stored in their
163 final destination.
163 final destination.
164
164
165 All data before the censored data can be blindly copied. The rest needs
165 All data before the censored data can be blindly copied. The rest needs
166 to be copied as we go and the associated index entry needs adjustement.
166 to be copied as we go and the associated index entry needs adjustement.
167 """
167 """
168 assert revlog._format_version != REVLOGV0, revlog._format_version
168 assert revlog._format_version != REVLOGV0, revlog._format_version
169 assert revlog._format_version != REVLOGV1, revlog._format_version
169 assert revlog._format_version != REVLOGV1, revlog._format_version
170
170
171 old_index = revlog.index
171 old_index = revlog.index
172 docket = revlog._docket
172 docket = revlog._docket
173
173
174 tombstone = storageutil.packmeta({b'censored': tombstone}, b'')
174 tombstone = storageutil.packmeta({b'censored': tombstone}, b'')
175
175
176 first_excl_rev = min(censor_revs)
176 first_excl_rev = min(censor_revs)
177
177
178 first_excl_entry = revlog.index[first_excl_rev]
178 first_excl_entry = revlog.index[first_excl_rev]
179 index_cutoff = revlog.index.entry_size * first_excl_rev
179 index_cutoff = revlog.index.entry_size * first_excl_rev
180 data_cutoff = first_excl_entry[ENTRY_DATA_OFFSET] >> 16
180 data_cutoff = first_excl_entry[ENTRY_DATA_OFFSET] >> 16
181 sidedata_cutoff = revlog.sidedata_cut_off(first_excl_rev)
181 sidedata_cutoff = revlog.sidedata_cut_off(first_excl_rev)
182
182
183 with pycompat.unnamedtempfile(mode=b"w+b") as tmp_storage:
183 with pycompat.unnamedtempfile(mode=b"w+b") as tmp_storage:
184 # rev β†’ (new_base, data_start, data_end, compression_mode)
184 # rev β†’ (new_base, data_start, data_end, compression_mode)
185 rewritten_entries = _precompute_rewritten_delta(
185 rewritten_entries = _precompute_rewritten_delta(
186 revlog,
186 revlog,
187 old_index,
187 old_index,
188 censor_revs,
188 censor_revs,
189 tmp_storage,
189 tmp_storage,
190 )
190 )
191
191
192 all_files = _setup_new_files(
192 all_files = _setup_new_files(
193 revlog,
193 revlog,
194 index_cutoff,
194 index_cutoff,
195 data_cutoff,
195 data_cutoff,
196 sidedata_cutoff,
196 sidedata_cutoff,
197 )
197 )
198
198
199 # we dont need to open the old index file since its content already
199 # we dont need to open the old index file since its content already
200 # exist in a usable form in `old_index`.
200 # exist in a usable form in `old_index`.
201 with all_files() as open_files:
201 with all_files() as open_files:
202 (
202 (
203 old_data_file,
203 old_data_file,
204 old_sidedata_file,
204 old_sidedata_file,
205 new_index_file,
205 new_index_file,
206 new_data_file,
206 new_data_file,
207 new_sidedata_file,
207 new_sidedata_file,
208 ) = open_files
208 ) = open_files
209
209
210 # writing the censored revision
210 # writing the censored revision
211
211
212 # Writing all subsequent revisions
212 # Writing all subsequent revisions
213 for rev in range(first_excl_rev, len(old_index)):
213 for rev in range(first_excl_rev, len(old_index)):
214 if rev in censor_revs:
214 if rev in censor_revs:
215 _rewrite_censor(
215 _rewrite_censor(
216 revlog,
216 revlog,
217 old_index,
217 old_index,
218 open_files,
218 open_files,
219 rev,
219 rev,
220 tombstone,
220 tombstone,
221 )
221 )
222 else:
222 else:
223 _rewrite_simple(
223 _rewrite_simple(
224 revlog,
224 revlog,
225 old_index,
225 old_index,
226 open_files,
226 open_files,
227 rev,
227 rev,
228 rewritten_entries,
228 rewritten_entries,
229 tmp_storage,
229 tmp_storage,
230 )
230 )
231 docket.write(transaction=None, stripping=True)
231 docket.write(transaction=None, stripping=True)
232
232
233
233
234 def _precompute_rewritten_delta(
234 def _precompute_rewritten_delta(
235 revlog,
235 revlog,
236 old_index,
236 old_index,
237 excluded_revs,
237 excluded_revs,
238 tmp_storage,
238 tmp_storage,
239 ):
239 ):
240 """Compute new delta for revisions whose delta is based on revision that
240 """Compute new delta for revisions whose delta is based on revision that
241 will not survive as is.
241 will not survive as is.
242
242
243 Return a mapping: {rev β†’ (new_base, data_start, data_end, compression_mode)}
243 Return a mapping: {rev β†’ (new_base, data_start, data_end, compression_mode)}
244 """
244 """
245 dc = deltas.deltacomputer(revlog)
245 dc = deltas.deltacomputer(revlog)
246 rewritten_entries = {}
246 rewritten_entries = {}
247 first_excl_rev = min(excluded_revs)
247 first_excl_rev = min(excluded_revs)
248 with revlog.reading():
248 with revlog.reading():
249 for rev in range(first_excl_rev, len(old_index)):
249 for rev in range(first_excl_rev, len(old_index)):
250 if rev in excluded_revs:
250 if rev in excluded_revs:
251 # this revision will be preserved as is, so we don't need to
251 # this revision will be preserved as is, so we don't need to
252 # consider recomputing a delta.
252 # consider recomputing a delta.
253 continue
253 continue
254 entry = old_index[rev]
254 entry = old_index[rev]
255 if entry[ENTRY_DELTA_BASE] not in excluded_revs:
255 if entry[ENTRY_DELTA_BASE] not in excluded_revs:
256 continue
256 continue
257 # This is a revision that use the censored revision as the base
257 # This is a revision that use the censored revision as the base
258 # for its delta. We need a need new deltas
258 # for its delta. We need a need new deltas
259 if entry[ENTRY_DATA_UNCOMPRESSED_LENGTH] == 0:
259 if entry[ENTRY_DATA_UNCOMPRESSED_LENGTH] == 0:
260 # this revision is empty, we can delta against nullrev
260 # this revision is empty, we can delta against nullrev
261 rewritten_entries[rev] = (nullrev, 0, 0, COMP_MODE_PLAIN)
261 rewritten_entries[rev] = (nullrev, 0, 0, COMP_MODE_PLAIN)
262 else:
262 else:
263 text = revlog.rawdata(rev)
263 text = revlog.rawdata(rev)
264 info = revlogutils.revisioninfo(
264 info = revlogutils.revisioninfo(
265 node=entry[ENTRY_NODE_ID],
265 node=entry[ENTRY_NODE_ID],
266 p1=revlog.node(entry[ENTRY_PARENT_1]),
266 p1=revlog.node(entry[ENTRY_PARENT_1]),
267 p2=revlog.node(entry[ENTRY_PARENT_2]),
267 p2=revlog.node(entry[ENTRY_PARENT_2]),
268 btext=[text],
268 btext=[text],
269 textlen=len(text),
269 textlen=len(text),
270 cachedelta=None,
270 cachedelta=None,
271 flags=entry[ENTRY_DATA_OFFSET] & 0xFFFF,
271 flags=entry[ENTRY_DATA_OFFSET] & 0xFFFF,
272 )
272 )
273 d = dc.finddeltainfo(
273 d = dc.finddeltainfo(
274 info, excluded_bases=excluded_revs, target_rev=rev
274 info, excluded_bases=excluded_revs, target_rev=rev
275 )
275 )
276 default_comp = revlog._docket.default_compression_header
276 default_comp = revlog._docket.default_compression_header
277 comp_mode, d = deltas.delta_compression(default_comp, d)
277 comp_mode, d = deltas.delta_compression(default_comp, d)
278 # using `tell` is a bit lazy, but we are not here for speed
278 # using `tell` is a bit lazy, but we are not here for speed
279 start = tmp_storage.tell()
279 start = tmp_storage.tell()
280 tmp_storage.write(d.data[1])
280 tmp_storage.write(d.data[1])
281 end = tmp_storage.tell()
281 end = tmp_storage.tell()
282 rewritten_entries[rev] = (d.base, start, end, comp_mode)
282 rewritten_entries[rev] = (d.base, start, end, comp_mode)
283 return rewritten_entries
283 return rewritten_entries
284
284
285
285
286 def _setup_new_files(
286 def _setup_new_files(
287 revlog,
287 revlog,
288 index_cutoff,
288 index_cutoff,
289 data_cutoff,
289 data_cutoff,
290 sidedata_cutoff,
290 sidedata_cutoff,
291 ):
291 ):
292 """
292 """
293
293
294 return a context manager to open all the relevant files:
294 return a context manager to open all the relevant files:
295 - old_data_file,
295 - old_data_file,
296 - old_sidedata_file,
296 - old_sidedata_file,
297 - new_index_file,
297 - new_index_file,
298 - new_data_file,
298 - new_data_file,
299 - new_sidedata_file,
299 - new_sidedata_file,
300
300
301 The old_index_file is not here because it is accessed through the
301 The old_index_file is not here because it is accessed through the
302 `old_index` object if the caller function.
302 `old_index` object if the caller function.
303 """
303 """
304 docket = revlog._docket
304 docket = revlog._docket
305 old_index_filepath = revlog.opener.join(docket.index_filepath())
305 old_index_filepath = revlog.opener.join(docket.index_filepath())
306 old_data_filepath = revlog.opener.join(docket.data_filepath())
306 old_data_filepath = revlog.opener.join(docket.data_filepath())
307 old_sidedata_filepath = revlog.opener.join(docket.sidedata_filepath())
307 old_sidedata_filepath = revlog.opener.join(docket.sidedata_filepath())
308
308
309 new_index_filepath = revlog.opener.join(docket.new_index_file())
309 new_index_filepath = revlog.opener.join(docket.new_index_file())
310 new_data_filepath = revlog.opener.join(docket.new_data_file())
310 new_data_filepath = revlog.opener.join(docket.new_data_file())
311 new_sidedata_filepath = revlog.opener.join(docket.new_sidedata_file())
311 new_sidedata_filepath = revlog.opener.join(docket.new_sidedata_file())
312
312
313 util.copyfile(old_index_filepath, new_index_filepath, nb_bytes=index_cutoff)
313 util.copyfile(old_index_filepath, new_index_filepath, nb_bytes=index_cutoff)
314 util.copyfile(old_data_filepath, new_data_filepath, nb_bytes=data_cutoff)
314 util.copyfile(old_data_filepath, new_data_filepath, nb_bytes=data_cutoff)
315 util.copyfile(
315 util.copyfile(
316 old_sidedata_filepath,
316 old_sidedata_filepath,
317 new_sidedata_filepath,
317 new_sidedata_filepath,
318 nb_bytes=sidedata_cutoff,
318 nb_bytes=sidedata_cutoff,
319 )
319 )
320 revlog.opener.register_file(docket.index_filepath())
320 revlog.opener.register_file(docket.index_filepath())
321 revlog.opener.register_file(docket.data_filepath())
321 revlog.opener.register_file(docket.data_filepath())
322 revlog.opener.register_file(docket.sidedata_filepath())
322 revlog.opener.register_file(docket.sidedata_filepath())
323
323
324 docket.index_end = index_cutoff
324 docket.index_end = index_cutoff
325 docket.data_end = data_cutoff
325 docket.data_end = data_cutoff
326 docket.sidedata_end = sidedata_cutoff
326 docket.sidedata_end = sidedata_cutoff
327
327
328 # reload the revlog internal information
328 # reload the revlog internal information
329 revlog.clearcaches()
329 revlog.clearcaches()
330 revlog._loadindex(docket=docket)
330 revlog._loadindex(docket=docket)
331
331
332 @contextlib.contextmanager
332 @contextlib.contextmanager
333 def all_files_opener():
333 def all_files_opener():
334 # hide opening in an helper function to please check-code, black
334 # hide opening in an helper function to please check-code, black
335 # and various python version at the same time
335 # and various python version at the same time
336 with open(old_data_filepath, 'rb') as old_data_file:
336 with open(old_data_filepath, 'rb') as old_data_file:
337 with open(old_sidedata_filepath, 'rb') as old_sidedata_file:
337 with open(old_sidedata_filepath, 'rb') as old_sidedata_file:
338 with open(new_index_filepath, 'r+b') as new_index_file:
338 with open(new_index_filepath, 'r+b') as new_index_file:
339 with open(new_data_filepath, 'r+b') as new_data_file:
339 with open(new_data_filepath, 'r+b') as new_data_file:
340 with open(
340 with open(
341 new_sidedata_filepath, 'r+b'
341 new_sidedata_filepath, 'r+b'
342 ) as new_sidedata_file:
342 ) as new_sidedata_file:
343 new_index_file.seek(0, os.SEEK_END)
343 new_index_file.seek(0, os.SEEK_END)
344 assert new_index_file.tell() == index_cutoff
344 assert new_index_file.tell() == index_cutoff
345 new_data_file.seek(0, os.SEEK_END)
345 new_data_file.seek(0, os.SEEK_END)
346 assert new_data_file.tell() == data_cutoff
346 assert new_data_file.tell() == data_cutoff
347 new_sidedata_file.seek(0, os.SEEK_END)
347 new_sidedata_file.seek(0, os.SEEK_END)
348 assert new_sidedata_file.tell() == sidedata_cutoff
348 assert new_sidedata_file.tell() == sidedata_cutoff
349 yield (
349 yield (
350 old_data_file,
350 old_data_file,
351 old_sidedata_file,
351 old_sidedata_file,
352 new_index_file,
352 new_index_file,
353 new_data_file,
353 new_data_file,
354 new_sidedata_file,
354 new_sidedata_file,
355 )
355 )
356
356
357 return all_files_opener
357 return all_files_opener
358
358
359
359
360 def _rewrite_simple(
360 def _rewrite_simple(
361 revlog,
361 revlog,
362 old_index,
362 old_index,
363 all_files,
363 all_files,
364 rev,
364 rev,
365 rewritten_entries,
365 rewritten_entries,
366 tmp_storage,
366 tmp_storage,
367 ):
367 ):
368 """append a normal revision to the index after the rewritten one(s)"""
368 """append a normal revision to the index after the rewritten one(s)"""
369 (
369 (
370 old_data_file,
370 old_data_file,
371 old_sidedata_file,
371 old_sidedata_file,
372 new_index_file,
372 new_index_file,
373 new_data_file,
373 new_data_file,
374 new_sidedata_file,
374 new_sidedata_file,
375 ) = all_files
375 ) = all_files
376 entry = old_index[rev]
376 entry = old_index[rev]
377 flags = entry[ENTRY_DATA_OFFSET] & 0xFFFF
377 flags = entry[ENTRY_DATA_OFFSET] & 0xFFFF
378 old_data_offset = entry[ENTRY_DATA_OFFSET] >> 16
378 old_data_offset = entry[ENTRY_DATA_OFFSET] >> 16
379
379
380 if rev not in rewritten_entries:
380 if rev not in rewritten_entries:
381 old_data_file.seek(old_data_offset)
381 old_data_file.seek(old_data_offset)
382 new_data_size = entry[ENTRY_DATA_COMPRESSED_LENGTH]
382 new_data_size = entry[ENTRY_DATA_COMPRESSED_LENGTH]
383 new_data = old_data_file.read(new_data_size)
383 new_data = old_data_file.read(new_data_size)
384 data_delta_base = entry[ENTRY_DELTA_BASE]
384 data_delta_base = entry[ENTRY_DELTA_BASE]
385 d_comp_mode = entry[ENTRY_DATA_COMPRESSION_MODE]
385 d_comp_mode = entry[ENTRY_DATA_COMPRESSION_MODE]
386 else:
386 else:
387 (
387 (
388 data_delta_base,
388 data_delta_base,
389 start,
389 start,
390 end,
390 end,
391 d_comp_mode,
391 d_comp_mode,
392 ) = rewritten_entries[rev]
392 ) = rewritten_entries[rev]
393 new_data_size = end - start
393 new_data_size = end - start
394 tmp_storage.seek(start)
394 tmp_storage.seek(start)
395 new_data = tmp_storage.read(new_data_size)
395 new_data = tmp_storage.read(new_data_size)
396
396
397 # It might be faster to group continuous read/write operation,
397 # It might be faster to group continuous read/write operation,
398 # however, this is censor, an operation that is not focussed
398 # however, this is censor, an operation that is not focussed
399 # around stellar performance. So I have not written this
399 # around stellar performance. So I have not written this
400 # optimisation yet.
400 # optimisation yet.
401 new_data_offset = new_data_file.tell()
401 new_data_offset = new_data_file.tell()
402 new_data_file.write(new_data)
402 new_data_file.write(new_data)
403
403
404 sidedata_size = entry[ENTRY_SIDEDATA_COMPRESSED_LENGTH]
404 sidedata_size = entry[ENTRY_SIDEDATA_COMPRESSED_LENGTH]
405 new_sidedata_offset = new_sidedata_file.tell()
405 new_sidedata_offset = new_sidedata_file.tell()
406 if 0 < sidedata_size:
406 if 0 < sidedata_size:
407 old_sidedata_offset = entry[ENTRY_SIDEDATA_OFFSET]
407 old_sidedata_offset = entry[ENTRY_SIDEDATA_OFFSET]
408 old_sidedata_file.seek(old_sidedata_offset)
408 old_sidedata_file.seek(old_sidedata_offset)
409 new_sidedata = old_sidedata_file.read(sidedata_size)
409 new_sidedata = old_sidedata_file.read(sidedata_size)
410 new_sidedata_file.write(new_sidedata)
410 new_sidedata_file.write(new_sidedata)
411
411
412 data_uncompressed_length = entry[ENTRY_DATA_UNCOMPRESSED_LENGTH]
412 data_uncompressed_length = entry[ENTRY_DATA_UNCOMPRESSED_LENGTH]
413 sd_com_mode = entry[ENTRY_SIDEDATA_COMPRESSION_MODE]
413 sd_com_mode = entry[ENTRY_SIDEDATA_COMPRESSION_MODE]
414 assert data_delta_base <= rev, (data_delta_base, rev)
414 assert data_delta_base <= rev, (data_delta_base, rev)
415
415
416 new_entry = revlogutils.entry(
416 new_entry = revlogutils.entry(
417 flags=flags,
417 flags=flags,
418 data_offset=new_data_offset,
418 data_offset=new_data_offset,
419 data_compressed_length=new_data_size,
419 data_compressed_length=new_data_size,
420 data_uncompressed_length=data_uncompressed_length,
420 data_uncompressed_length=data_uncompressed_length,
421 data_delta_base=data_delta_base,
421 data_delta_base=data_delta_base,
422 link_rev=entry[ENTRY_LINK_REV],
422 link_rev=entry[ENTRY_LINK_REV],
423 parent_rev_1=entry[ENTRY_PARENT_1],
423 parent_rev_1=entry[ENTRY_PARENT_1],
424 parent_rev_2=entry[ENTRY_PARENT_2],
424 parent_rev_2=entry[ENTRY_PARENT_2],
425 node_id=entry[ENTRY_NODE_ID],
425 node_id=entry[ENTRY_NODE_ID],
426 sidedata_offset=new_sidedata_offset,
426 sidedata_offset=new_sidedata_offset,
427 sidedata_compressed_length=sidedata_size,
427 sidedata_compressed_length=sidedata_size,
428 data_compression_mode=d_comp_mode,
428 data_compression_mode=d_comp_mode,
429 sidedata_compression_mode=sd_com_mode,
429 sidedata_compression_mode=sd_com_mode,
430 )
430 )
431 revlog.index.append(new_entry)
431 revlog.index.append(new_entry)
432 entry_bin = revlog.index.entry_binary(rev)
432 entry_bin = revlog.index.entry_binary(rev)
433 new_index_file.write(entry_bin)
433 new_index_file.write(entry_bin)
434
434
435 revlog._docket.index_end = new_index_file.tell()
435 revlog._docket.index_end = new_index_file.tell()
436 revlog._docket.data_end = new_data_file.tell()
436 revlog._docket.data_end = new_data_file.tell()
437 revlog._docket.sidedata_end = new_sidedata_file.tell()
437 revlog._docket.sidedata_end = new_sidedata_file.tell()
438
438
439
439
440 def _rewrite_censor(
440 def _rewrite_censor(
441 revlog,
441 revlog,
442 old_index,
442 old_index,
443 all_files,
443 all_files,
444 rev,
444 rev,
445 tombstone,
445 tombstone,
446 ):
446 ):
447 """rewrite and append a censored revision"""
447 """rewrite and append a censored revision"""
448 (
448 (
449 old_data_file,
449 old_data_file,
450 old_sidedata_file,
450 old_sidedata_file,
451 new_index_file,
451 new_index_file,
452 new_data_file,
452 new_data_file,
453 new_sidedata_file,
453 new_sidedata_file,
454 ) = all_files
454 ) = all_files
455 entry = old_index[rev]
455 entry = old_index[rev]
456
456
457 # XXX consider trying the default compression too
457 # XXX consider trying the default compression too
458 new_data_size = len(tombstone)
458 new_data_size = len(tombstone)
459 new_data_offset = new_data_file.tell()
459 new_data_offset = new_data_file.tell()
460 new_data_file.write(tombstone)
460 new_data_file.write(tombstone)
461
461
462 # we are not adding any sidedata as they might leak info about the censored version
462 # we are not adding any sidedata as they might leak info about the censored version
463
463
464 link_rev = entry[ENTRY_LINK_REV]
464 link_rev = entry[ENTRY_LINK_REV]
465
465
466 p1 = entry[ENTRY_PARENT_1]
466 p1 = entry[ENTRY_PARENT_1]
467 p2 = entry[ENTRY_PARENT_2]
467 p2 = entry[ENTRY_PARENT_2]
468
468
469 new_entry = revlogutils.entry(
469 new_entry = revlogutils.entry(
470 flags=constants.REVIDX_ISCENSORED,
470 flags=constants.REVIDX_ISCENSORED,
471 data_offset=new_data_offset,
471 data_offset=new_data_offset,
472 data_compressed_length=new_data_size,
472 data_compressed_length=new_data_size,
473 data_uncompressed_length=new_data_size,
473 data_uncompressed_length=new_data_size,
474 data_delta_base=rev,
474 data_delta_base=rev,
475 link_rev=link_rev,
475 link_rev=link_rev,
476 parent_rev_1=p1,
476 parent_rev_1=p1,
477 parent_rev_2=p2,
477 parent_rev_2=p2,
478 node_id=entry[ENTRY_NODE_ID],
478 node_id=entry[ENTRY_NODE_ID],
479 sidedata_offset=0,
479 sidedata_offset=0,
480 sidedata_compressed_length=0,
480 sidedata_compressed_length=0,
481 data_compression_mode=COMP_MODE_PLAIN,
481 data_compression_mode=COMP_MODE_PLAIN,
482 sidedata_compression_mode=COMP_MODE_PLAIN,
482 sidedata_compression_mode=COMP_MODE_PLAIN,
483 )
483 )
484 revlog.index.append(new_entry)
484 revlog.index.append(new_entry)
485 entry_bin = revlog.index.entry_binary(rev)
485 entry_bin = revlog.index.entry_binary(rev)
486 new_index_file.write(entry_bin)
486 new_index_file.write(entry_bin)
487 revlog._docket.index_end = new_index_file.tell()
487 revlog._docket.index_end = new_index_file.tell()
488 revlog._docket.data_end = new_data_file.tell()
488 revlog._docket.data_end = new_data_file.tell()
489
489
490
490
491 def _get_filename_from_filelog_index(path):
491 def _get_filename_from_filelog_index(path):
492 # Drop the extension and the `data/` prefix
492 # Drop the extension and the `data/` prefix
493 path_part = path.rsplit(b'.', 1)[0].split(b'/', 1)
493 path_part = path.rsplit(b'.', 1)[0].split(b'/', 1)
494 if len(path_part) < 2:
494 if len(path_part) < 2:
495 msg = _(b"cannot recognize filelog from filename: '%s'")
495 msg = _(b"cannot recognize filelog from filename: '%s'")
496 msg %= path
496 msg %= path
497 raise error.Abort(msg)
497 raise error.Abort(msg)
498
498
499 return path_part[1]
499 return path_part[1]
500
500
501
501
502 def _filelog_from_filename(repo, path):
502 def _filelog_from_filename(repo, path):
503 """Returns the filelog for the given `path`. Stolen from `engine.py`"""
503 """Returns the filelog for the given `path`. Stolen from `engine.py`"""
504
504
505 from .. import filelog # avoid cycle
505 from .. import filelog # avoid cycle
506
506
507 fl = filelog.filelog(repo.svfs, path)
507 fl = filelog.filelog(repo.svfs, path)
508 return fl
508 return fl
509
509
510
510
511 def _write_swapped_parents(repo, rl, rev, offset, fp):
511 def _write_swapped_parents(repo, rl, rev, offset, fp):
512 """Swaps p1 and p2 and overwrites the revlog entry for `rev` in `fp`"""
512 """Swaps p1 and p2 and overwrites the revlog entry for `rev` in `fp`"""
513 from ..pure import parsers # avoid cycle
513 from ..pure import parsers # avoid cycle
514
514
515 if repo._currentlock(repo._lockref) is None:
515 if repo._currentlock(repo._lockref) is None:
516 # Let's be paranoid about it
516 # Let's be paranoid about it
517 msg = "repo needs to be locked to rewrite parents"
517 msg = "repo needs to be locked to rewrite parents"
518 raise error.ProgrammingError(msg)
518 raise error.ProgrammingError(msg)
519
519
520 index_format = parsers.IndexObject.index_format
520 index_format = parsers.IndexObject.index_format
521 entry = rl.index[rev]
521 entry = rl.index[rev]
522 new_entry = list(entry)
522 new_entry = list(entry)
523 new_entry[5], new_entry[6] = entry[6], entry[5]
523 new_entry[5], new_entry[6] = entry[6], entry[5]
524 packed = index_format.pack(*new_entry[:8])
524 packed = index_format.pack(*new_entry[:8])
525 fp.seek(offset)
525 fp.seek(offset)
526 fp.write(packed)
526 fp.write(packed)
527
527
528
528
529 def _reorder_filelog_parents(repo, fl, to_fix):
529 def _reorder_filelog_parents(repo, fl, to_fix):
530 """
530 """
531 Swaps p1 and p2 for all `to_fix` revisions of filelog `fl` and writes the
531 Swaps p1 and p2 for all `to_fix` revisions of filelog `fl` and writes the
532 new version to disk, overwriting the old one with a rename.
532 new version to disk, overwriting the old one with a rename.
533 """
533 """
534 from ..pure import parsers # avoid cycle
534 from ..pure import parsers # avoid cycle
535
535
536 ui = repo.ui
536 ui = repo.ui
537 assert len(to_fix) > 0
537 assert len(to_fix) > 0
538 rl = fl._revlog
538 rl = fl._revlog
539 if rl._format_version != constants.REVLOGV1:
539 if rl._format_version != constants.REVLOGV1:
540 msg = "expected version 1 revlog, got version '%d'" % rl._format_version
540 msg = "expected version 1 revlog, got version '%d'" % rl._format_version
541 raise error.ProgrammingError(msg)
541 raise error.ProgrammingError(msg)
542
542
543 index_file = rl._indexfile
543 index_file = rl._indexfile
544 new_file_path = index_file + b'.tmp-parents-fix'
544 new_file_path = index_file + b'.tmp-parents-fix'
545 repaired_msg = _(b"repaired revision %d of 'filelog %s'\n")
545 repaired_msg = _(b"repaired revision %d of 'filelog %s'\n")
546
546
547 with ui.uninterruptible():
547 with ui.uninterruptible():
548 try:
548 try:
549 util.copyfile(
549 util.copyfile(
550 rl.opener.join(index_file),
550 rl.opener.join(index_file),
551 rl.opener.join(new_file_path),
551 rl.opener.join(new_file_path),
552 checkambig=rl.data_config.check_ambig,
552 checkambig=rl.data_config.check_ambig,
553 )
553 )
554
554
555 with rl.opener(new_file_path, mode=b"r+") as fp:
555 with rl.opener(new_file_path, mode=b"r+") as fp:
556 if rl._inline:
556 if rl._inline:
557 index = parsers.InlinedIndexObject(fp.read())
557 index = parsers.InlinedIndexObject(fp.read())
558 for rev in fl.revs():
558 for rev in fl.revs():
559 if rev in to_fix:
559 if rev in to_fix:
560 offset = index._calculate_index(rev)
560 offset = index._calculate_index(rev)
561 _write_swapped_parents(repo, rl, rev, offset, fp)
561 _write_swapped_parents(repo, rl, rev, offset, fp)
562 ui.write(repaired_msg % (rev, index_file))
562 ui.write(repaired_msg % (rev, index_file))
563 else:
563 else:
564 index_format = parsers.IndexObject.index_format
564 index_format = parsers.IndexObject.index_format
565 for rev in to_fix:
565 for rev in to_fix:
566 offset = rev * index_format.size
566 offset = rev * index_format.size
567 _write_swapped_parents(repo, rl, rev, offset, fp)
567 _write_swapped_parents(repo, rl, rev, offset, fp)
568 ui.write(repaired_msg % (rev, index_file))
568 ui.write(repaired_msg % (rev, index_file))
569
569
570 rl.opener.rename(new_file_path, index_file)
570 rl.opener.rename(new_file_path, index_file)
571 rl.clearcaches()
571 rl.clearcaches()
572 rl._loadindex()
572 rl._loadindex()
573 finally:
573 finally:
574 util.tryunlink(new_file_path)
574 util.tryunlink(new_file_path)
575
575
576
576
577 def _is_revision_affected(fl, filerev, metadata_cache=None):
577 def _is_revision_affected(fl, filerev, metadata_cache=None):
578 full_text = lambda: fl._revlog.rawdata(filerev)
578 full_text = lambda: fl._revlog.rawdata(filerev)
579 parent_revs = lambda: fl._revlog.parentrevs(filerev)
579 parent_revs = lambda: fl._revlog.parentrevs(filerev)
580 return _is_revision_affected_inner(
580 return _is_revision_affected_inner(
581 full_text, parent_revs, filerev, metadata_cache
581 full_text, parent_revs, filerev, metadata_cache
582 )
582 )
583
583
584
584
585 def _is_revision_affected_inner(
585 def _is_revision_affected_inner(
586 full_text,
586 full_text,
587 parents_revs,
587 parents_revs,
588 filerev,
588 filerev,
589 metadata_cache=None,
589 metadata_cache=None,
590 ):
590 ):
591 """Mercurial currently (5.9rc0) uses `p1 == nullrev and p2 != nullrev` as a
591 """Mercurial currently (5.9rc0) uses `p1 == nullrev and p2 != nullrev` as a
592 special meaning compared to the reverse in the context of filelog-based
592 special meaning compared to the reverse in the context of filelog-based
593 copytracing. issue6528 exists because new code assumed that parent ordering
593 copytracing. issue6528 exists because new code assumed that parent ordering
594 didn't matter, so this detects if the revision contains metadata (since
594 didn't matter, so this detects if the revision contains metadata (since
595 it's only used for filelog-based copytracing) and its parents are in the
595 it's only used for filelog-based copytracing) and its parents are in the
596 "wrong" order."""
596 "wrong" order."""
597 try:
597 try:
598 raw_text = full_text()
598 raw_text = full_text()
599 except error.CensoredNodeError:
599 except error.CensoredNodeError:
600 # We don't care about censored nodes as they never carry metadata
600 # We don't care about censored nodes as they never carry metadata
601 return False
601 return False
602
602
603 # raw text can be a `memoryview`, which doesn't implement `startswith`
603 # raw text can be a `memoryview`, which doesn't implement `startswith`
604 has_meta = bytes(raw_text[:2]) == b'\x01\n'
604 has_meta = bytes(raw_text[:2]) == b'\x01\n'
605 if metadata_cache is not None:
605 if metadata_cache is not None:
606 metadata_cache[filerev] = has_meta
606 metadata_cache[filerev] = has_meta
607 if has_meta:
607 if has_meta:
608 (p1, p2) = parents_revs()
608 (p1, p2) = parents_revs()
609 if p1 != nullrev and p2 == nullrev:
609 if p1 != nullrev and p2 == nullrev:
610 return True
610 return True
611 return False
611 return False
612
612
613
613
614 def _is_revision_affected_fast(repo, fl, filerev, metadata_cache):
614 def _is_revision_affected_fast(repo, fl, filerev, metadata_cache):
615 rl = fl._revlog
615 rl = fl._revlog
616 is_censored = lambda: rl.iscensored(filerev)
616 is_censored = lambda: rl.iscensored(filerev)
617 delta_base = lambda: rl.deltaparent(filerev)
617 delta_base = lambda: rl.deltaparent(filerev)
618 delta = lambda: rl._inner._chunk(filerev)
618 delta = lambda: rl._inner._chunk(filerev)
619 full_text = lambda: rl.rawdata(filerev)
619 full_text = lambda: rl.rawdata(filerev)
620 parent_revs = lambda: rl.parentrevs(filerev)
620 parent_revs = lambda: rl.parentrevs(filerev)
621 return _is_revision_affected_fast_inner(
621 return _is_revision_affected_fast_inner(
622 is_censored,
622 is_censored,
623 delta_base,
623 delta_base,
624 delta,
624 delta,
625 full_text,
625 full_text,
626 parent_revs,
626 parent_revs,
627 filerev,
627 filerev,
628 metadata_cache,
628 metadata_cache,
629 )
629 )
630
630
631
631
632 def _is_revision_affected_fast_inner(
632 def _is_revision_affected_fast_inner(
633 is_censored,
633 is_censored,
634 delta_base,
634 delta_base,
635 delta,
635 delta,
636 full_text,
636 full_text,
637 parent_revs,
637 parent_revs,
638 filerev,
638 filerev,
639 metadata_cache,
639 metadata_cache,
640 ):
640 ):
641 """Optimization fast-path for `_is_revision_affected`.
641 """Optimization fast-path for `_is_revision_affected`.
642
642
643 `metadata_cache` is a dict of `{rev: has_metadata}` which allows any
643 `metadata_cache` is a dict of `{rev: has_metadata}` which allows any
644 revision to check if its base has metadata, saving computation of the full
644 revision to check if its base has metadata, saving computation of the full
645 text, instead looking at the current delta.
645 text, instead looking at the current delta.
646
646
647 This optimization only works if the revisions are looked at in order."""
647 This optimization only works if the revisions are looked at in order."""
648
648
649 if is_censored():
649 if is_censored():
650 # Censored revisions don't contain metadata, so they cannot be affected
650 # Censored revisions don't contain metadata, so they cannot be affected
651 metadata_cache[filerev] = False
651 metadata_cache[filerev] = False
652 return False
652 return False
653
653
654 p1, p2 = parent_revs()
654 p1, p2 = parent_revs()
655 if p1 == nullrev or p2 != nullrev:
655 if p1 == nullrev or p2 != nullrev:
656 return False
656 return False
657
657
658 delta_parent = delta_base()
658 delta_parent = delta_base()
659 parent_has_metadata = metadata_cache.get(delta_parent)
659 parent_has_metadata = metadata_cache.get(delta_parent)
660 if parent_has_metadata is None:
660 if parent_has_metadata is None:
661 return _is_revision_affected_inner(
661 return _is_revision_affected_inner(
662 full_text,
662 full_text,
663 parent_revs,
663 parent_revs,
664 filerev,
664 filerev,
665 metadata_cache,
665 metadata_cache,
666 )
666 )
667
667
668 chunk = delta()
668 chunk = delta()
669 if not len(chunk):
669 if not len(chunk):
670 # No diff for this revision
670 # No diff for this revision
671 metadata_cache[filerev] = parent_has_metadata
671 metadata_cache[filerev] = parent_has_metadata
672 return parent_has_metadata
672 return parent_has_metadata
673
673
674 header_length = 12
674 header_length = 12
675 if len(chunk) < header_length:
675 if len(chunk) < header_length:
676 raise error.Abort(_(b"patch cannot be decoded"))
676 raise error.Abort(_(b"patch cannot be decoded"))
677
677
678 start, _end, _length = struct.unpack(b">lll", chunk[:header_length])
678 start, _end, _length = struct.unpack(b">lll", chunk[:header_length])
679
679
680 if start < 2: # len(b'\x01\n') == 2
680 if start < 2: # len(b'\x01\n') == 2
681 # This delta does *something* to the metadata marker (if any).
681 # This delta does *something* to the metadata marker (if any).
682 # Check it the slow way
682 # Check it the slow way
683 is_affected = _is_revision_affected_inner(
683 is_affected = _is_revision_affected_inner(
684 full_text,
684 full_text,
685 parent_revs,
685 parent_revs,
686 filerev,
686 filerev,
687 metadata_cache,
687 metadata_cache,
688 )
688 )
689 return is_affected
689 return is_affected
690
690
691 # The diff did not remove or add the metadata header, it's then in the same
691 # The diff did not remove or add the metadata header, it's then in the same
692 # situation as its parent
692 # situation as its parent
693 metadata_cache[filerev] = parent_has_metadata
693 metadata_cache[filerev] = parent_has_metadata
694 return parent_has_metadata
694 return parent_has_metadata
695
695
696
696
697 def _from_report(ui, repo, context, from_report, dry_run):
697 def _from_report(ui, repo, context, from_report, dry_run):
698 """
698 """
699 Fix the revisions given in the `from_report` file, but still checks if the
699 Fix the revisions given in the `from_report` file, but still checks if the
700 revisions are indeed affected to prevent an unfortunate cyclic situation
700 revisions are indeed affected to prevent an unfortunate cyclic situation
701 where we'd swap well-ordered parents again.
701 where we'd swap well-ordered parents again.
702
702
703 See the doc for `debug_fix_issue6528` for the format documentation.
703 See the doc for `debug_fix_issue6528` for the format documentation.
704 """
704 """
705 ui.write(_(b"loading report file '%s'\n") % from_report)
705 ui.write(_(b"loading report file '%s'\n") % from_report)
706
706
707 with context(), open(from_report, mode='rb') as f:
707 with context(), open(from_report, mode='rb') as f:
708 for line in f.read().split(b'\n'):
708 for line in f.read().split(b'\n'):
709 if not line:
709 if not line:
710 continue
710 continue
711 filenodes, filename = line.split(b' ', 1)
711 filenodes, filename = line.split(b' ', 1)
712 fl = _filelog_from_filename(repo, filename)
712 fl = _filelog_from_filename(repo, filename)
713 to_fix = set(
713 to_fix = set(
714 fl.rev(binascii.unhexlify(n)) for n in filenodes.split(b',')
714 fl.rev(binascii.unhexlify(n)) for n in filenodes.split(b',')
715 )
715 )
716 excluded = set()
716 excluded = set()
717
717
718 for filerev in to_fix:
718 for filerev in to_fix:
719 if _is_revision_affected(fl, filerev):
719 if _is_revision_affected(fl, filerev):
720 msg = b"found affected revision %d for filelog '%s'\n"
720 msg = b"found affected revision %d for filelog '%s'\n"
721 ui.warn(msg % (filerev, filename))
721 ui.warn(msg % (filerev, filename))
722 else:
722 else:
723 msg = _(b"revision %s of file '%s' is not affected\n")
723 msg = _(b"revision %s of file '%s' is not affected\n")
724 msg %= (binascii.hexlify(fl.node(filerev)), filename)
724 msg %= (binascii.hexlify(fl.node(filerev)), filename)
725 ui.warn(msg)
725 ui.warn(msg)
726 excluded.add(filerev)
726 excluded.add(filerev)
727
727
728 to_fix = to_fix - excluded
728 to_fix = to_fix - excluded
729 if not to_fix:
729 if not to_fix:
730 msg = _(b"no affected revisions were found for '%s'\n")
730 msg = _(b"no affected revisions were found for '%s'\n")
731 ui.write(msg % filename)
731 ui.write(msg % filename)
732 continue
732 continue
733 if not dry_run:
733 if not dry_run:
734 _reorder_filelog_parents(repo, fl, sorted(to_fix))
734 _reorder_filelog_parents(repo, fl, sorted(to_fix))
735
735
736
736
737 def filter_delta_issue6528(revlog, deltas_iter):
737 def filter_delta_issue6528(revlog, deltas_iter):
738 """filter incomind deltas to repaire issue 6528 on the fly"""
738 """filter incomind deltas to repaire issue 6528 on the fly"""
739 metadata_cache = {}
739 metadata_cache = {nullrev: False}
740
740
741 deltacomputer = deltas.deltacomputer(revlog)
741 deltacomputer = deltas.deltacomputer(revlog)
742
742
743 for rev, d in enumerate(deltas_iter, len(revlog)):
743 for rev, d in enumerate(deltas_iter, len(revlog)):
744 (
744 (
745 node,
745 node,
746 p1_node,
746 p1_node,
747 p2_node,
747 p2_node,
748 linknode,
748 linknode,
749 deltabase,
749 deltabase,
750 delta,
750 delta,
751 flags,
751 flags,
752 sidedata,
752 sidedata,
753 ) = d
753 ) = d
754
754
755 if not revlog.index.has_node(deltabase):
755 if not revlog.index.has_node(deltabase):
756 raise error.LookupError(
756 raise error.LookupError(
757 deltabase, revlog.radix, _(b'unknown parent')
757 deltabase, revlog.radix, _(b'unknown parent')
758 )
758 )
759 base_rev = revlog.rev(deltabase)
759 base_rev = revlog.rev(deltabase)
760 if not revlog.index.has_node(p1_node):
760 if not revlog.index.has_node(p1_node):
761 raise error.LookupError(p1_node, revlog.radix, _(b'unknown parent'))
761 raise error.LookupError(p1_node, revlog.radix, _(b'unknown parent'))
762 p1_rev = revlog.rev(p1_node)
762 p1_rev = revlog.rev(p1_node)
763 if not revlog.index.has_node(p2_node):
763 if not revlog.index.has_node(p2_node):
764 raise error.LookupError(p2_node, revlog.radix, _(b'unknown parent'))
764 raise error.LookupError(p2_node, revlog.radix, _(b'unknown parent'))
765 p2_rev = revlog.rev(p2_node)
765 p2_rev = revlog.rev(p2_node)
766
766
767 is_censored = lambda: bool(flags & REVIDX_ISCENSORED)
767 is_censored = lambda: bool(flags & REVIDX_ISCENSORED)
768 delta_base = lambda: revlog.rev(delta_base)
768 delta_base = lambda: revlog.rev(delta_base)
769 delta_base = lambda: base_rev
769 delta_base = lambda: base_rev
770 parent_revs = lambda: (p1_rev, p2_rev)
770 parent_revs = lambda: (p1_rev, p2_rev)
771
771
772 def full_text():
772 def full_text():
773 # note: being able to reuse the full text computation in the
773 # note: being able to reuse the full text computation in the
774 # underlying addrevision would be useful however this is a bit too
774 # underlying addrevision would be useful however this is a bit too
775 # intrusive the for the "quick" issue6528 we are writing before the
775 # intrusive the for the "quick" issue6528 we are writing before the
776 # 5.8 release
776 # 5.8 release
777 textlen = mdiff.patchedsize(revlog.size(base_rev), delta)
777 textlen = mdiff.patchedsize(revlog.size(base_rev), delta)
778
778
779 revinfo = revlogutils.revisioninfo(
779 revinfo = revlogutils.revisioninfo(
780 node,
780 node,
781 p1_node,
781 p1_node,
782 p2_node,
782 p2_node,
783 [None],
783 [None],
784 textlen,
784 textlen,
785 (base_rev, delta),
785 (base_rev, delta),
786 flags,
786 flags,
787 )
787 )
788 return deltacomputer.buildtext(revinfo)
788 return deltacomputer.buildtext(revinfo)
789
789
790 is_affected = _is_revision_affected_fast_inner(
790 is_affected = _is_revision_affected_fast_inner(
791 is_censored,
791 is_censored,
792 delta_base,
792 delta_base,
793 lambda: delta,
793 lambda: delta,
794 full_text,
794 full_text,
795 parent_revs,
795 parent_revs,
796 rev,
796 rev,
797 metadata_cache,
797 metadata_cache,
798 )
798 )
799 if is_affected:
799 if is_affected:
800 d = (
800 d = (
801 node,
801 node,
802 p2_node,
802 p2_node,
803 p1_node,
803 p1_node,
804 linknode,
804 linknode,
805 deltabase,
805 deltabase,
806 delta,
806 delta,
807 flags,
807 flags,
808 sidedata,
808 sidedata,
809 )
809 )
810 yield d
810 yield d
811
811
812
812
813 def repair_issue6528(
813 def repair_issue6528(
814 ui, repo, dry_run=False, to_report=None, from_report=None, paranoid=False
814 ui, repo, dry_run=False, to_report=None, from_report=None, paranoid=False
815 ):
815 ):
816 @contextlib.contextmanager
816 @contextlib.contextmanager
817 def context():
817 def context():
818 if dry_run or to_report: # No need for locking
818 if dry_run or to_report: # No need for locking
819 yield
819 yield
820 else:
820 else:
821 with repo.wlock(), repo.lock():
821 with repo.wlock(), repo.lock():
822 yield
822 yield
823
823
824 if from_report:
824 if from_report:
825 return _from_report(ui, repo, context, from_report, dry_run)
825 return _from_report(ui, repo, context, from_report, dry_run)
826
826
827 report_entries = []
827 report_entries = []
828
828
829 with context():
829 with context():
830 files = list(
830 files = list(
831 entry
831 entry
832 for entry in repo.store.data_entries()
832 for entry in repo.store.data_entries()
833 if entry.is_revlog and entry.is_filelog
833 if entry.is_revlog and entry.is_filelog
834 )
834 )
835
835
836 progress = ui.makeprogress(
836 progress = ui.makeprogress(
837 _(b"looking for affected revisions"),
837 _(b"looking for affected revisions"),
838 unit=_(b"filelogs"),
838 unit=_(b"filelogs"),
839 total=len(files),
839 total=len(files),
840 )
840 )
841 found_nothing = True
841 found_nothing = True
842
842
843 for entry in files:
843 for entry in files:
844 progress.increment()
844 progress.increment()
845 filename = entry.target_id
845 filename = entry.target_id
846 fl = _filelog_from_filename(repo, entry.target_id)
846 fl = _filelog_from_filename(repo, entry.target_id)
847
847
848 # Set of filerevs (or hex filenodes if `to_report`) that need fixing
848 # Set of filerevs (or hex filenodes if `to_report`) that need fixing
849 to_fix = set()
849 to_fix = set()
850 metadata_cache = {}
850 metadata_cache = {nullrev: False}
851 for filerev in fl.revs():
851 for filerev in fl.revs():
852 affected = _is_revision_affected_fast(
852 affected = _is_revision_affected_fast(
853 repo, fl, filerev, metadata_cache
853 repo, fl, filerev, metadata_cache
854 )
854 )
855 if paranoid:
855 if paranoid:
856 slow = _is_revision_affected(fl, filerev)
856 slow = _is_revision_affected(fl, filerev)
857 if slow != affected:
857 if slow != affected:
858 msg = _(b"paranoid check failed for '%s' at node %s")
858 msg = _(b"paranoid check failed for '%s' at node %s")
859 node = binascii.hexlify(fl.node(filerev))
859 node = binascii.hexlify(fl.node(filerev))
860 raise error.Abort(msg % (filename, node))
860 raise error.Abort(msg % (filename, node))
861 if affected:
861 if affected:
862 msg = b"found affected revision %d for file '%s'\n"
862 msg = b"found affected revision %d for file '%s'\n"
863 ui.warn(msg % (filerev, filename))
863 ui.warn(msg % (filerev, filename))
864 found_nothing = False
864 found_nothing = False
865 if not dry_run:
865 if not dry_run:
866 if to_report:
866 if to_report:
867 to_fix.add(binascii.hexlify(fl.node(filerev)))
867 to_fix.add(binascii.hexlify(fl.node(filerev)))
868 else:
868 else:
869 to_fix.add(filerev)
869 to_fix.add(filerev)
870
870
871 if to_fix:
871 if to_fix:
872 to_fix = sorted(to_fix)
872 to_fix = sorted(to_fix)
873 if to_report:
873 if to_report:
874 report_entries.append((filename, to_fix))
874 report_entries.append((filename, to_fix))
875 else:
875 else:
876 _reorder_filelog_parents(repo, fl, to_fix)
876 _reorder_filelog_parents(repo, fl, to_fix)
877
877
878 if found_nothing:
878 if found_nothing:
879 ui.write(_(b"no affected revisions were found\n"))
879 ui.write(_(b"no affected revisions were found\n"))
880
880
881 if to_report and report_entries:
881 if to_report and report_entries:
882 with open(to_report, mode="wb") as f:
882 with open(to_report, mode="wb") as f:
883 for path, to_fix in report_entries:
883 for path, to_fix in report_entries:
884 f.write(b"%s %s\n" % (b",".join(to_fix), path))
884 f.write(b"%s %s\n" % (b",".join(to_fix), path))
885
885
886 progress.complete()
886 progress.complete()
General Comments 0
You need to be logged in to leave comments. Login now