Show More
@@ -156,13 +156,31 b' class PersistentNodeMapIndexObject(Index' | |||
|
156 | 156 | index.""" |
|
157 | 157 | return nodemaputil.persistent_data(self) |
|
158 | 158 | |
|
159 | def nodemap_data_incremental(self): | |
|
160 | """Return bytes containing a incremental update to persistent nodemap | |
|
161 | ||
|
162 | This containst the data for an append-only update of the data provided | |
|
163 | in the last call to `update_nodemap_data`. | |
|
164 | """ | |
|
165 | if self._nm_root is None: | |
|
166 | return None | |
|
167 | data = nodemaputil.update_persistent_data( | |
|
168 | self, self._nm_root, self._nm_max_idx, self._nm_rev | |
|
169 | ) | |
|
170 | self._nm_root = self._nm_max_idx = self._nm_rev = None | |
|
171 | return data | |
|
172 | ||
|
159 | 173 | def update_nodemap_data(self, nm_data): |
|
160 | 174 | """provide full blokc of persisted binary data for a nodemap |
|
161 | 175 | |
|
162 | 176 | The data are expected to come from disk. See `nodemap_data_all` for a |
|
163 | 177 | produceur of such data.""" |
|
164 | 178 | if nm_data is not None: |
|
165 | nodemaputil.parse_data(nm_data) | |
|
179 | self._nm_root, self._nm_max_idx = nodemaputil.parse_data(nm_data) | |
|
180 | if self._nm_root: | |
|
181 | self._nm_rev = len(self) - 1 | |
|
182 | else: | |
|
183 | self._nm_root = self._nm_max_idx = self._nm_rev = None | |
|
166 | 184 | |
|
167 | 185 | |
|
168 | 186 | class InlinedIndexObject(BaseIndexObject): |
@@ -69,12 +69,41 b' def _persist_nodemap(tr, revlog):' | |||
|
69 | 69 | if revlog.nodemap_file is None: |
|
70 | 70 | msg = "calling persist nodemap on a revlog without the feature enableb" |
|
71 | 71 | raise error.ProgrammingError(msg) |
|
72 | if util.safehasattr(revlog.index, "nodemap_data_all"): | |
|
73 |
|
|
|
72 | ||
|
73 | can_incremental = util.safehasattr(revlog.index, "nodemap_data_incremental") | |
|
74 | ondisk_docket = revlog._nodemap_docket | |
|
75 | ||
|
76 | # first attemp an incremental update of the data | |
|
77 | if can_incremental and ondisk_docket is not None: | |
|
78 | target_docket = revlog._nodemap_docket.copy() | |
|
79 | data = revlog.index.nodemap_data_incremental() | |
|
80 | datafile = _rawdata_filepath(revlog, target_docket) | |
|
81 | # EXP-TODO: if this is a cache, this should use a cache vfs, not a | |
|
82 | # store vfs | |
|
83 | with revlog.opener(datafile, b'a') as fd: | |
|
84 | fd.write(data) | |
|
74 | 85 | else: |
|
75 | data = persistent_data(revlog.index) | |
|
76 | target_docket = NodeMapDocket() | |
|
77 | datafile = _rawdata_filepath(revlog, target_docket) | |
|
86 | # otherwise fallback to a full new export | |
|
87 | target_docket = NodeMapDocket() | |
|
88 | datafile = _rawdata_filepath(revlog, target_docket) | |
|
89 | if util.safehasattr(revlog.index, "nodemap_data_all"): | |
|
90 | data = revlog.index.nodemap_data_all() | |
|
91 | else: | |
|
92 | data = persistent_data(revlog.index) | |
|
93 | # EXP-TODO: if this is a cache, this should use a cache vfs, not a | |
|
94 | # store vfs | |
|
95 | with revlog.opener(datafile, b'w') as fd: | |
|
96 | fd.write(data) | |
|
97 | # EXP-TODO: if this is a cache, this should use a cache vfs, not a | |
|
98 | # store vfs | |
|
99 | with revlog.opener(revlog.nodemap_file, b'w', atomictemp=True) as fp: | |
|
100 | fp.write(target_docket.serialize()) | |
|
101 | revlog._nodemap_docket = target_docket | |
|
102 | # EXP-TODO: if the transaction abort, we should remove the new data and | |
|
103 | # reinstall the old one. | |
|
104 | ||
|
105 | # search for old index file in all cases, some older process might have | |
|
106 | # left one behind. | |
|
78 | 107 | olds = _other_rawdata_filepath(revlog, target_docket) |
|
79 | 108 | if olds: |
|
80 | 109 | realvfs = getattr(revlog, '_realopener', revlog.opener) |
@@ -85,17 +114,6 b' def _persist_nodemap(tr, revlog):' | |||
|
85 | 114 | |
|
86 | 115 | callback_id = b"revlog-cleanup-nodemap-%s" % revlog.nodemap_file |
|
87 | 116 | tr.addpostclose(callback_id, cleanup) |
|
88 | # EXP-TODO: if this is a cache, this should use a cache vfs, not a | |
|
89 | # store vfs | |
|
90 | with revlog.opener(datafile, b'w') as fd: | |
|
91 | fd.write(data) | |
|
92 | # EXP-TODO: if this is a cache, this should use a cache vfs, not a | |
|
93 | # store vfs | |
|
94 | with revlog.opener(revlog.nodemap_file, b'w', atomictemp=True) as fp: | |
|
95 | fp.write(target_docket.serialize()) | |
|
96 | revlog._nodemap_docket = target_docket | |
|
97 | # EXP-TODO: if the transaction abort, we should remove the new data and | |
|
98 | # reinstall the old one. | |
|
99 | 117 | |
|
100 | 118 | |
|
101 | 119 | ### Nodemap docket file |
@@ -208,6 +226,13 b' def persistent_data(index):' | |||
|
208 | 226 | return _persist_trie(trie) |
|
209 | 227 | |
|
210 | 228 | |
|
229 | def update_persistent_data(index, root, max_idx, last_rev): | |
|
230 | """return the incremental update for persistent nodemap from a given index | |
|
231 | """ | |
|
232 | trie = _update_trie(index, root, last_rev) | |
|
233 | return _persist_trie(trie, existing_idx=max_idx) | |
|
234 | ||
|
235 | ||
|
211 | 236 | S_BLOCK = struct.Struct(">" + ("l" * 16)) |
|
212 | 237 | |
|
213 | 238 | NO_ENTRY = -1 |
@@ -260,6 +285,14 b' def _build_trie(index):' | |||
|
260 | 285 | return root |
|
261 | 286 | |
|
262 | 287 | |
|
288 | def _update_trie(index, root, last_rev): | |
|
289 | """consume""" | |
|
290 | for rev in range(last_rev + 1, len(index)): | |
|
291 | hex = nodemod.hex(index[rev][7]) | |
|
292 | _insert_into_block(index, 0, root, rev, hex) | |
|
293 | return root | |
|
294 | ||
|
295 | ||
|
263 | 296 | def _insert_into_block(index, level, block, current_rev, current_hex): |
|
264 | 297 | """insert a new revision in a block |
|
265 | 298 | |
@@ -269,6 +302,8 b' def _insert_into_block(index, level, blo' | |||
|
269 | 302 | current_rev: the revision number we are adding |
|
270 | 303 | current_hex: the hexadecimal representation of the of that revision |
|
271 | 304 | """ |
|
305 | if block.ondisk_id is not None: | |
|
306 | block.ondisk_id = None | |
|
272 | 307 | hex_digit = _to_int(current_hex[level : level + 1]) |
|
273 | 308 | entry = block.get(hex_digit) |
|
274 | 309 | if entry is None: |
@@ -288,15 +323,22 b' def _insert_into_block(index, level, blo' | |||
|
288 | 323 | _insert_into_block(index, level + 1, new, current_rev, current_hex) |
|
289 | 324 | |
|
290 | 325 | |
|
291 | def _persist_trie(root): | |
|
326 | def _persist_trie(root, existing_idx=None): | |
|
292 | 327 | """turn a nodemap trie into persistent binary data |
|
293 | 328 | |
|
294 | 329 | See `_build_trie` for nodemap trie structure""" |
|
295 | 330 | block_map = {} |
|
331 | if existing_idx is not None: | |
|
332 | base_idx = existing_idx + 1 | |
|
333 | else: | |
|
334 | base_idx = 0 | |
|
296 | 335 | chunks = [] |
|
297 | 336 | for tn in _walk_trie(root): |
|
298 | block_map[id(tn)] = len(chunks) | |
|
299 | chunks.append(_persist_block(tn, block_map)) | |
|
337 | if tn.ondisk_id is not None: | |
|
338 | block_map[id(tn)] = tn.ondisk_id | |
|
339 | else: | |
|
340 | block_map[id(tn)] = len(chunks) + base_idx | |
|
341 | chunks.append(_persist_block(tn, block_map)) | |
|
300 | 342 | return b''.join(chunks) |
|
301 | 343 | |
|
302 | 344 | |
@@ -338,7 +380,7 b' def parse_data(data):' | |||
|
338 | 380 | msg = "nodemap data size is not a multiple of block size (%d): %d" |
|
339 | 381 | raise error.Abort(msg % (S_BLOCK.size, len(data))) |
|
340 | 382 | if not data: |
|
341 | return Block() | |
|
383 | return Block(), None | |
|
342 | 384 | block_map = {} |
|
343 | 385 | new_blocks = [] |
|
344 | 386 | for i in range(0, len(data), S_BLOCK.size): |
@@ -356,7 +398,7 b' def parse_data(data):' | |||
|
356 | 398 | b[idx] = block_map[v] |
|
357 | 399 | else: |
|
358 | 400 | b[idx] = _transform_rev(v) |
|
359 | return block | |
|
401 | return block, i // S_BLOCK.size | |
|
360 | 402 | |
|
361 | 403 | |
|
362 | 404 | # debug utility |
@@ -366,7 +408,7 b' def check_data(ui, index, data):' | |||
|
366 | 408 | """verify that the provided nodemap data are valid for the given idex""" |
|
367 | 409 | ret = 0 |
|
368 | 410 | ui.status((b"revision in index: %d\n") % len(index)) |
|
369 | root = parse_data(data) | |
|
411 | root, __ = parse_data(data) | |
|
370 | 412 | all_revs = set(_all_revisions(root)) |
|
371 | 413 | ui.status((b"revision in nodemap: %d\n") % len(all_revs)) |
|
372 | 414 | for r in range(len(index)): |
@@ -49,8 +49,19 b' add a new commit' | |||
|
49 | 49 | $ hg ci -m 'foo' |
|
50 | 50 | $ f --size .hg/store/00changelog.n |
|
51 | 51 | .hg/store/00changelog.n: size=18 |
|
52 | ||
|
53 | (The pure code use the debug code that perform incremental update, the C code reencode from scratch) | |
|
54 | ||
|
55 | #if pure | |
|
56 | $ f --sha256 .hg/store/00changelog-*.nd --size | |
|
57 | .hg/store/00changelog-????????????????.nd: size=123072, sha256=136472751566c8198ff09e306a7d2f9bd18bd32298d614752b73da4d6df23340 (glob) | |
|
58 | ||
|
59 | #else | |
|
52 | 60 | $ f --sha256 .hg/store/00changelog-*.nd --size |
|
53 | 61 | .hg/store/00changelog-????????????????.nd: size=122880, sha256=bfafebd751c4f6d116a76a37a1dee2a251747affe7efbcc4f4842ccc746d4db9 (glob) |
|
62 | ||
|
63 | #endif | |
|
64 | ||
|
54 | 65 | $ hg debugnodemap --check |
|
55 | 66 | revision in index: 5002 |
|
56 | 67 | revision in nodemap: 5002 |
General Comments 0
You need to be logged in to leave comments.
Login now