##// END OF EJS Templates
nodemap: introduce append-only incremental update of the persistent data...
marmoute -
r44805:50ad851e default
parent child Browse files
Show More
@@ -156,13 +156,31 b' class PersistentNodeMapIndexObject(Index'
156 156 index."""
157 157 return nodemaputil.persistent_data(self)
158 158
159 def nodemap_data_incremental(self):
160 """Return bytes containing a incremental update to persistent nodemap
161
162 This containst the data for an append-only update of the data provided
163 in the last call to `update_nodemap_data`.
164 """
165 if self._nm_root is None:
166 return None
167 data = nodemaputil.update_persistent_data(
168 self, self._nm_root, self._nm_max_idx, self._nm_rev
169 )
170 self._nm_root = self._nm_max_idx = self._nm_rev = None
171 return data
172
159 173 def update_nodemap_data(self, nm_data):
160 174 """provide full blokc of persisted binary data for a nodemap
161 175
162 176 The data are expected to come from disk. See `nodemap_data_all` for a
163 177 produceur of such data."""
164 178 if nm_data is not None:
165 nodemaputil.parse_data(nm_data)
179 self._nm_root, self._nm_max_idx = nodemaputil.parse_data(nm_data)
180 if self._nm_root:
181 self._nm_rev = len(self) - 1
182 else:
183 self._nm_root = self._nm_max_idx = self._nm_rev = None
166 184
167 185
168 186 class InlinedIndexObject(BaseIndexObject):
@@ -69,22 +69,27 b' def _persist_nodemap(tr, revlog):'
69 69 if revlog.nodemap_file is None:
70 70 msg = "calling persist nodemap on a revlog without the feature enableb"
71 71 raise error.ProgrammingError(msg)
72
73 can_incremental = util.safehasattr(revlog.index, "nodemap_data_incremental")
74 ondisk_docket = revlog._nodemap_docket
75
76 # first attemp an incremental update of the data
77 if can_incremental and ondisk_docket is not None:
78 target_docket = revlog._nodemap_docket.copy()
79 data = revlog.index.nodemap_data_incremental()
80 datafile = _rawdata_filepath(revlog, target_docket)
81 # EXP-TODO: if this is a cache, this should use a cache vfs, not a
82 # store vfs
83 with revlog.opener(datafile, b'a') as fd:
84 fd.write(data)
85 else:
86 # otherwise fallback to a full new export
87 target_docket = NodeMapDocket()
88 datafile = _rawdata_filepath(revlog, target_docket)
72 89 if util.safehasattr(revlog.index, "nodemap_data_all"):
73 90 data = revlog.index.nodemap_data_all()
74 91 else:
75 92 data = persistent_data(revlog.index)
76 target_docket = NodeMapDocket()
77 datafile = _rawdata_filepath(revlog, target_docket)
78 olds = _other_rawdata_filepath(revlog, target_docket)
79 if olds:
80 realvfs = getattr(revlog, '_realopener', revlog.opener)
81
82 def cleanup(tr):
83 for oldfile in olds:
84 realvfs.tryunlink(oldfile)
85
86 callback_id = b"revlog-cleanup-nodemap-%s" % revlog.nodemap_file
87 tr.addpostclose(callback_id, cleanup)
88 93 # EXP-TODO: if this is a cache, this should use a cache vfs, not a
89 94 # store vfs
90 95 with revlog.opener(datafile, b'w') as fd:
@@ -97,6 +102,19 b' def _persist_nodemap(tr, revlog):'
97 102 # EXP-TODO: if the transaction abort, we should remove the new data and
98 103 # reinstall the old one.
99 104
105 # search for old index file in all cases, some older process might have
106 # left one behind.
107 olds = _other_rawdata_filepath(revlog, target_docket)
108 if olds:
109 realvfs = getattr(revlog, '_realopener', revlog.opener)
110
111 def cleanup(tr):
112 for oldfile in olds:
113 realvfs.tryunlink(oldfile)
114
115 callback_id = b"revlog-cleanup-nodemap-%s" % revlog.nodemap_file
116 tr.addpostclose(callback_id, cleanup)
117
100 118
101 119 ### Nodemap docket file
102 120 #
@@ -208,6 +226,13 b' def persistent_data(index):'
208 226 return _persist_trie(trie)
209 227
210 228
229 def update_persistent_data(index, root, max_idx, last_rev):
230 """return the incremental update for persistent nodemap from a given index
231 """
232 trie = _update_trie(index, root, last_rev)
233 return _persist_trie(trie, existing_idx=max_idx)
234
235
211 236 S_BLOCK = struct.Struct(">" + ("l" * 16))
212 237
213 238 NO_ENTRY = -1
@@ -260,6 +285,14 b' def _build_trie(index):'
260 285 return root
261 286
262 287
288 def _update_trie(index, root, last_rev):
289 """consume"""
290 for rev in range(last_rev + 1, len(index)):
291 hex = nodemod.hex(index[rev][7])
292 _insert_into_block(index, 0, root, rev, hex)
293 return root
294
295
263 296 def _insert_into_block(index, level, block, current_rev, current_hex):
264 297 """insert a new revision in a block
265 298
@@ -269,6 +302,8 b' def _insert_into_block(index, level, blo'
269 302 current_rev: the revision number we are adding
270 303 current_hex: the hexadecimal representation of the of that revision
271 304 """
305 if block.ondisk_id is not None:
306 block.ondisk_id = None
272 307 hex_digit = _to_int(current_hex[level : level + 1])
273 308 entry = block.get(hex_digit)
274 309 if entry is None:
@@ -288,14 +323,21 b' def _insert_into_block(index, level, blo'
288 323 _insert_into_block(index, level + 1, new, current_rev, current_hex)
289 324
290 325
291 def _persist_trie(root):
326 def _persist_trie(root, existing_idx=None):
292 327 """turn a nodemap trie into persistent binary data
293 328
294 329 See `_build_trie` for nodemap trie structure"""
295 330 block_map = {}
331 if existing_idx is not None:
332 base_idx = existing_idx + 1
333 else:
334 base_idx = 0
296 335 chunks = []
297 336 for tn in _walk_trie(root):
298 block_map[id(tn)] = len(chunks)
337 if tn.ondisk_id is not None:
338 block_map[id(tn)] = tn.ondisk_id
339 else:
340 block_map[id(tn)] = len(chunks) + base_idx
299 341 chunks.append(_persist_block(tn, block_map))
300 342 return b''.join(chunks)
301 343
@@ -338,7 +380,7 b' def parse_data(data):'
338 380 msg = "nodemap data size is not a multiple of block size (%d): %d"
339 381 raise error.Abort(msg % (S_BLOCK.size, len(data)))
340 382 if not data:
341 return Block()
383 return Block(), None
342 384 block_map = {}
343 385 new_blocks = []
344 386 for i in range(0, len(data), S_BLOCK.size):
@@ -356,7 +398,7 b' def parse_data(data):'
356 398 b[idx] = block_map[v]
357 399 else:
358 400 b[idx] = _transform_rev(v)
359 return block
401 return block, i // S_BLOCK.size
360 402
361 403
362 404 # debug utility
@@ -366,7 +408,7 b' def check_data(ui, index, data):'
366 408 """verify that the provided nodemap data are valid for the given idex"""
367 409 ret = 0
368 410 ui.status((b"revision in index: %d\n") % len(index))
369 root = parse_data(data)
411 root, __ = parse_data(data)
370 412 all_revs = set(_all_revisions(root))
371 413 ui.status((b"revision in nodemap: %d\n") % len(all_revs))
372 414 for r in range(len(index)):
@@ -49,8 +49,19 b' add a new commit'
49 49 $ hg ci -m 'foo'
50 50 $ f --size .hg/store/00changelog.n
51 51 .hg/store/00changelog.n: size=18
52
53 (The pure code use the debug code that perform incremental update, the C code reencode from scratch)
54
55 #if pure
56 $ f --sha256 .hg/store/00changelog-*.nd --size
57 .hg/store/00changelog-????????????????.nd: size=123072, sha256=136472751566c8198ff09e306a7d2f9bd18bd32298d614752b73da4d6df23340 (glob)
58
59 #else
52 60 $ f --sha256 .hg/store/00changelog-*.nd --size
53 61 .hg/store/00changelog-????????????????.nd: size=122880, sha256=bfafebd751c4f6d116a76a37a1dee2a251747affe7efbcc4f4842ccc746d4db9 (glob)
62
63 #endif
64
54 65 $ hg debugnodemap --check
55 66 revision in index: 5002
56 67 revision in nodemap: 5002
General Comments 0
You need to be logged in to leave comments. Login now