##// END OF EJS Templates
nodemap: introduce append-only incremental update of the persistent data...
marmoute -
r44805:50ad851e default
parent child Browse files
Show More
@@ -156,13 +156,31 b' class PersistentNodeMapIndexObject(Index'
156 index."""
156 index."""
157 return nodemaputil.persistent_data(self)
157 return nodemaputil.persistent_data(self)
158
158
159 def nodemap_data_incremental(self):
160 """Return bytes containing a incremental update to persistent nodemap
161
162 This containst the data for an append-only update of the data provided
163 in the last call to `update_nodemap_data`.
164 """
165 if self._nm_root is None:
166 return None
167 data = nodemaputil.update_persistent_data(
168 self, self._nm_root, self._nm_max_idx, self._nm_rev
169 )
170 self._nm_root = self._nm_max_idx = self._nm_rev = None
171 return data
172
159 def update_nodemap_data(self, nm_data):
173 def update_nodemap_data(self, nm_data):
160 """provide full blokc of persisted binary data for a nodemap
174 """provide full blokc of persisted binary data for a nodemap
161
175
162 The data are expected to come from disk. See `nodemap_data_all` for a
176 The data are expected to come from disk. See `nodemap_data_all` for a
163 produceur of such data."""
177 produceur of such data."""
164 if nm_data is not None:
178 if nm_data is not None:
165 nodemaputil.parse_data(nm_data)
179 self._nm_root, self._nm_max_idx = nodemaputil.parse_data(nm_data)
180 if self._nm_root:
181 self._nm_rev = len(self) - 1
182 else:
183 self._nm_root = self._nm_max_idx = self._nm_rev = None
166
184
167
185
168 class InlinedIndexObject(BaseIndexObject):
186 class InlinedIndexObject(BaseIndexObject):
@@ -69,12 +69,41 b' def _persist_nodemap(tr, revlog):'
69 if revlog.nodemap_file is None:
69 if revlog.nodemap_file is None:
70 msg = "calling persist nodemap on a revlog without the feature enableb"
70 msg = "calling persist nodemap on a revlog without the feature enableb"
71 raise error.ProgrammingError(msg)
71 raise error.ProgrammingError(msg)
72 if util.safehasattr(revlog.index, "nodemap_data_all"):
72
73 data = revlog.index.nodemap_data_all()
73 can_incremental = util.safehasattr(revlog.index, "nodemap_data_incremental")
74 ondisk_docket = revlog._nodemap_docket
75
76 # first attemp an incremental update of the data
77 if can_incremental and ondisk_docket is not None:
78 target_docket = revlog._nodemap_docket.copy()
79 data = revlog.index.nodemap_data_incremental()
80 datafile = _rawdata_filepath(revlog, target_docket)
81 # EXP-TODO: if this is a cache, this should use a cache vfs, not a
82 # store vfs
83 with revlog.opener(datafile, b'a') as fd:
84 fd.write(data)
74 else:
85 else:
75 data = persistent_data(revlog.index)
86 # otherwise fallback to a full new export
76 target_docket = NodeMapDocket()
87 target_docket = NodeMapDocket()
77 datafile = _rawdata_filepath(revlog, target_docket)
88 datafile = _rawdata_filepath(revlog, target_docket)
89 if util.safehasattr(revlog.index, "nodemap_data_all"):
90 data = revlog.index.nodemap_data_all()
91 else:
92 data = persistent_data(revlog.index)
93 # EXP-TODO: if this is a cache, this should use a cache vfs, not a
94 # store vfs
95 with revlog.opener(datafile, b'w') as fd:
96 fd.write(data)
97 # EXP-TODO: if this is a cache, this should use a cache vfs, not a
98 # store vfs
99 with revlog.opener(revlog.nodemap_file, b'w', atomictemp=True) as fp:
100 fp.write(target_docket.serialize())
101 revlog._nodemap_docket = target_docket
102 # EXP-TODO: if the transaction abort, we should remove the new data and
103 # reinstall the old one.
104
105 # search for old index file in all cases, some older process might have
106 # left one behind.
78 olds = _other_rawdata_filepath(revlog, target_docket)
107 olds = _other_rawdata_filepath(revlog, target_docket)
79 if olds:
108 if olds:
80 realvfs = getattr(revlog, '_realopener', revlog.opener)
109 realvfs = getattr(revlog, '_realopener', revlog.opener)
@@ -85,17 +114,6 b' def _persist_nodemap(tr, revlog):'
85
114
86 callback_id = b"revlog-cleanup-nodemap-%s" % revlog.nodemap_file
115 callback_id = b"revlog-cleanup-nodemap-%s" % revlog.nodemap_file
87 tr.addpostclose(callback_id, cleanup)
116 tr.addpostclose(callback_id, cleanup)
88 # EXP-TODO: if this is a cache, this should use a cache vfs, not a
89 # store vfs
90 with revlog.opener(datafile, b'w') as fd:
91 fd.write(data)
92 # EXP-TODO: if this is a cache, this should use a cache vfs, not a
93 # store vfs
94 with revlog.opener(revlog.nodemap_file, b'w', atomictemp=True) as fp:
95 fp.write(target_docket.serialize())
96 revlog._nodemap_docket = target_docket
97 # EXP-TODO: if the transaction abort, we should remove the new data and
98 # reinstall the old one.
99
117
100
118
101 ### Nodemap docket file
119 ### Nodemap docket file
@@ -208,6 +226,13 b' def persistent_data(index):'
208 return _persist_trie(trie)
226 return _persist_trie(trie)
209
227
210
228
229 def update_persistent_data(index, root, max_idx, last_rev):
230 """return the incremental update for persistent nodemap from a given index
231 """
232 trie = _update_trie(index, root, last_rev)
233 return _persist_trie(trie, existing_idx=max_idx)
234
235
211 S_BLOCK = struct.Struct(">" + ("l" * 16))
236 S_BLOCK = struct.Struct(">" + ("l" * 16))
212
237
213 NO_ENTRY = -1
238 NO_ENTRY = -1
@@ -260,6 +285,14 b' def _build_trie(index):'
260 return root
285 return root
261
286
262
287
288 def _update_trie(index, root, last_rev):
289 """consume"""
290 for rev in range(last_rev + 1, len(index)):
291 hex = nodemod.hex(index[rev][7])
292 _insert_into_block(index, 0, root, rev, hex)
293 return root
294
295
263 def _insert_into_block(index, level, block, current_rev, current_hex):
296 def _insert_into_block(index, level, block, current_rev, current_hex):
264 """insert a new revision in a block
297 """insert a new revision in a block
265
298
@@ -269,6 +302,8 b' def _insert_into_block(index, level, blo'
269 current_rev: the revision number we are adding
302 current_rev: the revision number we are adding
270 current_hex: the hexadecimal representation of the of that revision
303 current_hex: the hexadecimal representation of the of that revision
271 """
304 """
305 if block.ondisk_id is not None:
306 block.ondisk_id = None
272 hex_digit = _to_int(current_hex[level : level + 1])
307 hex_digit = _to_int(current_hex[level : level + 1])
273 entry = block.get(hex_digit)
308 entry = block.get(hex_digit)
274 if entry is None:
309 if entry is None:
@@ -288,15 +323,22 b' def _insert_into_block(index, level, blo'
288 _insert_into_block(index, level + 1, new, current_rev, current_hex)
323 _insert_into_block(index, level + 1, new, current_rev, current_hex)
289
324
290
325
291 def _persist_trie(root):
326 def _persist_trie(root, existing_idx=None):
292 """turn a nodemap trie into persistent binary data
327 """turn a nodemap trie into persistent binary data
293
328
294 See `_build_trie` for nodemap trie structure"""
329 See `_build_trie` for nodemap trie structure"""
295 block_map = {}
330 block_map = {}
331 if existing_idx is not None:
332 base_idx = existing_idx + 1
333 else:
334 base_idx = 0
296 chunks = []
335 chunks = []
297 for tn in _walk_trie(root):
336 for tn in _walk_trie(root):
298 block_map[id(tn)] = len(chunks)
337 if tn.ondisk_id is not None:
299 chunks.append(_persist_block(tn, block_map))
338 block_map[id(tn)] = tn.ondisk_id
339 else:
340 block_map[id(tn)] = len(chunks) + base_idx
341 chunks.append(_persist_block(tn, block_map))
300 return b''.join(chunks)
342 return b''.join(chunks)
301
343
302
344
@@ -338,7 +380,7 b' def parse_data(data):'
338 msg = "nodemap data size is not a multiple of block size (%d): %d"
380 msg = "nodemap data size is not a multiple of block size (%d): %d"
339 raise error.Abort(msg % (S_BLOCK.size, len(data)))
381 raise error.Abort(msg % (S_BLOCK.size, len(data)))
340 if not data:
382 if not data:
341 return Block()
383 return Block(), None
342 block_map = {}
384 block_map = {}
343 new_blocks = []
385 new_blocks = []
344 for i in range(0, len(data), S_BLOCK.size):
386 for i in range(0, len(data), S_BLOCK.size):
@@ -356,7 +398,7 b' def parse_data(data):'
356 b[idx] = block_map[v]
398 b[idx] = block_map[v]
357 else:
399 else:
358 b[idx] = _transform_rev(v)
400 b[idx] = _transform_rev(v)
359 return block
401 return block, i // S_BLOCK.size
360
402
361
403
362 # debug utility
404 # debug utility
@@ -366,7 +408,7 b' def check_data(ui, index, data):'
366 """verify that the provided nodemap data are valid for the given idex"""
408 """verify that the provided nodemap data are valid for the given idex"""
367 ret = 0
409 ret = 0
368 ui.status((b"revision in index: %d\n") % len(index))
410 ui.status((b"revision in index: %d\n") % len(index))
369 root = parse_data(data)
411 root, __ = parse_data(data)
370 all_revs = set(_all_revisions(root))
412 all_revs = set(_all_revisions(root))
371 ui.status((b"revision in nodemap: %d\n") % len(all_revs))
413 ui.status((b"revision in nodemap: %d\n") % len(all_revs))
372 for r in range(len(index)):
414 for r in range(len(index)):
@@ -49,8 +49,19 b' add a new commit'
49 $ hg ci -m 'foo'
49 $ hg ci -m 'foo'
50 $ f --size .hg/store/00changelog.n
50 $ f --size .hg/store/00changelog.n
51 .hg/store/00changelog.n: size=18
51 .hg/store/00changelog.n: size=18
52
53 (The pure code use the debug code that perform incremental update, the C code reencode from scratch)
54
55 #if pure
56 $ f --sha256 .hg/store/00changelog-*.nd --size
57 .hg/store/00changelog-????????????????.nd: size=123072, sha256=136472751566c8198ff09e306a7d2f9bd18bd32298d614752b73da4d6df23340 (glob)
58
59 #else
52 $ f --sha256 .hg/store/00changelog-*.nd --size
60 $ f --sha256 .hg/store/00changelog-*.nd --size
53 .hg/store/00changelog-????????????????.nd: size=122880, sha256=bfafebd751c4f6d116a76a37a1dee2a251747affe7efbcc4f4842ccc746d4db9 (glob)
61 .hg/store/00changelog-????????????????.nd: size=122880, sha256=bfafebd751c4f6d116a76a37a1dee2a251747affe7efbcc4f4842ccc746d4db9 (glob)
62
63 #endif
64
54 $ hg debugnodemap --check
65 $ hg debugnodemap --check
55 revision in index: 5002
66 revision in index: 5002
56 revision in nodemap: 5002
67 revision in nodemap: 5002
General Comments 0
You need to be logged in to leave comments. Login now