##// END OF EJS Templates
revlogv2: track current index size in the docket...
marmoute -
r48012:6597255a default
parent child Browse files
Show More
@@ -1157,6 +1157,7 b' coreconfigitem('
1157 1157 # - for censoring operation
1158 1158 # - for stripping operation
1159 1159 # - for rollback operation
1160 # * proper streaming (race free) of the docket file
1160 1161 # * store the data size in the docket to simplify sidedata rewrite.
1161 1162 # * track garbage data to evemtually allow rewriting -existing- sidedata.
1162 1163 # * Exchange-wise, we will also need to do something more efficient than
@@ -739,6 +739,8 b' def makelocalrepository(baseui, path, in'
739 739
740 740 if requirementsmod.REVLOGV2_REQUIREMENT in requirements:
741 741 features.add(repository.REPO_FEATURE_SIDE_DATA)
742 # the revlogv2 docket introduced race condition that we need to fix
743 features.discard(repository.REPO_FEATURE_STREAM_CLONE)
742 744
743 745 # The cache vfs is used to manage cache files.
744 746 cachevfs = vfsmod.vfs(cachepath, cacheaudited=True)
@@ -453,7 +453,7 b' class revlog(object):'
453 453 force_nodemap = opts.get(b'devel-force-nodemap', False)
454 454 return new_header, mmapindexthreshold, force_nodemap
455 455
456 def _get_data(self, filepath, mmap_threshold):
456 def _get_data(self, filepath, mmap_threshold, size=None):
457 457 """return a file content with or without mmap
458 458
459 459 If the file is missing return the empty string"""
@@ -462,10 +462,19 b' class revlog(object):'
462 462 if mmap_threshold is not None:
463 463 file_size = self.opener.fstat(fp).st_size
464 464 if file_size >= mmap_threshold:
465 if size is not None:
466 # avoid potentiel mmap crash
467 size = min(file_size, size)
465 468 # TODO: should .close() to release resources without
466 469 # relying on Python GC
467 return util.buffer(util.mmapread(fp))
468 return fp.read()
470 if size is None:
471 return util.buffer(util.mmapread(fp))
472 else:
473 return util.buffer(util.mmapread(fp, size))
474 if size is None:
475 return fp.read()
476 else:
477 return fp.read(size)
469 478 except IOError as inst:
470 479 if inst.errno != errno.ENOENT:
471 480 raise
@@ -518,7 +527,17 b' class revlog(object):'
518 527 else:
519 528 self._docket = docketutil.parse_docket(self, entry_data)
520 529 self._indexfile = self._docket.index_filepath()
521 index_data = self._get_data(self._indexfile, mmapindexthreshold)
530 index_data = b''
531 index_size = self._docket.index_end
532 if index_size > 0:
533 index_data = self._get_data(
534 self._indexfile, mmapindexthreshold, size=index_size
535 )
536 if len(index_data) < index_size:
537 msg = _(b'too few index data for %s: got %d, expected %d')
538 msg %= (self.display_id, len(index_data), index_size)
539 raise error.RevlogError(msg)
540
522 541 self._inline = False
523 542 # generaldelta implied by version 2 revlogs.
524 543 self._generaldelta = True
@@ -619,7 +638,10 b' class revlog(object):'
619 638 f = self.opener(
620 639 self._indexfile, mode=b"r+", checkambig=self._checkambig
621 640 )
622 f.seek(0, os.SEEK_END)
641 if self._docket is None:
642 f.seek(0, os.SEEK_END)
643 else:
644 f.seek(self._docket.index_end, os.SEEK_SET)
623 645 return f
624 646 except IOError as inst:
625 647 if inst.errno != errno.ENOENT:
@@ -2022,6 +2044,8 b' class revlog(object):'
2022 2044 header = self.index.pack_header(header)
2023 2045 e = header + e
2024 2046 fp.write(e)
2047 if self._docket is not None:
2048 self._docket.index_end = fp.tell()
2025 2049 # the temp file replace the real index when we exit the context
2026 2050 # manager
2027 2051
@@ -2440,7 +2464,10 b' class revlog(object):'
2440 2464 msg = b'adding revision outside `revlog._writing` context'
2441 2465 raise error.ProgrammingError(msg)
2442 2466 ifh, dfh = self._writinghandles
2443 ifh.seek(0, os.SEEK_END)
2467 if self._docket is None:
2468 ifh.seek(0, os.SEEK_END)
2469 else:
2470 ifh.seek(self._docket.index_end, os.SEEK_SET)
2444 2471 if dfh:
2445 2472 dfh.seek(0, os.SEEK_END)
2446 2473
@@ -2463,6 +2490,9 b' class revlog(object):'
2463 2490 if sidedata:
2464 2491 ifh.write(sidedata)
2465 2492 self._enforceinlinesize(transaction)
2493 if self._docket is not None:
2494 self._docket.index_end = self._writinghandles[0].tell()
2495
2466 2496 nodemaputil.setup_persistent_nodemap(transaction, self)
2467 2497
2468 2498 def addgroup(
@@ -2632,6 +2662,11 b' class revlog(object):'
2632 2662 end += rev * self.index.entry_size
2633 2663
2634 2664 transaction.add(self._indexfile, end)
2665 if self._docket is not None:
2666 # XXX we could, leverage the docket while stripping. However it is
2667 # not powerfull enough at the time of this comment
2668 self._docket.index_end = end
2669 self._docket.write(transaction, stripping=True)
2635 2670
2636 2671 # then reset internal state in memory to forget those revisions
2637 2672 self._revisioncache = None
@@ -28,36 +28,55 b' from . import ('
28 28 # * 4 bytes: revlog version
29 29 # | This is mandatory as docket must be compatible with the previous
30 30 # | revlog index header.
31 S_HEADER = struct.Struct(constants.INDEX_HEADER.format)
31 # * 8 bytes: size of index data
32 S_HEADER = struct.Struct(constants.INDEX_HEADER.format + 'L')
32 33
33 34
34 35 class RevlogDocket(object):
35 36 """metadata associated with revlog"""
36 37
37 def __init__(self, revlog, version_header=None):
38 def __init__(self, revlog, version_header=None, index_end=0):
38 39 self._version_header = version_header
39 40 self._dirty = False
40 41 self._radix = revlog.radix
41 42 self._path = revlog._docket_file
42 43 self._opener = revlog.opener
44 self._index_end = index_end
43 45
44 46 def index_filepath(self):
45 47 """file path to the current index file associated to this docket"""
46 48 # very simplistic version at first
47 49 return b"%s.idx" % self._radix
48 50
49 def write(self, transaction):
51 @property
52 def index_end(self):
53 return self._index_end
54
55 @index_end.setter
56 def index_end(self, new_size):
57 if new_size != self._index_end:
58 self._index_end = new_size
59 self._dirty = True
60
61 def write(self, transaction, stripping=False):
50 62 """write the modification of disk if any
51 63
52 64 This make the new content visible to all process"""
53 65 if self._dirty:
54 transaction.addbackup(self._path, location=b'store')
66 if not stripping:
67 # XXX we could, leverage the docket while stripping. However it
68 # is not powerfull enough at the time of this comment
69 transaction.addbackup(self._path, location=b'store')
55 70 with self._opener(self._path, mode=b'w', atomictemp=True) as f:
56 71 f.write(self._serialize())
57 72 self._dirty = False
58 73
59 74 def _serialize(self):
60 return S_HEADER.pack(self._version_header)
75 data = (
76 self._version_header,
77 self._index_end,
78 )
79 return S_HEADER.pack(*data)
61 80
62 81
63 82 def default_docket(revlog, version_header):
@@ -72,9 +91,10 b' def default_docket(revlog, version_heade'
72 91 def parse_docket(revlog, data):
73 92 """given some docket data return a docket object for the given revlog"""
74 93 header = S_HEADER.unpack(data[: S_HEADER.size])
75 (version_header,) = header
94 version_header, index_size = header
76 95 docket = RevlogDocket(
77 96 revlog,
78 97 version_header=version_header,
98 index_end=index_size,
79 99 )
80 100 return docket
General Comments 0
You need to be logged in to leave comments. Login now