Show More
@@ -1156,7 +1156,6 b' coreconfigitem(' | |||||
1156 | # - for stripping operation |
|
1156 | # - for stripping operation | |
1157 | # - for rollback operation |
|
1157 | # - for rollback operation | |
1158 | # * proper streaming (race free) of the docket file |
|
1158 | # * proper streaming (race free) of the docket file | |
1159 | # * store the data size in the docket to simplify sidedata rewrite. |
|
|||
1160 | # * track garbage data to evemtually allow rewriting -existing- sidedata. |
|
1159 | # * track garbage data to evemtually allow rewriting -existing- sidedata. | |
1161 | # * Exchange-wise, we will also need to do something more efficient than |
|
1160 | # * Exchange-wise, we will also need to do something more efficient than | |
1162 | # keeping references to the affected revlogs, especially memory-wise when |
|
1161 | # keeping references to the affected revlogs, especially memory-wise when |
@@ -2088,7 +2088,10 b' class revlog(object):' | |||||
2088 | if not self._inline: |
|
2088 | if not self._inline: | |
2089 | try: |
|
2089 | try: | |
2090 | dfh = self._datafp(b"r+") |
|
2090 | dfh = self._datafp(b"r+") | |
|
2091 | if self._docket is None: | |||
2091 | dfh.seek(0, os.SEEK_END) |
|
2092 | dfh.seek(0, os.SEEK_END) | |
|
2093 | else: | |||
|
2094 | dfh.seek(self._docket.data_end, os.SEEK_SET) | |||
2092 | except IOError as inst: |
|
2095 | except IOError as inst: | |
2093 | if inst.errno != errno.ENOENT: |
|
2096 | if inst.errno != errno.ENOENT: | |
2094 | raise |
|
2097 | raise | |
@@ -2455,16 +2458,10 b' class revlog(object):' | |||||
2455 | to `n - 1`'s sidedata being written after `n`'s data. |
|
2458 | to `n - 1`'s sidedata being written after `n`'s data. | |
2456 |
|
2459 | |||
2457 | TODO cache this in a docket file before getting out of experimental.""" |
|
2460 | TODO cache this in a docket file before getting out of experimental.""" | |
2458 |
if self._ |
|
2461 | if self._docket is None: | |
2459 | return self.end(prev) |
|
2462 | return self.end(prev) | |
2460 |
|
2463 | else: | ||
2461 | offset = 0 |
|
2464 | return self._docket.data_end | |
2462 | for rev, entry in enumerate(self.index): |
|
|||
2463 | sidedata_end = entry[8] + entry[9] |
|
|||
2464 | # Sidedata for a previous rev has potentially been written after |
|
|||
2465 | # this rev's end, so take the max. |
|
|||
2466 | offset = max(self.end(rev), offset, sidedata_end) |
|
|||
2467 | return offset |
|
|||
2468 |
|
2465 | |||
2469 | def _writeentry(self, transaction, entry, data, link, offset, sidedata): |
|
2466 | def _writeentry(self, transaction, entry, data, link, offset, sidedata): | |
2470 | # Files opened in a+ mode have inconsistent behavior on various |
|
2467 | # Files opened in a+ mode have inconsistent behavior on various | |
@@ -2488,7 +2485,10 b' class revlog(object):' | |||||
2488 | else: |
|
2485 | else: | |
2489 | ifh.seek(self._docket.index_end, os.SEEK_SET) |
|
2486 | ifh.seek(self._docket.index_end, os.SEEK_SET) | |
2490 | if dfh: |
|
2487 | if dfh: | |
|
2488 | if self._docket is None: | |||
2491 | dfh.seek(0, os.SEEK_END) |
|
2489 | dfh.seek(0, os.SEEK_END) | |
|
2490 | else: | |||
|
2491 | dfh.seek(self._docket.data_end, os.SEEK_SET) | |||
2492 |
|
2492 | |||
2493 | curr = len(self) - 1 |
|
2493 | curr = len(self) - 1 | |
2494 | if not self._inline: |
|
2494 | if not self._inline: | |
@@ -2511,6 +2511,7 b' class revlog(object):' | |||||
2511 | self._enforceinlinesize(transaction) |
|
2511 | self._enforceinlinesize(transaction) | |
2512 | if self._docket is not None: |
|
2512 | if self._docket is not None: | |
2513 | self._docket.index_end = self._writinghandles[0].tell() |
|
2513 | self._docket.index_end = self._writinghandles[0].tell() | |
|
2514 | self._docket.data_end = self._writinghandles[1].tell() | |||
2514 |
|
2515 | |||
2515 | nodemaputil.setup_persistent_nodemap(transaction, self) |
|
2516 | nodemaputil.setup_persistent_nodemap(transaction, self) | |
2516 |
|
2517 | |||
@@ -2673,18 +2674,19 b' class revlog(object):' | |||||
2673 | return |
|
2674 | return | |
2674 |
|
2675 | |||
2675 | # first truncate the files on disk |
|
2676 | # first truncate the files on disk | |
2676 | end = self.start(rev) |
|
2677 | data_end = self.start(rev) | |
2677 | if not self._inline: |
|
2678 | if not self._inline: | |
2678 | transaction.add(self._datafile, end) |
|
2679 | transaction.add(self._datafile, data_end) | |
2679 | end = rev * self.index.entry_size |
|
2680 | end = rev * self.index.entry_size | |
2680 | else: |
|
2681 | else: | |
2681 |
end |
|
2682 | end = data_end + (rev * self.index.entry_size) | |
2682 |
|
2683 | |||
2683 | transaction.add(self._indexfile, end) |
|
2684 | transaction.add(self._indexfile, end) | |
2684 | if self._docket is not None: |
|
2685 | if self._docket is not None: | |
2685 | # XXX we could, leverage the docket while stripping. However it is |
|
2686 | # XXX we could, leverage the docket while stripping. However it is | |
2686 | # not powerfull enough at the time of this comment |
|
2687 | # not powerfull enough at the time of this comment | |
2687 | self._docket.index_end = end |
|
2688 | self._docket.index_end = end | |
|
2689 | self._docket.data_end = data_end | |||
2688 | self._docket.write(transaction, stripping=True) |
|
2690 | self._docket.write(transaction, stripping=True) | |
2689 |
|
2691 | |||
2690 | # then reset internal state in memory to forget those revisions |
|
2692 | # then reset internal state in memory to forget those revisions | |
@@ -3210,7 +3212,11 b' class revlog(object):' | |||||
3210 | # append the new sidedata |
|
3212 | # append the new sidedata | |
3211 | with self._writing(transaction): |
|
3213 | with self._writing(transaction): | |
3212 | ifh, dfh = self._writinghandles |
|
3214 | ifh, dfh = self._writinghandles | |
|
3215 | if self._docket is not None: | |||
|
3216 | dfh.seek(self._docket.data_end, os.SEEK_SET) | |||
|
3217 | else: | |||
3213 | dfh.seek(0, os.SEEK_END) |
|
3218 | dfh.seek(0, os.SEEK_END) | |
|
3219 | ||||
3214 | current_offset = dfh.tell() |
|
3220 | current_offset = dfh.tell() | |
3215 | for rev in range(startrev, endrev + 1): |
|
3221 | for rev in range(startrev, endrev + 1): | |
3216 | entry = self.index[rev] |
|
3222 | entry = self.index[rev] | |
@@ -3242,6 +3248,8 b' class revlog(object):' | |||||
3242 | dfh.write(serialized_sidedata) |
|
3248 | dfh.write(serialized_sidedata) | |
3243 | new_entries.append(entry) |
|
3249 | new_entries.append(entry) | |
3244 | current_offset += len(serialized_sidedata) |
|
3250 | current_offset += len(serialized_sidedata) | |
|
3251 | if self._docket is not None: | |||
|
3252 | self._docket.data_end = dfh.tell() | |||
3245 |
|
3253 | |||
3246 | # rewrite the new index entries |
|
3254 | # rewrite the new index entries | |
3247 | ifh.seek(startrev * self.index.entry_size) |
|
3255 | ifh.seek(startrev * self.index.entry_size) |
@@ -32,9 +32,11 b' from . import (' | |||||
32 | # * 4 bytes: revlog version |
|
32 | # * 4 bytes: revlog version | |
33 | # | This is mandatory as docket must be compatible with the previous |
|
33 | # | This is mandatory as docket must be compatible with the previous | |
34 | # | revlog index header. |
|
34 | # | revlog index header. | |
35 |
# * 8 bytes: size of index |
|
35 | # * 8 bytes: size of index-data | |
36 |
# * 8 bytes: pending size of index |
|
36 | # * 8 bytes: pending size of index-data | |
37 | S_HEADER = struct.Struct(constants.INDEX_HEADER.format + 'LL') |
|
37 | # * 8 bytes: size of data | |
|
38 | # * 8 bytes: pending size of data | |||
|
39 | S_HEADER = struct.Struct(constants.INDEX_HEADER.format + 'LLLL') | |||
38 |
|
40 | |||
39 |
|
41 | |||
40 | class RevlogDocket(object): |
|
42 | class RevlogDocket(object): | |
@@ -47,6 +49,8 b' class RevlogDocket(object):' | |||||
47 | version_header=None, |
|
49 | version_header=None, | |
48 | index_end=0, |
|
50 | index_end=0, | |
49 | pending_index_end=0, |
|
51 | pending_index_end=0, | |
|
52 | data_end=0, | |||
|
53 | pending_data_end=0, | |||
50 | ): |
|
54 | ): | |
51 | self._version_header = version_header |
|
55 | self._version_header = version_header | |
52 | self._read_only = bool(use_pending) |
|
56 | self._read_only = bool(use_pending) | |
@@ -54,14 +58,19 b' class RevlogDocket(object):' | |||||
54 | self._radix = revlog.radix |
|
58 | self._radix = revlog.radix | |
55 | self._path = revlog._docket_file |
|
59 | self._path = revlog._docket_file | |
56 | self._opener = revlog.opener |
|
60 | self._opener = revlog.opener | |
57 |
# th |
|
61 | # thes asserts should be True as long as we have a single index filename | |
58 | assert index_end <= pending_index_end |
|
62 | assert index_end <= pending_index_end | |
|
63 | assert data_end <= pending_data_end | |||
59 | self._initial_index_end = index_end |
|
64 | self._initial_index_end = index_end | |
60 | self._pending_index_end = pending_index_end |
|
65 | self._pending_index_end = pending_index_end | |
|
66 | self._initial_data_end = data_end | |||
|
67 | self._pending_data_end = pending_data_end | |||
61 | if use_pending: |
|
68 | if use_pending: | |
62 | self._index_end = self._pending_index_end |
|
69 | self._index_end = self._pending_index_end | |
|
70 | self._data_end = self._pending_data_end | |||
63 | else: |
|
71 | else: | |
64 | self._index_end = self._initial_index_end |
|
72 | self._index_end = self._initial_index_end | |
|
73 | self._data_end = self._initial_data_end | |||
65 |
|
74 | |||
66 | def index_filepath(self): |
|
75 | def index_filepath(self): | |
67 | """file path to the current index file associated to this docket""" |
|
76 | """file path to the current index file associated to this docket""" | |
@@ -78,6 +87,16 b' class RevlogDocket(object):' | |||||
78 | self._index_end = new_size |
|
87 | self._index_end = new_size | |
79 | self._dirty = True |
|
88 | self._dirty = True | |
80 |
|
89 | |||
|
90 | @property | |||
|
91 | def data_end(self): | |||
|
92 | return self._data_end | |||
|
93 | ||||
|
94 | @data_end.setter | |||
|
95 | def data_end(self, new_size): | |||
|
96 | if new_size != self._data_end: | |||
|
97 | self._data_end = new_size | |||
|
98 | self._dirty = True | |||
|
99 | ||||
81 | def write(self, transaction, pending=False, stripping=False): |
|
100 | def write(self, transaction, pending=False, stripping=False): | |
82 | """write the modification of disk if any |
|
101 | """write the modification of disk if any | |
83 |
|
102 | |||
@@ -102,15 +121,19 b' class RevlogDocket(object):' | |||||
102 | def _serialize(self, pending=False): |
|
121 | def _serialize(self, pending=False): | |
103 | if pending: |
|
122 | if pending: | |
104 | official_index_end = self._initial_index_end |
|
123 | official_index_end = self._initial_index_end | |
|
124 | official_data_end = self._initial_data_end | |||
105 | else: |
|
125 | else: | |
106 | official_index_end = self._index_end |
|
126 | official_index_end = self._index_end | |
|
127 | official_data_end = self._data_end | |||
107 |
|
128 | |||
108 | # this assert should be True as long as we have a single index filename |
|
129 | # this assert should be True as long as we have a single index filename | |
109 |
assert official_ |
|
130 | assert official_data_end <= self._data_end | |
110 | data = ( |
|
131 | data = ( | |
111 | self._version_header, |
|
132 | self._version_header, | |
112 | official_index_end, |
|
133 | official_index_end, | |
113 | self._index_end, |
|
134 | self._index_end, | |
|
135 | official_data_end, | |||
|
136 | self._data_end, | |||
114 | ) |
|
137 | ) | |
115 | return S_HEADER.pack(*data) |
|
138 | return S_HEADER.pack(*data) | |
116 |
|
139 | |||
@@ -127,12 +150,18 b' def default_docket(revlog, version_heade' | |||||
127 | def parse_docket(revlog, data, use_pending=False): |
|
150 | def parse_docket(revlog, data, use_pending=False): | |
128 | """given some docket data return a docket object for the given revlog""" |
|
151 | """given some docket data return a docket object for the given revlog""" | |
129 | header = S_HEADER.unpack(data[: S_HEADER.size]) |
|
152 | header = S_HEADER.unpack(data[: S_HEADER.size]) | |
130 |
version_header |
|
153 | version_header = header[0] | |
|
154 | index_size = header[1] | |||
|
155 | pending_index_size = header[2] | |||
|
156 | data_size = header[3] | |||
|
157 | pending_data_size = header[4] | |||
131 | docket = RevlogDocket( |
|
158 | docket = RevlogDocket( | |
132 | revlog, |
|
159 | revlog, | |
133 | use_pending=use_pending, |
|
160 | use_pending=use_pending, | |
134 | version_header=version_header, |
|
161 | version_header=version_header, | |
135 | index_end=index_size, |
|
162 | index_end=index_size, | |
136 | pending_index_end=pending_index_size, |
|
163 | pending_index_end=pending_index_size, | |
|
164 | data_end=data_size, | |||
|
165 | pending_data_end=pending_data_size, | |||
137 | ) |
|
166 | ) | |
138 | return docket |
|
167 | return docket |
General Comments 0
You need to be logged in to leave comments.
Login now