##// END OF EJS Templates
typing: suppress a false error in mercurial/revlogutils/docket.py on py2...
Matt Harbison -
r48216:be903d04 default
parent child Browse files
Show More
@@ -1,333 +1,335 b''
1 # docket - code related to revlog "docket"
1 # docket - code related to revlog "docket"
2 #
2 #
3 # Copyright 2021 Pierre-Yves David <pierre-yves.david@octobus.net>
3 # Copyright 2021 Pierre-Yves David <pierre-yves.david@octobus.net>
4 #
4 #
5 # This software may be used and distributed according to the terms of the
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
6 # GNU General Public License version 2 or any later version.
7
7
8 ### Revlog docket file
8 ### Revlog docket file
9 #
9 #
10 # The revlog is stored on disk using multiple files:
10 # The revlog is stored on disk using multiple files:
11 #
11 #
12 # * a small docket file, containing metadata and a pointer,
12 # * a small docket file, containing metadata and a pointer,
13 #
13 #
14 # * an index file, containing fixed width information about revisions,
14 # * an index file, containing fixed width information about revisions,
15 #
15 #
16 # * a data file, containing variable width data for these revisions,
16 # * a data file, containing variable width data for these revisions,
17
17
18 from __future__ import absolute_import
18 from __future__ import absolute_import
19
19
20 import errno
20 import errno
21 import os
21 import os
22 import random
22 import random
23 import struct
23 import struct
24
24
25 from .. import (
25 from .. import (
26 encoding,
26 encoding,
27 error,
27 error,
28 node,
28 node,
29 pycompat,
29 pycompat,
30 util,
30 util,
31 )
31 )
32
32
33 from . import (
33 from . import (
34 constants,
34 constants,
35 )
35 )
36
36
37
37
38 def make_uid(id_size=8):
38 def make_uid(id_size=8):
39 """return a new unique identifier.
39 """return a new unique identifier.
40
40
41 The identifier is random and composed of ascii characters."""
41 The identifier is random and composed of ascii characters."""
42 # size we "hex" the result we need half the number of bits to have a final
42 # size we "hex" the result we need half the number of bits to have a final
43 # uuid of size ID_SIZE
43 # uuid of size ID_SIZE
44 return node.hex(os.urandom(id_size // 2))
44 return node.hex(os.urandom(id_size // 2))
45
45
46
46
47 # some special test logic to avoid anoying random output in the test
47 # some special test logic to avoid anoying random output in the test
48 stable_docket_file = encoding.environ.get(b'HGTEST_UUIDFILE')
48 stable_docket_file = encoding.environ.get(b'HGTEST_UUIDFILE')
49
49
50 if stable_docket_file:
50 if stable_docket_file:
51
51
52 def make_uid(id_size=8):
52 def make_uid(id_size=8):
53 try:
53 try:
54 with open(stable_docket_file, mode='rb') as f:
54 with open(stable_docket_file, mode='rb') as f:
55 seed = f.read().strip()
55 seed = f.read().strip()
56 except IOError as inst:
56 except IOError as inst:
57 if inst.errno != errno.ENOENT:
57 if inst.errno != errno.ENOENT:
58 raise
58 raise
59 seed = b'04' # chosen by a fair dice roll. garanteed to be random
59 seed = b'04' # chosen by a fair dice roll. garanteed to be random
60 if pycompat.ispy3:
60 if pycompat.ispy3:
61 iter_seed = iter(seed)
61 iter_seed = iter(seed)
62 else:
62 else:
63 # pytype: disable=wrong-arg-types
63 iter_seed = (ord(c) for c in seed)
64 iter_seed = (ord(c) for c in seed)
65 # pytype: enable=wrong-arg-types
64 # some basic circular sum hashing on 64 bits
66 # some basic circular sum hashing on 64 bits
65 int_seed = 0
67 int_seed = 0
66 low_mask = int('1' * 35, 2)
68 low_mask = int('1' * 35, 2)
67 for i in iter_seed:
69 for i in iter_seed:
68 high_part = int_seed >> 35
70 high_part = int_seed >> 35
69 low_part = (int_seed & low_mask) << 28
71 low_part = (int_seed & low_mask) << 28
70 int_seed = high_part + low_part + i
72 int_seed = high_part + low_part + i
71 r = random.Random()
73 r = random.Random()
72 if pycompat.ispy3:
74 if pycompat.ispy3:
73 r.seed(int_seed, version=1)
75 r.seed(int_seed, version=1)
74 else:
76 else:
75 r.seed(int_seed)
77 r.seed(int_seed)
76 # once we drop python 3.8 support we can simply use r.randbytes
78 # once we drop python 3.8 support we can simply use r.randbytes
77 raw = r.getrandbits(id_size * 4)
79 raw = r.getrandbits(id_size * 4)
78 assert id_size == 8
80 assert id_size == 8
79 p = struct.pack('>L', raw)
81 p = struct.pack('>L', raw)
80 new = node.hex(p)
82 new = node.hex(p)
81 with open(stable_docket_file, 'wb') as f:
83 with open(stable_docket_file, 'wb') as f:
82 f.write(new)
84 f.write(new)
83 return new
85 return new
84
86
85
87
86 # Docket format
88 # Docket format
87 #
89 #
88 # * 4 bytes: revlog version
90 # * 4 bytes: revlog version
89 # | This is mandatory as docket must be compatible with the previous
91 # | This is mandatory as docket must be compatible with the previous
90 # | revlog index header.
92 # | revlog index header.
91 # * 1 bytes: size of index uuid
93 # * 1 bytes: size of index uuid
92 # * 1 bytes: size of data uuid
94 # * 1 bytes: size of data uuid
93 # * 1 bytes: size of sizedata uuid
95 # * 1 bytes: size of sizedata uuid
94 # * 8 bytes: size of index-data
96 # * 8 bytes: size of index-data
95 # * 8 bytes: pending size of index-data
97 # * 8 bytes: pending size of index-data
96 # * 8 bytes: size of data
98 # * 8 bytes: size of data
97 # * 8 bytes: size of sidedata
99 # * 8 bytes: size of sidedata
98 # * 8 bytes: pending size of data
100 # * 8 bytes: pending size of data
99 # * 8 bytes: pending size of sidedata
101 # * 8 bytes: pending size of sidedata
100 # * 1 bytes: default compression header
102 # * 1 bytes: default compression header
101 S_HEADER = struct.Struct(constants.INDEX_HEADER_FMT + b'BBBLLLLLLc')
103 S_HEADER = struct.Struct(constants.INDEX_HEADER_FMT + b'BBBLLLLLLc')
102
104
103
105
104 class RevlogDocket(object):
106 class RevlogDocket(object):
105 """metadata associated with revlog"""
107 """metadata associated with revlog"""
106
108
107 def __init__(
109 def __init__(
108 self,
110 self,
109 revlog,
111 revlog,
110 use_pending=False,
112 use_pending=False,
111 version_header=None,
113 version_header=None,
112 index_uuid=None,
114 index_uuid=None,
113 data_uuid=None,
115 data_uuid=None,
114 sidedata_uuid=None,
116 sidedata_uuid=None,
115 index_end=0,
117 index_end=0,
116 pending_index_end=0,
118 pending_index_end=0,
117 data_end=0,
119 data_end=0,
118 pending_data_end=0,
120 pending_data_end=0,
119 sidedata_end=0,
121 sidedata_end=0,
120 pending_sidedata_end=0,
122 pending_sidedata_end=0,
121 default_compression_header=None,
123 default_compression_header=None,
122 ):
124 ):
123 self._version_header = version_header
125 self._version_header = version_header
124 self._read_only = bool(use_pending)
126 self._read_only = bool(use_pending)
125 self._dirty = False
127 self._dirty = False
126 self._radix = revlog.radix
128 self._radix = revlog.radix
127 self._path = revlog._docket_file
129 self._path = revlog._docket_file
128 self._opener = revlog.opener
130 self._opener = revlog.opener
129 self._index_uuid = index_uuid
131 self._index_uuid = index_uuid
130 self._data_uuid = data_uuid
132 self._data_uuid = data_uuid
131 self._sidedata_uuid = sidedata_uuid
133 self._sidedata_uuid = sidedata_uuid
132 # thes asserts should be True as long as we have a single index filename
134 # thes asserts should be True as long as we have a single index filename
133 assert index_end <= pending_index_end
135 assert index_end <= pending_index_end
134 assert data_end <= pending_data_end
136 assert data_end <= pending_data_end
135 assert sidedata_end <= pending_sidedata_end
137 assert sidedata_end <= pending_sidedata_end
136 self._initial_index_end = index_end
138 self._initial_index_end = index_end
137 self._pending_index_end = pending_index_end
139 self._pending_index_end = pending_index_end
138 self._initial_data_end = data_end
140 self._initial_data_end = data_end
139 self._pending_data_end = pending_data_end
141 self._pending_data_end = pending_data_end
140 self._initial_sidedata_end = sidedata_end
142 self._initial_sidedata_end = sidedata_end
141 self._pending_sidedata_end = pending_sidedata_end
143 self._pending_sidedata_end = pending_sidedata_end
142 if use_pending:
144 if use_pending:
143 self._index_end = self._pending_index_end
145 self._index_end = self._pending_index_end
144 self._data_end = self._pending_data_end
146 self._data_end = self._pending_data_end
145 self._sidedata_end = self._pending_sidedata_end
147 self._sidedata_end = self._pending_sidedata_end
146 else:
148 else:
147 self._index_end = self._initial_index_end
149 self._index_end = self._initial_index_end
148 self._data_end = self._initial_data_end
150 self._data_end = self._initial_data_end
149 self._sidedata_end = self._initial_sidedata_end
151 self._sidedata_end = self._initial_sidedata_end
150 self.default_compression_header = default_compression_header
152 self.default_compression_header = default_compression_header
151
153
152 def index_filepath(self):
154 def index_filepath(self):
153 """file path to the current index file associated to this docket"""
155 """file path to the current index file associated to this docket"""
154 # very simplistic version at first
156 # very simplistic version at first
155 if self._index_uuid is None:
157 if self._index_uuid is None:
156 self._index_uuid = make_uid()
158 self._index_uuid = make_uid()
157 return b"%s-%s.idx" % (self._radix, self._index_uuid)
159 return b"%s-%s.idx" % (self._radix, self._index_uuid)
158
160
159 def data_filepath(self):
161 def data_filepath(self):
160 """file path to the current data file associated to this docket"""
162 """file path to the current data file associated to this docket"""
161 # very simplistic version at first
163 # very simplistic version at first
162 if self._data_uuid is None:
164 if self._data_uuid is None:
163 self._data_uuid = make_uid()
165 self._data_uuid = make_uid()
164 return b"%s-%s.dat" % (self._radix, self._data_uuid)
166 return b"%s-%s.dat" % (self._radix, self._data_uuid)
165
167
166 def sidedata_filepath(self):
168 def sidedata_filepath(self):
167 """file path to the current sidedata file associated to this docket"""
169 """file path to the current sidedata file associated to this docket"""
168 # very simplistic version at first
170 # very simplistic version at first
169 if self._sidedata_uuid is None:
171 if self._sidedata_uuid is None:
170 self._sidedata_uuid = make_uid()
172 self._sidedata_uuid = make_uid()
171 return b"%s-%s.sda" % (self._radix, self._sidedata_uuid)
173 return b"%s-%s.sda" % (self._radix, self._sidedata_uuid)
172
174
173 @property
175 @property
174 def index_end(self):
176 def index_end(self):
175 return self._index_end
177 return self._index_end
176
178
177 @index_end.setter
179 @index_end.setter
178 def index_end(self, new_size):
180 def index_end(self, new_size):
179 if new_size != self._index_end:
181 if new_size != self._index_end:
180 self._index_end = new_size
182 self._index_end = new_size
181 self._dirty = True
183 self._dirty = True
182
184
183 @property
185 @property
184 def data_end(self):
186 def data_end(self):
185 return self._data_end
187 return self._data_end
186
188
187 @data_end.setter
189 @data_end.setter
188 def data_end(self, new_size):
190 def data_end(self, new_size):
189 if new_size != self._data_end:
191 if new_size != self._data_end:
190 self._data_end = new_size
192 self._data_end = new_size
191 self._dirty = True
193 self._dirty = True
192
194
193 @property
195 @property
194 def sidedata_end(self):
196 def sidedata_end(self):
195 return self._sidedata_end
197 return self._sidedata_end
196
198
197 @sidedata_end.setter
199 @sidedata_end.setter
198 def sidedata_end(self, new_size):
200 def sidedata_end(self, new_size):
199 if new_size != self._sidedata_end:
201 if new_size != self._sidedata_end:
200 self._sidedata_end = new_size
202 self._sidedata_end = new_size
201 self._dirty = True
203 self._dirty = True
202
204
203 def write(self, transaction, pending=False, stripping=False):
205 def write(self, transaction, pending=False, stripping=False):
204 """write the modification of disk if any
206 """write the modification of disk if any
205
207
206 This make the new content visible to all process"""
208 This make the new content visible to all process"""
207 if not self._dirty:
209 if not self._dirty:
208 return False
210 return False
209 else:
211 else:
210 if self._read_only:
212 if self._read_only:
211 msg = b'writing read-only docket: %s'
213 msg = b'writing read-only docket: %s'
212 msg %= self._path
214 msg %= self._path
213 raise error.ProgrammingError(msg)
215 raise error.ProgrammingError(msg)
214 if not stripping:
216 if not stripping:
215 # XXX we could, leverage the docket while stripping. However it
217 # XXX we could, leverage the docket while stripping. However it
216 # is not powerfull enough at the time of this comment
218 # is not powerfull enough at the time of this comment
217 transaction.addbackup(self._path, location=b'store')
219 transaction.addbackup(self._path, location=b'store')
218 with self._opener(self._path, mode=b'w', atomictemp=True) as f:
220 with self._opener(self._path, mode=b'w', atomictemp=True) as f:
219 f.write(self._serialize(pending=pending))
221 f.write(self._serialize(pending=pending))
220 # if pending we still need to the write final data eventually
222 # if pending we still need to the write final data eventually
221 self._dirty = pending
223 self._dirty = pending
222 return True
224 return True
223
225
224 def _serialize(self, pending=False):
226 def _serialize(self, pending=False):
225 if pending:
227 if pending:
226 official_index_end = self._initial_index_end
228 official_index_end = self._initial_index_end
227 official_data_end = self._initial_data_end
229 official_data_end = self._initial_data_end
228 official_sidedata_end = self._initial_sidedata_end
230 official_sidedata_end = self._initial_sidedata_end
229 else:
231 else:
230 official_index_end = self._index_end
232 official_index_end = self._index_end
231 official_data_end = self._data_end
233 official_data_end = self._data_end
232 official_sidedata_end = self._sidedata_end
234 official_sidedata_end = self._sidedata_end
233
235
234 # this assert should be True as long as we have a single index filename
236 # this assert should be True as long as we have a single index filename
235 assert official_data_end <= self._data_end
237 assert official_data_end <= self._data_end
236 assert official_sidedata_end <= self._sidedata_end
238 assert official_sidedata_end <= self._sidedata_end
237 data = (
239 data = (
238 self._version_header,
240 self._version_header,
239 len(self._index_uuid),
241 len(self._index_uuid),
240 len(self._data_uuid),
242 len(self._data_uuid),
241 len(self._sidedata_uuid),
243 len(self._sidedata_uuid),
242 official_index_end,
244 official_index_end,
243 self._index_end,
245 self._index_end,
244 official_data_end,
246 official_data_end,
245 self._data_end,
247 self._data_end,
246 official_sidedata_end,
248 official_sidedata_end,
247 self._sidedata_end,
249 self._sidedata_end,
248 self.default_compression_header,
250 self.default_compression_header,
249 )
251 )
250 s = []
252 s = []
251 s.append(S_HEADER.pack(*data))
253 s.append(S_HEADER.pack(*data))
252 s.append(self._index_uuid)
254 s.append(self._index_uuid)
253 s.append(self._data_uuid)
255 s.append(self._data_uuid)
254 s.append(self._sidedata_uuid)
256 s.append(self._sidedata_uuid)
255 return b''.join(s)
257 return b''.join(s)
256
258
257
259
258 def default_docket(revlog, version_header):
260 def default_docket(revlog, version_header):
259 """given a revlog version a new docket object for the given revlog"""
261 """given a revlog version a new docket object for the given revlog"""
260 rl_version = version_header & 0xFFFF
262 rl_version = version_header & 0xFFFF
261 if rl_version not in (constants.REVLOGV2, constants.CHANGELOGV2):
263 if rl_version not in (constants.REVLOGV2, constants.CHANGELOGV2):
262 return None
264 return None
263 comp = util.compengines[revlog._compengine].revlogheader()
265 comp = util.compengines[revlog._compengine].revlogheader()
264 docket = RevlogDocket(
266 docket = RevlogDocket(
265 revlog,
267 revlog,
266 version_header=version_header,
268 version_header=version_header,
267 default_compression_header=comp,
269 default_compression_header=comp,
268 )
270 )
269 docket._dirty = True
271 docket._dirty = True
270 return docket
272 return docket
271
273
272
274
273 def parse_docket(revlog, data, use_pending=False):
275 def parse_docket(revlog, data, use_pending=False):
274 """given some docket data return a docket object for the given revlog"""
276 """given some docket data return a docket object for the given revlog"""
275 header = S_HEADER.unpack(data[: S_HEADER.size])
277 header = S_HEADER.unpack(data[: S_HEADER.size])
276
278
277 # this is a mutable closure capture used in `get_data`
279 # this is a mutable closure capture used in `get_data`
278 offset = [S_HEADER.size]
280 offset = [S_HEADER.size]
279
281
280 def get_data(size):
282 def get_data(size):
281 """utility closure to access the `size` next bytes"""
283 """utility closure to access the `size` next bytes"""
282 if offset[0] + size > len(data):
284 if offset[0] + size > len(data):
283 # XXX better class
285 # XXX better class
284 msg = b"docket is too short, expected %d got %d"
286 msg = b"docket is too short, expected %d got %d"
285 msg %= (offset[0] + size, len(data))
287 msg %= (offset[0] + size, len(data))
286 raise error.Abort(msg)
288 raise error.Abort(msg)
287 raw = data[offset[0] : offset[0] + size]
289 raw = data[offset[0] : offset[0] + size]
288 offset[0] += size
290 offset[0] += size
289 return raw
291 return raw
290
292
291 iheader = iter(header)
293 iheader = iter(header)
292
294
293 version_header = next(iheader)
295 version_header = next(iheader)
294
296
295 index_uuid_size = next(iheader)
297 index_uuid_size = next(iheader)
296 index_uuid = get_data(index_uuid_size)
298 index_uuid = get_data(index_uuid_size)
297
299
298 data_uuid_size = next(iheader)
300 data_uuid_size = next(iheader)
299 data_uuid = get_data(data_uuid_size)
301 data_uuid = get_data(data_uuid_size)
300
302
301 sidedata_uuid_size = next(iheader)
303 sidedata_uuid_size = next(iheader)
302 sidedata_uuid = get_data(sidedata_uuid_size)
304 sidedata_uuid = get_data(sidedata_uuid_size)
303
305
304 index_size = next(iheader)
306 index_size = next(iheader)
305
307
306 pending_index_size = next(iheader)
308 pending_index_size = next(iheader)
307
309
308 data_size = next(iheader)
310 data_size = next(iheader)
309
311
310 pending_data_size = next(iheader)
312 pending_data_size = next(iheader)
311
313
312 sidedata_size = next(iheader)
314 sidedata_size = next(iheader)
313
315
314 pending_sidedata_size = next(iheader)
316 pending_sidedata_size = next(iheader)
315
317
316 default_compression_header = next(iheader)
318 default_compression_header = next(iheader)
317
319
318 docket = RevlogDocket(
320 docket = RevlogDocket(
319 revlog,
321 revlog,
320 use_pending=use_pending,
322 use_pending=use_pending,
321 version_header=version_header,
323 version_header=version_header,
322 index_uuid=index_uuid,
324 index_uuid=index_uuid,
323 data_uuid=data_uuid,
325 data_uuid=data_uuid,
324 sidedata_uuid=sidedata_uuid,
326 sidedata_uuid=sidedata_uuid,
325 index_end=index_size,
327 index_end=index_size,
326 pending_index_end=pending_index_size,
328 pending_index_end=pending_index_size,
327 data_end=data_size,
329 data_end=data_size,
328 pending_data_end=pending_data_size,
330 pending_data_end=pending_data_size,
329 sidedata_end=sidedata_size,
331 sidedata_end=sidedata_size,
330 pending_sidedata_end=pending_sidedata_size,
332 pending_sidedata_end=pending_sidedata_size,
331 default_compression_header=default_compression_header,
333 default_compression_header=default_compression_header,
332 )
334 )
333 return docket
335 return docket
General Comments 0
You need to be logged in to leave comments. Login now