##// END OF EJS Templates
typing: suppress a false error in mercurial/revlogutils/docket.py on py2...
Matt Harbison -
r48216:be903d04 default
parent child Browse files
Show More
@@ -1,333 +1,335 b''
1 1 # docket - code related to revlog "docket"
2 2 #
3 3 # Copyright 2021 Pierre-Yves David <pierre-yves.david@octobus.net>
4 4 #
5 5 # This software may be used and distributed according to the terms of the
6 6 # GNU General Public License version 2 or any later version.
7 7
8 8 ### Revlog docket file
9 9 #
10 10 # The revlog is stored on disk using multiple files:
11 11 #
12 12 # * a small docket file, containing metadata and a pointer,
13 13 #
14 14 # * an index file, containing fixed width information about revisions,
15 15 #
16 16 # * a data file, containing variable width data for these revisions,
17 17
18 18 from __future__ import absolute_import
19 19
20 20 import errno
21 21 import os
22 22 import random
23 23 import struct
24 24
25 25 from .. import (
26 26 encoding,
27 27 error,
28 28 node,
29 29 pycompat,
30 30 util,
31 31 )
32 32
33 33 from . import (
34 34 constants,
35 35 )
36 36
37 37
38 38 def make_uid(id_size=8):
39 39 """return a new unique identifier.
40 40
41 41 The identifier is random and composed of ascii characters."""
42 42 # size we "hex" the result we need half the number of bits to have a final
43 43 # uuid of size ID_SIZE
44 44 return node.hex(os.urandom(id_size // 2))
45 45
46 46
47 47 # some special test logic to avoid anoying random output in the test
48 48 stable_docket_file = encoding.environ.get(b'HGTEST_UUIDFILE')
49 49
50 50 if stable_docket_file:
51 51
52 52 def make_uid(id_size=8):
53 53 try:
54 54 with open(stable_docket_file, mode='rb') as f:
55 55 seed = f.read().strip()
56 56 except IOError as inst:
57 57 if inst.errno != errno.ENOENT:
58 58 raise
59 59 seed = b'04' # chosen by a fair dice roll. garanteed to be random
60 60 if pycompat.ispy3:
61 61 iter_seed = iter(seed)
62 62 else:
63 # pytype: disable=wrong-arg-types
63 64 iter_seed = (ord(c) for c in seed)
65 # pytype: enable=wrong-arg-types
64 66 # some basic circular sum hashing on 64 bits
65 67 int_seed = 0
66 68 low_mask = int('1' * 35, 2)
67 69 for i in iter_seed:
68 70 high_part = int_seed >> 35
69 71 low_part = (int_seed & low_mask) << 28
70 72 int_seed = high_part + low_part + i
71 73 r = random.Random()
72 74 if pycompat.ispy3:
73 75 r.seed(int_seed, version=1)
74 76 else:
75 77 r.seed(int_seed)
76 78 # once we drop python 3.8 support we can simply use r.randbytes
77 79 raw = r.getrandbits(id_size * 4)
78 80 assert id_size == 8
79 81 p = struct.pack('>L', raw)
80 82 new = node.hex(p)
81 83 with open(stable_docket_file, 'wb') as f:
82 84 f.write(new)
83 85 return new
84 86
85 87
86 88 # Docket format
87 89 #
88 90 # * 4 bytes: revlog version
89 91 # | This is mandatory as docket must be compatible with the previous
90 92 # | revlog index header.
91 93 # * 1 bytes: size of index uuid
92 94 # * 1 bytes: size of data uuid
93 95 # * 1 bytes: size of sizedata uuid
94 96 # * 8 bytes: size of index-data
95 97 # * 8 bytes: pending size of index-data
96 98 # * 8 bytes: size of data
97 99 # * 8 bytes: size of sidedata
98 100 # * 8 bytes: pending size of data
99 101 # * 8 bytes: pending size of sidedata
100 102 # * 1 bytes: default compression header
101 103 S_HEADER = struct.Struct(constants.INDEX_HEADER_FMT + b'BBBLLLLLLc')
102 104
103 105
104 106 class RevlogDocket(object):
105 107 """metadata associated with revlog"""
106 108
107 109 def __init__(
108 110 self,
109 111 revlog,
110 112 use_pending=False,
111 113 version_header=None,
112 114 index_uuid=None,
113 115 data_uuid=None,
114 116 sidedata_uuid=None,
115 117 index_end=0,
116 118 pending_index_end=0,
117 119 data_end=0,
118 120 pending_data_end=0,
119 121 sidedata_end=0,
120 122 pending_sidedata_end=0,
121 123 default_compression_header=None,
122 124 ):
123 125 self._version_header = version_header
124 126 self._read_only = bool(use_pending)
125 127 self._dirty = False
126 128 self._radix = revlog.radix
127 129 self._path = revlog._docket_file
128 130 self._opener = revlog.opener
129 131 self._index_uuid = index_uuid
130 132 self._data_uuid = data_uuid
131 133 self._sidedata_uuid = sidedata_uuid
132 134 # thes asserts should be True as long as we have a single index filename
133 135 assert index_end <= pending_index_end
134 136 assert data_end <= pending_data_end
135 137 assert sidedata_end <= pending_sidedata_end
136 138 self._initial_index_end = index_end
137 139 self._pending_index_end = pending_index_end
138 140 self._initial_data_end = data_end
139 141 self._pending_data_end = pending_data_end
140 142 self._initial_sidedata_end = sidedata_end
141 143 self._pending_sidedata_end = pending_sidedata_end
142 144 if use_pending:
143 145 self._index_end = self._pending_index_end
144 146 self._data_end = self._pending_data_end
145 147 self._sidedata_end = self._pending_sidedata_end
146 148 else:
147 149 self._index_end = self._initial_index_end
148 150 self._data_end = self._initial_data_end
149 151 self._sidedata_end = self._initial_sidedata_end
150 152 self.default_compression_header = default_compression_header
151 153
152 154 def index_filepath(self):
153 155 """file path to the current index file associated to this docket"""
154 156 # very simplistic version at first
155 157 if self._index_uuid is None:
156 158 self._index_uuid = make_uid()
157 159 return b"%s-%s.idx" % (self._radix, self._index_uuid)
158 160
159 161 def data_filepath(self):
160 162 """file path to the current data file associated to this docket"""
161 163 # very simplistic version at first
162 164 if self._data_uuid is None:
163 165 self._data_uuid = make_uid()
164 166 return b"%s-%s.dat" % (self._radix, self._data_uuid)
165 167
166 168 def sidedata_filepath(self):
167 169 """file path to the current sidedata file associated to this docket"""
168 170 # very simplistic version at first
169 171 if self._sidedata_uuid is None:
170 172 self._sidedata_uuid = make_uid()
171 173 return b"%s-%s.sda" % (self._radix, self._sidedata_uuid)
172 174
173 175 @property
174 176 def index_end(self):
175 177 return self._index_end
176 178
177 179 @index_end.setter
178 180 def index_end(self, new_size):
179 181 if new_size != self._index_end:
180 182 self._index_end = new_size
181 183 self._dirty = True
182 184
183 185 @property
184 186 def data_end(self):
185 187 return self._data_end
186 188
187 189 @data_end.setter
188 190 def data_end(self, new_size):
189 191 if new_size != self._data_end:
190 192 self._data_end = new_size
191 193 self._dirty = True
192 194
193 195 @property
194 196 def sidedata_end(self):
195 197 return self._sidedata_end
196 198
197 199 @sidedata_end.setter
198 200 def sidedata_end(self, new_size):
199 201 if new_size != self._sidedata_end:
200 202 self._sidedata_end = new_size
201 203 self._dirty = True
202 204
203 205 def write(self, transaction, pending=False, stripping=False):
204 206 """write the modification of disk if any
205 207
206 208 This make the new content visible to all process"""
207 209 if not self._dirty:
208 210 return False
209 211 else:
210 212 if self._read_only:
211 213 msg = b'writing read-only docket: %s'
212 214 msg %= self._path
213 215 raise error.ProgrammingError(msg)
214 216 if not stripping:
215 217 # XXX we could, leverage the docket while stripping. However it
216 218 # is not powerfull enough at the time of this comment
217 219 transaction.addbackup(self._path, location=b'store')
218 220 with self._opener(self._path, mode=b'w', atomictemp=True) as f:
219 221 f.write(self._serialize(pending=pending))
220 222 # if pending we still need to the write final data eventually
221 223 self._dirty = pending
222 224 return True
223 225
224 226 def _serialize(self, pending=False):
225 227 if pending:
226 228 official_index_end = self._initial_index_end
227 229 official_data_end = self._initial_data_end
228 230 official_sidedata_end = self._initial_sidedata_end
229 231 else:
230 232 official_index_end = self._index_end
231 233 official_data_end = self._data_end
232 234 official_sidedata_end = self._sidedata_end
233 235
234 236 # this assert should be True as long as we have a single index filename
235 237 assert official_data_end <= self._data_end
236 238 assert official_sidedata_end <= self._sidedata_end
237 239 data = (
238 240 self._version_header,
239 241 len(self._index_uuid),
240 242 len(self._data_uuid),
241 243 len(self._sidedata_uuid),
242 244 official_index_end,
243 245 self._index_end,
244 246 official_data_end,
245 247 self._data_end,
246 248 official_sidedata_end,
247 249 self._sidedata_end,
248 250 self.default_compression_header,
249 251 )
250 252 s = []
251 253 s.append(S_HEADER.pack(*data))
252 254 s.append(self._index_uuid)
253 255 s.append(self._data_uuid)
254 256 s.append(self._sidedata_uuid)
255 257 return b''.join(s)
256 258
257 259
258 260 def default_docket(revlog, version_header):
259 261 """given a revlog version a new docket object for the given revlog"""
260 262 rl_version = version_header & 0xFFFF
261 263 if rl_version not in (constants.REVLOGV2, constants.CHANGELOGV2):
262 264 return None
263 265 comp = util.compengines[revlog._compengine].revlogheader()
264 266 docket = RevlogDocket(
265 267 revlog,
266 268 version_header=version_header,
267 269 default_compression_header=comp,
268 270 )
269 271 docket._dirty = True
270 272 return docket
271 273
272 274
273 275 def parse_docket(revlog, data, use_pending=False):
274 276 """given some docket data return a docket object for the given revlog"""
275 277 header = S_HEADER.unpack(data[: S_HEADER.size])
276 278
277 279 # this is a mutable closure capture used in `get_data`
278 280 offset = [S_HEADER.size]
279 281
280 282 def get_data(size):
281 283 """utility closure to access the `size` next bytes"""
282 284 if offset[0] + size > len(data):
283 285 # XXX better class
284 286 msg = b"docket is too short, expected %d got %d"
285 287 msg %= (offset[0] + size, len(data))
286 288 raise error.Abort(msg)
287 289 raw = data[offset[0] : offset[0] + size]
288 290 offset[0] += size
289 291 return raw
290 292
291 293 iheader = iter(header)
292 294
293 295 version_header = next(iheader)
294 296
295 297 index_uuid_size = next(iheader)
296 298 index_uuid = get_data(index_uuid_size)
297 299
298 300 data_uuid_size = next(iheader)
299 301 data_uuid = get_data(data_uuid_size)
300 302
301 303 sidedata_uuid_size = next(iheader)
302 304 sidedata_uuid = get_data(sidedata_uuid_size)
303 305
304 306 index_size = next(iheader)
305 307
306 308 pending_index_size = next(iheader)
307 309
308 310 data_size = next(iheader)
309 311
310 312 pending_data_size = next(iheader)
311 313
312 314 sidedata_size = next(iheader)
313 315
314 316 pending_sidedata_size = next(iheader)
315 317
316 318 default_compression_header = next(iheader)
317 319
318 320 docket = RevlogDocket(
319 321 revlog,
320 322 use_pending=use_pending,
321 323 version_header=version_header,
322 324 index_uuid=index_uuid,
323 325 data_uuid=data_uuid,
324 326 sidedata_uuid=sidedata_uuid,
325 327 index_end=index_size,
326 328 pending_index_end=pending_index_size,
327 329 data_end=data_size,
328 330 pending_data_end=pending_data_size,
329 331 sidedata_end=sidedata_size,
330 332 pending_sidedata_end=pending_sidedata_size,
331 333 default_compression_header=default_compression_header,
332 334 )
333 335 return docket
General Comments 0
You need to be logged in to leave comments. Login now