##// END OF EJS Templates
flagprocessors: have the read transform function return side data (API)...
marmoute -
r43255:bd5858c2 default
parent child Browse files
Show More
@@ -1,449 +1,449 b''
1 1 # wrapper.py - methods wrapping core mercurial logic
2 2 #
3 3 # Copyright 2017 Facebook, Inc.
4 4 #
5 5 # This software may be used and distributed according to the terms of the
6 6 # GNU General Public License version 2 or any later version.
7 7
8 8 from __future__ import absolute_import
9 9
10 10 import hashlib
11 11
12 12 from mercurial.i18n import _
13 13 from mercurial.node import bin, hex, nullid, short
14 14
15 15 from mercurial import (
16 16 bundle2,
17 17 changegroup,
18 18 cmdutil,
19 19 context,
20 20 error,
21 21 exchange,
22 22 exthelper,
23 23 localrepo,
24 24 revlog,
25 25 scmutil,
26 26 upgrade,
27 27 util,
28 28 vfs as vfsmod,
29 29 wireprotov1server,
30 30 )
31 31
32 32 from mercurial.interfaces import (
33 33 repository,
34 34 )
35 35
36 36 from mercurial.utils import (
37 37 storageutil,
38 38 stringutil,
39 39 )
40 40
41 41 from ..largefiles import lfutil
42 42
43 43 from . import (
44 44 blobstore,
45 45 pointer,
46 46 )
47 47
48 48 eh = exthelper.exthelper()
49 49
50 50 @eh.wrapfunction(localrepo, 'makefilestorage')
51 51 def localrepomakefilestorage(orig, requirements, features, **kwargs):
52 52 if b'lfs' in requirements:
53 53 features.add(repository.REPO_FEATURE_LFS)
54 54
55 55 return orig(requirements=requirements, features=features, **kwargs)
56 56
57 57 @eh.wrapfunction(changegroup, 'allsupportedversions')
58 58 def allsupportedversions(orig, ui):
59 59 versions = orig(ui)
60 60 versions.add('03')
61 61 return versions
62 62
63 63 @eh.wrapfunction(wireprotov1server, '_capabilities')
64 64 def _capabilities(orig, repo, proto):
65 65 '''Wrap server command to announce lfs server capability'''
66 66 caps = orig(repo, proto)
67 67 if util.safehasattr(repo.svfs, 'lfslocalblobstore'):
68 68 # Advertise a slightly different capability when lfs is *required*, so
69 69 # that the client knows it MUST load the extension. If lfs is not
70 70 # required on the server, there's no reason to autoload the extension
71 71 # on the client.
72 72 if b'lfs' in repo.requirements:
73 73 caps.append('lfs-serve')
74 74
75 75 caps.append('lfs')
76 76 return caps
77 77
78 78 def bypasscheckhash(self, text):
79 79 return False
80 80
81 81 def readfromstore(self, text):
82 82 """Read filelog content from local blobstore transform for flagprocessor.
83 83
84 84 Default tranform for flagprocessor, returning contents from blobstore.
85 85 Returns a 2-typle (text, validatehash) where validatehash is True as the
86 86 contents of the blobstore should be checked using checkhash.
87 87 """
88 88 p = pointer.deserialize(text)
89 89 oid = p.oid()
90 90 store = self.opener.lfslocalblobstore
91 91 if not store.has(oid):
92 92 p.filename = self.filename
93 93 self.opener.lfsremoteblobstore.readbatch([p], store)
94 94
95 95 # The caller will validate the content
96 96 text = store.read(oid, verify=False)
97 97
98 98 # pack hg filelog metadata
99 99 hgmeta = {}
100 100 for k in p.keys():
101 101 if k.startswith('x-hg-'):
102 102 name = k[len('x-hg-'):]
103 103 hgmeta[name] = p[k]
104 104 if hgmeta or text.startswith('\1\n'):
105 105 text = storageutil.packmeta(hgmeta, text)
106 106
107 return (text, True)
107 return (text, True, {})
108 108
109 109 def writetostore(self, text):
110 110 # hg filelog metadata (includes rename, etc)
111 111 hgmeta, offset = storageutil.parsemeta(text)
112 112 if offset and offset > 0:
113 113 # lfs blob does not contain hg filelog metadata
114 114 text = text[offset:]
115 115
116 116 # git-lfs only supports sha256
117 117 oid = hex(hashlib.sha256(text).digest())
118 118 self.opener.lfslocalblobstore.write(oid, text)
119 119
120 120 # replace contents with metadata
121 121 longoid = 'sha256:%s' % oid
122 122 metadata = pointer.gitlfspointer(oid=longoid, size='%d' % len(text))
123 123
124 124 # by default, we expect the content to be binary. however, LFS could also
125 125 # be used for non-binary content. add a special entry for non-binary data.
126 126 # this will be used by filectx.isbinary().
127 127 if not stringutil.binary(text):
128 128 # not hg filelog metadata (affecting commit hash), no "x-hg-" prefix
129 129 metadata['x-is-binary'] = '0'
130 130
131 131 # translate hg filelog metadata to lfs metadata with "x-hg-" prefix
132 132 if hgmeta is not None:
133 133 for k, v in hgmeta.iteritems():
134 134 metadata['x-hg-%s' % k] = v
135 135
136 136 rawtext = metadata.serialize()
137 137 return (rawtext, False)
138 138
139 139 def _islfs(rlog, node=None, rev=None):
140 140 if rev is None:
141 141 if node is None:
142 142 # both None - likely working copy content where node is not ready
143 143 return False
144 144 rev = rlog._revlog.rev(node)
145 145 else:
146 146 node = rlog._revlog.node(rev)
147 147 if node == nullid:
148 148 return False
149 149 flags = rlog._revlog.flags(rev)
150 150 return bool(flags & revlog.REVIDX_EXTSTORED)
151 151
152 152 # Wrapping may also be applied by remotefilelog
153 153 def filelogaddrevision(orig, self, text, transaction, link, p1, p2,
154 154 cachedelta=None, node=None,
155 155 flags=revlog.REVIDX_DEFAULT_FLAGS, **kwds):
156 156 # The matcher isn't available if reposetup() wasn't called.
157 157 lfstrack = self._revlog.opener.options.get('lfstrack')
158 158
159 159 if lfstrack:
160 160 textlen = len(text)
161 161 # exclude hg rename meta from file size
162 162 meta, offset = storageutil.parsemeta(text)
163 163 if offset:
164 164 textlen -= offset
165 165
166 166 if lfstrack(self._revlog.filename, textlen):
167 167 flags |= revlog.REVIDX_EXTSTORED
168 168
169 169 return orig(self, text, transaction, link, p1, p2, cachedelta=cachedelta,
170 170 node=node, flags=flags, **kwds)
171 171
172 172 # Wrapping may also be applied by remotefilelog
173 173 def filelogrenamed(orig, self, node):
174 174 if _islfs(self, node):
175 175 rawtext = self._revlog.rawdata(node)
176 176 if not rawtext:
177 177 return False
178 178 metadata = pointer.deserialize(rawtext)
179 179 if 'x-hg-copy' in metadata and 'x-hg-copyrev' in metadata:
180 180 return metadata['x-hg-copy'], bin(metadata['x-hg-copyrev'])
181 181 else:
182 182 return False
183 183 return orig(self, node)
184 184
185 185 # Wrapping may also be applied by remotefilelog
186 186 def filelogsize(orig, self, rev):
187 187 if _islfs(self, rev=rev):
188 188 # fast path: use lfs metadata to answer size
189 189 rawtext = self._revlog.rawdata(rev)
190 190 metadata = pointer.deserialize(rawtext)
191 191 return int(metadata['size'])
192 192 return orig(self, rev)
193 193
194 194 @eh.wrapfunction(context.basefilectx, 'cmp')
195 195 def filectxcmp(orig, self, fctx):
196 196 """returns True if text is different than fctx"""
197 197 # some fctx (ex. hg-git) is not based on basefilectx and do not have islfs
198 198 if self.islfs() and getattr(fctx, 'islfs', lambda: False)():
199 199 # fast path: check LFS oid
200 200 p1 = pointer.deserialize(self.rawdata())
201 201 p2 = pointer.deserialize(fctx.rawdata())
202 202 return p1.oid() != p2.oid()
203 203 return orig(self, fctx)
204 204
205 205 @eh.wrapfunction(context.basefilectx, 'isbinary')
206 206 def filectxisbinary(orig, self):
207 207 if self.islfs():
208 208 # fast path: use lfs metadata to answer isbinary
209 209 metadata = pointer.deserialize(self.rawdata())
210 210 # if lfs metadata says nothing, assume it's binary by default
211 211 return bool(int(metadata.get('x-is-binary', 1)))
212 212 return orig(self)
213 213
214 214 def filectxislfs(self):
215 215 return _islfs(self.filelog(), self.filenode())
216 216
217 217 @eh.wrapfunction(cmdutil, '_updatecatformatter')
218 218 def _updatecatformatter(orig, fm, ctx, matcher, path, decode):
219 219 orig(fm, ctx, matcher, path, decode)
220 220 fm.data(rawdata=ctx[path].rawdata())
221 221
222 222 @eh.wrapfunction(scmutil, 'wrapconvertsink')
223 223 def convertsink(orig, sink):
224 224 sink = orig(sink)
225 225 if sink.repotype == 'hg':
226 226 class lfssink(sink.__class__):
227 227 def putcommit(self, files, copies, parents, commit, source, revmap,
228 228 full, cleanp2):
229 229 pc = super(lfssink, self).putcommit
230 230 node = pc(files, copies, parents, commit, source, revmap, full,
231 231 cleanp2)
232 232
233 233 if 'lfs' not in self.repo.requirements:
234 234 ctx = self.repo[node]
235 235
236 236 # The file list may contain removed files, so check for
237 237 # membership before assuming it is in the context.
238 238 if any(f in ctx and ctx[f].islfs() for f, n in files):
239 239 self.repo.requirements.add('lfs')
240 240 self.repo._writerequirements()
241 241
242 242 return node
243 243
244 244 sink.__class__ = lfssink
245 245
246 246 return sink
247 247
248 248 # bundlerepo uses "vfsmod.readonlyvfs(othervfs)", we need to make sure lfs
249 249 # options and blob stores are passed from othervfs to the new readonlyvfs.
250 250 @eh.wrapfunction(vfsmod.readonlyvfs, '__init__')
251 251 def vfsinit(orig, self, othervfs):
252 252 orig(self, othervfs)
253 253 # copy lfs related options
254 254 for k, v in othervfs.options.items():
255 255 if k.startswith('lfs'):
256 256 self.options[k] = v
257 257 # also copy lfs blobstores. note: this can run before reposetup, so lfs
258 258 # blobstore attributes are not always ready at this time.
259 259 for name in ['lfslocalblobstore', 'lfsremoteblobstore']:
260 260 if util.safehasattr(othervfs, name):
261 261 setattr(self, name, getattr(othervfs, name))
262 262
263 263 def _prefetchfiles(repo, revs, match):
264 264 """Ensure that required LFS blobs are present, fetching them as a group if
265 265 needed."""
266 266 if not util.safehasattr(repo.svfs, 'lfslocalblobstore'):
267 267 return
268 268
269 269 pointers = []
270 270 oids = set()
271 271 localstore = repo.svfs.lfslocalblobstore
272 272
273 273 for rev in revs:
274 274 ctx = repo[rev]
275 275 for f in ctx.walk(match):
276 276 p = pointerfromctx(ctx, f)
277 277 if p and p.oid() not in oids and not localstore.has(p.oid()):
278 278 p.filename = f
279 279 pointers.append(p)
280 280 oids.add(p.oid())
281 281
282 282 if pointers:
283 283 # Recalculating the repo store here allows 'paths.default' that is set
284 284 # on the repo by a clone command to be used for the update.
285 285 blobstore.remote(repo).readbatch(pointers, localstore)
286 286
287 287 def _canskipupload(repo):
288 288 # Skip if this hasn't been passed to reposetup()
289 289 if not util.safehasattr(repo.svfs, 'lfsremoteblobstore'):
290 290 return True
291 291
292 292 # if remotestore is a null store, upload is a no-op and can be skipped
293 293 return isinstance(repo.svfs.lfsremoteblobstore, blobstore._nullremote)
294 294
295 295 def candownload(repo):
296 296 # Skip if this hasn't been passed to reposetup()
297 297 if not util.safehasattr(repo.svfs, 'lfsremoteblobstore'):
298 298 return False
299 299
300 300 # if remotestore is a null store, downloads will lead to nothing
301 301 return not isinstance(repo.svfs.lfsremoteblobstore, blobstore._nullremote)
302 302
303 303 def uploadblobsfromrevs(repo, revs):
304 304 '''upload lfs blobs introduced by revs
305 305
306 306 Note: also used by other extensions e. g. infinitepush. avoid renaming.
307 307 '''
308 308 if _canskipupload(repo):
309 309 return
310 310 pointers = extractpointers(repo, revs)
311 311 uploadblobs(repo, pointers)
312 312
313 313 def prepush(pushop):
314 314 """Prepush hook.
315 315
316 316 Read through the revisions to push, looking for filelog entries that can be
317 317 deserialized into metadata so that we can block the push on their upload to
318 318 the remote blobstore.
319 319 """
320 320 return uploadblobsfromrevs(pushop.repo, pushop.outgoing.missing)
321 321
322 322 @eh.wrapfunction(exchange, 'push')
323 323 def push(orig, repo, remote, *args, **kwargs):
324 324 """bail on push if the extension isn't enabled on remote when needed, and
325 325 update the remote store based on the destination path."""
326 326 if 'lfs' in repo.requirements:
327 327 # If the remote peer is for a local repo, the requirement tests in the
328 328 # base class method enforce lfs support. Otherwise, some revisions in
329 329 # this repo use lfs, and the remote repo needs the extension loaded.
330 330 if not remote.local() and not remote.capable('lfs'):
331 331 # This is a copy of the message in exchange.push() when requirements
332 332 # are missing between local repos.
333 333 m = _("required features are not supported in the destination: %s")
334 334 raise error.Abort(m % 'lfs',
335 335 hint=_('enable the lfs extension on the server'))
336 336
337 337 # Repositories where this extension is disabled won't have the field.
338 338 # But if there's a requirement, then the extension must be loaded AND
339 339 # there may be blobs to push.
340 340 remotestore = repo.svfs.lfsremoteblobstore
341 341 try:
342 342 repo.svfs.lfsremoteblobstore = blobstore.remote(repo, remote.url())
343 343 return orig(repo, remote, *args, **kwargs)
344 344 finally:
345 345 repo.svfs.lfsremoteblobstore = remotestore
346 346 else:
347 347 return orig(repo, remote, *args, **kwargs)
348 348
349 349 # when writing a bundle via "hg bundle" command, upload related LFS blobs
350 350 @eh.wrapfunction(bundle2, 'writenewbundle')
351 351 def writenewbundle(orig, ui, repo, source, filename, bundletype, outgoing,
352 352 *args, **kwargs):
353 353 """upload LFS blobs added by outgoing revisions on 'hg bundle'"""
354 354 uploadblobsfromrevs(repo, outgoing.missing)
355 355 return orig(ui, repo, source, filename, bundletype, outgoing, *args,
356 356 **kwargs)
357 357
358 358 def extractpointers(repo, revs):
359 359 """return a list of lfs pointers added by given revs"""
360 360 repo.ui.debug('lfs: computing set of blobs to upload\n')
361 361 pointers = {}
362 362
363 363 makeprogress = repo.ui.makeprogress
364 364 with makeprogress(_('lfs search'), _('changesets'), len(revs)) as progress:
365 365 for r in revs:
366 366 ctx = repo[r]
367 367 for p in pointersfromctx(ctx).values():
368 368 pointers[p.oid()] = p
369 369 progress.increment()
370 370 return sorted(pointers.values(), key=lambda p: p.oid())
371 371
372 372 def pointerfromctx(ctx, f, removed=False):
373 373 """return a pointer for the named file from the given changectx, or None if
374 374 the file isn't LFS.
375 375
376 376 Optionally, the pointer for a file deleted from the context can be returned.
377 377 Since no such pointer is actually stored, and to distinguish from a non LFS
378 378 file, this pointer is represented by an empty dict.
379 379 """
380 380 _ctx = ctx
381 381 if f not in ctx:
382 382 if not removed:
383 383 return None
384 384 if f in ctx.p1():
385 385 _ctx = ctx.p1()
386 386 elif f in ctx.p2():
387 387 _ctx = ctx.p2()
388 388 else:
389 389 return None
390 390 fctx = _ctx[f]
391 391 if not _islfs(fctx.filelog(), fctx.filenode()):
392 392 return None
393 393 try:
394 394 p = pointer.deserialize(fctx.rawdata())
395 395 if ctx == _ctx:
396 396 return p
397 397 return {}
398 398 except pointer.InvalidPointer as ex:
399 399 raise error.Abort(_('lfs: corrupted pointer (%s@%s): %s\n')
400 400 % (f, short(_ctx.node()), ex))
401 401
402 402 def pointersfromctx(ctx, removed=False):
403 403 """return a dict {path: pointer} for given single changectx.
404 404
405 405 If ``removed`` == True and the LFS file was removed from ``ctx``, the value
406 406 stored for the path is an empty dict.
407 407 """
408 408 result = {}
409 409 m = ctx.repo().narrowmatch()
410 410
411 411 # TODO: consider manifest.fastread() instead
412 412 for f in ctx.files():
413 413 if not m(f):
414 414 continue
415 415 p = pointerfromctx(ctx, f, removed=removed)
416 416 if p is not None:
417 417 result[f] = p
418 418 return result
419 419
420 420 def uploadblobs(repo, pointers):
421 421 """upload given pointers from local blobstore"""
422 422 if not pointers:
423 423 return
424 424
425 425 remoteblob = repo.svfs.lfsremoteblobstore
426 426 remoteblob.writebatch(pointers, repo.svfs.lfslocalblobstore)
427 427
428 428 @eh.wrapfunction(upgrade, '_finishdatamigration')
429 429 def upgradefinishdatamigration(orig, ui, srcrepo, dstrepo, requirements):
430 430 orig(ui, srcrepo, dstrepo, requirements)
431 431
432 432 # Skip if this hasn't been passed to reposetup()
433 433 if (util.safehasattr(srcrepo.svfs, 'lfslocalblobstore') and
434 434 util.safehasattr(dstrepo.svfs, 'lfslocalblobstore')):
435 435 srclfsvfs = srcrepo.svfs.lfslocalblobstore.vfs
436 436 dstlfsvfs = dstrepo.svfs.lfslocalblobstore.vfs
437 437
438 438 for dirpath, dirs, files in srclfsvfs.walk():
439 439 for oid in files:
440 440 ui.write(_('copying lfs blob %s\n') % oid)
441 441 lfutil.link(srclfsvfs.join(oid), dstlfsvfs.join(oid))
442 442
443 443 @eh.wrapfunction(upgrade, 'preservedrequirements')
444 444 @eh.wrapfunction(upgrade, 'supporteddestrequirements')
445 445 def upgraderequirements(orig, repo):
446 446 reqs = orig(repo)
447 447 if 'lfs' in repo.requirements:
448 448 reqs.add('lfs')
449 449 return reqs
@@ -1,2634 +1,2634 b''
1 1 # revlog.py - storage back-end for mercurial
2 2 #
3 3 # Copyright 2005-2007 Matt Mackall <mpm@selenic.com>
4 4 #
5 5 # This software may be used and distributed according to the terms of the
6 6 # GNU General Public License version 2 or any later version.
7 7
8 8 """Storage back-end for Mercurial.
9 9
10 10 This provides efficient delta storage with O(1) retrieve and append
11 11 and O(changes) merge between branches.
12 12 """
13 13
14 14 from __future__ import absolute_import
15 15
16 16 import collections
17 17 import contextlib
18 18 import errno
19 19 import io
20 20 import os
21 21 import struct
22 22 import zlib
23 23
24 24 # import stuff from node for others to import from revlog
25 25 from .node import (
26 26 bin,
27 27 hex,
28 28 nullhex,
29 29 nullid,
30 30 nullrev,
31 31 short,
32 32 wdirfilenodeids,
33 33 wdirhex,
34 34 wdirid,
35 35 wdirrev,
36 36 )
37 37 from .i18n import _
38 38 from .revlogutils.constants import (
39 39 FLAG_GENERALDELTA,
40 40 FLAG_INLINE_DATA,
41 41 REVLOGV0,
42 42 REVLOGV1,
43 43 REVLOGV1_FLAGS,
44 44 REVLOGV2,
45 45 REVLOGV2_FLAGS,
46 46 REVLOG_DEFAULT_FLAGS,
47 47 REVLOG_DEFAULT_FORMAT,
48 48 REVLOG_DEFAULT_VERSION,
49 49 )
50 50 from .revlogutils.flagutil import (
51 51 REVIDX_DEFAULT_FLAGS,
52 52 REVIDX_ELLIPSIS,
53 53 REVIDX_EXTSTORED,
54 54 REVIDX_FLAGS_ORDER,
55 55 REVIDX_ISCENSORED,
56 56 REVIDX_RAWTEXT_CHANGING_FLAGS,
57 57 )
58 58 from .thirdparty import (
59 59 attr,
60 60 )
61 61 from . import (
62 62 ancestor,
63 63 dagop,
64 64 error,
65 65 mdiff,
66 66 policy,
67 67 pycompat,
68 68 templatefilters,
69 69 util,
70 70 )
71 71 from .interfaces import (
72 72 repository,
73 73 util as interfaceutil,
74 74 )
75 75 from .revlogutils import (
76 76 deltas as deltautil,
77 77 flagutil,
78 78 )
79 79 from .utils import (
80 80 storageutil,
81 81 stringutil,
82 82 )
83 83
84 84 # blanked usage of all the name to prevent pyflakes constraints
85 85 # We need these name available in the module for extensions.
86 86 REVLOGV0
87 87 REVLOGV1
88 88 REVLOGV2
89 89 FLAG_INLINE_DATA
90 90 FLAG_GENERALDELTA
91 91 REVLOG_DEFAULT_FLAGS
92 92 REVLOG_DEFAULT_FORMAT
93 93 REVLOG_DEFAULT_VERSION
94 94 REVLOGV1_FLAGS
95 95 REVLOGV2_FLAGS
96 96 REVIDX_ISCENSORED
97 97 REVIDX_ELLIPSIS
98 98 REVIDX_EXTSTORED
99 99 REVIDX_DEFAULT_FLAGS
100 100 REVIDX_FLAGS_ORDER
101 101 REVIDX_RAWTEXT_CHANGING_FLAGS
102 102
103 103 parsers = policy.importmod(r'parsers')
104 104 rustancestor = policy.importrust(r'ancestor')
105 105 rustdagop = policy.importrust(r'dagop')
106 106
107 107 # Aliased for performance.
108 108 _zlibdecompress = zlib.decompress
109 109
110 110 # max size of revlog with inline data
111 111 _maxinline = 131072
112 112 _chunksize = 1048576
113 113
114 114 # Flag processors for REVIDX_ELLIPSIS.
115 115 def ellipsisreadprocessor(rl, text):
116 return text, False
116 return text, False, {}
117 117
118 118 def ellipsiswriteprocessor(rl, text):
119 119 return text, False
120 120
121 121 def ellipsisrawprocessor(rl, text):
122 122 return False
123 123
124 124 ellipsisprocessor = (
125 125 ellipsisreadprocessor,
126 126 ellipsiswriteprocessor,
127 127 ellipsisrawprocessor,
128 128 )
129 129
130 130 def getoffset(q):
131 131 return int(q >> 16)
132 132
133 133 def gettype(q):
134 134 return int(q & 0xFFFF)
135 135
136 136 def offset_type(offset, type):
137 137 if (type & ~flagutil.REVIDX_KNOWN_FLAGS) != 0:
138 138 raise ValueError('unknown revlog index flags')
139 139 return int(int(offset) << 16 | type)
140 140
141 141 @attr.s(slots=True, frozen=True)
142 142 class _revisioninfo(object):
143 143 """Information about a revision that allows building its fulltext
144 144 node: expected hash of the revision
145 145 p1, p2: parent revs of the revision
146 146 btext: built text cache consisting of a one-element list
147 147 cachedelta: (baserev, uncompressed_delta) or None
148 148 flags: flags associated to the revision storage
149 149
150 150 One of btext[0] or cachedelta must be set.
151 151 """
152 152 node = attr.ib()
153 153 p1 = attr.ib()
154 154 p2 = attr.ib()
155 155 btext = attr.ib()
156 156 textlen = attr.ib()
157 157 cachedelta = attr.ib()
158 158 flags = attr.ib()
159 159
160 160 @interfaceutil.implementer(repository.irevisiondelta)
161 161 @attr.s(slots=True)
162 162 class revlogrevisiondelta(object):
163 163 node = attr.ib()
164 164 p1node = attr.ib()
165 165 p2node = attr.ib()
166 166 basenode = attr.ib()
167 167 flags = attr.ib()
168 168 baserevisionsize = attr.ib()
169 169 revision = attr.ib()
170 170 delta = attr.ib()
171 171 linknode = attr.ib(default=None)
172 172
173 173 @interfaceutil.implementer(repository.iverifyproblem)
174 174 @attr.s(frozen=True)
175 175 class revlogproblem(object):
176 176 warning = attr.ib(default=None)
177 177 error = attr.ib(default=None)
178 178 node = attr.ib(default=None)
179 179
180 180 # index v0:
181 181 # 4 bytes: offset
182 182 # 4 bytes: compressed length
183 183 # 4 bytes: base rev
184 184 # 4 bytes: link rev
185 185 # 20 bytes: parent 1 nodeid
186 186 # 20 bytes: parent 2 nodeid
187 187 # 20 bytes: nodeid
188 188 indexformatv0 = struct.Struct(">4l20s20s20s")
189 189 indexformatv0_pack = indexformatv0.pack
190 190 indexformatv0_unpack = indexformatv0.unpack
191 191
192 192 class revlogoldindex(list):
193 193 def __getitem__(self, i):
194 194 if i == -1:
195 195 return (0, 0, 0, -1, -1, -1, -1, nullid)
196 196 return list.__getitem__(self, i)
197 197
198 198 class revlogoldio(object):
199 199 def __init__(self):
200 200 self.size = indexformatv0.size
201 201
202 202 def parseindex(self, data, inline):
203 203 s = self.size
204 204 index = []
205 205 nodemap = {nullid: nullrev}
206 206 n = off = 0
207 207 l = len(data)
208 208 while off + s <= l:
209 209 cur = data[off:off + s]
210 210 off += s
211 211 e = indexformatv0_unpack(cur)
212 212 # transform to revlogv1 format
213 213 e2 = (offset_type(e[0], 0), e[1], -1, e[2], e[3],
214 214 nodemap.get(e[4], nullrev), nodemap.get(e[5], nullrev), e[6])
215 215 index.append(e2)
216 216 nodemap[e[6]] = n
217 217 n += 1
218 218
219 219 return revlogoldindex(index), nodemap, None
220 220
221 221 def packentry(self, entry, node, version, rev):
222 222 if gettype(entry[0]):
223 223 raise error.RevlogError(_('index entry flags need revlog '
224 224 'version 1'))
225 225 e2 = (getoffset(entry[0]), entry[1], entry[3], entry[4],
226 226 node(entry[5]), node(entry[6]), entry[7])
227 227 return indexformatv0_pack(*e2)
228 228
229 229 # index ng:
230 230 # 6 bytes: offset
231 231 # 2 bytes: flags
232 232 # 4 bytes: compressed length
233 233 # 4 bytes: uncompressed length
234 234 # 4 bytes: base rev
235 235 # 4 bytes: link rev
236 236 # 4 bytes: parent 1 rev
237 237 # 4 bytes: parent 2 rev
238 238 # 32 bytes: nodeid
239 239 indexformatng = struct.Struct(">Qiiiiii20s12x")
240 240 indexformatng_pack = indexformatng.pack
241 241 versionformat = struct.Struct(">I")
242 242 versionformat_pack = versionformat.pack
243 243 versionformat_unpack = versionformat.unpack
244 244
245 245 # corresponds to uncompressed length of indexformatng (2 gigs, 4-byte
246 246 # signed integer)
247 247 _maxentrysize = 0x7fffffff
248 248
249 249 class revlogio(object):
250 250 def __init__(self):
251 251 self.size = indexformatng.size
252 252
253 253 def parseindex(self, data, inline):
254 254 # call the C implementation to parse the index data
255 255 index, cache = parsers.parse_index2(data, inline)
256 256 return index, getattr(index, 'nodemap', None), cache
257 257
258 258 def packentry(self, entry, node, version, rev):
259 259 p = indexformatng_pack(*entry)
260 260 if rev == 0:
261 261 p = versionformat_pack(version) + p[4:]
262 262 return p
263 263
264 264 class revlog(flagutil.flagprocessorsmixin):
265 265 """
266 266 the underlying revision storage object
267 267
268 268 A revlog consists of two parts, an index and the revision data.
269 269
270 270 The index is a file with a fixed record size containing
271 271 information on each revision, including its nodeid (hash), the
272 272 nodeids of its parents, the position and offset of its data within
273 273 the data file, and the revision it's based on. Finally, each entry
274 274 contains a linkrev entry that can serve as a pointer to external
275 275 data.
276 276
277 277 The revision data itself is a linear collection of data chunks.
278 278 Each chunk represents a revision and is usually represented as a
279 279 delta against the previous chunk. To bound lookup time, runs of
280 280 deltas are limited to about 2 times the length of the original
281 281 version data. This makes retrieval of a version proportional to
282 282 its size, or O(1) relative to the number of revisions.
283 283
284 284 Both pieces of the revlog are written to in an append-only
285 285 fashion, which means we never need to rewrite a file to insert or
286 286 remove data, and can use some simple techniques to avoid the need
287 287 for locking while reading.
288 288
289 289 If checkambig, indexfile is opened with checkambig=True at
290 290 writing, to avoid file stat ambiguity.
291 291
292 292 If mmaplargeindex is True, and an mmapindexthreshold is set, the
293 293 index will be mmapped rather than read if it is larger than the
294 294 configured threshold.
295 295
296 296 If censorable is True, the revlog can have censored revisions.
297 297
298 298 If `upperboundcomp` is not None, this is the expected maximal gain from
299 299 compression for the data content.
300 300 """
301 301 def __init__(self, opener, indexfile, datafile=None, checkambig=False,
302 302 mmaplargeindex=False, censorable=False,
303 303 upperboundcomp=None):
304 304 """
305 305 create a revlog object
306 306
307 307 opener is a function that abstracts the file opening operation
308 308 and can be used to implement COW semantics or the like.
309 309
310 310 """
311 311 self.upperboundcomp = upperboundcomp
312 312 self.indexfile = indexfile
313 313 self.datafile = datafile or (indexfile[:-2] + ".d")
314 314 self.opener = opener
315 315 # When True, indexfile is opened with checkambig=True at writing, to
316 316 # avoid file stat ambiguity.
317 317 self._checkambig = checkambig
318 318 self._mmaplargeindex = mmaplargeindex
319 319 self._censorable = censorable
320 320 # 3-tuple of (node, rev, text) for a raw revision.
321 321 self._revisioncache = None
322 322 # Maps rev to chain base rev.
323 323 self._chainbasecache = util.lrucachedict(100)
324 324 # 2-tuple of (offset, data) of raw data from the revlog at an offset.
325 325 self._chunkcache = (0, '')
326 326 # How much data to read and cache into the raw revlog data cache.
327 327 self._chunkcachesize = 65536
328 328 self._maxchainlen = None
329 329 self._deltabothparents = True
330 330 self.index = []
331 331 # Mapping of partial identifiers to full nodes.
332 332 self._pcache = {}
333 333 # Mapping of revision integer to full node.
334 334 self._nodecache = {nullid: nullrev}
335 335 self._nodepos = None
336 336 self._compengine = 'zlib'
337 337 self._compengineopts = {}
338 338 self._maxdeltachainspan = -1
339 339 self._withsparseread = False
340 340 self._sparserevlog = False
341 341 self._srdensitythreshold = 0.50
342 342 self._srmingapsize = 262144
343 343
344 344 # Make copy of flag processors so each revlog instance can support
345 345 # custom flags.
346 346 self._flagprocessors = dict(flagutil.flagprocessors)
347 347
348 348 # 2-tuple of file handles being used for active writing.
349 349 self._writinghandles = None
350 350
351 351 self._loadindex()
352 352
353 353 def _loadindex(self):
354 354 mmapindexthreshold = None
355 355 opts = getattr(self.opener, 'options', {}) or {}
356 356
357 357 if 'revlogv2' in opts:
358 358 newversionflags = REVLOGV2 | FLAG_INLINE_DATA
359 359 elif 'revlogv1' in opts:
360 360 newversionflags = REVLOGV1 | FLAG_INLINE_DATA
361 361 if 'generaldelta' in opts:
362 362 newversionflags |= FLAG_GENERALDELTA
363 363 elif getattr(self.opener, 'options', None) is not None:
364 364 # If options provided but no 'revlog*' found, the repository
365 365 # would have no 'requires' file in it, which means we have to
366 366 # stick to the old format.
367 367 newversionflags = REVLOGV0
368 368 else:
369 369 newversionflags = REVLOG_DEFAULT_VERSION
370 370
371 371 if 'chunkcachesize' in opts:
372 372 self._chunkcachesize = opts['chunkcachesize']
373 373 if 'maxchainlen' in opts:
374 374 self._maxchainlen = opts['maxchainlen']
375 375 if 'deltabothparents' in opts:
376 376 self._deltabothparents = opts['deltabothparents']
377 377 self._lazydelta = bool(opts.get('lazydelta', True))
378 378 self._lazydeltabase = False
379 379 if self._lazydelta:
380 380 self._lazydeltabase = bool(opts.get('lazydeltabase', False))
381 381 if 'compengine' in opts:
382 382 self._compengine = opts['compengine']
383 383 if 'zlib.level' in opts:
384 384 self._compengineopts['zlib.level'] = opts['zlib.level']
385 385 if 'zstd.level' in opts:
386 386 self._compengineopts['zstd.level'] = opts['zstd.level']
387 387 if 'maxdeltachainspan' in opts:
388 388 self._maxdeltachainspan = opts['maxdeltachainspan']
389 389 if self._mmaplargeindex and 'mmapindexthreshold' in opts:
390 390 mmapindexthreshold = opts['mmapindexthreshold']
391 391 self._sparserevlog = bool(opts.get('sparse-revlog', False))
392 392 withsparseread = bool(opts.get('with-sparse-read', False))
393 393 # sparse-revlog forces sparse-read
394 394 self._withsparseread = self._sparserevlog or withsparseread
395 395 if 'sparse-read-density-threshold' in opts:
396 396 self._srdensitythreshold = opts['sparse-read-density-threshold']
397 397 if 'sparse-read-min-gap-size' in opts:
398 398 self._srmingapsize = opts['sparse-read-min-gap-size']
399 399 if opts.get('enableellipsis'):
400 400 self._flagprocessors[REVIDX_ELLIPSIS] = ellipsisprocessor
401 401
402 402 # revlog v0 doesn't have flag processors
403 403 for flag, processor in opts.get(b'flagprocessors', {}).iteritems():
404 404 flagutil.insertflagprocessor(flag, processor, self._flagprocessors)
405 405
406 406 if self._chunkcachesize <= 0:
407 407 raise error.RevlogError(_('revlog chunk cache size %r is not '
408 408 'greater than 0') % self._chunkcachesize)
409 409 elif self._chunkcachesize & (self._chunkcachesize - 1):
410 410 raise error.RevlogError(_('revlog chunk cache size %r is not a '
411 411 'power of 2') % self._chunkcachesize)
412 412
413 413 indexdata = ''
414 414 self._initempty = True
415 415 try:
416 416 with self._indexfp() as f:
417 417 if (mmapindexthreshold is not None and
418 418 self.opener.fstat(f).st_size >= mmapindexthreshold):
419 419 # TODO: should .close() to release resources without
420 420 # relying on Python GC
421 421 indexdata = util.buffer(util.mmapread(f))
422 422 else:
423 423 indexdata = f.read()
424 424 if len(indexdata) > 0:
425 425 versionflags = versionformat_unpack(indexdata[:4])[0]
426 426 self._initempty = False
427 427 else:
428 428 versionflags = newversionflags
429 429 except IOError as inst:
430 430 if inst.errno != errno.ENOENT:
431 431 raise
432 432
433 433 versionflags = newversionflags
434 434
435 435 self.version = versionflags
436 436
437 437 flags = versionflags & ~0xFFFF
438 438 fmt = versionflags & 0xFFFF
439 439
440 440 if fmt == REVLOGV0:
441 441 if flags:
442 442 raise error.RevlogError(_('unknown flags (%#04x) in version %d '
443 443 'revlog %s') %
444 444 (flags >> 16, fmt, self.indexfile))
445 445
446 446 self._inline = False
447 447 self._generaldelta = False
448 448
449 449 elif fmt == REVLOGV1:
450 450 if flags & ~REVLOGV1_FLAGS:
451 451 raise error.RevlogError(_('unknown flags (%#04x) in version %d '
452 452 'revlog %s') %
453 453 (flags >> 16, fmt, self.indexfile))
454 454
455 455 self._inline = versionflags & FLAG_INLINE_DATA
456 456 self._generaldelta = versionflags & FLAG_GENERALDELTA
457 457
458 458 elif fmt == REVLOGV2:
459 459 if flags & ~REVLOGV2_FLAGS:
460 460 raise error.RevlogError(_('unknown flags (%#04x) in version %d '
461 461 'revlog %s') %
462 462 (flags >> 16, fmt, self.indexfile))
463 463
464 464 self._inline = versionflags & FLAG_INLINE_DATA
465 465 # generaldelta implied by version 2 revlogs.
466 466 self._generaldelta = True
467 467
468 468 else:
469 469 raise error.RevlogError(_('unknown version (%d) in revlog %s') %
470 470 (fmt, self.indexfile))
471 471 # sparse-revlog can't be on without general-delta (issue6056)
472 472 if not self._generaldelta:
473 473 self._sparserevlog = False
474 474
475 475 self._storedeltachains = True
476 476
477 477 self._io = revlogio()
478 478 if self.version == REVLOGV0:
479 479 self._io = revlogoldio()
480 480 try:
481 481 d = self._io.parseindex(indexdata, self._inline)
482 482 except (ValueError, IndexError):
483 483 raise error.RevlogError(_("index %s is corrupted") %
484 484 self.indexfile)
485 485 self.index, nodemap, self._chunkcache = d
486 486 if nodemap is not None:
487 487 self.nodemap = self._nodecache = nodemap
488 488 if not self._chunkcache:
489 489 self._chunkclear()
490 490 # revnum -> (chain-length, sum-delta-length)
491 491 self._chaininfocache = {}
492 492 # revlog header -> revlog compressor
493 493 self._decompressors = {}
494 494
495 495 @util.propertycache
496 496 def _compressor(self):
497 497 engine = util.compengines[self._compengine]
498 498 return engine.revlogcompressor(self._compengineopts)
499 499
500 500 def _indexfp(self, mode='r'):
501 501 """file object for the revlog's index file"""
502 502 args = {r'mode': mode}
503 503 if mode != 'r':
504 504 args[r'checkambig'] = self._checkambig
505 505 if mode == 'w':
506 506 args[r'atomictemp'] = True
507 507 return self.opener(self.indexfile, **args)
508 508
509 509 def _datafp(self, mode='r'):
510 510 """file object for the revlog's data file"""
511 511 return self.opener(self.datafile, mode=mode)
512 512
513 513 @contextlib.contextmanager
514 514 def _datareadfp(self, existingfp=None):
515 515 """file object suitable to read data"""
516 516 # Use explicit file handle, if given.
517 517 if existingfp is not None:
518 518 yield existingfp
519 519
520 520 # Use a file handle being actively used for writes, if available.
521 521 # There is some danger to doing this because reads will seek the
522 522 # file. However, _writeentry() performs a SEEK_END before all writes,
523 523 # so we should be safe.
524 524 elif self._writinghandles:
525 525 if self._inline:
526 526 yield self._writinghandles[0]
527 527 else:
528 528 yield self._writinghandles[1]
529 529
530 530 # Otherwise open a new file handle.
531 531 else:
532 532 if self._inline:
533 533 func = self._indexfp
534 534 else:
535 535 func = self._datafp
536 536 with func() as fp:
537 537 yield fp
538 538
539 539 def tip(self):
540 540 return self.node(len(self.index) - 1)
541 541 def __contains__(self, rev):
542 542 return 0 <= rev < len(self)
543 543 def __len__(self):
544 544 return len(self.index)
545 545 def __iter__(self):
546 546 return iter(pycompat.xrange(len(self)))
547 547 def revs(self, start=0, stop=None):
548 548 """iterate over all rev in this revlog (from start to stop)"""
549 549 return storageutil.iterrevs(len(self), start=start, stop=stop)
550 550
551 551 @util.propertycache
552 552 def nodemap(self):
553 553 if self.index:
554 554 # populate mapping down to the initial node
555 555 node0 = self.index[0][7] # get around changelog filtering
556 556 self.rev(node0)
557 557 return self._nodecache
558 558
559 559 def hasnode(self, node):
560 560 try:
561 561 self.rev(node)
562 562 return True
563 563 except KeyError:
564 564 return False
565 565
566 566 def candelta(self, baserev, rev):
567 567 """whether two revisions (baserev, rev) can be delta-ed or not"""
568 568 # Disable delta if either rev requires a content-changing flag
569 569 # processor (ex. LFS). This is because such flag processor can alter
570 570 # the rawtext content that the delta will be based on, and two clients
571 571 # could have a same revlog node with different flags (i.e. different
572 572 # rawtext contents) and the delta could be incompatible.
573 573 if ((self.flags(baserev) & REVIDX_RAWTEXT_CHANGING_FLAGS)
574 574 or (self.flags(rev) & REVIDX_RAWTEXT_CHANGING_FLAGS)):
575 575 return False
576 576 return True
577 577
578 578 def clearcaches(self):
579 579 self._revisioncache = None
580 580 self._chainbasecache.clear()
581 581 self._chunkcache = (0, '')
582 582 self._pcache = {}
583 583
584 584 try:
585 585 # If we are using the native C version, you are in a fun case
586 586 # where self.index, self.nodemap and self._nodecaches is the same
587 587 # object.
588 588 self._nodecache.clearcaches()
589 589 except AttributeError:
590 590 self._nodecache = {nullid: nullrev}
591 591 self._nodepos = None
592 592
593 593 def rev(self, node):
594 594 try:
595 595 return self._nodecache[node]
596 596 except TypeError:
597 597 raise
598 598 except error.RevlogError:
599 599 # parsers.c radix tree lookup failed
600 600 if node == wdirid or node in wdirfilenodeids:
601 601 raise error.WdirUnsupported
602 602 raise error.LookupError(node, self.indexfile, _('no node'))
603 603 except KeyError:
604 604 # pure python cache lookup failed
605 605 n = self._nodecache
606 606 i = self.index
607 607 p = self._nodepos
608 608 if p is None:
609 609 p = len(i) - 1
610 610 else:
611 611 assert p < len(i)
612 612 for r in pycompat.xrange(p, -1, -1):
613 613 v = i[r][7]
614 614 n[v] = r
615 615 if v == node:
616 616 self._nodepos = r - 1
617 617 return r
618 618 if node == wdirid or node in wdirfilenodeids:
619 619 raise error.WdirUnsupported
620 620 raise error.LookupError(node, self.indexfile, _('no node'))
621 621
622 622 # Accessors for index entries.
623 623
624 624 # First tuple entry is 8 bytes. First 6 bytes are offset. Last 2 bytes
625 625 # are flags.
626 626 def start(self, rev):
627 627 return int(self.index[rev][0] >> 16)
628 628
629 629 def flags(self, rev):
630 630 return self.index[rev][0] & 0xFFFF
631 631
632 632 def length(self, rev):
633 633 return self.index[rev][1]
634 634
635 635 def rawsize(self, rev):
636 636 """return the length of the uncompressed text for a given revision"""
637 637 l = self.index[rev][2]
638 638 if l >= 0:
639 639 return l
640 640
641 641 t = self.rawdata(rev)
642 642 return len(t)
643 643
644 644 def size(self, rev):
645 645 """length of non-raw text (processed by a "read" flag processor)"""
646 646 # fast path: if no "read" flag processor could change the content,
647 647 # size is rawsize. note: ELLIPSIS is known to not change the content.
648 648 flags = self.flags(rev)
649 649 if flags & (flagutil.REVIDX_KNOWN_FLAGS ^ REVIDX_ELLIPSIS) == 0:
650 650 return self.rawsize(rev)
651 651
652 652 return len(self.revision(rev, raw=False))
653 653
654 654 def chainbase(self, rev):
655 655 base = self._chainbasecache.get(rev)
656 656 if base is not None:
657 657 return base
658 658
659 659 index = self.index
660 660 iterrev = rev
661 661 base = index[iterrev][3]
662 662 while base != iterrev:
663 663 iterrev = base
664 664 base = index[iterrev][3]
665 665
666 666 self._chainbasecache[rev] = base
667 667 return base
668 668
669 669 def linkrev(self, rev):
670 670 return self.index[rev][4]
671 671
672 672 def parentrevs(self, rev):
673 673 try:
674 674 entry = self.index[rev]
675 675 except IndexError:
676 676 if rev == wdirrev:
677 677 raise error.WdirUnsupported
678 678 raise
679 679
680 680 return entry[5], entry[6]
681 681
682 682 # fast parentrevs(rev) where rev isn't filtered
683 683 _uncheckedparentrevs = parentrevs
684 684
685 685 def node(self, rev):
686 686 try:
687 687 return self.index[rev][7]
688 688 except IndexError:
689 689 if rev == wdirrev:
690 690 raise error.WdirUnsupported
691 691 raise
692 692
693 693 # Derived from index values.
694 694
695 695 def end(self, rev):
696 696 return self.start(rev) + self.length(rev)
697 697
698 698 def parents(self, node):
699 699 i = self.index
700 700 d = i[self.rev(node)]
701 701 return i[d[5]][7], i[d[6]][7] # map revisions to nodes inline
702 702
703 703 def chainlen(self, rev):
704 704 return self._chaininfo(rev)[0]
705 705
706 706 def _chaininfo(self, rev):
707 707 chaininfocache = self._chaininfocache
708 708 if rev in chaininfocache:
709 709 return chaininfocache[rev]
710 710 index = self.index
711 711 generaldelta = self._generaldelta
712 712 iterrev = rev
713 713 e = index[iterrev]
714 714 clen = 0
715 715 compresseddeltalen = 0
716 716 while iterrev != e[3]:
717 717 clen += 1
718 718 compresseddeltalen += e[1]
719 719 if generaldelta:
720 720 iterrev = e[3]
721 721 else:
722 722 iterrev -= 1
723 723 if iterrev in chaininfocache:
724 724 t = chaininfocache[iterrev]
725 725 clen += t[0]
726 726 compresseddeltalen += t[1]
727 727 break
728 728 e = index[iterrev]
729 729 else:
730 730 # Add text length of base since decompressing that also takes
731 731 # work. For cache hits the length is already included.
732 732 compresseddeltalen += e[1]
733 733 r = (clen, compresseddeltalen)
734 734 chaininfocache[rev] = r
735 735 return r
736 736
737 737 def _deltachain(self, rev, stoprev=None):
738 738 """Obtain the delta chain for a revision.
739 739
740 740 ``stoprev`` specifies a revision to stop at. If not specified, we
741 741 stop at the base of the chain.
742 742
743 743 Returns a 2-tuple of (chain, stopped) where ``chain`` is a list of
744 744 revs in ascending order and ``stopped`` is a bool indicating whether
745 745 ``stoprev`` was hit.
746 746 """
747 747 # Try C implementation.
748 748 try:
749 749 return self.index.deltachain(rev, stoprev, self._generaldelta)
750 750 except AttributeError:
751 751 pass
752 752
753 753 chain = []
754 754
755 755 # Alias to prevent attribute lookup in tight loop.
756 756 index = self.index
757 757 generaldelta = self._generaldelta
758 758
759 759 iterrev = rev
760 760 e = index[iterrev]
761 761 while iterrev != e[3] and iterrev != stoprev:
762 762 chain.append(iterrev)
763 763 if generaldelta:
764 764 iterrev = e[3]
765 765 else:
766 766 iterrev -= 1
767 767 e = index[iterrev]
768 768
769 769 if iterrev == stoprev:
770 770 stopped = True
771 771 else:
772 772 chain.append(iterrev)
773 773 stopped = False
774 774
775 775 chain.reverse()
776 776 return chain, stopped
777 777
778 778 def ancestors(self, revs, stoprev=0, inclusive=False):
779 779 """Generate the ancestors of 'revs' in reverse revision order.
780 780 Does not generate revs lower than stoprev.
781 781
782 782 See the documentation for ancestor.lazyancestors for more details."""
783 783
784 784 # first, make sure start revisions aren't filtered
785 785 revs = list(revs)
786 786 checkrev = self.node
787 787 for r in revs:
788 788 checkrev(r)
789 789 # and we're sure ancestors aren't filtered as well
790 790
791 791 if rustancestor is not None:
792 792 lazyancestors = rustancestor.LazyAncestors
793 793 arg = self.index
794 794 elif util.safehasattr(parsers, 'rustlazyancestors'):
795 795 lazyancestors = ancestor.rustlazyancestors
796 796 arg = self.index
797 797 else:
798 798 lazyancestors = ancestor.lazyancestors
799 799 arg = self._uncheckedparentrevs
800 800 return lazyancestors(arg, revs, stoprev=stoprev, inclusive=inclusive)
801 801
802 802 def descendants(self, revs):
803 803 return dagop.descendantrevs(revs, self.revs, self.parentrevs)
804 804
805 805 def findcommonmissing(self, common=None, heads=None):
806 806 """Return a tuple of the ancestors of common and the ancestors of heads
807 807 that are not ancestors of common. In revset terminology, we return the
808 808 tuple:
809 809
810 810 ::common, (::heads) - (::common)
811 811
812 812 The list is sorted by revision number, meaning it is
813 813 topologically sorted.
814 814
815 815 'heads' and 'common' are both lists of node IDs. If heads is
816 816 not supplied, uses all of the revlog's heads. If common is not
817 817 supplied, uses nullid."""
818 818 if common is None:
819 819 common = [nullid]
820 820 if heads is None:
821 821 heads = self.heads()
822 822
823 823 common = [self.rev(n) for n in common]
824 824 heads = [self.rev(n) for n in heads]
825 825
826 826 # we want the ancestors, but inclusive
827 827 class lazyset(object):
828 828 def __init__(self, lazyvalues):
829 829 self.addedvalues = set()
830 830 self.lazyvalues = lazyvalues
831 831
832 832 def __contains__(self, value):
833 833 return value in self.addedvalues or value in self.lazyvalues
834 834
835 835 def __iter__(self):
836 836 added = self.addedvalues
837 837 for r in added:
838 838 yield r
839 839 for r in self.lazyvalues:
840 840 if not r in added:
841 841 yield r
842 842
843 843 def add(self, value):
844 844 self.addedvalues.add(value)
845 845
846 846 def update(self, values):
847 847 self.addedvalues.update(values)
848 848
849 849 has = lazyset(self.ancestors(common))
850 850 has.add(nullrev)
851 851 has.update(common)
852 852
853 853 # take all ancestors from heads that aren't in has
854 854 missing = set()
855 855 visit = collections.deque(r for r in heads if r not in has)
856 856 while visit:
857 857 r = visit.popleft()
858 858 if r in missing:
859 859 continue
860 860 else:
861 861 missing.add(r)
862 862 for p in self.parentrevs(r):
863 863 if p not in has:
864 864 visit.append(p)
865 865 missing = list(missing)
866 866 missing.sort()
867 867 return has, [self.node(miss) for miss in missing]
868 868
869 869 def incrementalmissingrevs(self, common=None):
870 870 """Return an object that can be used to incrementally compute the
871 871 revision numbers of the ancestors of arbitrary sets that are not
872 872 ancestors of common. This is an ancestor.incrementalmissingancestors
873 873 object.
874 874
875 875 'common' is a list of revision numbers. If common is not supplied, uses
876 876 nullrev.
877 877 """
878 878 if common is None:
879 879 common = [nullrev]
880 880
881 881 if rustancestor is not None:
882 882 return rustancestor.MissingAncestors(self.index, common)
883 883 return ancestor.incrementalmissingancestors(self.parentrevs, common)
884 884
885 885 def findmissingrevs(self, common=None, heads=None):
886 886 """Return the revision numbers of the ancestors of heads that
887 887 are not ancestors of common.
888 888
889 889 More specifically, return a list of revision numbers corresponding to
890 890 nodes N such that every N satisfies the following constraints:
891 891
892 892 1. N is an ancestor of some node in 'heads'
893 893 2. N is not an ancestor of any node in 'common'
894 894
895 895 The list is sorted by revision number, meaning it is
896 896 topologically sorted.
897 897
898 898 'heads' and 'common' are both lists of revision numbers. If heads is
899 899 not supplied, uses all of the revlog's heads. If common is not
900 900 supplied, uses nullid."""
901 901 if common is None:
902 902 common = [nullrev]
903 903 if heads is None:
904 904 heads = self.headrevs()
905 905
906 906 inc = self.incrementalmissingrevs(common=common)
907 907 return inc.missingancestors(heads)
908 908
909 909 def findmissing(self, common=None, heads=None):
910 910 """Return the ancestors of heads that are not ancestors of common.
911 911
912 912 More specifically, return a list of nodes N such that every N
913 913 satisfies the following constraints:
914 914
915 915 1. N is an ancestor of some node in 'heads'
916 916 2. N is not an ancestor of any node in 'common'
917 917
918 918 The list is sorted by revision number, meaning it is
919 919 topologically sorted.
920 920
921 921 'heads' and 'common' are both lists of node IDs. If heads is
922 922 not supplied, uses all of the revlog's heads. If common is not
923 923 supplied, uses nullid."""
924 924 if common is None:
925 925 common = [nullid]
926 926 if heads is None:
927 927 heads = self.heads()
928 928
929 929 common = [self.rev(n) for n in common]
930 930 heads = [self.rev(n) for n in heads]
931 931
932 932 inc = self.incrementalmissingrevs(common=common)
933 933 return [self.node(r) for r in inc.missingancestors(heads)]
934 934
935 935 def nodesbetween(self, roots=None, heads=None):
936 936 """Return a topological path from 'roots' to 'heads'.
937 937
938 938 Return a tuple (nodes, outroots, outheads) where 'nodes' is a
939 939 topologically sorted list of all nodes N that satisfy both of
940 940 these constraints:
941 941
942 942 1. N is a descendant of some node in 'roots'
943 943 2. N is an ancestor of some node in 'heads'
944 944
945 945 Every node is considered to be both a descendant and an ancestor
946 946 of itself, so every reachable node in 'roots' and 'heads' will be
947 947 included in 'nodes'.
948 948
949 949 'outroots' is the list of reachable nodes in 'roots', i.e., the
950 950 subset of 'roots' that is returned in 'nodes'. Likewise,
951 951 'outheads' is the subset of 'heads' that is also in 'nodes'.
952 952
953 953 'roots' and 'heads' are both lists of node IDs. If 'roots' is
954 954 unspecified, uses nullid as the only root. If 'heads' is
955 955 unspecified, uses list of all of the revlog's heads."""
956 956 nonodes = ([], [], [])
957 957 if roots is not None:
958 958 roots = list(roots)
959 959 if not roots:
960 960 return nonodes
961 961 lowestrev = min([self.rev(n) for n in roots])
962 962 else:
963 963 roots = [nullid] # Everybody's a descendant of nullid
964 964 lowestrev = nullrev
965 965 if (lowestrev == nullrev) and (heads is None):
966 966 # We want _all_ the nodes!
967 967 return ([self.node(r) for r in self], [nullid], list(self.heads()))
968 968 if heads is None:
969 969 # All nodes are ancestors, so the latest ancestor is the last
970 970 # node.
971 971 highestrev = len(self) - 1
972 972 # Set ancestors to None to signal that every node is an ancestor.
973 973 ancestors = None
974 974 # Set heads to an empty dictionary for later discovery of heads
975 975 heads = {}
976 976 else:
977 977 heads = list(heads)
978 978 if not heads:
979 979 return nonodes
980 980 ancestors = set()
981 981 # Turn heads into a dictionary so we can remove 'fake' heads.
982 982 # Also, later we will be using it to filter out the heads we can't
983 983 # find from roots.
984 984 heads = dict.fromkeys(heads, False)
985 985 # Start at the top and keep marking parents until we're done.
986 986 nodestotag = set(heads)
987 987 # Remember where the top was so we can use it as a limit later.
988 988 highestrev = max([self.rev(n) for n in nodestotag])
989 989 while nodestotag:
990 990 # grab a node to tag
991 991 n = nodestotag.pop()
992 992 # Never tag nullid
993 993 if n == nullid:
994 994 continue
995 995 # A node's revision number represents its place in a
996 996 # topologically sorted list of nodes.
997 997 r = self.rev(n)
998 998 if r >= lowestrev:
999 999 if n not in ancestors:
1000 1000 # If we are possibly a descendant of one of the roots
1001 1001 # and we haven't already been marked as an ancestor
1002 1002 ancestors.add(n) # Mark as ancestor
1003 1003 # Add non-nullid parents to list of nodes to tag.
1004 1004 nodestotag.update([p for p in self.parents(n) if
1005 1005 p != nullid])
1006 1006 elif n in heads: # We've seen it before, is it a fake head?
1007 1007 # So it is, real heads should not be the ancestors of
1008 1008 # any other heads.
1009 1009 heads.pop(n)
1010 1010 if not ancestors:
1011 1011 return nonodes
1012 1012 # Now that we have our set of ancestors, we want to remove any
1013 1013 # roots that are not ancestors.
1014 1014
1015 1015 # If one of the roots was nullid, everything is included anyway.
1016 1016 if lowestrev > nullrev:
1017 1017 # But, since we weren't, let's recompute the lowest rev to not
1018 1018 # include roots that aren't ancestors.
1019 1019
1020 1020 # Filter out roots that aren't ancestors of heads
1021 1021 roots = [root for root in roots if root in ancestors]
1022 1022 # Recompute the lowest revision
1023 1023 if roots:
1024 1024 lowestrev = min([self.rev(root) for root in roots])
1025 1025 else:
1026 1026 # No more roots? Return empty list
1027 1027 return nonodes
1028 1028 else:
1029 1029 # We are descending from nullid, and don't need to care about
1030 1030 # any other roots.
1031 1031 lowestrev = nullrev
1032 1032 roots = [nullid]
1033 1033 # Transform our roots list into a set.
1034 1034 descendants = set(roots)
1035 1035 # Also, keep the original roots so we can filter out roots that aren't
1036 1036 # 'real' roots (i.e. are descended from other roots).
1037 1037 roots = descendants.copy()
1038 1038 # Our topologically sorted list of output nodes.
1039 1039 orderedout = []
1040 1040 # Don't start at nullid since we don't want nullid in our output list,
1041 1041 # and if nullid shows up in descendants, empty parents will look like
1042 1042 # they're descendants.
1043 1043 for r in self.revs(start=max(lowestrev, 0), stop=highestrev + 1):
1044 1044 n = self.node(r)
1045 1045 isdescendant = False
1046 1046 if lowestrev == nullrev: # Everybody is a descendant of nullid
1047 1047 isdescendant = True
1048 1048 elif n in descendants:
1049 1049 # n is already a descendant
1050 1050 isdescendant = True
1051 1051 # This check only needs to be done here because all the roots
1052 1052 # will start being marked is descendants before the loop.
1053 1053 if n in roots:
1054 1054 # If n was a root, check if it's a 'real' root.
1055 1055 p = tuple(self.parents(n))
1056 1056 # If any of its parents are descendants, it's not a root.
1057 1057 if (p[0] in descendants) or (p[1] in descendants):
1058 1058 roots.remove(n)
1059 1059 else:
1060 1060 p = tuple(self.parents(n))
1061 1061 # A node is a descendant if either of its parents are
1062 1062 # descendants. (We seeded the dependents list with the roots
1063 1063 # up there, remember?)
1064 1064 if (p[0] in descendants) or (p[1] in descendants):
1065 1065 descendants.add(n)
1066 1066 isdescendant = True
1067 1067 if isdescendant and ((ancestors is None) or (n in ancestors)):
1068 1068 # Only include nodes that are both descendants and ancestors.
1069 1069 orderedout.append(n)
1070 1070 if (ancestors is not None) and (n in heads):
1071 1071 # We're trying to figure out which heads are reachable
1072 1072 # from roots.
1073 1073 # Mark this head as having been reached
1074 1074 heads[n] = True
1075 1075 elif ancestors is None:
1076 1076 # Otherwise, we're trying to discover the heads.
1077 1077 # Assume this is a head because if it isn't, the next step
1078 1078 # will eventually remove it.
1079 1079 heads[n] = True
1080 1080 # But, obviously its parents aren't.
1081 1081 for p in self.parents(n):
1082 1082 heads.pop(p, None)
1083 1083 heads = [head for head, flag in heads.iteritems() if flag]
1084 1084 roots = list(roots)
1085 1085 assert orderedout
1086 1086 assert roots
1087 1087 assert heads
1088 1088 return (orderedout, roots, heads)
1089 1089
1090 1090 def headrevs(self, revs=None):
1091 1091 if revs is None:
1092 1092 try:
1093 1093 return self.index.headrevs()
1094 1094 except AttributeError:
1095 1095 return self._headrevs()
1096 1096 if rustdagop is not None:
1097 1097 return rustdagop.headrevs(self.index, revs)
1098 1098 return dagop.headrevs(revs, self._uncheckedparentrevs)
1099 1099
1100 1100 def computephases(self, roots):
1101 1101 return self.index.computephasesmapsets(roots)
1102 1102
1103 1103 def _headrevs(self):
1104 1104 count = len(self)
1105 1105 if not count:
1106 1106 return [nullrev]
1107 1107 # we won't iter over filtered rev so nobody is a head at start
1108 1108 ishead = [0] * (count + 1)
1109 1109 index = self.index
1110 1110 for r in self:
1111 1111 ishead[r] = 1 # I may be an head
1112 1112 e = index[r]
1113 1113 ishead[e[5]] = ishead[e[6]] = 0 # my parent are not
1114 1114 return [r for r, val in enumerate(ishead) if val]
1115 1115
1116 1116 def heads(self, start=None, stop=None):
1117 1117 """return the list of all nodes that have no children
1118 1118
1119 1119 if start is specified, only heads that are descendants of
1120 1120 start will be returned
1121 1121 if stop is specified, it will consider all the revs from stop
1122 1122 as if they had no children
1123 1123 """
1124 1124 if start is None and stop is None:
1125 1125 if not len(self):
1126 1126 return [nullid]
1127 1127 return [self.node(r) for r in self.headrevs()]
1128 1128
1129 1129 if start is None:
1130 1130 start = nullrev
1131 1131 else:
1132 1132 start = self.rev(start)
1133 1133
1134 1134 stoprevs = set(self.rev(n) for n in stop or [])
1135 1135
1136 1136 revs = dagop.headrevssubset(self.revs, self.parentrevs, startrev=start,
1137 1137 stoprevs=stoprevs)
1138 1138
1139 1139 return [self.node(rev) for rev in revs]
1140 1140
1141 1141 def children(self, node):
1142 1142 """find the children of a given node"""
1143 1143 c = []
1144 1144 p = self.rev(node)
1145 1145 for r in self.revs(start=p + 1):
1146 1146 prevs = [pr for pr in self.parentrevs(r) if pr != nullrev]
1147 1147 if prevs:
1148 1148 for pr in prevs:
1149 1149 if pr == p:
1150 1150 c.append(self.node(r))
1151 1151 elif p == nullrev:
1152 1152 c.append(self.node(r))
1153 1153 return c
1154 1154
1155 1155 def commonancestorsheads(self, a, b):
1156 1156 """calculate all the heads of the common ancestors of nodes a and b"""
1157 1157 a, b = self.rev(a), self.rev(b)
1158 1158 ancs = self._commonancestorsheads(a, b)
1159 1159 return pycompat.maplist(self.node, ancs)
1160 1160
1161 1161 def _commonancestorsheads(self, *revs):
1162 1162 """calculate all the heads of the common ancestors of revs"""
1163 1163 try:
1164 1164 ancs = self.index.commonancestorsheads(*revs)
1165 1165 except (AttributeError, OverflowError): # C implementation failed
1166 1166 ancs = ancestor.commonancestorsheads(self.parentrevs, *revs)
1167 1167 return ancs
1168 1168
1169 1169 def isancestor(self, a, b):
1170 1170 """return True if node a is an ancestor of node b
1171 1171
1172 1172 A revision is considered an ancestor of itself."""
1173 1173 a, b = self.rev(a), self.rev(b)
1174 1174 return self.isancestorrev(a, b)
1175 1175
1176 1176 def isancestorrev(self, a, b):
1177 1177 """return True if revision a is an ancestor of revision b
1178 1178
1179 1179 A revision is considered an ancestor of itself.
1180 1180
1181 1181 The implementation of this is trivial but the use of
1182 1182 reachableroots is not."""
1183 1183 if a == nullrev:
1184 1184 return True
1185 1185 elif a == b:
1186 1186 return True
1187 1187 elif a > b:
1188 1188 return False
1189 1189 return bool(self.reachableroots(a, [b], [a], includepath=False))
1190 1190
1191 1191 def reachableroots(self, minroot, heads, roots, includepath=False):
1192 1192 """return (heads(::<roots> and <roots>::<heads>))
1193 1193
1194 1194 If includepath is True, return (<roots>::<heads>)."""
1195 1195 try:
1196 1196 return self.index.reachableroots2(minroot, heads, roots,
1197 1197 includepath)
1198 1198 except AttributeError:
1199 1199 return dagop._reachablerootspure(self.parentrevs,
1200 1200 minroot, roots, heads, includepath)
1201 1201
1202 1202 def ancestor(self, a, b):
1203 1203 """calculate the "best" common ancestor of nodes a and b"""
1204 1204
1205 1205 a, b = self.rev(a), self.rev(b)
1206 1206 try:
1207 1207 ancs = self.index.ancestors(a, b)
1208 1208 except (AttributeError, OverflowError):
1209 1209 ancs = ancestor.ancestors(self.parentrevs, a, b)
1210 1210 if ancs:
1211 1211 # choose a consistent winner when there's a tie
1212 1212 return min(map(self.node, ancs))
1213 1213 return nullid
1214 1214
1215 1215 def _match(self, id):
1216 1216 if isinstance(id, int):
1217 1217 # rev
1218 1218 return self.node(id)
1219 1219 if len(id) == 20:
1220 1220 # possibly a binary node
1221 1221 # odds of a binary node being all hex in ASCII are 1 in 10**25
1222 1222 try:
1223 1223 node = id
1224 1224 self.rev(node) # quick search the index
1225 1225 return node
1226 1226 except error.LookupError:
1227 1227 pass # may be partial hex id
1228 1228 try:
1229 1229 # str(rev)
1230 1230 rev = int(id)
1231 1231 if "%d" % rev != id:
1232 1232 raise ValueError
1233 1233 if rev < 0:
1234 1234 rev = len(self) + rev
1235 1235 if rev < 0 or rev >= len(self):
1236 1236 raise ValueError
1237 1237 return self.node(rev)
1238 1238 except (ValueError, OverflowError):
1239 1239 pass
1240 1240 if len(id) == 40:
1241 1241 try:
1242 1242 # a full hex nodeid?
1243 1243 node = bin(id)
1244 1244 self.rev(node)
1245 1245 return node
1246 1246 except (TypeError, error.LookupError):
1247 1247 pass
1248 1248
1249 1249 def _partialmatch(self, id):
1250 1250 # we don't care wdirfilenodeids as they should be always full hash
1251 1251 maybewdir = wdirhex.startswith(id)
1252 1252 try:
1253 1253 partial = self.index.partialmatch(id)
1254 1254 if partial and self.hasnode(partial):
1255 1255 if maybewdir:
1256 1256 # single 'ff...' match in radix tree, ambiguous with wdir
1257 1257 raise error.RevlogError
1258 1258 return partial
1259 1259 if maybewdir:
1260 1260 # no 'ff...' match in radix tree, wdir identified
1261 1261 raise error.WdirUnsupported
1262 1262 return None
1263 1263 except error.RevlogError:
1264 1264 # parsers.c radix tree lookup gave multiple matches
1265 1265 # fast path: for unfiltered changelog, radix tree is accurate
1266 1266 if not getattr(self, 'filteredrevs', None):
1267 1267 raise error.AmbiguousPrefixLookupError(
1268 1268 id, self.indexfile, _('ambiguous identifier'))
1269 1269 # fall through to slow path that filters hidden revisions
1270 1270 except (AttributeError, ValueError):
1271 1271 # we are pure python, or key was too short to search radix tree
1272 1272 pass
1273 1273
1274 1274 if id in self._pcache:
1275 1275 return self._pcache[id]
1276 1276
1277 1277 if len(id) <= 40:
1278 1278 try:
1279 1279 # hex(node)[:...]
1280 1280 l = len(id) // 2 # grab an even number of digits
1281 1281 prefix = bin(id[:l * 2])
1282 1282 nl = [e[7] for e in self.index if e[7].startswith(prefix)]
1283 1283 nl = [n for n in nl if hex(n).startswith(id) and
1284 1284 self.hasnode(n)]
1285 1285 if nullhex.startswith(id):
1286 1286 nl.append(nullid)
1287 1287 if len(nl) > 0:
1288 1288 if len(nl) == 1 and not maybewdir:
1289 1289 self._pcache[id] = nl[0]
1290 1290 return nl[0]
1291 1291 raise error.AmbiguousPrefixLookupError(
1292 1292 id, self.indexfile, _('ambiguous identifier'))
1293 1293 if maybewdir:
1294 1294 raise error.WdirUnsupported
1295 1295 return None
1296 1296 except TypeError:
1297 1297 pass
1298 1298
1299 1299 def lookup(self, id):
1300 1300 """locate a node based on:
1301 1301 - revision number or str(revision number)
1302 1302 - nodeid or subset of hex nodeid
1303 1303 """
1304 1304 n = self._match(id)
1305 1305 if n is not None:
1306 1306 return n
1307 1307 n = self._partialmatch(id)
1308 1308 if n:
1309 1309 return n
1310 1310
1311 1311 raise error.LookupError(id, self.indexfile, _('no match found'))
1312 1312
1313 1313 def shortest(self, node, minlength=1):
1314 1314 """Find the shortest unambiguous prefix that matches node."""
1315 1315 def isvalid(prefix):
1316 1316 try:
1317 1317 matchednode = self._partialmatch(prefix)
1318 1318 except error.AmbiguousPrefixLookupError:
1319 1319 return False
1320 1320 except error.WdirUnsupported:
1321 1321 # single 'ff...' match
1322 1322 return True
1323 1323 if matchednode is None:
1324 1324 raise error.LookupError(node, self.indexfile, _('no node'))
1325 1325 return True
1326 1326
1327 1327 def maybewdir(prefix):
1328 1328 return all(c == 'f' for c in pycompat.iterbytestr(prefix))
1329 1329
1330 1330 hexnode = hex(node)
1331 1331
1332 1332 def disambiguate(hexnode, minlength):
1333 1333 """Disambiguate against wdirid."""
1334 1334 for length in range(minlength, 41):
1335 1335 prefix = hexnode[:length]
1336 1336 if not maybewdir(prefix):
1337 1337 return prefix
1338 1338
1339 1339 if not getattr(self, 'filteredrevs', None):
1340 1340 try:
1341 1341 length = max(self.index.shortest(node), minlength)
1342 1342 return disambiguate(hexnode, length)
1343 1343 except error.RevlogError:
1344 1344 if node != wdirid:
1345 1345 raise error.LookupError(node, self.indexfile, _('no node'))
1346 1346 except AttributeError:
1347 1347 # Fall through to pure code
1348 1348 pass
1349 1349
1350 1350 if node == wdirid:
1351 1351 for length in range(minlength, 41):
1352 1352 prefix = hexnode[:length]
1353 1353 if isvalid(prefix):
1354 1354 return prefix
1355 1355
1356 1356 for length in range(minlength, 41):
1357 1357 prefix = hexnode[:length]
1358 1358 if isvalid(prefix):
1359 1359 return disambiguate(hexnode, length)
1360 1360
1361 1361 def cmp(self, node, text):
1362 1362 """compare text with a given file revision
1363 1363
1364 1364 returns True if text is different than what is stored.
1365 1365 """
1366 1366 p1, p2 = self.parents(node)
1367 1367 return storageutil.hashrevisionsha1(text, p1, p2) != node
1368 1368
1369 1369 def _cachesegment(self, offset, data):
1370 1370 """Add a segment to the revlog cache.
1371 1371
1372 1372 Accepts an absolute offset and the data that is at that location.
1373 1373 """
1374 1374 o, d = self._chunkcache
1375 1375 # try to add to existing cache
1376 1376 if o + len(d) == offset and len(d) + len(data) < _chunksize:
1377 1377 self._chunkcache = o, d + data
1378 1378 else:
1379 1379 self._chunkcache = offset, data
1380 1380
1381 1381 def _readsegment(self, offset, length, df=None):
1382 1382 """Load a segment of raw data from the revlog.
1383 1383
1384 1384 Accepts an absolute offset, length to read, and an optional existing
1385 1385 file handle to read from.
1386 1386
1387 1387 If an existing file handle is passed, it will be seeked and the
1388 1388 original seek position will NOT be restored.
1389 1389
1390 1390 Returns a str or buffer of raw byte data.
1391 1391
1392 1392 Raises if the requested number of bytes could not be read.
1393 1393 """
1394 1394 # Cache data both forward and backward around the requested
1395 1395 # data, in a fixed size window. This helps speed up operations
1396 1396 # involving reading the revlog backwards.
1397 1397 cachesize = self._chunkcachesize
1398 1398 realoffset = offset & ~(cachesize - 1)
1399 1399 reallength = (((offset + length + cachesize) & ~(cachesize - 1))
1400 1400 - realoffset)
1401 1401 with self._datareadfp(df) as df:
1402 1402 df.seek(realoffset)
1403 1403 d = df.read(reallength)
1404 1404
1405 1405 self._cachesegment(realoffset, d)
1406 1406 if offset != realoffset or reallength != length:
1407 1407 startoffset = offset - realoffset
1408 1408 if len(d) - startoffset < length:
1409 1409 raise error.RevlogError(
1410 1410 _('partial read of revlog %s; expected %d bytes from '
1411 1411 'offset %d, got %d') %
1412 1412 (self.indexfile if self._inline else self.datafile,
1413 1413 length, realoffset, len(d) - startoffset))
1414 1414
1415 1415 return util.buffer(d, startoffset, length)
1416 1416
1417 1417 if len(d) < length:
1418 1418 raise error.RevlogError(
1419 1419 _('partial read of revlog %s; expected %d bytes from offset '
1420 1420 '%d, got %d') %
1421 1421 (self.indexfile if self._inline else self.datafile,
1422 1422 length, offset, len(d)))
1423 1423
1424 1424 return d
1425 1425
1426 1426 def _getsegment(self, offset, length, df=None):
1427 1427 """Obtain a segment of raw data from the revlog.
1428 1428
1429 1429 Accepts an absolute offset, length of bytes to obtain, and an
1430 1430 optional file handle to the already-opened revlog. If the file
1431 1431 handle is used, it's original seek position will not be preserved.
1432 1432
1433 1433 Requests for data may be returned from a cache.
1434 1434
1435 1435 Returns a str or a buffer instance of raw byte data.
1436 1436 """
1437 1437 o, d = self._chunkcache
1438 1438 l = len(d)
1439 1439
1440 1440 # is it in the cache?
1441 1441 cachestart = offset - o
1442 1442 cacheend = cachestart + length
1443 1443 if cachestart >= 0 and cacheend <= l:
1444 1444 if cachestart == 0 and cacheend == l:
1445 1445 return d # avoid a copy
1446 1446 return util.buffer(d, cachestart, cacheend - cachestart)
1447 1447
1448 1448 return self._readsegment(offset, length, df=df)
1449 1449
1450 1450 def _getsegmentforrevs(self, startrev, endrev, df=None):
1451 1451 """Obtain a segment of raw data corresponding to a range of revisions.
1452 1452
1453 1453 Accepts the start and end revisions and an optional already-open
1454 1454 file handle to be used for reading. If the file handle is read, its
1455 1455 seek position will not be preserved.
1456 1456
1457 1457 Requests for data may be satisfied by a cache.
1458 1458
1459 1459 Returns a 2-tuple of (offset, data) for the requested range of
1460 1460 revisions. Offset is the integer offset from the beginning of the
1461 1461 revlog and data is a str or buffer of the raw byte data.
1462 1462
1463 1463 Callers will need to call ``self.start(rev)`` and ``self.length(rev)``
1464 1464 to determine where each revision's data begins and ends.
1465 1465 """
1466 1466 # Inlined self.start(startrev) & self.end(endrev) for perf reasons
1467 1467 # (functions are expensive).
1468 1468 index = self.index
1469 1469 istart = index[startrev]
1470 1470 start = int(istart[0] >> 16)
1471 1471 if startrev == endrev:
1472 1472 end = start + istart[1]
1473 1473 else:
1474 1474 iend = index[endrev]
1475 1475 end = int(iend[0] >> 16) + iend[1]
1476 1476
1477 1477 if self._inline:
1478 1478 start += (startrev + 1) * self._io.size
1479 1479 end += (endrev + 1) * self._io.size
1480 1480 length = end - start
1481 1481
1482 1482 return start, self._getsegment(start, length, df=df)
1483 1483
1484 1484 def _chunk(self, rev, df=None):
1485 1485 """Obtain a single decompressed chunk for a revision.
1486 1486
1487 1487 Accepts an integer revision and an optional already-open file handle
1488 1488 to be used for reading. If used, the seek position of the file will not
1489 1489 be preserved.
1490 1490
1491 1491 Returns a str holding uncompressed data for the requested revision.
1492 1492 """
1493 1493 return self.decompress(self._getsegmentforrevs(rev, rev, df=df)[1])
1494 1494
1495 1495 def _chunks(self, revs, df=None, targetsize=None):
1496 1496 """Obtain decompressed chunks for the specified revisions.
1497 1497
1498 1498 Accepts an iterable of numeric revisions that are assumed to be in
1499 1499 ascending order. Also accepts an optional already-open file handle
1500 1500 to be used for reading. If used, the seek position of the file will
1501 1501 not be preserved.
1502 1502
1503 1503 This function is similar to calling ``self._chunk()`` multiple times,
1504 1504 but is faster.
1505 1505
1506 1506 Returns a list with decompressed data for each requested revision.
1507 1507 """
1508 1508 if not revs:
1509 1509 return []
1510 1510 start = self.start
1511 1511 length = self.length
1512 1512 inline = self._inline
1513 1513 iosize = self._io.size
1514 1514 buffer = util.buffer
1515 1515
1516 1516 l = []
1517 1517 ladd = l.append
1518 1518
1519 1519 if not self._withsparseread:
1520 1520 slicedchunks = (revs,)
1521 1521 else:
1522 1522 slicedchunks = deltautil.slicechunk(self, revs,
1523 1523 targetsize=targetsize)
1524 1524
1525 1525 for revschunk in slicedchunks:
1526 1526 firstrev = revschunk[0]
1527 1527 # Skip trailing revisions with empty diff
1528 1528 for lastrev in revschunk[::-1]:
1529 1529 if length(lastrev) != 0:
1530 1530 break
1531 1531
1532 1532 try:
1533 1533 offset, data = self._getsegmentforrevs(firstrev, lastrev, df=df)
1534 1534 except OverflowError:
1535 1535 # issue4215 - we can't cache a run of chunks greater than
1536 1536 # 2G on Windows
1537 1537 return [self._chunk(rev, df=df) for rev in revschunk]
1538 1538
1539 1539 decomp = self.decompress
1540 1540 for rev in revschunk:
1541 1541 chunkstart = start(rev)
1542 1542 if inline:
1543 1543 chunkstart += (rev + 1) * iosize
1544 1544 chunklength = length(rev)
1545 1545 ladd(decomp(buffer(data, chunkstart - offset, chunklength)))
1546 1546
1547 1547 return l
1548 1548
1549 1549 def _chunkclear(self):
1550 1550 """Clear the raw chunk cache."""
1551 1551 self._chunkcache = (0, '')
1552 1552
1553 1553 def deltaparent(self, rev):
1554 1554 """return deltaparent of the given revision"""
1555 1555 base = self.index[rev][3]
1556 1556 if base == rev:
1557 1557 return nullrev
1558 1558 elif self._generaldelta:
1559 1559 return base
1560 1560 else:
1561 1561 return rev - 1
1562 1562
1563 1563 def issnapshot(self, rev):
1564 1564 """tells whether rev is a snapshot
1565 1565 """
1566 1566 if not self._sparserevlog:
1567 1567 return self.deltaparent(rev) == nullrev
1568 1568 elif util.safehasattr(self.index, 'issnapshot'):
1569 1569 # directly assign the method to cache the testing and access
1570 1570 self.issnapshot = self.index.issnapshot
1571 1571 return self.issnapshot(rev)
1572 1572 if rev == nullrev:
1573 1573 return True
1574 1574 entry = self.index[rev]
1575 1575 base = entry[3]
1576 1576 if base == rev:
1577 1577 return True
1578 1578 if base == nullrev:
1579 1579 return True
1580 1580 p1 = entry[5]
1581 1581 p2 = entry[6]
1582 1582 if base == p1 or base == p2:
1583 1583 return False
1584 1584 return self.issnapshot(base)
1585 1585
1586 1586 def snapshotdepth(self, rev):
1587 1587 """number of snapshot in the chain before this one"""
1588 1588 if not self.issnapshot(rev):
1589 1589 raise error.ProgrammingError('revision %d not a snapshot')
1590 1590 return len(self._deltachain(rev)[0]) - 1
1591 1591
1592 1592 def revdiff(self, rev1, rev2):
1593 1593 """return or calculate a delta between two revisions
1594 1594
1595 1595 The delta calculated is in binary form and is intended to be written to
1596 1596 revlog data directly. So this function needs raw revision data.
1597 1597 """
1598 1598 if rev1 != nullrev and self.deltaparent(rev2) == rev1:
1599 1599 return bytes(self._chunk(rev2))
1600 1600
1601 1601 return mdiff.textdiff(self.rawdata(rev1),
1602 1602 self.rawdata(rev2))
1603 1603
1604 1604 def revision(self, nodeorrev, _df=None, raw=False):
1605 1605 """return an uncompressed revision of a given node or revision
1606 1606 number.
1607 1607
1608 1608 _df - an existing file handle to read from. (internal-only)
1609 1609 raw - an optional argument specifying if the revision data is to be
1610 1610 treated as raw data when applying flag transforms. 'raw' should be set
1611 1611 to True when generating changegroups or in debug commands.
1612 1612 """
1613 1613 if raw:
1614 1614 msg = ('revlog.revision(..., raw=True) is deprecated, '
1615 1615 'use revlog.rawdata(...)')
1616 1616 util.nouideprecwarn(msg, '5.2', stacklevel=2)
1617 1617 return self._revisiondata(nodeorrev, _df, raw=raw)[0]
1618 1618
1619 1619 def sidedata(self, nodeorrev, _df=None):
1620 1620 """a map of extra data related to the changeset but not part of the hash
1621 1621
1622 1622 This function currently return a dictionary. However, more advanced
1623 1623 mapping object will likely be used in the future for a more
1624 1624 efficient/lazy code.
1625 1625 """
1626 1626 return self._revisiondata(nodeorrev, _df)[1]
1627 1627
1628 1628 def _revisiondata(self, nodeorrev, _df=None, raw=False):
1629 1629 # deal with <nodeorrev> argument type
1630 1630 if isinstance(nodeorrev, int):
1631 1631 rev = nodeorrev
1632 1632 node = self.node(rev)
1633 1633 else:
1634 1634 node = nodeorrev
1635 1635 rev = None
1636 1636
1637 1637 # fast path the special `nullid` rev
1638 1638 if node == nullid:
1639 1639 return "", {}
1640 1640
1641 1641 # The text as stored inside the revlog. Might be the revision or might
1642 1642 # need to be processed to retrieve the revision.
1643 1643 rawtext = None
1644 1644
1645 1645 rev, rawtext, validated = self._rawtext(node, rev, _df=_df)
1646 1646
1647 1647 if raw and validated:
1648 1648 # if we don't want to process the raw text and that raw
1649 1649 # text is cached, we can exit early.
1650 1650 return rawtext, {}
1651 1651 if rev is None:
1652 1652 rev = self.rev(node)
1653 1653 # the revlog's flag for this revision
1654 1654 # (usually alter its state or content)
1655 1655 flags = self.flags(rev)
1656 1656
1657 1657 if validated and flags == REVIDX_DEFAULT_FLAGS:
1658 1658 # no extra flags set, no flag processor runs, text = rawtext
1659 1659 return rawtext, {}
1660 1660
1661 1661 sidedata = {}
1662 1662 if raw:
1663 1663 validatehash = self._processflagsraw(rawtext, flags)
1664 1664 text = rawtext
1665 1665 else:
1666 1666 r = self._processflagsread(rawtext, flags)
1667 1667 text, validatehash, sidedata = r
1668 1668 if validatehash:
1669 1669 self.checkhash(text, node, rev=rev)
1670 1670 if not validated:
1671 1671 self._revisioncache = (node, rev, rawtext)
1672 1672
1673 1673 return text, sidedata
1674 1674
1675 1675 def _rawtext(self, node, rev, _df=None):
1676 1676 """return the possibly unvalidated rawtext for a revision
1677 1677
1678 1678 returns (rev, rawtext, validated)
1679 1679 """
1680 1680
1681 1681 # revision in the cache (could be useful to apply delta)
1682 1682 cachedrev = None
1683 1683 # An intermediate text to apply deltas to
1684 1684 basetext = None
1685 1685
1686 1686 # Check if we have the entry in cache
1687 1687 # The cache entry looks like (node, rev, rawtext)
1688 1688 if self._revisioncache:
1689 1689 if self._revisioncache[0] == node:
1690 1690 return (rev, self._revisioncache[2], True)
1691 1691 cachedrev = self._revisioncache[1]
1692 1692
1693 1693 if rev is None:
1694 1694 rev = self.rev(node)
1695 1695
1696 1696 chain, stopped = self._deltachain(rev, stoprev=cachedrev)
1697 1697 if stopped:
1698 1698 basetext = self._revisioncache[2]
1699 1699
1700 1700 # drop cache to save memory, the caller is expected to
1701 1701 # update self._revisioncache after validating the text
1702 1702 self._revisioncache = None
1703 1703
1704 1704 targetsize = None
1705 1705 rawsize = self.index[rev][2]
1706 1706 if 0 <= rawsize:
1707 1707 targetsize = 4 * rawsize
1708 1708
1709 1709 bins = self._chunks(chain, df=_df, targetsize=targetsize)
1710 1710 if basetext is None:
1711 1711 basetext = bytes(bins[0])
1712 1712 bins = bins[1:]
1713 1713
1714 1714 rawtext = mdiff.patches(basetext, bins)
1715 1715 del basetext # let us have a chance to free memory early
1716 1716 return (rev, rawtext, False)
1717 1717
1718 1718 def rawdata(self, nodeorrev, _df=None):
1719 1719 """return an uncompressed raw data of a given node or revision number.
1720 1720
1721 1721 _df - an existing file handle to read from. (internal-only)
1722 1722 """
1723 1723 return self._revisiondata(nodeorrev, _df, raw=True)[0]
1724 1724
1725 1725 def hash(self, text, p1, p2):
1726 1726 """Compute a node hash.
1727 1727
1728 1728 Available as a function so that subclasses can replace the hash
1729 1729 as needed.
1730 1730 """
1731 1731 return storageutil.hashrevisionsha1(text, p1, p2)
1732 1732
1733 1733 def checkhash(self, text, node, p1=None, p2=None, rev=None):
1734 1734 """Check node hash integrity.
1735 1735
1736 1736 Available as a function so that subclasses can extend hash mismatch
1737 1737 behaviors as needed.
1738 1738 """
1739 1739 try:
1740 1740 if p1 is None and p2 is None:
1741 1741 p1, p2 = self.parents(node)
1742 1742 if node != self.hash(text, p1, p2):
1743 1743 # Clear the revision cache on hash failure. The revision cache
1744 1744 # only stores the raw revision and clearing the cache does have
1745 1745 # the side-effect that we won't have a cache hit when the raw
1746 1746 # revision data is accessed. But this case should be rare and
1747 1747 # it is extra work to teach the cache about the hash
1748 1748 # verification state.
1749 1749 if self._revisioncache and self._revisioncache[0] == node:
1750 1750 self._revisioncache = None
1751 1751
1752 1752 revornode = rev
1753 1753 if revornode is None:
1754 1754 revornode = templatefilters.short(hex(node))
1755 1755 raise error.RevlogError(_("integrity check failed on %s:%s")
1756 1756 % (self.indexfile, pycompat.bytestr(revornode)))
1757 1757 except error.RevlogError:
1758 1758 if self._censorable and storageutil.iscensoredtext(text):
1759 1759 raise error.CensoredNodeError(self.indexfile, node, text)
1760 1760 raise
1761 1761
1762 1762 def _enforceinlinesize(self, tr, fp=None):
1763 1763 """Check if the revlog is too big for inline and convert if so.
1764 1764
1765 1765 This should be called after revisions are added to the revlog. If the
1766 1766 revlog has grown too large to be an inline revlog, it will convert it
1767 1767 to use multiple index and data files.
1768 1768 """
1769 1769 tiprev = len(self) - 1
1770 1770 if (not self._inline or
1771 1771 (self.start(tiprev) + self.length(tiprev)) < _maxinline):
1772 1772 return
1773 1773
1774 1774 trinfo = tr.find(self.indexfile)
1775 1775 if trinfo is None:
1776 1776 raise error.RevlogError(_("%s not found in the transaction")
1777 1777 % self.indexfile)
1778 1778
1779 1779 trindex = trinfo[2]
1780 1780 if trindex is not None:
1781 1781 dataoff = self.start(trindex)
1782 1782 else:
1783 1783 # revlog was stripped at start of transaction, use all leftover data
1784 1784 trindex = len(self) - 1
1785 1785 dataoff = self.end(tiprev)
1786 1786
1787 1787 tr.add(self.datafile, dataoff)
1788 1788
1789 1789 if fp:
1790 1790 fp.flush()
1791 1791 fp.close()
1792 1792 # We can't use the cached file handle after close(). So prevent
1793 1793 # its usage.
1794 1794 self._writinghandles = None
1795 1795
1796 1796 with self._indexfp('r') as ifh, self._datafp('w') as dfh:
1797 1797 for r in self:
1798 1798 dfh.write(self._getsegmentforrevs(r, r, df=ifh)[1])
1799 1799
1800 1800 with self._indexfp('w') as fp:
1801 1801 self.version &= ~FLAG_INLINE_DATA
1802 1802 self._inline = False
1803 1803 io = self._io
1804 1804 for i in self:
1805 1805 e = io.packentry(self.index[i], self.node, self.version, i)
1806 1806 fp.write(e)
1807 1807
1808 1808 # the temp file replace the real index when we exit the context
1809 1809 # manager
1810 1810
1811 1811 tr.replace(self.indexfile, trindex * self._io.size)
1812 1812 self._chunkclear()
1813 1813
1814 1814 def _nodeduplicatecallback(self, transaction, node):
1815 1815 """called when trying to add a node already stored.
1816 1816 """
1817 1817
1818 1818 def addrevision(self, text, transaction, link, p1, p2, cachedelta=None,
1819 1819 node=None, flags=REVIDX_DEFAULT_FLAGS, deltacomputer=None):
1820 1820 """add a revision to the log
1821 1821
1822 1822 text - the revision data to add
1823 1823 transaction - the transaction object used for rollback
1824 1824 link - the linkrev data to add
1825 1825 p1, p2 - the parent nodeids of the revision
1826 1826 cachedelta - an optional precomputed delta
1827 1827 node - nodeid of revision; typically node is not specified, and it is
1828 1828 computed by default as hash(text, p1, p2), however subclasses might
1829 1829 use different hashing method (and override checkhash() in such case)
1830 1830 flags - the known flags to set on the revision
1831 1831 deltacomputer - an optional deltacomputer instance shared between
1832 1832 multiple calls
1833 1833 """
1834 1834 if link == nullrev:
1835 1835 raise error.RevlogError(_("attempted to add linkrev -1 to %s")
1836 1836 % self.indexfile)
1837 1837
1838 1838 if flags:
1839 1839 node = node or self.hash(text, p1, p2)
1840 1840
1841 1841 rawtext, validatehash = self._processflagswrite(text, flags)
1842 1842
1843 1843 # If the flag processor modifies the revision data, ignore any provided
1844 1844 # cachedelta.
1845 1845 if rawtext != text:
1846 1846 cachedelta = None
1847 1847
1848 1848 if len(rawtext) > _maxentrysize:
1849 1849 raise error.RevlogError(
1850 1850 _("%s: size of %d bytes exceeds maximum revlog storage of 2GiB")
1851 1851 % (self.indexfile, len(rawtext)))
1852 1852
1853 1853 node = node or self.hash(rawtext, p1, p2)
1854 1854 if node in self.nodemap:
1855 1855 return node
1856 1856
1857 1857 if validatehash:
1858 1858 self.checkhash(rawtext, node, p1=p1, p2=p2)
1859 1859
1860 1860 return self.addrawrevision(rawtext, transaction, link, p1, p2, node,
1861 1861 flags, cachedelta=cachedelta,
1862 1862 deltacomputer=deltacomputer)
1863 1863
1864 1864 def addrawrevision(self, rawtext, transaction, link, p1, p2, node, flags,
1865 1865 cachedelta=None, deltacomputer=None):
1866 1866 """add a raw revision with known flags, node and parents
1867 1867 useful when reusing a revision not stored in this revlog (ex: received
1868 1868 over wire, or read from an external bundle).
1869 1869 """
1870 1870 dfh = None
1871 1871 if not self._inline:
1872 1872 dfh = self._datafp("a+")
1873 1873 ifh = self._indexfp("a+")
1874 1874 try:
1875 1875 return self._addrevision(node, rawtext, transaction, link, p1, p2,
1876 1876 flags, cachedelta, ifh, dfh,
1877 1877 deltacomputer=deltacomputer)
1878 1878 finally:
1879 1879 if dfh:
1880 1880 dfh.close()
1881 1881 ifh.close()
1882 1882
1883 1883 def compress(self, data):
1884 1884 """Generate a possibly-compressed representation of data."""
1885 1885 if not data:
1886 1886 return '', data
1887 1887
1888 1888 compressed = self._compressor.compress(data)
1889 1889
1890 1890 if compressed:
1891 1891 # The revlog compressor added the header in the returned data.
1892 1892 return '', compressed
1893 1893
1894 1894 if data[0:1] == '\0':
1895 1895 return '', data
1896 1896 return 'u', data
1897 1897
1898 1898 def decompress(self, data):
1899 1899 """Decompress a revlog chunk.
1900 1900
1901 1901 The chunk is expected to begin with a header identifying the
1902 1902 format type so it can be routed to an appropriate decompressor.
1903 1903 """
1904 1904 if not data:
1905 1905 return data
1906 1906
1907 1907 # Revlogs are read much more frequently than they are written and many
1908 1908 # chunks only take microseconds to decompress, so performance is
1909 1909 # important here.
1910 1910 #
1911 1911 # We can make a few assumptions about revlogs:
1912 1912 #
1913 1913 # 1) the majority of chunks will be compressed (as opposed to inline
1914 1914 # raw data).
1915 1915 # 2) decompressing *any* data will likely by at least 10x slower than
1916 1916 # returning raw inline data.
1917 1917 # 3) we want to prioritize common and officially supported compression
1918 1918 # engines
1919 1919 #
1920 1920 # It follows that we want to optimize for "decompress compressed data
1921 1921 # when encoded with common and officially supported compression engines"
1922 1922 # case over "raw data" and "data encoded by less common or non-official
1923 1923 # compression engines." That is why we have the inline lookup first
1924 1924 # followed by the compengines lookup.
1925 1925 #
1926 1926 # According to `hg perfrevlogchunks`, this is ~0.5% faster for zlib
1927 1927 # compressed chunks. And this matters for changelog and manifest reads.
1928 1928 t = data[0:1]
1929 1929
1930 1930 if t == 'x':
1931 1931 try:
1932 1932 return _zlibdecompress(data)
1933 1933 except zlib.error as e:
1934 1934 raise error.RevlogError(_('revlog decompress error: %s') %
1935 1935 stringutil.forcebytestr(e))
1936 1936 # '\0' is more common than 'u' so it goes first.
1937 1937 elif t == '\0':
1938 1938 return data
1939 1939 elif t == 'u':
1940 1940 return util.buffer(data, 1)
1941 1941
1942 1942 try:
1943 1943 compressor = self._decompressors[t]
1944 1944 except KeyError:
1945 1945 try:
1946 1946 engine = util.compengines.forrevlogheader(t)
1947 1947 compressor = engine.revlogcompressor(self._compengineopts)
1948 1948 self._decompressors[t] = compressor
1949 1949 except KeyError:
1950 1950 raise error.RevlogError(_('unknown compression type %r') % t)
1951 1951
1952 1952 return compressor.decompress(data)
1953 1953
1954 1954 def _addrevision(self, node, rawtext, transaction, link, p1, p2, flags,
1955 1955 cachedelta, ifh, dfh, alwayscache=False,
1956 1956 deltacomputer=None):
1957 1957 """internal function to add revisions to the log
1958 1958
1959 1959 see addrevision for argument descriptions.
1960 1960
1961 1961 note: "addrevision" takes non-raw text, "_addrevision" takes raw text.
1962 1962
1963 1963 if "deltacomputer" is not provided or None, a defaultdeltacomputer will
1964 1964 be used.
1965 1965
1966 1966 invariants:
1967 1967 - rawtext is optional (can be None); if not set, cachedelta must be set.
1968 1968 if both are set, they must correspond to each other.
1969 1969 """
1970 1970 if node == nullid:
1971 1971 raise error.RevlogError(_("%s: attempt to add null revision") %
1972 1972 self.indexfile)
1973 1973 if node == wdirid or node in wdirfilenodeids:
1974 1974 raise error.RevlogError(_("%s: attempt to add wdir revision") %
1975 1975 self.indexfile)
1976 1976
1977 1977 if self._inline:
1978 1978 fh = ifh
1979 1979 else:
1980 1980 fh = dfh
1981 1981
1982 1982 btext = [rawtext]
1983 1983
1984 1984 curr = len(self)
1985 1985 prev = curr - 1
1986 1986 offset = self.end(prev)
1987 1987 p1r, p2r = self.rev(p1), self.rev(p2)
1988 1988
1989 1989 # full versions are inserted when the needed deltas
1990 1990 # become comparable to the uncompressed text
1991 1991 if rawtext is None:
1992 1992 # need rawtext size, before changed by flag processors, which is
1993 1993 # the non-raw size. use revlog explicitly to avoid filelog's extra
1994 1994 # logic that might remove metadata size.
1995 1995 textlen = mdiff.patchedsize(revlog.size(self, cachedelta[0]),
1996 1996 cachedelta[1])
1997 1997 else:
1998 1998 textlen = len(rawtext)
1999 1999
2000 2000 if deltacomputer is None:
2001 2001 deltacomputer = deltautil.deltacomputer(self)
2002 2002
2003 2003 revinfo = _revisioninfo(node, p1, p2, btext, textlen, cachedelta, flags)
2004 2004
2005 2005 deltainfo = deltacomputer.finddeltainfo(revinfo, fh)
2006 2006
2007 2007 e = (offset_type(offset, flags), deltainfo.deltalen, textlen,
2008 2008 deltainfo.base, link, p1r, p2r, node)
2009 2009 self.index.append(e)
2010 2010 self.nodemap[node] = curr
2011 2011
2012 2012 # Reset the pure node cache start lookup offset to account for new
2013 2013 # revision.
2014 2014 if self._nodepos is not None:
2015 2015 self._nodepos = curr
2016 2016
2017 2017 entry = self._io.packentry(e, self.node, self.version, curr)
2018 2018 self._writeentry(transaction, ifh, dfh, entry, deltainfo.data,
2019 2019 link, offset)
2020 2020
2021 2021 rawtext = btext[0]
2022 2022
2023 2023 if alwayscache and rawtext is None:
2024 2024 rawtext = deltacomputer.buildtext(revinfo, fh)
2025 2025
2026 2026 if type(rawtext) == bytes: # only accept immutable objects
2027 2027 self._revisioncache = (node, curr, rawtext)
2028 2028 self._chainbasecache[curr] = deltainfo.chainbase
2029 2029 return node
2030 2030
2031 2031 def _writeentry(self, transaction, ifh, dfh, entry, data, link, offset):
2032 2032 # Files opened in a+ mode have inconsistent behavior on various
2033 2033 # platforms. Windows requires that a file positioning call be made
2034 2034 # when the file handle transitions between reads and writes. See
2035 2035 # 3686fa2b8eee and the mixedfilemodewrapper in windows.py. On other
2036 2036 # platforms, Python or the platform itself can be buggy. Some versions
2037 2037 # of Solaris have been observed to not append at the end of the file
2038 2038 # if the file was seeked to before the end. See issue4943 for more.
2039 2039 #
2040 2040 # We work around this issue by inserting a seek() before writing.
2041 2041 # Note: This is likely not necessary on Python 3. However, because
2042 2042 # the file handle is reused for reads and may be seeked there, we need
2043 2043 # to be careful before changing this.
2044 2044 ifh.seek(0, os.SEEK_END)
2045 2045 if dfh:
2046 2046 dfh.seek(0, os.SEEK_END)
2047 2047
2048 2048 curr = len(self) - 1
2049 2049 if not self._inline:
2050 2050 transaction.add(self.datafile, offset)
2051 2051 transaction.add(self.indexfile, curr * len(entry))
2052 2052 if data[0]:
2053 2053 dfh.write(data[0])
2054 2054 dfh.write(data[1])
2055 2055 ifh.write(entry)
2056 2056 else:
2057 2057 offset += curr * self._io.size
2058 2058 transaction.add(self.indexfile, offset, curr)
2059 2059 ifh.write(entry)
2060 2060 ifh.write(data[0])
2061 2061 ifh.write(data[1])
2062 2062 self._enforceinlinesize(transaction, ifh)
2063 2063
2064 2064 def addgroup(self, deltas, linkmapper, transaction, addrevisioncb=None):
2065 2065 """
2066 2066 add a delta group
2067 2067
2068 2068 given a set of deltas, add them to the revision log. the
2069 2069 first delta is against its parent, which should be in our
2070 2070 log, the rest are against the previous delta.
2071 2071
2072 2072 If ``addrevisioncb`` is defined, it will be called with arguments of
2073 2073 this revlog and the node that was added.
2074 2074 """
2075 2075
2076 2076 if self._writinghandles:
2077 2077 raise error.ProgrammingError('cannot nest addgroup() calls')
2078 2078
2079 2079 nodes = []
2080 2080
2081 2081 r = len(self)
2082 2082 end = 0
2083 2083 if r:
2084 2084 end = self.end(r - 1)
2085 2085 ifh = self._indexfp("a+")
2086 2086 isize = r * self._io.size
2087 2087 if self._inline:
2088 2088 transaction.add(self.indexfile, end + isize, r)
2089 2089 dfh = None
2090 2090 else:
2091 2091 transaction.add(self.indexfile, isize, r)
2092 2092 transaction.add(self.datafile, end)
2093 2093 dfh = self._datafp("a+")
2094 2094 def flush():
2095 2095 if dfh:
2096 2096 dfh.flush()
2097 2097 ifh.flush()
2098 2098
2099 2099 self._writinghandles = (ifh, dfh)
2100 2100
2101 2101 try:
2102 2102 deltacomputer = deltautil.deltacomputer(self)
2103 2103 # loop through our set of deltas
2104 2104 for data in deltas:
2105 2105 node, p1, p2, linknode, deltabase, delta, flags = data
2106 2106 link = linkmapper(linknode)
2107 2107 flags = flags or REVIDX_DEFAULT_FLAGS
2108 2108
2109 2109 nodes.append(node)
2110 2110
2111 2111 if node in self.nodemap:
2112 2112 self._nodeduplicatecallback(transaction, node)
2113 2113 # this can happen if two branches make the same change
2114 2114 continue
2115 2115
2116 2116 for p in (p1, p2):
2117 2117 if p not in self.nodemap:
2118 2118 raise error.LookupError(p, self.indexfile,
2119 2119 _('unknown parent'))
2120 2120
2121 2121 if deltabase not in self.nodemap:
2122 2122 raise error.LookupError(deltabase, self.indexfile,
2123 2123 _('unknown delta base'))
2124 2124
2125 2125 baserev = self.rev(deltabase)
2126 2126
2127 2127 if baserev != nullrev and self.iscensored(baserev):
2128 2128 # if base is censored, delta must be full replacement in a
2129 2129 # single patch operation
2130 2130 hlen = struct.calcsize(">lll")
2131 2131 oldlen = self.rawsize(baserev)
2132 2132 newlen = len(delta) - hlen
2133 2133 if delta[:hlen] != mdiff.replacediffheader(oldlen, newlen):
2134 2134 raise error.CensoredBaseError(self.indexfile,
2135 2135 self.node(baserev))
2136 2136
2137 2137 if not flags and self._peek_iscensored(baserev, delta, flush):
2138 2138 flags |= REVIDX_ISCENSORED
2139 2139
2140 2140 # We assume consumers of addrevisioncb will want to retrieve
2141 2141 # the added revision, which will require a call to
2142 2142 # revision(). revision() will fast path if there is a cache
2143 2143 # hit. So, we tell _addrevision() to always cache in this case.
2144 2144 # We're only using addgroup() in the context of changegroup
2145 2145 # generation so the revision data can always be handled as raw
2146 2146 # by the flagprocessor.
2147 2147 self._addrevision(node, None, transaction, link,
2148 2148 p1, p2, flags, (baserev, delta),
2149 2149 ifh, dfh,
2150 2150 alwayscache=bool(addrevisioncb),
2151 2151 deltacomputer=deltacomputer)
2152 2152
2153 2153 if addrevisioncb:
2154 2154 addrevisioncb(self, node)
2155 2155
2156 2156 if not dfh and not self._inline:
2157 2157 # addrevision switched from inline to conventional
2158 2158 # reopen the index
2159 2159 ifh.close()
2160 2160 dfh = self._datafp("a+")
2161 2161 ifh = self._indexfp("a+")
2162 2162 self._writinghandles = (ifh, dfh)
2163 2163 finally:
2164 2164 self._writinghandles = None
2165 2165
2166 2166 if dfh:
2167 2167 dfh.close()
2168 2168 ifh.close()
2169 2169
2170 2170 return nodes
2171 2171
2172 2172 def iscensored(self, rev):
2173 2173 """Check if a file revision is censored."""
2174 2174 if not self._censorable:
2175 2175 return False
2176 2176
2177 2177 return self.flags(rev) & REVIDX_ISCENSORED
2178 2178
2179 2179 def _peek_iscensored(self, baserev, delta, flush):
2180 2180 """Quickly check if a delta produces a censored revision."""
2181 2181 if not self._censorable:
2182 2182 return False
2183 2183
2184 2184 return storageutil.deltaiscensored(delta, baserev, self.rawsize)
2185 2185
2186 2186 def getstrippoint(self, minlink):
2187 2187 """find the minimum rev that must be stripped to strip the linkrev
2188 2188
2189 2189 Returns a tuple containing the minimum rev and a set of all revs that
2190 2190 have linkrevs that will be broken by this strip.
2191 2191 """
2192 2192 return storageutil.resolvestripinfo(minlink, len(self) - 1,
2193 2193 self.headrevs(),
2194 2194 self.linkrev, self.parentrevs)
2195 2195
2196 2196 def strip(self, minlink, transaction):
2197 2197 """truncate the revlog on the first revision with a linkrev >= minlink
2198 2198
2199 2199 This function is called when we're stripping revision minlink and
2200 2200 its descendants from the repository.
2201 2201
2202 2202 We have to remove all revisions with linkrev >= minlink, because
2203 2203 the equivalent changelog revisions will be renumbered after the
2204 2204 strip.
2205 2205
2206 2206 So we truncate the revlog on the first of these revisions, and
2207 2207 trust that the caller has saved the revisions that shouldn't be
2208 2208 removed and that it'll re-add them after this truncation.
2209 2209 """
2210 2210 if len(self) == 0:
2211 2211 return
2212 2212
2213 2213 rev, _ = self.getstrippoint(minlink)
2214 2214 if rev == len(self):
2215 2215 return
2216 2216
2217 2217 # first truncate the files on disk
2218 2218 end = self.start(rev)
2219 2219 if not self._inline:
2220 2220 transaction.add(self.datafile, end)
2221 2221 end = rev * self._io.size
2222 2222 else:
2223 2223 end += rev * self._io.size
2224 2224
2225 2225 transaction.add(self.indexfile, end)
2226 2226
2227 2227 # then reset internal state in memory to forget those revisions
2228 2228 self._revisioncache = None
2229 2229 self._chaininfocache = {}
2230 2230 self._chunkclear()
2231 2231 for x in pycompat.xrange(rev, len(self)):
2232 2232 del self.nodemap[self.node(x)]
2233 2233
2234 2234 del self.index[rev:-1]
2235 2235 self._nodepos = None
2236 2236
2237 2237 def checksize(self):
2238 2238 """Check size of index and data files
2239 2239
2240 2240 return a (dd, di) tuple.
2241 2241 - dd: extra bytes for the "data" file
2242 2242 - di: extra bytes for the "index" file
2243 2243
2244 2244 A healthy revlog will return (0, 0).
2245 2245 """
2246 2246 expected = 0
2247 2247 if len(self):
2248 2248 expected = max(0, self.end(len(self) - 1))
2249 2249
2250 2250 try:
2251 2251 with self._datafp() as f:
2252 2252 f.seek(0, io.SEEK_END)
2253 2253 actual = f.tell()
2254 2254 dd = actual - expected
2255 2255 except IOError as inst:
2256 2256 if inst.errno != errno.ENOENT:
2257 2257 raise
2258 2258 dd = 0
2259 2259
2260 2260 try:
2261 2261 f = self.opener(self.indexfile)
2262 2262 f.seek(0, io.SEEK_END)
2263 2263 actual = f.tell()
2264 2264 f.close()
2265 2265 s = self._io.size
2266 2266 i = max(0, actual // s)
2267 2267 di = actual - (i * s)
2268 2268 if self._inline:
2269 2269 databytes = 0
2270 2270 for r in self:
2271 2271 databytes += max(0, self.length(r))
2272 2272 dd = 0
2273 2273 di = actual - len(self) * s - databytes
2274 2274 except IOError as inst:
2275 2275 if inst.errno != errno.ENOENT:
2276 2276 raise
2277 2277 di = 0
2278 2278
2279 2279 return (dd, di)
2280 2280
2281 2281 def files(self):
2282 2282 res = [self.indexfile]
2283 2283 if not self._inline:
2284 2284 res.append(self.datafile)
2285 2285 return res
2286 2286
2287 2287 def emitrevisions(self, nodes, nodesorder=None, revisiondata=False,
2288 2288 assumehaveparentrevisions=False,
2289 2289 deltamode=repository.CG_DELTAMODE_STD):
2290 2290 if nodesorder not in ('nodes', 'storage', 'linear', None):
2291 2291 raise error.ProgrammingError('unhandled value for nodesorder: %s' %
2292 2292 nodesorder)
2293 2293
2294 2294 if nodesorder is None and not self._generaldelta:
2295 2295 nodesorder = 'storage'
2296 2296
2297 2297 if (not self._storedeltachains and
2298 2298 deltamode != repository.CG_DELTAMODE_PREV):
2299 2299 deltamode = repository.CG_DELTAMODE_FULL
2300 2300
2301 2301 return storageutil.emitrevisions(
2302 2302 self, nodes, nodesorder, revlogrevisiondelta,
2303 2303 deltaparentfn=self.deltaparent,
2304 2304 candeltafn=self.candelta,
2305 2305 rawsizefn=self.rawsize,
2306 2306 revdifffn=self.revdiff,
2307 2307 flagsfn=self.flags,
2308 2308 deltamode=deltamode,
2309 2309 revisiondata=revisiondata,
2310 2310 assumehaveparentrevisions=assumehaveparentrevisions)
2311 2311
2312 2312 DELTAREUSEALWAYS = 'always'
2313 2313 DELTAREUSESAMEREVS = 'samerevs'
2314 2314 DELTAREUSENEVER = 'never'
2315 2315
2316 2316 DELTAREUSEFULLADD = 'fulladd'
2317 2317
2318 2318 DELTAREUSEALL = {'always', 'samerevs', 'never', 'fulladd'}
2319 2319
2320 2320 def clone(self, tr, destrevlog, addrevisioncb=None,
2321 2321 deltareuse=DELTAREUSESAMEREVS, forcedeltabothparents=None):
2322 2322 """Copy this revlog to another, possibly with format changes.
2323 2323
2324 2324 The destination revlog will contain the same revisions and nodes.
2325 2325 However, it may not be bit-for-bit identical due to e.g. delta encoding
2326 2326 differences.
2327 2327
2328 2328 The ``deltareuse`` argument control how deltas from the existing revlog
2329 2329 are preserved in the destination revlog. The argument can have the
2330 2330 following values:
2331 2331
2332 2332 DELTAREUSEALWAYS
2333 2333 Deltas will always be reused (if possible), even if the destination
2334 2334 revlog would not select the same revisions for the delta. This is the
2335 2335 fastest mode of operation.
2336 2336 DELTAREUSESAMEREVS
2337 2337 Deltas will be reused if the destination revlog would pick the same
2338 2338 revisions for the delta. This mode strikes a balance between speed
2339 2339 and optimization.
2340 2340 DELTAREUSENEVER
2341 2341 Deltas will never be reused. This is the slowest mode of execution.
2342 2342 This mode can be used to recompute deltas (e.g. if the diff/delta
2343 2343 algorithm changes).
2344 2344
2345 2345 Delta computation can be slow, so the choice of delta reuse policy can
2346 2346 significantly affect run time.
2347 2347
2348 2348 The default policy (``DELTAREUSESAMEREVS``) strikes a balance between
2349 2349 two extremes. Deltas will be reused if they are appropriate. But if the
2350 2350 delta could choose a better revision, it will do so. This means if you
2351 2351 are converting a non-generaldelta revlog to a generaldelta revlog,
2352 2352 deltas will be recomputed if the delta's parent isn't a parent of the
2353 2353 revision.
2354 2354
2355 2355 In addition to the delta policy, the ``forcedeltabothparents``
2356 2356 argument controls whether to force compute deltas against both parents
2357 2357 for merges. By default, the current default is used.
2358 2358 """
2359 2359 if deltareuse not in self.DELTAREUSEALL:
2360 2360 raise ValueError(_('value for deltareuse invalid: %s') % deltareuse)
2361 2361
2362 2362 if len(destrevlog):
2363 2363 raise ValueError(_('destination revlog is not empty'))
2364 2364
2365 2365 if getattr(self, 'filteredrevs', None):
2366 2366 raise ValueError(_('source revlog has filtered revisions'))
2367 2367 if getattr(destrevlog, 'filteredrevs', None):
2368 2368 raise ValueError(_('destination revlog has filtered revisions'))
2369 2369
2370 2370 # lazydelta and lazydeltabase controls whether to reuse a cached delta,
2371 2371 # if possible.
2372 2372 oldlazydelta = destrevlog._lazydelta
2373 2373 oldlazydeltabase = destrevlog._lazydeltabase
2374 2374 oldamd = destrevlog._deltabothparents
2375 2375
2376 2376 try:
2377 2377 if deltareuse == self.DELTAREUSEALWAYS:
2378 2378 destrevlog._lazydeltabase = True
2379 2379 destrevlog._lazydelta = True
2380 2380 elif deltareuse == self.DELTAREUSESAMEREVS:
2381 2381 destrevlog._lazydeltabase = False
2382 2382 destrevlog._lazydelta = True
2383 2383 elif deltareuse == self.DELTAREUSENEVER:
2384 2384 destrevlog._lazydeltabase = False
2385 2385 destrevlog._lazydelta = False
2386 2386
2387 2387 destrevlog._deltabothparents = forcedeltabothparents or oldamd
2388 2388
2389 2389 deltacomputer = deltautil.deltacomputer(destrevlog)
2390 2390 index = self.index
2391 2391 for rev in self:
2392 2392 entry = index[rev]
2393 2393
2394 2394 # Some classes override linkrev to take filtered revs into
2395 2395 # account. Use raw entry from index.
2396 2396 flags = entry[0] & 0xffff
2397 2397 linkrev = entry[4]
2398 2398 p1 = index[entry[5]][7]
2399 2399 p2 = index[entry[6]][7]
2400 2400 node = entry[7]
2401 2401
2402 2402 # (Possibly) reuse the delta from the revlog if allowed and
2403 2403 # the revlog chunk is a delta.
2404 2404 cachedelta = None
2405 2405 rawtext = None
2406 2406 if (deltareuse != self.DELTAREUSEFULLADD
2407 2407 and destrevlog._lazydelta):
2408 2408 dp = self.deltaparent(rev)
2409 2409 if dp != nullrev:
2410 2410 cachedelta = (dp, bytes(self._chunk(rev)))
2411 2411
2412 2412 if not cachedelta:
2413 2413 rawtext = self.rawdata(rev)
2414 2414
2415 2415
2416 2416 if deltareuse == self.DELTAREUSEFULLADD:
2417 2417 destrevlog.addrevision(rawtext, tr, linkrev, p1, p2,
2418 2418 cachedelta=cachedelta,
2419 2419 node=node, flags=flags,
2420 2420 deltacomputer=deltacomputer)
2421 2421 else:
2422 2422 ifh = destrevlog.opener(destrevlog.indexfile, 'a+',
2423 2423 checkambig=False)
2424 2424 dfh = None
2425 2425 if not destrevlog._inline:
2426 2426 dfh = destrevlog.opener(destrevlog.datafile, 'a+')
2427 2427 try:
2428 2428 destrevlog._addrevision(node, rawtext, tr, linkrev, p1,
2429 2429 p2, flags, cachedelta, ifh, dfh,
2430 2430 deltacomputer=deltacomputer)
2431 2431 finally:
2432 2432 if dfh:
2433 2433 dfh.close()
2434 2434 ifh.close()
2435 2435
2436 2436 if addrevisioncb:
2437 2437 addrevisioncb(self, rev, node)
2438 2438 finally:
2439 2439 destrevlog._lazydelta = oldlazydelta
2440 2440 destrevlog._lazydeltabase = oldlazydeltabase
2441 2441 destrevlog._deltabothparents = oldamd
2442 2442
2443 2443 def censorrevision(self, tr, censornode, tombstone=b''):
2444 2444 if (self.version & 0xFFFF) == REVLOGV0:
2445 2445 raise error.RevlogError(_('cannot censor with version %d revlogs') %
2446 2446 self.version)
2447 2447
2448 2448 censorrev = self.rev(censornode)
2449 2449 tombstone = storageutil.packmeta({b'censored': tombstone}, b'')
2450 2450
2451 2451 if len(tombstone) > self.rawsize(censorrev):
2452 2452 raise error.Abort(_('censor tombstone must be no longer than '
2453 2453 'censored data'))
2454 2454
2455 2455 # Rewriting the revlog in place is hard. Our strategy for censoring is
2456 2456 # to create a new revlog, copy all revisions to it, then replace the
2457 2457 # revlogs on transaction close.
2458 2458
2459 2459 newindexfile = self.indexfile + b'.tmpcensored'
2460 2460 newdatafile = self.datafile + b'.tmpcensored'
2461 2461
2462 2462 # This is a bit dangerous. We could easily have a mismatch of state.
2463 2463 newrl = revlog(self.opener, newindexfile, newdatafile,
2464 2464 censorable=True)
2465 2465 newrl.version = self.version
2466 2466 newrl._generaldelta = self._generaldelta
2467 2467 newrl._io = self._io
2468 2468
2469 2469 for rev in self.revs():
2470 2470 node = self.node(rev)
2471 2471 p1, p2 = self.parents(node)
2472 2472
2473 2473 if rev == censorrev:
2474 2474 newrl.addrawrevision(tombstone, tr, self.linkrev(censorrev),
2475 2475 p1, p2, censornode, REVIDX_ISCENSORED)
2476 2476
2477 2477 if newrl.deltaparent(rev) != nullrev:
2478 2478 raise error.Abort(_('censored revision stored as delta; '
2479 2479 'cannot censor'),
2480 2480 hint=_('censoring of revlogs is not '
2481 2481 'fully implemented; please report '
2482 2482 'this bug'))
2483 2483 continue
2484 2484
2485 2485 if self.iscensored(rev):
2486 2486 if self.deltaparent(rev) != nullrev:
2487 2487 raise error.Abort(_('cannot censor due to censored '
2488 2488 'revision having delta stored'))
2489 2489 rawtext = self._chunk(rev)
2490 2490 else:
2491 2491 rawtext = self.rawdata(rev)
2492 2492
2493 2493 newrl.addrawrevision(rawtext, tr, self.linkrev(rev), p1, p2, node,
2494 2494 self.flags(rev))
2495 2495
2496 2496 tr.addbackup(self.indexfile, location='store')
2497 2497 if not self._inline:
2498 2498 tr.addbackup(self.datafile, location='store')
2499 2499
2500 2500 self.opener.rename(newrl.indexfile, self.indexfile)
2501 2501 if not self._inline:
2502 2502 self.opener.rename(newrl.datafile, self.datafile)
2503 2503
2504 2504 self.clearcaches()
2505 2505 self._loadindex()
2506 2506
2507 2507 def verifyintegrity(self, state):
2508 2508 """Verifies the integrity of the revlog.
2509 2509
2510 2510 Yields ``revlogproblem`` instances describing problems that are
2511 2511 found.
2512 2512 """
2513 2513 dd, di = self.checksize()
2514 2514 if dd:
2515 2515 yield revlogproblem(error=_('data length off by %d bytes') % dd)
2516 2516 if di:
2517 2517 yield revlogproblem(error=_('index contains %d extra bytes') % di)
2518 2518
2519 2519 version = self.version & 0xFFFF
2520 2520
2521 2521 # The verifier tells us what version revlog we should be.
2522 2522 if version != state['expectedversion']:
2523 2523 yield revlogproblem(
2524 2524 warning=_("warning: '%s' uses revlog format %d; expected %d") %
2525 2525 (self.indexfile, version, state['expectedversion']))
2526 2526
2527 2527 state['skipread'] = set()
2528 2528
2529 2529 for rev in self:
2530 2530 node = self.node(rev)
2531 2531
2532 2532 # Verify contents. 4 cases to care about:
2533 2533 #
2534 2534 # common: the most common case
2535 2535 # rename: with a rename
2536 2536 # meta: file content starts with b'\1\n', the metadata
2537 2537 # header defined in filelog.py, but without a rename
2538 2538 # ext: content stored externally
2539 2539 #
2540 2540 # More formally, their differences are shown below:
2541 2541 #
2542 2542 # | common | rename | meta | ext
2543 2543 # -------------------------------------------------------
2544 2544 # flags() | 0 | 0 | 0 | not 0
2545 2545 # renamed() | False | True | False | ?
2546 2546 # rawtext[0:2]=='\1\n'| False | True | True | ?
2547 2547 #
2548 2548 # "rawtext" means the raw text stored in revlog data, which
2549 2549 # could be retrieved by "rawdata(rev)". "text"
2550 2550 # mentioned below is "revision(rev)".
2551 2551 #
2552 2552 # There are 3 different lengths stored physically:
2553 2553 # 1. L1: rawsize, stored in revlog index
2554 2554 # 2. L2: len(rawtext), stored in revlog data
2555 2555 # 3. L3: len(text), stored in revlog data if flags==0, or
2556 2556 # possibly somewhere else if flags!=0
2557 2557 #
2558 2558 # L1 should be equal to L2. L3 could be different from them.
2559 2559 # "text" may or may not affect commit hash depending on flag
2560 2560 # processors (see flagutil.addflagprocessor).
2561 2561 #
2562 2562 # | common | rename | meta | ext
2563 2563 # -------------------------------------------------
2564 2564 # rawsize() | L1 | L1 | L1 | L1
2565 2565 # size() | L1 | L2-LM | L1(*) | L1 (?)
2566 2566 # len(rawtext) | L2 | L2 | L2 | L2
2567 2567 # len(text) | L2 | L2 | L2 | L3
2568 2568 # len(read()) | L2 | L2-LM | L2-LM | L3 (?)
2569 2569 #
2570 2570 # LM: length of metadata, depending on rawtext
2571 2571 # (*): not ideal, see comment in filelog.size
2572 2572 # (?): could be "- len(meta)" if the resolved content has
2573 2573 # rename metadata
2574 2574 #
2575 2575 # Checks needed to be done:
2576 2576 # 1. length check: L1 == L2, in all cases.
2577 2577 # 2. hash check: depending on flag processor, we may need to
2578 2578 # use either "text" (external), or "rawtext" (in revlog).
2579 2579
2580 2580 try:
2581 2581 skipflags = state.get('skipflags', 0)
2582 2582 if skipflags:
2583 2583 skipflags &= self.flags(rev)
2584 2584
2585 2585 if skipflags:
2586 2586 state['skipread'].add(node)
2587 2587 else:
2588 2588 # Side-effect: read content and verify hash.
2589 2589 self.revision(node)
2590 2590
2591 2591 l1 = self.rawsize(rev)
2592 2592 l2 = len(self.rawdata(node))
2593 2593
2594 2594 if l1 != l2:
2595 2595 yield revlogproblem(
2596 2596 error=_('unpacked size is %d, %d expected') % (l2, l1),
2597 2597 node=node)
2598 2598
2599 2599 except error.CensoredNodeError:
2600 2600 if state['erroroncensored']:
2601 2601 yield revlogproblem(error=_('censored file data'),
2602 2602 node=node)
2603 2603 state['skipread'].add(node)
2604 2604 except Exception as e:
2605 2605 yield revlogproblem(
2606 2606 error=_('unpacking %s: %s') % (short(node),
2607 2607 stringutil.forcebytestr(e)),
2608 2608 node=node)
2609 2609 state['skipread'].add(node)
2610 2610
2611 2611 def storageinfo(self, exclusivefiles=False, sharedfiles=False,
2612 2612 revisionscount=False, trackedsize=False,
2613 2613 storedsize=False):
2614 2614 d = {}
2615 2615
2616 2616 if exclusivefiles:
2617 2617 d['exclusivefiles'] = [(self.opener, self.indexfile)]
2618 2618 if not self._inline:
2619 2619 d['exclusivefiles'].append((self.opener, self.datafile))
2620 2620
2621 2621 if sharedfiles:
2622 2622 d['sharedfiles'] = []
2623 2623
2624 2624 if revisionscount:
2625 2625 d['revisionscount'] = len(self)
2626 2626
2627 2627 if trackedsize:
2628 2628 d['trackedsize'] = sum(map(self.rawsize, iter(self)))
2629 2629
2630 2630 if storedsize:
2631 2631 d['storedsize'] = sum(self.opener.stat(path).st_size
2632 2632 for path in self.files())
2633 2633
2634 2634 return d
@@ -1,200 +1,201 b''
1 1 # flagutils.py - code to deal with revlog flags and their processors
2 2 #
3 3 # Copyright 2016 Remi Chaintron <remi@fb.com>
4 4 # Copyright 2016-2019 Pierre-Yves David <pierre-yves.david@ens-lyon.org>
5 5 #
6 6 # This software may be used and distributed according to the terms of the
7 7 # GNU General Public License version 2 or any later version.
8 8
9 9 from __future__ import absolute_import
10 10
11 11 from ..i18n import _
12 12
13 13 from .constants import (
14 14 REVIDX_DEFAULT_FLAGS,
15 15 REVIDX_ELLIPSIS,
16 16 REVIDX_EXTSTORED,
17 17 REVIDX_FLAGS_ORDER,
18 18 REVIDX_ISCENSORED,
19 19 REVIDX_RAWTEXT_CHANGING_FLAGS,
20 20 )
21 21
22 22 from .. import (
23 23 error,
24 24 util
25 25 )
26 26
27 27 # blanked usage of all the name to prevent pyflakes constraints
28 28 # We need these name available in the module for extensions.
29 29 REVIDX_ISCENSORED
30 30 REVIDX_ELLIPSIS
31 31 REVIDX_EXTSTORED
32 32 REVIDX_DEFAULT_FLAGS
33 33 REVIDX_FLAGS_ORDER
34 34 REVIDX_RAWTEXT_CHANGING_FLAGS
35 35
36 36 REVIDX_KNOWN_FLAGS = util.bitsfrom(REVIDX_FLAGS_ORDER)
37 37
38 38 # Store flag processors (cf. 'addflagprocessor()' to register)
39 39 flagprocessors = {
40 40 REVIDX_ISCENSORED: None,
41 41 }
42 42
43 43 def addflagprocessor(flag, processor):
44 44 """Register a flag processor on a revision data flag.
45 45
46 46 Invariant:
47 47 - Flags need to be defined in REVIDX_KNOWN_FLAGS and REVIDX_FLAGS_ORDER,
48 48 and REVIDX_RAWTEXT_CHANGING_FLAGS if they can alter rawtext.
49 49 - Only one flag processor can be registered on a specific flag.
50 50 - flagprocessors must be 3-tuples of functions (read, write, raw) with the
51 51 following signatures:
52 52 - (read) f(self, rawtext) -> text, bool
53 53 - (write) f(self, text) -> rawtext, bool
54 54 - (raw) f(self, rawtext) -> bool
55 55 "text" is presented to the user. "rawtext" is stored in revlog data, not
56 56 directly visible to the user.
57 57 The boolean returned by these transforms is used to determine whether
58 58 the returned text can be used for hash integrity checking. For example,
59 59 if "write" returns False, then "text" is used to generate hash. If
60 60 "write" returns True, that basically means "rawtext" returned by "write"
61 61 should be used to generate hash. Usually, "write" and "read" return
62 62 different booleans. And "raw" returns a same boolean as "write".
63 63
64 64 Note: The 'raw' transform is used for changegroup generation and in some
65 65 debug commands. In this case the transform only indicates whether the
66 66 contents can be used for hash integrity checks.
67 67 """
68 68 insertflagprocessor(flag, processor, flagprocessors)
69 69
70 70 def insertflagprocessor(flag, processor, flagprocessors):
71 71 if not flag & REVIDX_KNOWN_FLAGS:
72 72 msg = _("cannot register processor on unknown flag '%#x'.") % (flag)
73 73 raise error.ProgrammingError(msg)
74 74 if flag not in REVIDX_FLAGS_ORDER:
75 75 msg = _("flag '%#x' undefined in REVIDX_FLAGS_ORDER.") % (flag)
76 76 raise error.ProgrammingError(msg)
77 77 if flag in flagprocessors:
78 78 msg = _("cannot register multiple processors on flag '%#x'.") % (flag)
79 79 raise error.Abort(msg)
80 80 flagprocessors[flag] = processor
81 81
82 82 class flagprocessorsmixin(object):
83 83 """basic mixin to support revlog flag processing
84 84
85 85 Make sure the `_flagprocessors` attribute is set at ``__init__`` time.
86 86
87 87 See the documentation of the ``_processflags`` method for details.
88 88 """
89 89
90 90 _flagserrorclass = error.RevlogError
91 91
92 92 def _processflags(self, text, flags, operation, raw=False):
93 93 """deprecated entry point to access flag processors"""
94 94 msg = ('_processflag(...) use the specialized variant')
95 95 util.nouideprecwarn(msg, '5.2', stacklevel=2)
96 96 if raw:
97 97 return text, self._processflagsraw(text, flags)
98 98 elif operation == 'read':
99 99 return self._processflagsread(text, flags)
100 100 else: # write operation
101 101 return self._processflagswrite(text, flags)
102 102
103 103 def _processflagsread(self, text, flags):
104 104 """Inspect revision data flags and applies read transformations defined
105 105 by registered flag processors.
106 106
107 107 ``text`` - the revision data to process
108 108 ``flags`` - the revision flags
109 109 ``raw`` - an optional argument describing if the raw transform should be
110 110 applied.
111 111
112 112 This method processes the flags in the order (or reverse order if
113 113 ``operation`` is 'write') defined by REVIDX_FLAGS_ORDER, applying the
114 114 flag processors registered for present flags. The order of flags defined
115 115 in REVIDX_FLAGS_ORDER needs to be stable to allow non-commutativity.
116 116
117 117 Returns a 2-tuple of ``(text, validatehash)`` where ``text`` is the
118 118 processed text and ``validatehash`` is a bool indicating whether the
119 119 returned text should be checked for hash integrity.
120 120 """
121 121 return self._processflagsfunc(text, flags, 'read')
122 122
123 123 def _processflagswrite(self, text, flags):
124 124 """Inspect revision data flags and applies write transformations defined
125 125 by registered flag processors.
126 126
127 127 ``text`` - the revision data to process
128 128 ``flags`` - the revision flags
129 129
130 130 This method processes the flags in the order (or reverse order if
131 131 ``operation`` is 'write') defined by REVIDX_FLAGS_ORDER, applying the
132 132 flag processors registered for present flags. The order of flags defined
133 133 in REVIDX_FLAGS_ORDER needs to be stable to allow non-commutativity.
134 134
135 135 Returns a 2-tuple of ``(text, validatehash)`` where ``text`` is the
136 136 processed text and ``validatehash`` is a bool indicating whether the
137 137 returned text should be checked for hash integrity.
138 138 """
139 139 return self._processflagsfunc(text, flags, 'write')[:2]
140 140
141 141 def _processflagsraw(self, text, flags):
142 142 """Inspect revision data flags to check is the content hash should be
143 143 validated.
144 144
145 145 ``text`` - the revision data to process
146 146 ``flags`` - the revision flags
147 147
148 148 This method processes the flags in the order (or reverse order if
149 149 ``operation`` is 'write') defined by REVIDX_FLAGS_ORDER, applying the
150 150 flag processors registered for present flags. The order of flags defined
151 151 in REVIDX_FLAGS_ORDER needs to be stable to allow non-commutativity.
152 152
153 153 Returns a 2-tuple of ``(text, validatehash)`` where ``text`` is the
154 154 processed text and ``validatehash`` is a bool indicating whether the
155 155 returned text should be checked for hash integrity.
156 156 """
157 157 return self._processflagsfunc(text, flags, 'raw')[1]
158 158
159 159 def _processflagsfunc(self, text, flags, operation):
160 160 # fast path: no flag processors will run
161 161 if flags == 0:
162 162 return text, True, {}
163 163 if operation not in ('read', 'write', 'raw'):
164 164 raise error.ProgrammingError(_("invalid '%s' operation") %
165 165 operation)
166 166 # Check all flags are known.
167 167 if flags & ~REVIDX_KNOWN_FLAGS:
168 168 raise self._flagserrorclass(_("incompatible revision flag '%#x'") %
169 169 (flags & ~REVIDX_KNOWN_FLAGS))
170 170 validatehash = True
171 171 # Depending on the operation (read or write), the order might be
172 172 # reversed due to non-commutative transforms.
173 173 orderedflags = REVIDX_FLAGS_ORDER
174 174 if operation == 'write':
175 175 orderedflags = reversed(orderedflags)
176 176
177 177 outsidedata = {}
178 178 for flag in orderedflags:
179 179 # If a flagprocessor has been registered for a known flag, apply the
180 180 # related operation transform and update result tuple.
181 181 if flag & flags:
182 182 vhash = True
183 183
184 184 if flag not in self._flagprocessors:
185 185 message = _("missing processor for flag '%#x'") % (flag)
186 186 raise self._flagserrorclass(message)
187 187
188 188 processor = self._flagprocessors[flag]
189 189 if processor is not None:
190 190 readtransform, writetransform, rawtransform = processor
191 191
192 192 if operation == 'raw':
193 193 vhash = rawtransform(self, text)
194 194 elif operation == 'read':
195 text, vhash = readtransform(self, text)
195 text, vhash, s = readtransform(self, text)
196 outsidedata.update(s)
196 197 else: # write operation
197 198 text, vhash = writetransform(self, text)
198 199 validatehash = validatehash and vhash
199 200
200 201 return text, validatehash, outsidedata
@@ -1,139 +1,142 b''
1 1 # coding=UTF-8
2 2
3 3 from __future__ import absolute_import
4 4
5 5 import base64
6 6 import zlib
7 7
8 8 from mercurial import (
9 9 changegroup,
10 10 exchange,
11 11 extensions,
12 12 revlog,
13 13 util,
14 14 )
15 15 from mercurial.revlogutils import (
16 16 flagutil,
17 17 )
18 18
19 19 # Test only: These flags are defined here only in the context of testing the
20 20 # behavior of the flag processor. The canonical way to add flags is to get in
21 21 # touch with the community and make them known in revlog.
22 22 REVIDX_NOOP = (1 << 3)
23 23 REVIDX_BASE64 = (1 << 2)
24 24 REVIDX_GZIP = (1 << 1)
25 25 REVIDX_FAIL = 1
26 26
27 27 def validatehash(self, text):
28 28 return True
29 29
30 30 def bypass(self, text):
31 31 return False
32 32
33 33 def noopdonothing(self, text):
34 34 return (text, True)
35 35
36 def noopdonothingread(self, text):
37 return (text, True, {})
38
36 39 def b64encode(self, text):
37 40 return (base64.b64encode(text), False)
38 41
39 42 def b64decode(self, text):
40 return (base64.b64decode(text), True)
43 return (base64.b64decode(text), True, {})
41 44
42 45 def gzipcompress(self, text):
43 46 return (zlib.compress(text), False)
44 47
45 48 def gzipdecompress(self, text):
46 return (zlib.decompress(text), True)
49 return (zlib.decompress(text), True, {})
47 50
48 51 def supportedoutgoingversions(orig, repo):
49 52 versions = orig(repo)
50 53 versions.discard(b'01')
51 54 versions.discard(b'02')
52 55 versions.add(b'03')
53 56 return versions
54 57
55 58 def allsupportedversions(orig, ui):
56 59 versions = orig(ui)
57 60 versions.add(b'03')
58 61 return versions
59 62
60 63 def makewrappedfile(obj):
61 64 class wrappedfile(obj.__class__):
62 65 def addrevision(self, text, transaction, link, p1, p2,
63 66 cachedelta=None, node=None,
64 67 flags=flagutil.REVIDX_DEFAULT_FLAGS):
65 68 if b'[NOOP]' in text:
66 69 flags |= REVIDX_NOOP
67 70
68 71 if b'[BASE64]' in text:
69 72 flags |= REVIDX_BASE64
70 73
71 74 if b'[GZIP]' in text:
72 75 flags |= REVIDX_GZIP
73 76
74 77 # This addrevision wrapper is meant to add a flag we will not have
75 78 # transforms registered for, ensuring we handle this error case.
76 79 if b'[FAIL]' in text:
77 80 flags |= REVIDX_FAIL
78 81
79 82 return super(wrappedfile, self).addrevision(text, transaction, link,
80 83 p1, p2,
81 84 cachedelta=cachedelta,
82 85 node=node,
83 86 flags=flags)
84 87
85 88 obj.__class__ = wrappedfile
86 89
87 90 def reposetup(ui, repo):
88 91 class wrappingflagprocessorrepo(repo.__class__):
89 92 def file(self, f):
90 93 orig = super(wrappingflagprocessorrepo, self).file(f)
91 94 makewrappedfile(orig)
92 95 return orig
93 96
94 97 repo.__class__ = wrappingflagprocessorrepo
95 98
96 99 def extsetup(ui):
97 100 # Enable changegroup3 for flags to be sent over the wire
98 101 wrapfunction = extensions.wrapfunction
99 102 wrapfunction(changegroup,
100 103 'supportedoutgoingversions',
101 104 supportedoutgoingversions)
102 105 wrapfunction(changegroup,
103 106 'allsupportedversions',
104 107 allsupportedversions)
105 108
106 109 # Teach revlog about our test flags
107 110 flags = [REVIDX_NOOP, REVIDX_BASE64, REVIDX_GZIP, REVIDX_FAIL]
108 111 flagutil.REVIDX_KNOWN_FLAGS |= util.bitsfrom(flags)
109 112 revlog.REVIDX_FLAGS_ORDER.extend(flags)
110 113
111 114 # Teach exchange to use changegroup 3
112 115 for k in exchange._bundlespeccontentopts.keys():
113 116 exchange._bundlespeccontentopts[k][b"cg.version"] = b"03"
114 117
115 118 # Register flag processors for each extension
116 119 flagutil.addflagprocessor(
117 120 REVIDX_NOOP,
118 121 (
119 noopdonothing,
122 noopdonothingread,
120 123 noopdonothing,
121 124 validatehash,
122 125 )
123 126 )
124 127 flagutil.addflagprocessor(
125 128 REVIDX_BASE64,
126 129 (
127 130 b64decode,
128 131 b64encode,
129 132 bypass,
130 133 ),
131 134 )
132 135 flagutil.addflagprocessor(
133 136 REVIDX_GZIP,
134 137 (
135 138 gzipdecompress,
136 139 gzipcompress,
137 140 bypass
138 141 )
139 142 )
@@ -1,455 +1,455 b''
1 1 # test revlog interaction about raw data (flagprocessor)
2 2
3 3 from __future__ import absolute_import, print_function
4 4
5 5 import collections
6 6 import hashlib
7 7 import sys
8 8
9 9 from mercurial import (
10 10 encoding,
11 11 node,
12 12 revlog,
13 13 transaction,
14 14 vfs,
15 15 )
16 16
17 17 from mercurial.revlogutils import (
18 18 deltas,
19 19 flagutil,
20 20 )
21 21
22 22 # TESTTMP is optional. This makes it convenient to run without run-tests.py
23 23 tvfs = vfs.vfs(encoding.environ.get(b'TESTTMP', b'/tmp'))
24 24
25 25 # Enable generaldelta otherwise revlog won't use delta as expected by the test
26 26 tvfs.options = {b'generaldelta': True, b'revlogv1': True,
27 27 b'sparse-revlog': True}
28 28
29 29 # The test wants to control whether to use delta explicitly, based on
30 30 # "storedeltachains".
31 31 revlog.revlog._isgooddeltainfo = lambda self, d, textlen: self._storedeltachains
32 32
33 33 def abort(msg):
34 34 print('abort: %s' % msg)
35 35 # Return 0 so run-tests.py could compare the output.
36 36 sys.exit()
37 37
38 38 # Register a revlog processor for flag EXTSTORED.
39 39 #
40 40 # It simply prepends a fixed header, and replaces '1' to 'i'. So it has
41 41 # insertion and replacement, and may be interesting to test revlog's line-based
42 42 # deltas.
43 43 _extheader = b'E\n'
44 44
45 45 def readprocessor(self, rawtext):
46 46 # True: the returned text could be used to verify hash
47 47 text = rawtext[len(_extheader):].replace(b'i', b'1')
48 return text, True
48 return text, True, {}
49 49
50 50 def writeprocessor(self, text):
51 51 # False: the returned rawtext shouldn't be used to verify hash
52 52 rawtext = _extheader + text.replace(b'1', b'i')
53 53 return rawtext, False
54 54
55 55 def rawprocessor(self, rawtext):
56 56 # False: do not verify hash. Only the content returned by "readprocessor"
57 57 # can be used to verify hash.
58 58 return False
59 59
60 60 flagutil.addflagprocessor(revlog.REVIDX_EXTSTORED,
61 61 (readprocessor, writeprocessor, rawprocessor))
62 62
63 63 # Utilities about reading and appending revlog
64 64
65 65 def newtransaction():
66 66 # A transaction is required to write revlogs
67 67 report = lambda msg: None
68 68 return transaction.transaction(report, tvfs, {'plain': tvfs}, b'journal')
69 69
70 70 def newrevlog(name=b'_testrevlog.i', recreate=False):
71 71 if recreate:
72 72 tvfs.tryunlink(name)
73 73 rlog = revlog.revlog(tvfs, name)
74 74 return rlog
75 75
76 76 def appendrev(rlog, text, tr, isext=False, isdelta=True):
77 77 '''Append a revision. If isext is True, set the EXTSTORED flag so flag
78 78 processor will be used (and rawtext is different from text). If isdelta is
79 79 True, force the revision to be a delta, otherwise it's full text.
80 80 '''
81 81 nextrev = len(rlog)
82 82 p1 = rlog.node(nextrev - 1)
83 83 p2 = node.nullid
84 84 if isext:
85 85 flags = revlog.REVIDX_EXTSTORED
86 86 else:
87 87 flags = revlog.REVIDX_DEFAULT_FLAGS
88 88 # Change storedeltachains temporarily, to override revlog's delta decision
89 89 rlog._storedeltachains = isdelta
90 90 try:
91 91 rlog.addrevision(text, tr, nextrev, p1, p2, flags=flags)
92 92 return nextrev
93 93 except Exception as ex:
94 94 abort('rev %d: failed to append: %s' % (nextrev, ex))
95 95 finally:
96 96 # Restore storedeltachains. It is always True, see revlog.__init__
97 97 rlog._storedeltachains = True
98 98
99 99 def addgroupcopy(rlog, tr, destname=b'_destrevlog.i', optimaldelta=True):
100 100 '''Copy revlog to destname using revlog.addgroup. Return the copied revlog.
101 101
102 102 This emulates push or pull. They use changegroup. Changegroup requires
103 103 repo to work. We don't have a repo, so a dummy changegroup is used.
104 104
105 105 If optimaldelta is True, use optimized delta parent, so the destination
106 106 revlog could probably reuse it. Otherwise it builds sub-optimal delta, and
107 107 the destination revlog needs more work to use it.
108 108
109 109 This exercises some revlog.addgroup (and revlog._addrevision(text=None))
110 110 code path, which is not covered by "appendrev" alone.
111 111 '''
112 112 class dummychangegroup(object):
113 113 @staticmethod
114 114 def deltachunk(pnode):
115 115 pnode = pnode or node.nullid
116 116 parentrev = rlog.rev(pnode)
117 117 r = parentrev + 1
118 118 if r >= len(rlog):
119 119 return {}
120 120 if optimaldelta:
121 121 deltaparent = parentrev
122 122 else:
123 123 # suboptimal deltaparent
124 124 deltaparent = min(0, parentrev)
125 125 if not rlog.candelta(deltaparent, r):
126 126 deltaparent = -1
127 127 return {b'node': rlog.node(r), b'p1': pnode, b'p2': node.nullid,
128 128 b'cs': rlog.node(rlog.linkrev(r)), b'flags': rlog.flags(r),
129 129 b'deltabase': rlog.node(deltaparent),
130 130 b'delta': rlog.revdiff(deltaparent, r)}
131 131
132 132 def deltaiter(self):
133 133 chain = None
134 134 for chunkdata in iter(lambda: self.deltachunk(chain), {}):
135 135 node = chunkdata[b'node']
136 136 p1 = chunkdata[b'p1']
137 137 p2 = chunkdata[b'p2']
138 138 cs = chunkdata[b'cs']
139 139 deltabase = chunkdata[b'deltabase']
140 140 delta = chunkdata[b'delta']
141 141 flags = chunkdata[b'flags']
142 142
143 143 chain = node
144 144
145 145 yield (node, p1, p2, cs, deltabase, delta, flags)
146 146
147 147 def linkmap(lnode):
148 148 return rlog.rev(lnode)
149 149
150 150 dlog = newrevlog(destname, recreate=True)
151 151 dummydeltas = dummychangegroup().deltaiter()
152 152 dlog.addgroup(dummydeltas, linkmap, tr)
153 153 return dlog
154 154
155 155 def lowlevelcopy(rlog, tr, destname=b'_destrevlog.i'):
156 156 '''Like addgroupcopy, but use the low level revlog._addrevision directly.
157 157
158 158 It exercises some code paths that are hard to reach easily otherwise.
159 159 '''
160 160 dlog = newrevlog(destname, recreate=True)
161 161 for r in rlog:
162 162 p1 = rlog.node(r - 1)
163 163 p2 = node.nullid
164 164 if r == 0 or (rlog.flags(r) & revlog.REVIDX_EXTSTORED):
165 165 text = rlog.rawdata(r)
166 166 cachedelta = None
167 167 else:
168 168 # deltaparent cannot have EXTSTORED flag.
169 169 deltaparent = max([-1] +
170 170 [p for p in range(r)
171 171 if rlog.flags(p) & revlog.REVIDX_EXTSTORED == 0])
172 172 text = None
173 173 cachedelta = (deltaparent, rlog.revdiff(deltaparent, r))
174 174 flags = rlog.flags(r)
175 175 ifh = dfh = None
176 176 try:
177 177 ifh = dlog.opener(dlog.indexfile, b'a+')
178 178 if not dlog._inline:
179 179 dfh = dlog.opener(dlog.datafile, b'a+')
180 180 dlog._addrevision(rlog.node(r), text, tr, r, p1, p2, flags,
181 181 cachedelta, ifh, dfh)
182 182 finally:
183 183 if dfh is not None:
184 184 dfh.close()
185 185 if ifh is not None:
186 186 ifh.close()
187 187 return dlog
188 188
189 189 # Utilities to generate revisions for testing
190 190
191 191 def genbits(n):
192 192 '''Given a number n, generate (2 ** (n * 2) + 1) numbers in range(2 ** n).
193 193 i.e. the generated numbers have a width of n bits.
194 194
195 195 The combination of two adjacent numbers will cover all possible cases.
196 196 That is to say, given any x, y where both x, and y are in range(2 ** n),
197 197 there is an x followed immediately by y in the generated sequence.
198 198 '''
199 199 m = 2 ** n
200 200
201 201 # Gray Code. See https://en.wikipedia.org/wiki/Gray_code
202 202 gray = lambda x: x ^ (x >> 1)
203 203 reversegray = dict((gray(i), i) for i in range(m))
204 204
205 205 # Generate (n * 2) bit gray code, yield lower n bits as X, and look for
206 206 # the next unused gray code where higher n bits equal to X.
207 207
208 208 # For gray codes whose higher bits are X, a[X] of them have been used.
209 209 a = [0] * m
210 210
211 211 # Iterate from 0.
212 212 x = 0
213 213 yield x
214 214 for i in range(m * m):
215 215 x = reversegray[x]
216 216 y = gray(a[x] + x * m) & (m - 1)
217 217 assert a[x] < m
218 218 a[x] += 1
219 219 x = y
220 220 yield x
221 221
222 222 def gentext(rev):
223 223 '''Given a revision number, generate dummy text'''
224 224 return b''.join(b'%d\n' % j for j in range(-1, rev % 5))
225 225
226 226 def writecases(rlog, tr):
227 227 '''Write some revisions interested to the test.
228 228
229 229 The test is interested in 3 properties of a revision:
230 230
231 231 - Is it a delta or a full text? (isdelta)
232 232 This is to catch some delta application issues.
233 233 - Does it have a flag of EXTSTORED? (isext)
234 234 This is to catch some flag processor issues. Especially when
235 235 interacted with revlog deltas.
236 236 - Is its text empty? (isempty)
237 237 This is less important. It is intended to try to catch some careless
238 238 checks like "if text" instead of "if text is None". Note: if flag
239 239 processor is involved, raw text may be not empty.
240 240
241 241 Write 65 revisions. So that all combinations of the above flags for
242 242 adjacent revisions are covered. That is to say,
243 243
244 244 len(set(
245 245 (r.delta, r.ext, r.empty, (r+1).delta, (r+1).ext, (r+1).empty)
246 246 for r in range(len(rlog) - 1)
247 247 )) is 64.
248 248
249 249 Where "r.delta", "r.ext", and "r.empty" are booleans matching properties
250 250 mentioned above.
251 251
252 252 Return expected [(text, rawtext)].
253 253 '''
254 254 result = []
255 255 for i, x in enumerate(genbits(3)):
256 256 isdelta, isext, isempty = bool(x & 1), bool(x & 2), bool(x & 4)
257 257 if isempty:
258 258 text = b''
259 259 else:
260 260 text = gentext(i)
261 261 rev = appendrev(rlog, text, tr, isext=isext, isdelta=isdelta)
262 262
263 263 # Verify text, rawtext, and rawsize
264 264 if isext:
265 265 rawtext = writeprocessor(None, text)[0]
266 266 else:
267 267 rawtext = text
268 268 if rlog.rawsize(rev) != len(rawtext):
269 269 abort('rev %d: wrong rawsize' % rev)
270 270 if rlog.revision(rev, raw=False) != text:
271 271 abort('rev %d: wrong text' % rev)
272 272 if rlog.rawdata(rev) != rawtext:
273 273 abort('rev %d: wrong rawtext' % rev)
274 274 result.append((text, rawtext))
275 275
276 276 # Verify flags like isdelta, isext work as expected
277 277 # isdelta can be overridden to False if this or p1 has isext set
278 278 if bool(rlog.deltaparent(rev) > -1) and not isdelta:
279 279 abort('rev %d: isdelta is unexpected' % rev)
280 280 if bool(rlog.flags(rev)) != isext:
281 281 abort('rev %d: isext is ineffective' % rev)
282 282 return result
283 283
284 284 # Main test and checking
285 285
286 286 def checkrevlog(rlog, expected):
287 287 '''Check if revlog has expected contents. expected is [(text, rawtext)]'''
288 288 # Test using different access orders. This could expose some issues
289 289 # depending on revlog caching (see revlog._cache).
290 290 for r0 in range(len(rlog) - 1):
291 291 r1 = r0 + 1
292 292 for revorder in [[r0, r1], [r1, r0]]:
293 293 for raworder in [[True], [False], [True, False], [False, True]]:
294 294 nlog = newrevlog()
295 295 for rev in revorder:
296 296 for raw in raworder:
297 297 if raw:
298 298 t = nlog.rawdata(rev)
299 299 else:
300 300 t = nlog.revision(rev)
301 301 if t != expected[rev][int(raw)]:
302 302 abort('rev %d: corrupted %stext'
303 303 % (rev, raw and 'raw' or ''))
304 304
305 305 slicingdata = [
306 306 ([0, 1, 2, 3, 55, 56, 58, 59, 60],
307 307 [[0, 1], [2], [58], [59, 60]],
308 308 10),
309 309 ([0, 1, 2, 3, 55, 56, 58, 59, 60],
310 310 [[0, 1], [2], [58], [59, 60]],
311 311 10),
312 312 ([-1, 0, 1, 2, 3, 55, 56, 58, 59, 60],
313 313 [[-1, 0, 1], [2], [58], [59, 60]],
314 314 10),
315 315 ]
316 316
317 317 def slicingtest(rlog):
318 318 oldmin = rlog._srmingapsize
319 319 try:
320 320 # the test revlog is small, we remove the floor under which we
321 321 # slicing is diregarded.
322 322 rlog._srmingapsize = 0
323 323 for item in slicingdata:
324 324 chain, expected, target = item
325 325 result = deltas.slicechunk(rlog, chain, targetsize=target)
326 326 result = list(result)
327 327 if result != expected:
328 328 print('slicing differ:')
329 329 print(' chain: %s' % chain)
330 330 print(' target: %s' % target)
331 331 print(' expected: %s' % expected)
332 332 print(' result: %s' % result)
333 333 finally:
334 334 rlog._srmingapsize = oldmin
335 335
336 336 def md5sum(s):
337 337 return hashlib.md5(s).digest()
338 338
339 339 def _maketext(*coord):
340 340 """create piece of text according to range of integers
341 341
342 342 The test returned use a md5sum of the integer to make it less
343 343 compressible"""
344 344 pieces = []
345 345 for start, size in coord:
346 346 num = range(start, start + size)
347 347 p = [md5sum(b'%d' % r) for r in num]
348 348 pieces.append(b'\n'.join(p))
349 349 return b'\n'.join(pieces) + b'\n'
350 350
351 351 data = [
352 352 _maketext((0, 120), (456, 60)),
353 353 _maketext((0, 120), (345, 60)),
354 354 _maketext((0, 120), (734, 60)),
355 355 _maketext((0, 120), (734, 60), (923, 45)),
356 356 _maketext((0, 120), (734, 60), (234, 45)),
357 357 _maketext((0, 120), (734, 60), (564, 45)),
358 358 _maketext((0, 120), (734, 60), (361, 45)),
359 359 _maketext((0, 120), (734, 60), (489, 45)),
360 360 _maketext((0, 120), (123, 60)),
361 361 _maketext((0, 120), (145, 60)),
362 362 _maketext((0, 120), (104, 60)),
363 363 _maketext((0, 120), (430, 60)),
364 364 _maketext((0, 120), (430, 60), (923, 45)),
365 365 _maketext((0, 120), (430, 60), (234, 45)),
366 366 _maketext((0, 120), (430, 60), (564, 45)),
367 367 _maketext((0, 120), (430, 60), (361, 45)),
368 368 _maketext((0, 120), (430, 60), (489, 45)),
369 369 _maketext((0, 120), (249, 60)),
370 370 _maketext((0, 120), (832, 60)),
371 371 _maketext((0, 120), (891, 60)),
372 372 _maketext((0, 120), (543, 60)),
373 373 _maketext((0, 120), (120, 60)),
374 374 _maketext((0, 120), (60, 60), (768, 30)),
375 375 _maketext((0, 120), (60, 60), (260, 30)),
376 376 _maketext((0, 120), (60, 60), (450, 30)),
377 377 _maketext((0, 120), (60, 60), (361, 30)),
378 378 _maketext((0, 120), (60, 60), (886, 30)),
379 379 _maketext((0, 120), (60, 60), (116, 30)),
380 380 _maketext((0, 120), (60, 60), (567, 30), (629, 40)),
381 381 _maketext((0, 120), (60, 60), (569, 30), (745, 40)),
382 382 _maketext((0, 120), (60, 60), (777, 30), (700, 40)),
383 383 _maketext((0, 120), (60, 60), (618, 30), (398, 40), (158, 10)),
384 384 ]
385 385
386 386 def makesnapshot(tr):
387 387 rl = newrevlog(name=b'_snaprevlog3.i', recreate=True)
388 388 for i in data:
389 389 appendrev(rl, i, tr)
390 390 return rl
391 391
392 392 snapshots = [-1, 0, 6, 8, 11, 17, 19, 21, 25, 30]
393 393 def issnapshottest(rlog):
394 394 result = []
395 395 if rlog.issnapshot(-1):
396 396 result.append(-1)
397 397 for rev in rlog:
398 398 if rlog.issnapshot(rev):
399 399 result.append(rev)
400 400 if snapshots != result:
401 401 print('snapshot differ:')
402 402 print(' expected: %s' % snapshots)
403 403 print(' got: %s' % result)
404 404
405 405 snapshotmapall = {0: [6, 8, 11, 17, 19, 25], 8: [21], -1: [0, 30]}
406 406 snapshotmap15 = {0: [17, 19, 25], 8: [21], -1: [30]}
407 407 def findsnapshottest(rlog):
408 408 resultall = collections.defaultdict(list)
409 409 deltas._findsnapshots(rlog, resultall, 0)
410 410 resultall = dict(resultall.items())
411 411 if resultall != snapshotmapall:
412 412 print('snapshot map differ:')
413 413 print(' expected: %s' % snapshotmapall)
414 414 print(' got: %s' % resultall)
415 415 result15 = collections.defaultdict(list)
416 416 deltas._findsnapshots(rlog, result15, 15)
417 417 result15 = dict(result15.items())
418 418 if result15 != snapshotmap15:
419 419 print('snapshot map differ:')
420 420 print(' expected: %s' % snapshotmap15)
421 421 print(' got: %s' % result15)
422 422
423 423 def maintest():
424 424 with newtransaction() as tr:
425 425 rl = newrevlog(recreate=True)
426 426 expected = writecases(rl, tr)
427 427 checkrevlog(rl, expected)
428 428 print('local test passed')
429 429 # Copy via revlog.addgroup
430 430 rl1 = addgroupcopy(rl, tr)
431 431 checkrevlog(rl1, expected)
432 432 rl2 = addgroupcopy(rl, tr, optimaldelta=False)
433 433 checkrevlog(rl2, expected)
434 434 print('addgroupcopy test passed')
435 435 # Copy via revlog.clone
436 436 rl3 = newrevlog(name=b'_destrevlog3.i', recreate=True)
437 437 rl.clone(tr, rl3)
438 438 checkrevlog(rl3, expected)
439 439 print('clone test passed')
440 440 # Copy via low-level revlog._addrevision
441 441 rl4 = lowlevelcopy(rl, tr)
442 442 checkrevlog(rl4, expected)
443 443 print('lowlevelcopy test passed')
444 444 slicingtest(rl)
445 445 print('slicing test passed')
446 446 rl5 = makesnapshot(tr)
447 447 issnapshottest(rl5)
448 448 print('issnapshot test passed')
449 449 findsnapshottest(rl5)
450 450 print('findsnapshot test passed')
451 451
452 452 try:
453 453 maintest()
454 454 except Exception as ex:
455 455 abort('crashed: %s' % ex)
General Comments 0
You need to be logged in to leave comments. Login now