##// END OF EJS Templates
flagprocessors: have the read transform function return side data (API)...
marmoute -
r43255:bd5858c2 default
parent child Browse files
Show More
@@ -1,449 +1,449 b''
1 # wrapper.py - methods wrapping core mercurial logic
1 # wrapper.py - methods wrapping core mercurial logic
2 #
2 #
3 # Copyright 2017 Facebook, Inc.
3 # Copyright 2017 Facebook, Inc.
4 #
4 #
5 # This software may be used and distributed according to the terms of the
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
6 # GNU General Public License version 2 or any later version.
7
7
8 from __future__ import absolute_import
8 from __future__ import absolute_import
9
9
10 import hashlib
10 import hashlib
11
11
12 from mercurial.i18n import _
12 from mercurial.i18n import _
13 from mercurial.node import bin, hex, nullid, short
13 from mercurial.node import bin, hex, nullid, short
14
14
15 from mercurial import (
15 from mercurial import (
16 bundle2,
16 bundle2,
17 changegroup,
17 changegroup,
18 cmdutil,
18 cmdutil,
19 context,
19 context,
20 error,
20 error,
21 exchange,
21 exchange,
22 exthelper,
22 exthelper,
23 localrepo,
23 localrepo,
24 revlog,
24 revlog,
25 scmutil,
25 scmutil,
26 upgrade,
26 upgrade,
27 util,
27 util,
28 vfs as vfsmod,
28 vfs as vfsmod,
29 wireprotov1server,
29 wireprotov1server,
30 )
30 )
31
31
32 from mercurial.interfaces import (
32 from mercurial.interfaces import (
33 repository,
33 repository,
34 )
34 )
35
35
36 from mercurial.utils import (
36 from mercurial.utils import (
37 storageutil,
37 storageutil,
38 stringutil,
38 stringutil,
39 )
39 )
40
40
41 from ..largefiles import lfutil
41 from ..largefiles import lfutil
42
42
43 from . import (
43 from . import (
44 blobstore,
44 blobstore,
45 pointer,
45 pointer,
46 )
46 )
47
47
48 eh = exthelper.exthelper()
48 eh = exthelper.exthelper()
49
49
50 @eh.wrapfunction(localrepo, 'makefilestorage')
50 @eh.wrapfunction(localrepo, 'makefilestorage')
51 def localrepomakefilestorage(orig, requirements, features, **kwargs):
51 def localrepomakefilestorage(orig, requirements, features, **kwargs):
52 if b'lfs' in requirements:
52 if b'lfs' in requirements:
53 features.add(repository.REPO_FEATURE_LFS)
53 features.add(repository.REPO_FEATURE_LFS)
54
54
55 return orig(requirements=requirements, features=features, **kwargs)
55 return orig(requirements=requirements, features=features, **kwargs)
56
56
57 @eh.wrapfunction(changegroup, 'allsupportedversions')
57 @eh.wrapfunction(changegroup, 'allsupportedversions')
58 def allsupportedversions(orig, ui):
58 def allsupportedversions(orig, ui):
59 versions = orig(ui)
59 versions = orig(ui)
60 versions.add('03')
60 versions.add('03')
61 return versions
61 return versions
62
62
63 @eh.wrapfunction(wireprotov1server, '_capabilities')
63 @eh.wrapfunction(wireprotov1server, '_capabilities')
64 def _capabilities(orig, repo, proto):
64 def _capabilities(orig, repo, proto):
65 '''Wrap server command to announce lfs server capability'''
65 '''Wrap server command to announce lfs server capability'''
66 caps = orig(repo, proto)
66 caps = orig(repo, proto)
67 if util.safehasattr(repo.svfs, 'lfslocalblobstore'):
67 if util.safehasattr(repo.svfs, 'lfslocalblobstore'):
68 # Advertise a slightly different capability when lfs is *required*, so
68 # Advertise a slightly different capability when lfs is *required*, so
69 # that the client knows it MUST load the extension. If lfs is not
69 # that the client knows it MUST load the extension. If lfs is not
70 # required on the server, there's no reason to autoload the extension
70 # required on the server, there's no reason to autoload the extension
71 # on the client.
71 # on the client.
72 if b'lfs' in repo.requirements:
72 if b'lfs' in repo.requirements:
73 caps.append('lfs-serve')
73 caps.append('lfs-serve')
74
74
75 caps.append('lfs')
75 caps.append('lfs')
76 return caps
76 return caps
77
77
78 def bypasscheckhash(self, text):
78 def bypasscheckhash(self, text):
79 return False
79 return False
80
80
81 def readfromstore(self, text):
81 def readfromstore(self, text):
82 """Read filelog content from local blobstore transform for flagprocessor.
82 """Read filelog content from local blobstore transform for flagprocessor.
83
83
84 Default tranform for flagprocessor, returning contents from blobstore.
84 Default tranform for flagprocessor, returning contents from blobstore.
85 Returns a 2-typle (text, validatehash) where validatehash is True as the
85 Returns a 2-typle (text, validatehash) where validatehash is True as the
86 contents of the blobstore should be checked using checkhash.
86 contents of the blobstore should be checked using checkhash.
87 """
87 """
88 p = pointer.deserialize(text)
88 p = pointer.deserialize(text)
89 oid = p.oid()
89 oid = p.oid()
90 store = self.opener.lfslocalblobstore
90 store = self.opener.lfslocalblobstore
91 if not store.has(oid):
91 if not store.has(oid):
92 p.filename = self.filename
92 p.filename = self.filename
93 self.opener.lfsremoteblobstore.readbatch([p], store)
93 self.opener.lfsremoteblobstore.readbatch([p], store)
94
94
95 # The caller will validate the content
95 # The caller will validate the content
96 text = store.read(oid, verify=False)
96 text = store.read(oid, verify=False)
97
97
98 # pack hg filelog metadata
98 # pack hg filelog metadata
99 hgmeta = {}
99 hgmeta = {}
100 for k in p.keys():
100 for k in p.keys():
101 if k.startswith('x-hg-'):
101 if k.startswith('x-hg-'):
102 name = k[len('x-hg-'):]
102 name = k[len('x-hg-'):]
103 hgmeta[name] = p[k]
103 hgmeta[name] = p[k]
104 if hgmeta or text.startswith('\1\n'):
104 if hgmeta or text.startswith('\1\n'):
105 text = storageutil.packmeta(hgmeta, text)
105 text = storageutil.packmeta(hgmeta, text)
106
106
107 return (text, True)
107 return (text, True, {})
108
108
109 def writetostore(self, text):
109 def writetostore(self, text):
110 # hg filelog metadata (includes rename, etc)
110 # hg filelog metadata (includes rename, etc)
111 hgmeta, offset = storageutil.parsemeta(text)
111 hgmeta, offset = storageutil.parsemeta(text)
112 if offset and offset > 0:
112 if offset and offset > 0:
113 # lfs blob does not contain hg filelog metadata
113 # lfs blob does not contain hg filelog metadata
114 text = text[offset:]
114 text = text[offset:]
115
115
116 # git-lfs only supports sha256
116 # git-lfs only supports sha256
117 oid = hex(hashlib.sha256(text).digest())
117 oid = hex(hashlib.sha256(text).digest())
118 self.opener.lfslocalblobstore.write(oid, text)
118 self.opener.lfslocalblobstore.write(oid, text)
119
119
120 # replace contents with metadata
120 # replace contents with metadata
121 longoid = 'sha256:%s' % oid
121 longoid = 'sha256:%s' % oid
122 metadata = pointer.gitlfspointer(oid=longoid, size='%d' % len(text))
122 metadata = pointer.gitlfspointer(oid=longoid, size='%d' % len(text))
123
123
124 # by default, we expect the content to be binary. however, LFS could also
124 # by default, we expect the content to be binary. however, LFS could also
125 # be used for non-binary content. add a special entry for non-binary data.
125 # be used for non-binary content. add a special entry for non-binary data.
126 # this will be used by filectx.isbinary().
126 # this will be used by filectx.isbinary().
127 if not stringutil.binary(text):
127 if not stringutil.binary(text):
128 # not hg filelog metadata (affecting commit hash), no "x-hg-" prefix
128 # not hg filelog metadata (affecting commit hash), no "x-hg-" prefix
129 metadata['x-is-binary'] = '0'
129 metadata['x-is-binary'] = '0'
130
130
131 # translate hg filelog metadata to lfs metadata with "x-hg-" prefix
131 # translate hg filelog metadata to lfs metadata with "x-hg-" prefix
132 if hgmeta is not None:
132 if hgmeta is not None:
133 for k, v in hgmeta.iteritems():
133 for k, v in hgmeta.iteritems():
134 metadata['x-hg-%s' % k] = v
134 metadata['x-hg-%s' % k] = v
135
135
136 rawtext = metadata.serialize()
136 rawtext = metadata.serialize()
137 return (rawtext, False)
137 return (rawtext, False)
138
138
139 def _islfs(rlog, node=None, rev=None):
139 def _islfs(rlog, node=None, rev=None):
140 if rev is None:
140 if rev is None:
141 if node is None:
141 if node is None:
142 # both None - likely working copy content where node is not ready
142 # both None - likely working copy content where node is not ready
143 return False
143 return False
144 rev = rlog._revlog.rev(node)
144 rev = rlog._revlog.rev(node)
145 else:
145 else:
146 node = rlog._revlog.node(rev)
146 node = rlog._revlog.node(rev)
147 if node == nullid:
147 if node == nullid:
148 return False
148 return False
149 flags = rlog._revlog.flags(rev)
149 flags = rlog._revlog.flags(rev)
150 return bool(flags & revlog.REVIDX_EXTSTORED)
150 return bool(flags & revlog.REVIDX_EXTSTORED)
151
151
152 # Wrapping may also be applied by remotefilelog
152 # Wrapping may also be applied by remotefilelog
153 def filelogaddrevision(orig, self, text, transaction, link, p1, p2,
153 def filelogaddrevision(orig, self, text, transaction, link, p1, p2,
154 cachedelta=None, node=None,
154 cachedelta=None, node=None,
155 flags=revlog.REVIDX_DEFAULT_FLAGS, **kwds):
155 flags=revlog.REVIDX_DEFAULT_FLAGS, **kwds):
156 # The matcher isn't available if reposetup() wasn't called.
156 # The matcher isn't available if reposetup() wasn't called.
157 lfstrack = self._revlog.opener.options.get('lfstrack')
157 lfstrack = self._revlog.opener.options.get('lfstrack')
158
158
159 if lfstrack:
159 if lfstrack:
160 textlen = len(text)
160 textlen = len(text)
161 # exclude hg rename meta from file size
161 # exclude hg rename meta from file size
162 meta, offset = storageutil.parsemeta(text)
162 meta, offset = storageutil.parsemeta(text)
163 if offset:
163 if offset:
164 textlen -= offset
164 textlen -= offset
165
165
166 if lfstrack(self._revlog.filename, textlen):
166 if lfstrack(self._revlog.filename, textlen):
167 flags |= revlog.REVIDX_EXTSTORED
167 flags |= revlog.REVIDX_EXTSTORED
168
168
169 return orig(self, text, transaction, link, p1, p2, cachedelta=cachedelta,
169 return orig(self, text, transaction, link, p1, p2, cachedelta=cachedelta,
170 node=node, flags=flags, **kwds)
170 node=node, flags=flags, **kwds)
171
171
172 # Wrapping may also be applied by remotefilelog
172 # Wrapping may also be applied by remotefilelog
173 def filelogrenamed(orig, self, node):
173 def filelogrenamed(orig, self, node):
174 if _islfs(self, node):
174 if _islfs(self, node):
175 rawtext = self._revlog.rawdata(node)
175 rawtext = self._revlog.rawdata(node)
176 if not rawtext:
176 if not rawtext:
177 return False
177 return False
178 metadata = pointer.deserialize(rawtext)
178 metadata = pointer.deserialize(rawtext)
179 if 'x-hg-copy' in metadata and 'x-hg-copyrev' in metadata:
179 if 'x-hg-copy' in metadata and 'x-hg-copyrev' in metadata:
180 return metadata['x-hg-copy'], bin(metadata['x-hg-copyrev'])
180 return metadata['x-hg-copy'], bin(metadata['x-hg-copyrev'])
181 else:
181 else:
182 return False
182 return False
183 return orig(self, node)
183 return orig(self, node)
184
184
185 # Wrapping may also be applied by remotefilelog
185 # Wrapping may also be applied by remotefilelog
186 def filelogsize(orig, self, rev):
186 def filelogsize(orig, self, rev):
187 if _islfs(self, rev=rev):
187 if _islfs(self, rev=rev):
188 # fast path: use lfs metadata to answer size
188 # fast path: use lfs metadata to answer size
189 rawtext = self._revlog.rawdata(rev)
189 rawtext = self._revlog.rawdata(rev)
190 metadata = pointer.deserialize(rawtext)
190 metadata = pointer.deserialize(rawtext)
191 return int(metadata['size'])
191 return int(metadata['size'])
192 return orig(self, rev)
192 return orig(self, rev)
193
193
194 @eh.wrapfunction(context.basefilectx, 'cmp')
194 @eh.wrapfunction(context.basefilectx, 'cmp')
195 def filectxcmp(orig, self, fctx):
195 def filectxcmp(orig, self, fctx):
196 """returns True if text is different than fctx"""
196 """returns True if text is different than fctx"""
197 # some fctx (ex. hg-git) is not based on basefilectx and do not have islfs
197 # some fctx (ex. hg-git) is not based on basefilectx and do not have islfs
198 if self.islfs() and getattr(fctx, 'islfs', lambda: False)():
198 if self.islfs() and getattr(fctx, 'islfs', lambda: False)():
199 # fast path: check LFS oid
199 # fast path: check LFS oid
200 p1 = pointer.deserialize(self.rawdata())
200 p1 = pointer.deserialize(self.rawdata())
201 p2 = pointer.deserialize(fctx.rawdata())
201 p2 = pointer.deserialize(fctx.rawdata())
202 return p1.oid() != p2.oid()
202 return p1.oid() != p2.oid()
203 return orig(self, fctx)
203 return orig(self, fctx)
204
204
205 @eh.wrapfunction(context.basefilectx, 'isbinary')
205 @eh.wrapfunction(context.basefilectx, 'isbinary')
206 def filectxisbinary(orig, self):
206 def filectxisbinary(orig, self):
207 if self.islfs():
207 if self.islfs():
208 # fast path: use lfs metadata to answer isbinary
208 # fast path: use lfs metadata to answer isbinary
209 metadata = pointer.deserialize(self.rawdata())
209 metadata = pointer.deserialize(self.rawdata())
210 # if lfs metadata says nothing, assume it's binary by default
210 # if lfs metadata says nothing, assume it's binary by default
211 return bool(int(metadata.get('x-is-binary', 1)))
211 return bool(int(metadata.get('x-is-binary', 1)))
212 return orig(self)
212 return orig(self)
213
213
214 def filectxislfs(self):
214 def filectxislfs(self):
215 return _islfs(self.filelog(), self.filenode())
215 return _islfs(self.filelog(), self.filenode())
216
216
217 @eh.wrapfunction(cmdutil, '_updatecatformatter')
217 @eh.wrapfunction(cmdutil, '_updatecatformatter')
218 def _updatecatformatter(orig, fm, ctx, matcher, path, decode):
218 def _updatecatformatter(orig, fm, ctx, matcher, path, decode):
219 orig(fm, ctx, matcher, path, decode)
219 orig(fm, ctx, matcher, path, decode)
220 fm.data(rawdata=ctx[path].rawdata())
220 fm.data(rawdata=ctx[path].rawdata())
221
221
222 @eh.wrapfunction(scmutil, 'wrapconvertsink')
222 @eh.wrapfunction(scmutil, 'wrapconvertsink')
223 def convertsink(orig, sink):
223 def convertsink(orig, sink):
224 sink = orig(sink)
224 sink = orig(sink)
225 if sink.repotype == 'hg':
225 if sink.repotype == 'hg':
226 class lfssink(sink.__class__):
226 class lfssink(sink.__class__):
227 def putcommit(self, files, copies, parents, commit, source, revmap,
227 def putcommit(self, files, copies, parents, commit, source, revmap,
228 full, cleanp2):
228 full, cleanp2):
229 pc = super(lfssink, self).putcommit
229 pc = super(lfssink, self).putcommit
230 node = pc(files, copies, parents, commit, source, revmap, full,
230 node = pc(files, copies, parents, commit, source, revmap, full,
231 cleanp2)
231 cleanp2)
232
232
233 if 'lfs' not in self.repo.requirements:
233 if 'lfs' not in self.repo.requirements:
234 ctx = self.repo[node]
234 ctx = self.repo[node]
235
235
236 # The file list may contain removed files, so check for
236 # The file list may contain removed files, so check for
237 # membership before assuming it is in the context.
237 # membership before assuming it is in the context.
238 if any(f in ctx and ctx[f].islfs() for f, n in files):
238 if any(f in ctx and ctx[f].islfs() for f, n in files):
239 self.repo.requirements.add('lfs')
239 self.repo.requirements.add('lfs')
240 self.repo._writerequirements()
240 self.repo._writerequirements()
241
241
242 return node
242 return node
243
243
244 sink.__class__ = lfssink
244 sink.__class__ = lfssink
245
245
246 return sink
246 return sink
247
247
248 # bundlerepo uses "vfsmod.readonlyvfs(othervfs)", we need to make sure lfs
248 # bundlerepo uses "vfsmod.readonlyvfs(othervfs)", we need to make sure lfs
249 # options and blob stores are passed from othervfs to the new readonlyvfs.
249 # options and blob stores are passed from othervfs to the new readonlyvfs.
250 @eh.wrapfunction(vfsmod.readonlyvfs, '__init__')
250 @eh.wrapfunction(vfsmod.readonlyvfs, '__init__')
251 def vfsinit(orig, self, othervfs):
251 def vfsinit(orig, self, othervfs):
252 orig(self, othervfs)
252 orig(self, othervfs)
253 # copy lfs related options
253 # copy lfs related options
254 for k, v in othervfs.options.items():
254 for k, v in othervfs.options.items():
255 if k.startswith('lfs'):
255 if k.startswith('lfs'):
256 self.options[k] = v
256 self.options[k] = v
257 # also copy lfs blobstores. note: this can run before reposetup, so lfs
257 # also copy lfs blobstores. note: this can run before reposetup, so lfs
258 # blobstore attributes are not always ready at this time.
258 # blobstore attributes are not always ready at this time.
259 for name in ['lfslocalblobstore', 'lfsremoteblobstore']:
259 for name in ['lfslocalblobstore', 'lfsremoteblobstore']:
260 if util.safehasattr(othervfs, name):
260 if util.safehasattr(othervfs, name):
261 setattr(self, name, getattr(othervfs, name))
261 setattr(self, name, getattr(othervfs, name))
262
262
263 def _prefetchfiles(repo, revs, match):
263 def _prefetchfiles(repo, revs, match):
264 """Ensure that required LFS blobs are present, fetching them as a group if
264 """Ensure that required LFS blobs are present, fetching them as a group if
265 needed."""
265 needed."""
266 if not util.safehasattr(repo.svfs, 'lfslocalblobstore'):
266 if not util.safehasattr(repo.svfs, 'lfslocalblobstore'):
267 return
267 return
268
268
269 pointers = []
269 pointers = []
270 oids = set()
270 oids = set()
271 localstore = repo.svfs.lfslocalblobstore
271 localstore = repo.svfs.lfslocalblobstore
272
272
273 for rev in revs:
273 for rev in revs:
274 ctx = repo[rev]
274 ctx = repo[rev]
275 for f in ctx.walk(match):
275 for f in ctx.walk(match):
276 p = pointerfromctx(ctx, f)
276 p = pointerfromctx(ctx, f)
277 if p and p.oid() not in oids and not localstore.has(p.oid()):
277 if p and p.oid() not in oids and not localstore.has(p.oid()):
278 p.filename = f
278 p.filename = f
279 pointers.append(p)
279 pointers.append(p)
280 oids.add(p.oid())
280 oids.add(p.oid())
281
281
282 if pointers:
282 if pointers:
283 # Recalculating the repo store here allows 'paths.default' that is set
283 # Recalculating the repo store here allows 'paths.default' that is set
284 # on the repo by a clone command to be used for the update.
284 # on the repo by a clone command to be used for the update.
285 blobstore.remote(repo).readbatch(pointers, localstore)
285 blobstore.remote(repo).readbatch(pointers, localstore)
286
286
287 def _canskipupload(repo):
287 def _canskipupload(repo):
288 # Skip if this hasn't been passed to reposetup()
288 # Skip if this hasn't been passed to reposetup()
289 if not util.safehasattr(repo.svfs, 'lfsremoteblobstore'):
289 if not util.safehasattr(repo.svfs, 'lfsremoteblobstore'):
290 return True
290 return True
291
291
292 # if remotestore is a null store, upload is a no-op and can be skipped
292 # if remotestore is a null store, upload is a no-op and can be skipped
293 return isinstance(repo.svfs.lfsremoteblobstore, blobstore._nullremote)
293 return isinstance(repo.svfs.lfsremoteblobstore, blobstore._nullremote)
294
294
295 def candownload(repo):
295 def candownload(repo):
296 # Skip if this hasn't been passed to reposetup()
296 # Skip if this hasn't been passed to reposetup()
297 if not util.safehasattr(repo.svfs, 'lfsremoteblobstore'):
297 if not util.safehasattr(repo.svfs, 'lfsremoteblobstore'):
298 return False
298 return False
299
299
300 # if remotestore is a null store, downloads will lead to nothing
300 # if remotestore is a null store, downloads will lead to nothing
301 return not isinstance(repo.svfs.lfsremoteblobstore, blobstore._nullremote)
301 return not isinstance(repo.svfs.lfsremoteblobstore, blobstore._nullremote)
302
302
303 def uploadblobsfromrevs(repo, revs):
303 def uploadblobsfromrevs(repo, revs):
304 '''upload lfs blobs introduced by revs
304 '''upload lfs blobs introduced by revs
305
305
306 Note: also used by other extensions e. g. infinitepush. avoid renaming.
306 Note: also used by other extensions e. g. infinitepush. avoid renaming.
307 '''
307 '''
308 if _canskipupload(repo):
308 if _canskipupload(repo):
309 return
309 return
310 pointers = extractpointers(repo, revs)
310 pointers = extractpointers(repo, revs)
311 uploadblobs(repo, pointers)
311 uploadblobs(repo, pointers)
312
312
313 def prepush(pushop):
313 def prepush(pushop):
314 """Prepush hook.
314 """Prepush hook.
315
315
316 Read through the revisions to push, looking for filelog entries that can be
316 Read through the revisions to push, looking for filelog entries that can be
317 deserialized into metadata so that we can block the push on their upload to
317 deserialized into metadata so that we can block the push on their upload to
318 the remote blobstore.
318 the remote blobstore.
319 """
319 """
320 return uploadblobsfromrevs(pushop.repo, pushop.outgoing.missing)
320 return uploadblobsfromrevs(pushop.repo, pushop.outgoing.missing)
321
321
322 @eh.wrapfunction(exchange, 'push')
322 @eh.wrapfunction(exchange, 'push')
323 def push(orig, repo, remote, *args, **kwargs):
323 def push(orig, repo, remote, *args, **kwargs):
324 """bail on push if the extension isn't enabled on remote when needed, and
324 """bail on push if the extension isn't enabled on remote when needed, and
325 update the remote store based on the destination path."""
325 update the remote store based on the destination path."""
326 if 'lfs' in repo.requirements:
326 if 'lfs' in repo.requirements:
327 # If the remote peer is for a local repo, the requirement tests in the
327 # If the remote peer is for a local repo, the requirement tests in the
328 # base class method enforce lfs support. Otherwise, some revisions in
328 # base class method enforce lfs support. Otherwise, some revisions in
329 # this repo use lfs, and the remote repo needs the extension loaded.
329 # this repo use lfs, and the remote repo needs the extension loaded.
330 if not remote.local() and not remote.capable('lfs'):
330 if not remote.local() and not remote.capable('lfs'):
331 # This is a copy of the message in exchange.push() when requirements
331 # This is a copy of the message in exchange.push() when requirements
332 # are missing between local repos.
332 # are missing between local repos.
333 m = _("required features are not supported in the destination: %s")
333 m = _("required features are not supported in the destination: %s")
334 raise error.Abort(m % 'lfs',
334 raise error.Abort(m % 'lfs',
335 hint=_('enable the lfs extension on the server'))
335 hint=_('enable the lfs extension on the server'))
336
336
337 # Repositories where this extension is disabled won't have the field.
337 # Repositories where this extension is disabled won't have the field.
338 # But if there's a requirement, then the extension must be loaded AND
338 # But if there's a requirement, then the extension must be loaded AND
339 # there may be blobs to push.
339 # there may be blobs to push.
340 remotestore = repo.svfs.lfsremoteblobstore
340 remotestore = repo.svfs.lfsremoteblobstore
341 try:
341 try:
342 repo.svfs.lfsremoteblobstore = blobstore.remote(repo, remote.url())
342 repo.svfs.lfsremoteblobstore = blobstore.remote(repo, remote.url())
343 return orig(repo, remote, *args, **kwargs)
343 return orig(repo, remote, *args, **kwargs)
344 finally:
344 finally:
345 repo.svfs.lfsremoteblobstore = remotestore
345 repo.svfs.lfsremoteblobstore = remotestore
346 else:
346 else:
347 return orig(repo, remote, *args, **kwargs)
347 return orig(repo, remote, *args, **kwargs)
348
348
349 # when writing a bundle via "hg bundle" command, upload related LFS blobs
349 # when writing a bundle via "hg bundle" command, upload related LFS blobs
350 @eh.wrapfunction(bundle2, 'writenewbundle')
350 @eh.wrapfunction(bundle2, 'writenewbundle')
351 def writenewbundle(orig, ui, repo, source, filename, bundletype, outgoing,
351 def writenewbundle(orig, ui, repo, source, filename, bundletype, outgoing,
352 *args, **kwargs):
352 *args, **kwargs):
353 """upload LFS blobs added by outgoing revisions on 'hg bundle'"""
353 """upload LFS blobs added by outgoing revisions on 'hg bundle'"""
354 uploadblobsfromrevs(repo, outgoing.missing)
354 uploadblobsfromrevs(repo, outgoing.missing)
355 return orig(ui, repo, source, filename, bundletype, outgoing, *args,
355 return orig(ui, repo, source, filename, bundletype, outgoing, *args,
356 **kwargs)
356 **kwargs)
357
357
358 def extractpointers(repo, revs):
358 def extractpointers(repo, revs):
359 """return a list of lfs pointers added by given revs"""
359 """return a list of lfs pointers added by given revs"""
360 repo.ui.debug('lfs: computing set of blobs to upload\n')
360 repo.ui.debug('lfs: computing set of blobs to upload\n')
361 pointers = {}
361 pointers = {}
362
362
363 makeprogress = repo.ui.makeprogress
363 makeprogress = repo.ui.makeprogress
364 with makeprogress(_('lfs search'), _('changesets'), len(revs)) as progress:
364 with makeprogress(_('lfs search'), _('changesets'), len(revs)) as progress:
365 for r in revs:
365 for r in revs:
366 ctx = repo[r]
366 ctx = repo[r]
367 for p in pointersfromctx(ctx).values():
367 for p in pointersfromctx(ctx).values():
368 pointers[p.oid()] = p
368 pointers[p.oid()] = p
369 progress.increment()
369 progress.increment()
370 return sorted(pointers.values(), key=lambda p: p.oid())
370 return sorted(pointers.values(), key=lambda p: p.oid())
371
371
372 def pointerfromctx(ctx, f, removed=False):
372 def pointerfromctx(ctx, f, removed=False):
373 """return a pointer for the named file from the given changectx, or None if
373 """return a pointer for the named file from the given changectx, or None if
374 the file isn't LFS.
374 the file isn't LFS.
375
375
376 Optionally, the pointer for a file deleted from the context can be returned.
376 Optionally, the pointer for a file deleted from the context can be returned.
377 Since no such pointer is actually stored, and to distinguish from a non LFS
377 Since no such pointer is actually stored, and to distinguish from a non LFS
378 file, this pointer is represented by an empty dict.
378 file, this pointer is represented by an empty dict.
379 """
379 """
380 _ctx = ctx
380 _ctx = ctx
381 if f not in ctx:
381 if f not in ctx:
382 if not removed:
382 if not removed:
383 return None
383 return None
384 if f in ctx.p1():
384 if f in ctx.p1():
385 _ctx = ctx.p1()
385 _ctx = ctx.p1()
386 elif f in ctx.p2():
386 elif f in ctx.p2():
387 _ctx = ctx.p2()
387 _ctx = ctx.p2()
388 else:
388 else:
389 return None
389 return None
390 fctx = _ctx[f]
390 fctx = _ctx[f]
391 if not _islfs(fctx.filelog(), fctx.filenode()):
391 if not _islfs(fctx.filelog(), fctx.filenode()):
392 return None
392 return None
393 try:
393 try:
394 p = pointer.deserialize(fctx.rawdata())
394 p = pointer.deserialize(fctx.rawdata())
395 if ctx == _ctx:
395 if ctx == _ctx:
396 return p
396 return p
397 return {}
397 return {}
398 except pointer.InvalidPointer as ex:
398 except pointer.InvalidPointer as ex:
399 raise error.Abort(_('lfs: corrupted pointer (%s@%s): %s\n')
399 raise error.Abort(_('lfs: corrupted pointer (%s@%s): %s\n')
400 % (f, short(_ctx.node()), ex))
400 % (f, short(_ctx.node()), ex))
401
401
402 def pointersfromctx(ctx, removed=False):
402 def pointersfromctx(ctx, removed=False):
403 """return a dict {path: pointer} for given single changectx.
403 """return a dict {path: pointer} for given single changectx.
404
404
405 If ``removed`` == True and the LFS file was removed from ``ctx``, the value
405 If ``removed`` == True and the LFS file was removed from ``ctx``, the value
406 stored for the path is an empty dict.
406 stored for the path is an empty dict.
407 """
407 """
408 result = {}
408 result = {}
409 m = ctx.repo().narrowmatch()
409 m = ctx.repo().narrowmatch()
410
410
411 # TODO: consider manifest.fastread() instead
411 # TODO: consider manifest.fastread() instead
412 for f in ctx.files():
412 for f in ctx.files():
413 if not m(f):
413 if not m(f):
414 continue
414 continue
415 p = pointerfromctx(ctx, f, removed=removed)
415 p = pointerfromctx(ctx, f, removed=removed)
416 if p is not None:
416 if p is not None:
417 result[f] = p
417 result[f] = p
418 return result
418 return result
419
419
420 def uploadblobs(repo, pointers):
420 def uploadblobs(repo, pointers):
421 """upload given pointers from local blobstore"""
421 """upload given pointers from local blobstore"""
422 if not pointers:
422 if not pointers:
423 return
423 return
424
424
425 remoteblob = repo.svfs.lfsremoteblobstore
425 remoteblob = repo.svfs.lfsremoteblobstore
426 remoteblob.writebatch(pointers, repo.svfs.lfslocalblobstore)
426 remoteblob.writebatch(pointers, repo.svfs.lfslocalblobstore)
427
427
428 @eh.wrapfunction(upgrade, '_finishdatamigration')
428 @eh.wrapfunction(upgrade, '_finishdatamigration')
429 def upgradefinishdatamigration(orig, ui, srcrepo, dstrepo, requirements):
429 def upgradefinishdatamigration(orig, ui, srcrepo, dstrepo, requirements):
430 orig(ui, srcrepo, dstrepo, requirements)
430 orig(ui, srcrepo, dstrepo, requirements)
431
431
432 # Skip if this hasn't been passed to reposetup()
432 # Skip if this hasn't been passed to reposetup()
433 if (util.safehasattr(srcrepo.svfs, 'lfslocalblobstore') and
433 if (util.safehasattr(srcrepo.svfs, 'lfslocalblobstore') and
434 util.safehasattr(dstrepo.svfs, 'lfslocalblobstore')):
434 util.safehasattr(dstrepo.svfs, 'lfslocalblobstore')):
435 srclfsvfs = srcrepo.svfs.lfslocalblobstore.vfs
435 srclfsvfs = srcrepo.svfs.lfslocalblobstore.vfs
436 dstlfsvfs = dstrepo.svfs.lfslocalblobstore.vfs
436 dstlfsvfs = dstrepo.svfs.lfslocalblobstore.vfs
437
437
438 for dirpath, dirs, files in srclfsvfs.walk():
438 for dirpath, dirs, files in srclfsvfs.walk():
439 for oid in files:
439 for oid in files:
440 ui.write(_('copying lfs blob %s\n') % oid)
440 ui.write(_('copying lfs blob %s\n') % oid)
441 lfutil.link(srclfsvfs.join(oid), dstlfsvfs.join(oid))
441 lfutil.link(srclfsvfs.join(oid), dstlfsvfs.join(oid))
442
442
443 @eh.wrapfunction(upgrade, 'preservedrequirements')
443 @eh.wrapfunction(upgrade, 'preservedrequirements')
444 @eh.wrapfunction(upgrade, 'supporteddestrequirements')
444 @eh.wrapfunction(upgrade, 'supporteddestrequirements')
445 def upgraderequirements(orig, repo):
445 def upgraderequirements(orig, repo):
446 reqs = orig(repo)
446 reqs = orig(repo)
447 if 'lfs' in repo.requirements:
447 if 'lfs' in repo.requirements:
448 reqs.add('lfs')
448 reqs.add('lfs')
449 return reqs
449 return reqs
@@ -1,2634 +1,2634 b''
1 # revlog.py - storage back-end for mercurial
1 # revlog.py - storage back-end for mercurial
2 #
2 #
3 # Copyright 2005-2007 Matt Mackall <mpm@selenic.com>
3 # Copyright 2005-2007 Matt Mackall <mpm@selenic.com>
4 #
4 #
5 # This software may be used and distributed according to the terms of the
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
6 # GNU General Public License version 2 or any later version.
7
7
8 """Storage back-end for Mercurial.
8 """Storage back-end for Mercurial.
9
9
10 This provides efficient delta storage with O(1) retrieve and append
10 This provides efficient delta storage with O(1) retrieve and append
11 and O(changes) merge between branches.
11 and O(changes) merge between branches.
12 """
12 """
13
13
14 from __future__ import absolute_import
14 from __future__ import absolute_import
15
15
16 import collections
16 import collections
17 import contextlib
17 import contextlib
18 import errno
18 import errno
19 import io
19 import io
20 import os
20 import os
21 import struct
21 import struct
22 import zlib
22 import zlib
23
23
24 # import stuff from node for others to import from revlog
24 # import stuff from node for others to import from revlog
25 from .node import (
25 from .node import (
26 bin,
26 bin,
27 hex,
27 hex,
28 nullhex,
28 nullhex,
29 nullid,
29 nullid,
30 nullrev,
30 nullrev,
31 short,
31 short,
32 wdirfilenodeids,
32 wdirfilenodeids,
33 wdirhex,
33 wdirhex,
34 wdirid,
34 wdirid,
35 wdirrev,
35 wdirrev,
36 )
36 )
37 from .i18n import _
37 from .i18n import _
38 from .revlogutils.constants import (
38 from .revlogutils.constants import (
39 FLAG_GENERALDELTA,
39 FLAG_GENERALDELTA,
40 FLAG_INLINE_DATA,
40 FLAG_INLINE_DATA,
41 REVLOGV0,
41 REVLOGV0,
42 REVLOGV1,
42 REVLOGV1,
43 REVLOGV1_FLAGS,
43 REVLOGV1_FLAGS,
44 REVLOGV2,
44 REVLOGV2,
45 REVLOGV2_FLAGS,
45 REVLOGV2_FLAGS,
46 REVLOG_DEFAULT_FLAGS,
46 REVLOG_DEFAULT_FLAGS,
47 REVLOG_DEFAULT_FORMAT,
47 REVLOG_DEFAULT_FORMAT,
48 REVLOG_DEFAULT_VERSION,
48 REVLOG_DEFAULT_VERSION,
49 )
49 )
50 from .revlogutils.flagutil import (
50 from .revlogutils.flagutil import (
51 REVIDX_DEFAULT_FLAGS,
51 REVIDX_DEFAULT_FLAGS,
52 REVIDX_ELLIPSIS,
52 REVIDX_ELLIPSIS,
53 REVIDX_EXTSTORED,
53 REVIDX_EXTSTORED,
54 REVIDX_FLAGS_ORDER,
54 REVIDX_FLAGS_ORDER,
55 REVIDX_ISCENSORED,
55 REVIDX_ISCENSORED,
56 REVIDX_RAWTEXT_CHANGING_FLAGS,
56 REVIDX_RAWTEXT_CHANGING_FLAGS,
57 )
57 )
58 from .thirdparty import (
58 from .thirdparty import (
59 attr,
59 attr,
60 )
60 )
61 from . import (
61 from . import (
62 ancestor,
62 ancestor,
63 dagop,
63 dagop,
64 error,
64 error,
65 mdiff,
65 mdiff,
66 policy,
66 policy,
67 pycompat,
67 pycompat,
68 templatefilters,
68 templatefilters,
69 util,
69 util,
70 )
70 )
71 from .interfaces import (
71 from .interfaces import (
72 repository,
72 repository,
73 util as interfaceutil,
73 util as interfaceutil,
74 )
74 )
75 from .revlogutils import (
75 from .revlogutils import (
76 deltas as deltautil,
76 deltas as deltautil,
77 flagutil,
77 flagutil,
78 )
78 )
79 from .utils import (
79 from .utils import (
80 storageutil,
80 storageutil,
81 stringutil,
81 stringutil,
82 )
82 )
83
83
84 # blanked usage of all the name to prevent pyflakes constraints
84 # blanked usage of all the name to prevent pyflakes constraints
85 # We need these name available in the module for extensions.
85 # We need these name available in the module for extensions.
86 REVLOGV0
86 REVLOGV0
87 REVLOGV1
87 REVLOGV1
88 REVLOGV2
88 REVLOGV2
89 FLAG_INLINE_DATA
89 FLAG_INLINE_DATA
90 FLAG_GENERALDELTA
90 FLAG_GENERALDELTA
91 REVLOG_DEFAULT_FLAGS
91 REVLOG_DEFAULT_FLAGS
92 REVLOG_DEFAULT_FORMAT
92 REVLOG_DEFAULT_FORMAT
93 REVLOG_DEFAULT_VERSION
93 REVLOG_DEFAULT_VERSION
94 REVLOGV1_FLAGS
94 REVLOGV1_FLAGS
95 REVLOGV2_FLAGS
95 REVLOGV2_FLAGS
96 REVIDX_ISCENSORED
96 REVIDX_ISCENSORED
97 REVIDX_ELLIPSIS
97 REVIDX_ELLIPSIS
98 REVIDX_EXTSTORED
98 REVIDX_EXTSTORED
99 REVIDX_DEFAULT_FLAGS
99 REVIDX_DEFAULT_FLAGS
100 REVIDX_FLAGS_ORDER
100 REVIDX_FLAGS_ORDER
101 REVIDX_RAWTEXT_CHANGING_FLAGS
101 REVIDX_RAWTEXT_CHANGING_FLAGS
102
102
103 parsers = policy.importmod(r'parsers')
103 parsers = policy.importmod(r'parsers')
104 rustancestor = policy.importrust(r'ancestor')
104 rustancestor = policy.importrust(r'ancestor')
105 rustdagop = policy.importrust(r'dagop')
105 rustdagop = policy.importrust(r'dagop')
106
106
107 # Aliased for performance.
107 # Aliased for performance.
108 _zlibdecompress = zlib.decompress
108 _zlibdecompress = zlib.decompress
109
109
110 # max size of revlog with inline data
110 # max size of revlog with inline data
111 _maxinline = 131072
111 _maxinline = 131072
112 _chunksize = 1048576
112 _chunksize = 1048576
113
113
114 # Flag processors for REVIDX_ELLIPSIS.
114 # Flag processors for REVIDX_ELLIPSIS.
115 def ellipsisreadprocessor(rl, text):
115 def ellipsisreadprocessor(rl, text):
116 return text, False
116 return text, False, {}
117
117
118 def ellipsiswriteprocessor(rl, text):
118 def ellipsiswriteprocessor(rl, text):
119 return text, False
119 return text, False
120
120
121 def ellipsisrawprocessor(rl, text):
121 def ellipsisrawprocessor(rl, text):
122 return False
122 return False
123
123
124 ellipsisprocessor = (
124 ellipsisprocessor = (
125 ellipsisreadprocessor,
125 ellipsisreadprocessor,
126 ellipsiswriteprocessor,
126 ellipsiswriteprocessor,
127 ellipsisrawprocessor,
127 ellipsisrawprocessor,
128 )
128 )
129
129
130 def getoffset(q):
130 def getoffset(q):
131 return int(q >> 16)
131 return int(q >> 16)
132
132
133 def gettype(q):
133 def gettype(q):
134 return int(q & 0xFFFF)
134 return int(q & 0xFFFF)
135
135
136 def offset_type(offset, type):
136 def offset_type(offset, type):
137 if (type & ~flagutil.REVIDX_KNOWN_FLAGS) != 0:
137 if (type & ~flagutil.REVIDX_KNOWN_FLAGS) != 0:
138 raise ValueError('unknown revlog index flags')
138 raise ValueError('unknown revlog index flags')
139 return int(int(offset) << 16 | type)
139 return int(int(offset) << 16 | type)
140
140
141 @attr.s(slots=True, frozen=True)
141 @attr.s(slots=True, frozen=True)
142 class _revisioninfo(object):
142 class _revisioninfo(object):
143 """Information about a revision that allows building its fulltext
143 """Information about a revision that allows building its fulltext
144 node: expected hash of the revision
144 node: expected hash of the revision
145 p1, p2: parent revs of the revision
145 p1, p2: parent revs of the revision
146 btext: built text cache consisting of a one-element list
146 btext: built text cache consisting of a one-element list
147 cachedelta: (baserev, uncompressed_delta) or None
147 cachedelta: (baserev, uncompressed_delta) or None
148 flags: flags associated to the revision storage
148 flags: flags associated to the revision storage
149
149
150 One of btext[0] or cachedelta must be set.
150 One of btext[0] or cachedelta must be set.
151 """
151 """
152 node = attr.ib()
152 node = attr.ib()
153 p1 = attr.ib()
153 p1 = attr.ib()
154 p2 = attr.ib()
154 p2 = attr.ib()
155 btext = attr.ib()
155 btext = attr.ib()
156 textlen = attr.ib()
156 textlen = attr.ib()
157 cachedelta = attr.ib()
157 cachedelta = attr.ib()
158 flags = attr.ib()
158 flags = attr.ib()
159
159
160 @interfaceutil.implementer(repository.irevisiondelta)
160 @interfaceutil.implementer(repository.irevisiondelta)
161 @attr.s(slots=True)
161 @attr.s(slots=True)
162 class revlogrevisiondelta(object):
162 class revlogrevisiondelta(object):
163 node = attr.ib()
163 node = attr.ib()
164 p1node = attr.ib()
164 p1node = attr.ib()
165 p2node = attr.ib()
165 p2node = attr.ib()
166 basenode = attr.ib()
166 basenode = attr.ib()
167 flags = attr.ib()
167 flags = attr.ib()
168 baserevisionsize = attr.ib()
168 baserevisionsize = attr.ib()
169 revision = attr.ib()
169 revision = attr.ib()
170 delta = attr.ib()
170 delta = attr.ib()
171 linknode = attr.ib(default=None)
171 linknode = attr.ib(default=None)
172
172
173 @interfaceutil.implementer(repository.iverifyproblem)
173 @interfaceutil.implementer(repository.iverifyproblem)
174 @attr.s(frozen=True)
174 @attr.s(frozen=True)
175 class revlogproblem(object):
175 class revlogproblem(object):
176 warning = attr.ib(default=None)
176 warning = attr.ib(default=None)
177 error = attr.ib(default=None)
177 error = attr.ib(default=None)
178 node = attr.ib(default=None)
178 node = attr.ib(default=None)
179
179
180 # index v0:
180 # index v0:
181 # 4 bytes: offset
181 # 4 bytes: offset
182 # 4 bytes: compressed length
182 # 4 bytes: compressed length
183 # 4 bytes: base rev
183 # 4 bytes: base rev
184 # 4 bytes: link rev
184 # 4 bytes: link rev
185 # 20 bytes: parent 1 nodeid
185 # 20 bytes: parent 1 nodeid
186 # 20 bytes: parent 2 nodeid
186 # 20 bytes: parent 2 nodeid
187 # 20 bytes: nodeid
187 # 20 bytes: nodeid
188 indexformatv0 = struct.Struct(">4l20s20s20s")
188 indexformatv0 = struct.Struct(">4l20s20s20s")
189 indexformatv0_pack = indexformatv0.pack
189 indexformatv0_pack = indexformatv0.pack
190 indexformatv0_unpack = indexformatv0.unpack
190 indexformatv0_unpack = indexformatv0.unpack
191
191
192 class revlogoldindex(list):
192 class revlogoldindex(list):
193 def __getitem__(self, i):
193 def __getitem__(self, i):
194 if i == -1:
194 if i == -1:
195 return (0, 0, 0, -1, -1, -1, -1, nullid)
195 return (0, 0, 0, -1, -1, -1, -1, nullid)
196 return list.__getitem__(self, i)
196 return list.__getitem__(self, i)
197
197
198 class revlogoldio(object):
198 class revlogoldio(object):
199 def __init__(self):
199 def __init__(self):
200 self.size = indexformatv0.size
200 self.size = indexformatv0.size
201
201
202 def parseindex(self, data, inline):
202 def parseindex(self, data, inline):
203 s = self.size
203 s = self.size
204 index = []
204 index = []
205 nodemap = {nullid: nullrev}
205 nodemap = {nullid: nullrev}
206 n = off = 0
206 n = off = 0
207 l = len(data)
207 l = len(data)
208 while off + s <= l:
208 while off + s <= l:
209 cur = data[off:off + s]
209 cur = data[off:off + s]
210 off += s
210 off += s
211 e = indexformatv0_unpack(cur)
211 e = indexformatv0_unpack(cur)
212 # transform to revlogv1 format
212 # transform to revlogv1 format
213 e2 = (offset_type(e[0], 0), e[1], -1, e[2], e[3],
213 e2 = (offset_type(e[0], 0), e[1], -1, e[2], e[3],
214 nodemap.get(e[4], nullrev), nodemap.get(e[5], nullrev), e[6])
214 nodemap.get(e[4], nullrev), nodemap.get(e[5], nullrev), e[6])
215 index.append(e2)
215 index.append(e2)
216 nodemap[e[6]] = n
216 nodemap[e[6]] = n
217 n += 1
217 n += 1
218
218
219 return revlogoldindex(index), nodemap, None
219 return revlogoldindex(index), nodemap, None
220
220
221 def packentry(self, entry, node, version, rev):
221 def packentry(self, entry, node, version, rev):
222 if gettype(entry[0]):
222 if gettype(entry[0]):
223 raise error.RevlogError(_('index entry flags need revlog '
223 raise error.RevlogError(_('index entry flags need revlog '
224 'version 1'))
224 'version 1'))
225 e2 = (getoffset(entry[0]), entry[1], entry[3], entry[4],
225 e2 = (getoffset(entry[0]), entry[1], entry[3], entry[4],
226 node(entry[5]), node(entry[6]), entry[7])
226 node(entry[5]), node(entry[6]), entry[7])
227 return indexformatv0_pack(*e2)
227 return indexformatv0_pack(*e2)
228
228
229 # index ng:
229 # index ng:
230 # 6 bytes: offset
230 # 6 bytes: offset
231 # 2 bytes: flags
231 # 2 bytes: flags
232 # 4 bytes: compressed length
232 # 4 bytes: compressed length
233 # 4 bytes: uncompressed length
233 # 4 bytes: uncompressed length
234 # 4 bytes: base rev
234 # 4 bytes: base rev
235 # 4 bytes: link rev
235 # 4 bytes: link rev
236 # 4 bytes: parent 1 rev
236 # 4 bytes: parent 1 rev
237 # 4 bytes: parent 2 rev
237 # 4 bytes: parent 2 rev
238 # 32 bytes: nodeid
238 # 32 bytes: nodeid
239 indexformatng = struct.Struct(">Qiiiiii20s12x")
239 indexformatng = struct.Struct(">Qiiiiii20s12x")
240 indexformatng_pack = indexformatng.pack
240 indexformatng_pack = indexformatng.pack
241 versionformat = struct.Struct(">I")
241 versionformat = struct.Struct(">I")
242 versionformat_pack = versionformat.pack
242 versionformat_pack = versionformat.pack
243 versionformat_unpack = versionformat.unpack
243 versionformat_unpack = versionformat.unpack
244
244
245 # corresponds to uncompressed length of indexformatng (2 gigs, 4-byte
245 # corresponds to uncompressed length of indexformatng (2 gigs, 4-byte
246 # signed integer)
246 # signed integer)
247 _maxentrysize = 0x7fffffff
247 _maxentrysize = 0x7fffffff
248
248
249 class revlogio(object):
249 class revlogio(object):
250 def __init__(self):
250 def __init__(self):
251 self.size = indexformatng.size
251 self.size = indexformatng.size
252
252
253 def parseindex(self, data, inline):
253 def parseindex(self, data, inline):
254 # call the C implementation to parse the index data
254 # call the C implementation to parse the index data
255 index, cache = parsers.parse_index2(data, inline)
255 index, cache = parsers.parse_index2(data, inline)
256 return index, getattr(index, 'nodemap', None), cache
256 return index, getattr(index, 'nodemap', None), cache
257
257
258 def packentry(self, entry, node, version, rev):
258 def packentry(self, entry, node, version, rev):
259 p = indexformatng_pack(*entry)
259 p = indexformatng_pack(*entry)
260 if rev == 0:
260 if rev == 0:
261 p = versionformat_pack(version) + p[4:]
261 p = versionformat_pack(version) + p[4:]
262 return p
262 return p
263
263
264 class revlog(flagutil.flagprocessorsmixin):
264 class revlog(flagutil.flagprocessorsmixin):
265 """
265 """
266 the underlying revision storage object
266 the underlying revision storage object
267
267
268 A revlog consists of two parts, an index and the revision data.
268 A revlog consists of two parts, an index and the revision data.
269
269
270 The index is a file with a fixed record size containing
270 The index is a file with a fixed record size containing
271 information on each revision, including its nodeid (hash), the
271 information on each revision, including its nodeid (hash), the
272 nodeids of its parents, the position and offset of its data within
272 nodeids of its parents, the position and offset of its data within
273 the data file, and the revision it's based on. Finally, each entry
273 the data file, and the revision it's based on. Finally, each entry
274 contains a linkrev entry that can serve as a pointer to external
274 contains a linkrev entry that can serve as a pointer to external
275 data.
275 data.
276
276
277 The revision data itself is a linear collection of data chunks.
277 The revision data itself is a linear collection of data chunks.
278 Each chunk represents a revision and is usually represented as a
278 Each chunk represents a revision and is usually represented as a
279 delta against the previous chunk. To bound lookup time, runs of
279 delta against the previous chunk. To bound lookup time, runs of
280 deltas are limited to about 2 times the length of the original
280 deltas are limited to about 2 times the length of the original
281 version data. This makes retrieval of a version proportional to
281 version data. This makes retrieval of a version proportional to
282 its size, or O(1) relative to the number of revisions.
282 its size, or O(1) relative to the number of revisions.
283
283
284 Both pieces of the revlog are written to in an append-only
284 Both pieces of the revlog are written to in an append-only
285 fashion, which means we never need to rewrite a file to insert or
285 fashion, which means we never need to rewrite a file to insert or
286 remove data, and can use some simple techniques to avoid the need
286 remove data, and can use some simple techniques to avoid the need
287 for locking while reading.
287 for locking while reading.
288
288
289 If checkambig, indexfile is opened with checkambig=True at
289 If checkambig, indexfile is opened with checkambig=True at
290 writing, to avoid file stat ambiguity.
290 writing, to avoid file stat ambiguity.
291
291
292 If mmaplargeindex is True, and an mmapindexthreshold is set, the
292 If mmaplargeindex is True, and an mmapindexthreshold is set, the
293 index will be mmapped rather than read if it is larger than the
293 index will be mmapped rather than read if it is larger than the
294 configured threshold.
294 configured threshold.
295
295
296 If censorable is True, the revlog can have censored revisions.
296 If censorable is True, the revlog can have censored revisions.
297
297
298 If `upperboundcomp` is not None, this is the expected maximal gain from
298 If `upperboundcomp` is not None, this is the expected maximal gain from
299 compression for the data content.
299 compression for the data content.
300 """
300 """
301 def __init__(self, opener, indexfile, datafile=None, checkambig=False,
301 def __init__(self, opener, indexfile, datafile=None, checkambig=False,
302 mmaplargeindex=False, censorable=False,
302 mmaplargeindex=False, censorable=False,
303 upperboundcomp=None):
303 upperboundcomp=None):
304 """
304 """
305 create a revlog object
305 create a revlog object
306
306
307 opener is a function that abstracts the file opening operation
307 opener is a function that abstracts the file opening operation
308 and can be used to implement COW semantics or the like.
308 and can be used to implement COW semantics or the like.
309
309
310 """
310 """
311 self.upperboundcomp = upperboundcomp
311 self.upperboundcomp = upperboundcomp
312 self.indexfile = indexfile
312 self.indexfile = indexfile
313 self.datafile = datafile or (indexfile[:-2] + ".d")
313 self.datafile = datafile or (indexfile[:-2] + ".d")
314 self.opener = opener
314 self.opener = opener
315 # When True, indexfile is opened with checkambig=True at writing, to
315 # When True, indexfile is opened with checkambig=True at writing, to
316 # avoid file stat ambiguity.
316 # avoid file stat ambiguity.
317 self._checkambig = checkambig
317 self._checkambig = checkambig
318 self._mmaplargeindex = mmaplargeindex
318 self._mmaplargeindex = mmaplargeindex
319 self._censorable = censorable
319 self._censorable = censorable
320 # 3-tuple of (node, rev, text) for a raw revision.
320 # 3-tuple of (node, rev, text) for a raw revision.
321 self._revisioncache = None
321 self._revisioncache = None
322 # Maps rev to chain base rev.
322 # Maps rev to chain base rev.
323 self._chainbasecache = util.lrucachedict(100)
323 self._chainbasecache = util.lrucachedict(100)
324 # 2-tuple of (offset, data) of raw data from the revlog at an offset.
324 # 2-tuple of (offset, data) of raw data from the revlog at an offset.
325 self._chunkcache = (0, '')
325 self._chunkcache = (0, '')
326 # How much data to read and cache into the raw revlog data cache.
326 # How much data to read and cache into the raw revlog data cache.
327 self._chunkcachesize = 65536
327 self._chunkcachesize = 65536
328 self._maxchainlen = None
328 self._maxchainlen = None
329 self._deltabothparents = True
329 self._deltabothparents = True
330 self.index = []
330 self.index = []
331 # Mapping of partial identifiers to full nodes.
331 # Mapping of partial identifiers to full nodes.
332 self._pcache = {}
332 self._pcache = {}
333 # Mapping of revision integer to full node.
333 # Mapping of revision integer to full node.
334 self._nodecache = {nullid: nullrev}
334 self._nodecache = {nullid: nullrev}
335 self._nodepos = None
335 self._nodepos = None
336 self._compengine = 'zlib'
336 self._compengine = 'zlib'
337 self._compengineopts = {}
337 self._compengineopts = {}
338 self._maxdeltachainspan = -1
338 self._maxdeltachainspan = -1
339 self._withsparseread = False
339 self._withsparseread = False
340 self._sparserevlog = False
340 self._sparserevlog = False
341 self._srdensitythreshold = 0.50
341 self._srdensitythreshold = 0.50
342 self._srmingapsize = 262144
342 self._srmingapsize = 262144
343
343
344 # Make copy of flag processors so each revlog instance can support
344 # Make copy of flag processors so each revlog instance can support
345 # custom flags.
345 # custom flags.
346 self._flagprocessors = dict(flagutil.flagprocessors)
346 self._flagprocessors = dict(flagutil.flagprocessors)
347
347
348 # 2-tuple of file handles being used for active writing.
348 # 2-tuple of file handles being used for active writing.
349 self._writinghandles = None
349 self._writinghandles = None
350
350
351 self._loadindex()
351 self._loadindex()
352
352
353 def _loadindex(self):
353 def _loadindex(self):
354 mmapindexthreshold = None
354 mmapindexthreshold = None
355 opts = getattr(self.opener, 'options', {}) or {}
355 opts = getattr(self.opener, 'options', {}) or {}
356
356
357 if 'revlogv2' in opts:
357 if 'revlogv2' in opts:
358 newversionflags = REVLOGV2 | FLAG_INLINE_DATA
358 newversionflags = REVLOGV2 | FLAG_INLINE_DATA
359 elif 'revlogv1' in opts:
359 elif 'revlogv1' in opts:
360 newversionflags = REVLOGV1 | FLAG_INLINE_DATA
360 newversionflags = REVLOGV1 | FLAG_INLINE_DATA
361 if 'generaldelta' in opts:
361 if 'generaldelta' in opts:
362 newversionflags |= FLAG_GENERALDELTA
362 newversionflags |= FLAG_GENERALDELTA
363 elif getattr(self.opener, 'options', None) is not None:
363 elif getattr(self.opener, 'options', None) is not None:
364 # If options provided but no 'revlog*' found, the repository
364 # If options provided but no 'revlog*' found, the repository
365 # would have no 'requires' file in it, which means we have to
365 # would have no 'requires' file in it, which means we have to
366 # stick to the old format.
366 # stick to the old format.
367 newversionflags = REVLOGV0
367 newversionflags = REVLOGV0
368 else:
368 else:
369 newversionflags = REVLOG_DEFAULT_VERSION
369 newversionflags = REVLOG_DEFAULT_VERSION
370
370
371 if 'chunkcachesize' in opts:
371 if 'chunkcachesize' in opts:
372 self._chunkcachesize = opts['chunkcachesize']
372 self._chunkcachesize = opts['chunkcachesize']
373 if 'maxchainlen' in opts:
373 if 'maxchainlen' in opts:
374 self._maxchainlen = opts['maxchainlen']
374 self._maxchainlen = opts['maxchainlen']
375 if 'deltabothparents' in opts:
375 if 'deltabothparents' in opts:
376 self._deltabothparents = opts['deltabothparents']
376 self._deltabothparents = opts['deltabothparents']
377 self._lazydelta = bool(opts.get('lazydelta', True))
377 self._lazydelta = bool(opts.get('lazydelta', True))
378 self._lazydeltabase = False
378 self._lazydeltabase = False
379 if self._lazydelta:
379 if self._lazydelta:
380 self._lazydeltabase = bool(opts.get('lazydeltabase', False))
380 self._lazydeltabase = bool(opts.get('lazydeltabase', False))
381 if 'compengine' in opts:
381 if 'compengine' in opts:
382 self._compengine = opts['compengine']
382 self._compengine = opts['compengine']
383 if 'zlib.level' in opts:
383 if 'zlib.level' in opts:
384 self._compengineopts['zlib.level'] = opts['zlib.level']
384 self._compengineopts['zlib.level'] = opts['zlib.level']
385 if 'zstd.level' in opts:
385 if 'zstd.level' in opts:
386 self._compengineopts['zstd.level'] = opts['zstd.level']
386 self._compengineopts['zstd.level'] = opts['zstd.level']
387 if 'maxdeltachainspan' in opts:
387 if 'maxdeltachainspan' in opts:
388 self._maxdeltachainspan = opts['maxdeltachainspan']
388 self._maxdeltachainspan = opts['maxdeltachainspan']
389 if self._mmaplargeindex and 'mmapindexthreshold' in opts:
389 if self._mmaplargeindex and 'mmapindexthreshold' in opts:
390 mmapindexthreshold = opts['mmapindexthreshold']
390 mmapindexthreshold = opts['mmapindexthreshold']
391 self._sparserevlog = bool(opts.get('sparse-revlog', False))
391 self._sparserevlog = bool(opts.get('sparse-revlog', False))
392 withsparseread = bool(opts.get('with-sparse-read', False))
392 withsparseread = bool(opts.get('with-sparse-read', False))
393 # sparse-revlog forces sparse-read
393 # sparse-revlog forces sparse-read
394 self._withsparseread = self._sparserevlog or withsparseread
394 self._withsparseread = self._sparserevlog or withsparseread
395 if 'sparse-read-density-threshold' in opts:
395 if 'sparse-read-density-threshold' in opts:
396 self._srdensitythreshold = opts['sparse-read-density-threshold']
396 self._srdensitythreshold = opts['sparse-read-density-threshold']
397 if 'sparse-read-min-gap-size' in opts:
397 if 'sparse-read-min-gap-size' in opts:
398 self._srmingapsize = opts['sparse-read-min-gap-size']
398 self._srmingapsize = opts['sparse-read-min-gap-size']
399 if opts.get('enableellipsis'):
399 if opts.get('enableellipsis'):
400 self._flagprocessors[REVIDX_ELLIPSIS] = ellipsisprocessor
400 self._flagprocessors[REVIDX_ELLIPSIS] = ellipsisprocessor
401
401
402 # revlog v0 doesn't have flag processors
402 # revlog v0 doesn't have flag processors
403 for flag, processor in opts.get(b'flagprocessors', {}).iteritems():
403 for flag, processor in opts.get(b'flagprocessors', {}).iteritems():
404 flagutil.insertflagprocessor(flag, processor, self._flagprocessors)
404 flagutil.insertflagprocessor(flag, processor, self._flagprocessors)
405
405
406 if self._chunkcachesize <= 0:
406 if self._chunkcachesize <= 0:
407 raise error.RevlogError(_('revlog chunk cache size %r is not '
407 raise error.RevlogError(_('revlog chunk cache size %r is not '
408 'greater than 0') % self._chunkcachesize)
408 'greater than 0') % self._chunkcachesize)
409 elif self._chunkcachesize & (self._chunkcachesize - 1):
409 elif self._chunkcachesize & (self._chunkcachesize - 1):
410 raise error.RevlogError(_('revlog chunk cache size %r is not a '
410 raise error.RevlogError(_('revlog chunk cache size %r is not a '
411 'power of 2') % self._chunkcachesize)
411 'power of 2') % self._chunkcachesize)
412
412
413 indexdata = ''
413 indexdata = ''
414 self._initempty = True
414 self._initempty = True
415 try:
415 try:
416 with self._indexfp() as f:
416 with self._indexfp() as f:
417 if (mmapindexthreshold is not None and
417 if (mmapindexthreshold is not None and
418 self.opener.fstat(f).st_size >= mmapindexthreshold):
418 self.opener.fstat(f).st_size >= mmapindexthreshold):
419 # TODO: should .close() to release resources without
419 # TODO: should .close() to release resources without
420 # relying on Python GC
420 # relying on Python GC
421 indexdata = util.buffer(util.mmapread(f))
421 indexdata = util.buffer(util.mmapread(f))
422 else:
422 else:
423 indexdata = f.read()
423 indexdata = f.read()
424 if len(indexdata) > 0:
424 if len(indexdata) > 0:
425 versionflags = versionformat_unpack(indexdata[:4])[0]
425 versionflags = versionformat_unpack(indexdata[:4])[0]
426 self._initempty = False
426 self._initempty = False
427 else:
427 else:
428 versionflags = newversionflags
428 versionflags = newversionflags
429 except IOError as inst:
429 except IOError as inst:
430 if inst.errno != errno.ENOENT:
430 if inst.errno != errno.ENOENT:
431 raise
431 raise
432
432
433 versionflags = newversionflags
433 versionflags = newversionflags
434
434
435 self.version = versionflags
435 self.version = versionflags
436
436
437 flags = versionflags & ~0xFFFF
437 flags = versionflags & ~0xFFFF
438 fmt = versionflags & 0xFFFF
438 fmt = versionflags & 0xFFFF
439
439
440 if fmt == REVLOGV0:
440 if fmt == REVLOGV0:
441 if flags:
441 if flags:
442 raise error.RevlogError(_('unknown flags (%#04x) in version %d '
442 raise error.RevlogError(_('unknown flags (%#04x) in version %d '
443 'revlog %s') %
443 'revlog %s') %
444 (flags >> 16, fmt, self.indexfile))
444 (flags >> 16, fmt, self.indexfile))
445
445
446 self._inline = False
446 self._inline = False
447 self._generaldelta = False
447 self._generaldelta = False
448
448
449 elif fmt == REVLOGV1:
449 elif fmt == REVLOGV1:
450 if flags & ~REVLOGV1_FLAGS:
450 if flags & ~REVLOGV1_FLAGS:
451 raise error.RevlogError(_('unknown flags (%#04x) in version %d '
451 raise error.RevlogError(_('unknown flags (%#04x) in version %d '
452 'revlog %s') %
452 'revlog %s') %
453 (flags >> 16, fmt, self.indexfile))
453 (flags >> 16, fmt, self.indexfile))
454
454
455 self._inline = versionflags & FLAG_INLINE_DATA
455 self._inline = versionflags & FLAG_INLINE_DATA
456 self._generaldelta = versionflags & FLAG_GENERALDELTA
456 self._generaldelta = versionflags & FLAG_GENERALDELTA
457
457
458 elif fmt == REVLOGV2:
458 elif fmt == REVLOGV2:
459 if flags & ~REVLOGV2_FLAGS:
459 if flags & ~REVLOGV2_FLAGS:
460 raise error.RevlogError(_('unknown flags (%#04x) in version %d '
460 raise error.RevlogError(_('unknown flags (%#04x) in version %d '
461 'revlog %s') %
461 'revlog %s') %
462 (flags >> 16, fmt, self.indexfile))
462 (flags >> 16, fmt, self.indexfile))
463
463
464 self._inline = versionflags & FLAG_INLINE_DATA
464 self._inline = versionflags & FLAG_INLINE_DATA
465 # generaldelta implied by version 2 revlogs.
465 # generaldelta implied by version 2 revlogs.
466 self._generaldelta = True
466 self._generaldelta = True
467
467
468 else:
468 else:
469 raise error.RevlogError(_('unknown version (%d) in revlog %s') %
469 raise error.RevlogError(_('unknown version (%d) in revlog %s') %
470 (fmt, self.indexfile))
470 (fmt, self.indexfile))
471 # sparse-revlog can't be on without general-delta (issue6056)
471 # sparse-revlog can't be on without general-delta (issue6056)
472 if not self._generaldelta:
472 if not self._generaldelta:
473 self._sparserevlog = False
473 self._sparserevlog = False
474
474
475 self._storedeltachains = True
475 self._storedeltachains = True
476
476
477 self._io = revlogio()
477 self._io = revlogio()
478 if self.version == REVLOGV0:
478 if self.version == REVLOGV0:
479 self._io = revlogoldio()
479 self._io = revlogoldio()
480 try:
480 try:
481 d = self._io.parseindex(indexdata, self._inline)
481 d = self._io.parseindex(indexdata, self._inline)
482 except (ValueError, IndexError):
482 except (ValueError, IndexError):
483 raise error.RevlogError(_("index %s is corrupted") %
483 raise error.RevlogError(_("index %s is corrupted") %
484 self.indexfile)
484 self.indexfile)
485 self.index, nodemap, self._chunkcache = d
485 self.index, nodemap, self._chunkcache = d
486 if nodemap is not None:
486 if nodemap is not None:
487 self.nodemap = self._nodecache = nodemap
487 self.nodemap = self._nodecache = nodemap
488 if not self._chunkcache:
488 if not self._chunkcache:
489 self._chunkclear()
489 self._chunkclear()
490 # revnum -> (chain-length, sum-delta-length)
490 # revnum -> (chain-length, sum-delta-length)
491 self._chaininfocache = {}
491 self._chaininfocache = {}
492 # revlog header -> revlog compressor
492 # revlog header -> revlog compressor
493 self._decompressors = {}
493 self._decompressors = {}
494
494
495 @util.propertycache
495 @util.propertycache
496 def _compressor(self):
496 def _compressor(self):
497 engine = util.compengines[self._compengine]
497 engine = util.compengines[self._compengine]
498 return engine.revlogcompressor(self._compengineopts)
498 return engine.revlogcompressor(self._compengineopts)
499
499
500 def _indexfp(self, mode='r'):
500 def _indexfp(self, mode='r'):
501 """file object for the revlog's index file"""
501 """file object for the revlog's index file"""
502 args = {r'mode': mode}
502 args = {r'mode': mode}
503 if mode != 'r':
503 if mode != 'r':
504 args[r'checkambig'] = self._checkambig
504 args[r'checkambig'] = self._checkambig
505 if mode == 'w':
505 if mode == 'w':
506 args[r'atomictemp'] = True
506 args[r'atomictemp'] = True
507 return self.opener(self.indexfile, **args)
507 return self.opener(self.indexfile, **args)
508
508
509 def _datafp(self, mode='r'):
509 def _datafp(self, mode='r'):
510 """file object for the revlog's data file"""
510 """file object for the revlog's data file"""
511 return self.opener(self.datafile, mode=mode)
511 return self.opener(self.datafile, mode=mode)
512
512
513 @contextlib.contextmanager
513 @contextlib.contextmanager
514 def _datareadfp(self, existingfp=None):
514 def _datareadfp(self, existingfp=None):
515 """file object suitable to read data"""
515 """file object suitable to read data"""
516 # Use explicit file handle, if given.
516 # Use explicit file handle, if given.
517 if existingfp is not None:
517 if existingfp is not None:
518 yield existingfp
518 yield existingfp
519
519
520 # Use a file handle being actively used for writes, if available.
520 # Use a file handle being actively used for writes, if available.
521 # There is some danger to doing this because reads will seek the
521 # There is some danger to doing this because reads will seek the
522 # file. However, _writeentry() performs a SEEK_END before all writes,
522 # file. However, _writeentry() performs a SEEK_END before all writes,
523 # so we should be safe.
523 # so we should be safe.
524 elif self._writinghandles:
524 elif self._writinghandles:
525 if self._inline:
525 if self._inline:
526 yield self._writinghandles[0]
526 yield self._writinghandles[0]
527 else:
527 else:
528 yield self._writinghandles[1]
528 yield self._writinghandles[1]
529
529
530 # Otherwise open a new file handle.
530 # Otherwise open a new file handle.
531 else:
531 else:
532 if self._inline:
532 if self._inline:
533 func = self._indexfp
533 func = self._indexfp
534 else:
534 else:
535 func = self._datafp
535 func = self._datafp
536 with func() as fp:
536 with func() as fp:
537 yield fp
537 yield fp
538
538
539 def tip(self):
539 def tip(self):
540 return self.node(len(self.index) - 1)
540 return self.node(len(self.index) - 1)
541 def __contains__(self, rev):
541 def __contains__(self, rev):
542 return 0 <= rev < len(self)
542 return 0 <= rev < len(self)
543 def __len__(self):
543 def __len__(self):
544 return len(self.index)
544 return len(self.index)
545 def __iter__(self):
545 def __iter__(self):
546 return iter(pycompat.xrange(len(self)))
546 return iter(pycompat.xrange(len(self)))
547 def revs(self, start=0, stop=None):
547 def revs(self, start=0, stop=None):
548 """iterate over all rev in this revlog (from start to stop)"""
548 """iterate over all rev in this revlog (from start to stop)"""
549 return storageutil.iterrevs(len(self), start=start, stop=stop)
549 return storageutil.iterrevs(len(self), start=start, stop=stop)
550
550
551 @util.propertycache
551 @util.propertycache
552 def nodemap(self):
552 def nodemap(self):
553 if self.index:
553 if self.index:
554 # populate mapping down to the initial node
554 # populate mapping down to the initial node
555 node0 = self.index[0][7] # get around changelog filtering
555 node0 = self.index[0][7] # get around changelog filtering
556 self.rev(node0)
556 self.rev(node0)
557 return self._nodecache
557 return self._nodecache
558
558
559 def hasnode(self, node):
559 def hasnode(self, node):
560 try:
560 try:
561 self.rev(node)
561 self.rev(node)
562 return True
562 return True
563 except KeyError:
563 except KeyError:
564 return False
564 return False
565
565
566 def candelta(self, baserev, rev):
566 def candelta(self, baserev, rev):
567 """whether two revisions (baserev, rev) can be delta-ed or not"""
567 """whether two revisions (baserev, rev) can be delta-ed or not"""
568 # Disable delta if either rev requires a content-changing flag
568 # Disable delta if either rev requires a content-changing flag
569 # processor (ex. LFS). This is because such flag processor can alter
569 # processor (ex. LFS). This is because such flag processor can alter
570 # the rawtext content that the delta will be based on, and two clients
570 # the rawtext content that the delta will be based on, and two clients
571 # could have a same revlog node with different flags (i.e. different
571 # could have a same revlog node with different flags (i.e. different
572 # rawtext contents) and the delta could be incompatible.
572 # rawtext contents) and the delta could be incompatible.
573 if ((self.flags(baserev) & REVIDX_RAWTEXT_CHANGING_FLAGS)
573 if ((self.flags(baserev) & REVIDX_RAWTEXT_CHANGING_FLAGS)
574 or (self.flags(rev) & REVIDX_RAWTEXT_CHANGING_FLAGS)):
574 or (self.flags(rev) & REVIDX_RAWTEXT_CHANGING_FLAGS)):
575 return False
575 return False
576 return True
576 return True
577
577
578 def clearcaches(self):
578 def clearcaches(self):
579 self._revisioncache = None
579 self._revisioncache = None
580 self._chainbasecache.clear()
580 self._chainbasecache.clear()
581 self._chunkcache = (0, '')
581 self._chunkcache = (0, '')
582 self._pcache = {}
582 self._pcache = {}
583
583
584 try:
584 try:
585 # If we are using the native C version, you are in a fun case
585 # If we are using the native C version, you are in a fun case
586 # where self.index, self.nodemap and self._nodecaches is the same
586 # where self.index, self.nodemap and self._nodecaches is the same
587 # object.
587 # object.
588 self._nodecache.clearcaches()
588 self._nodecache.clearcaches()
589 except AttributeError:
589 except AttributeError:
590 self._nodecache = {nullid: nullrev}
590 self._nodecache = {nullid: nullrev}
591 self._nodepos = None
591 self._nodepos = None
592
592
593 def rev(self, node):
593 def rev(self, node):
594 try:
594 try:
595 return self._nodecache[node]
595 return self._nodecache[node]
596 except TypeError:
596 except TypeError:
597 raise
597 raise
598 except error.RevlogError:
598 except error.RevlogError:
599 # parsers.c radix tree lookup failed
599 # parsers.c radix tree lookup failed
600 if node == wdirid or node in wdirfilenodeids:
600 if node == wdirid or node in wdirfilenodeids:
601 raise error.WdirUnsupported
601 raise error.WdirUnsupported
602 raise error.LookupError(node, self.indexfile, _('no node'))
602 raise error.LookupError(node, self.indexfile, _('no node'))
603 except KeyError:
603 except KeyError:
604 # pure python cache lookup failed
604 # pure python cache lookup failed
605 n = self._nodecache
605 n = self._nodecache
606 i = self.index
606 i = self.index
607 p = self._nodepos
607 p = self._nodepos
608 if p is None:
608 if p is None:
609 p = len(i) - 1
609 p = len(i) - 1
610 else:
610 else:
611 assert p < len(i)
611 assert p < len(i)
612 for r in pycompat.xrange(p, -1, -1):
612 for r in pycompat.xrange(p, -1, -1):
613 v = i[r][7]
613 v = i[r][7]
614 n[v] = r
614 n[v] = r
615 if v == node:
615 if v == node:
616 self._nodepos = r - 1
616 self._nodepos = r - 1
617 return r
617 return r
618 if node == wdirid or node in wdirfilenodeids:
618 if node == wdirid or node in wdirfilenodeids:
619 raise error.WdirUnsupported
619 raise error.WdirUnsupported
620 raise error.LookupError(node, self.indexfile, _('no node'))
620 raise error.LookupError(node, self.indexfile, _('no node'))
621
621
622 # Accessors for index entries.
622 # Accessors for index entries.
623
623
624 # First tuple entry is 8 bytes. First 6 bytes are offset. Last 2 bytes
624 # First tuple entry is 8 bytes. First 6 bytes are offset. Last 2 bytes
625 # are flags.
625 # are flags.
626 def start(self, rev):
626 def start(self, rev):
627 return int(self.index[rev][0] >> 16)
627 return int(self.index[rev][0] >> 16)
628
628
629 def flags(self, rev):
629 def flags(self, rev):
630 return self.index[rev][0] & 0xFFFF
630 return self.index[rev][0] & 0xFFFF
631
631
632 def length(self, rev):
632 def length(self, rev):
633 return self.index[rev][1]
633 return self.index[rev][1]
634
634
635 def rawsize(self, rev):
635 def rawsize(self, rev):
636 """return the length of the uncompressed text for a given revision"""
636 """return the length of the uncompressed text for a given revision"""
637 l = self.index[rev][2]
637 l = self.index[rev][2]
638 if l >= 0:
638 if l >= 0:
639 return l
639 return l
640
640
641 t = self.rawdata(rev)
641 t = self.rawdata(rev)
642 return len(t)
642 return len(t)
643
643
644 def size(self, rev):
644 def size(self, rev):
645 """length of non-raw text (processed by a "read" flag processor)"""
645 """length of non-raw text (processed by a "read" flag processor)"""
646 # fast path: if no "read" flag processor could change the content,
646 # fast path: if no "read" flag processor could change the content,
647 # size is rawsize. note: ELLIPSIS is known to not change the content.
647 # size is rawsize. note: ELLIPSIS is known to not change the content.
648 flags = self.flags(rev)
648 flags = self.flags(rev)
649 if flags & (flagutil.REVIDX_KNOWN_FLAGS ^ REVIDX_ELLIPSIS) == 0:
649 if flags & (flagutil.REVIDX_KNOWN_FLAGS ^ REVIDX_ELLIPSIS) == 0:
650 return self.rawsize(rev)
650 return self.rawsize(rev)
651
651
652 return len(self.revision(rev, raw=False))
652 return len(self.revision(rev, raw=False))
653
653
654 def chainbase(self, rev):
654 def chainbase(self, rev):
655 base = self._chainbasecache.get(rev)
655 base = self._chainbasecache.get(rev)
656 if base is not None:
656 if base is not None:
657 return base
657 return base
658
658
659 index = self.index
659 index = self.index
660 iterrev = rev
660 iterrev = rev
661 base = index[iterrev][3]
661 base = index[iterrev][3]
662 while base != iterrev:
662 while base != iterrev:
663 iterrev = base
663 iterrev = base
664 base = index[iterrev][3]
664 base = index[iterrev][3]
665
665
666 self._chainbasecache[rev] = base
666 self._chainbasecache[rev] = base
667 return base
667 return base
668
668
669 def linkrev(self, rev):
669 def linkrev(self, rev):
670 return self.index[rev][4]
670 return self.index[rev][4]
671
671
672 def parentrevs(self, rev):
672 def parentrevs(self, rev):
673 try:
673 try:
674 entry = self.index[rev]
674 entry = self.index[rev]
675 except IndexError:
675 except IndexError:
676 if rev == wdirrev:
676 if rev == wdirrev:
677 raise error.WdirUnsupported
677 raise error.WdirUnsupported
678 raise
678 raise
679
679
680 return entry[5], entry[6]
680 return entry[5], entry[6]
681
681
682 # fast parentrevs(rev) where rev isn't filtered
682 # fast parentrevs(rev) where rev isn't filtered
683 _uncheckedparentrevs = parentrevs
683 _uncheckedparentrevs = parentrevs
684
684
685 def node(self, rev):
685 def node(self, rev):
686 try:
686 try:
687 return self.index[rev][7]
687 return self.index[rev][7]
688 except IndexError:
688 except IndexError:
689 if rev == wdirrev:
689 if rev == wdirrev:
690 raise error.WdirUnsupported
690 raise error.WdirUnsupported
691 raise
691 raise
692
692
693 # Derived from index values.
693 # Derived from index values.
694
694
695 def end(self, rev):
695 def end(self, rev):
696 return self.start(rev) + self.length(rev)
696 return self.start(rev) + self.length(rev)
697
697
698 def parents(self, node):
698 def parents(self, node):
699 i = self.index
699 i = self.index
700 d = i[self.rev(node)]
700 d = i[self.rev(node)]
701 return i[d[5]][7], i[d[6]][7] # map revisions to nodes inline
701 return i[d[5]][7], i[d[6]][7] # map revisions to nodes inline
702
702
703 def chainlen(self, rev):
703 def chainlen(self, rev):
704 return self._chaininfo(rev)[0]
704 return self._chaininfo(rev)[0]
705
705
706 def _chaininfo(self, rev):
706 def _chaininfo(self, rev):
707 chaininfocache = self._chaininfocache
707 chaininfocache = self._chaininfocache
708 if rev in chaininfocache:
708 if rev in chaininfocache:
709 return chaininfocache[rev]
709 return chaininfocache[rev]
710 index = self.index
710 index = self.index
711 generaldelta = self._generaldelta
711 generaldelta = self._generaldelta
712 iterrev = rev
712 iterrev = rev
713 e = index[iterrev]
713 e = index[iterrev]
714 clen = 0
714 clen = 0
715 compresseddeltalen = 0
715 compresseddeltalen = 0
716 while iterrev != e[3]:
716 while iterrev != e[3]:
717 clen += 1
717 clen += 1
718 compresseddeltalen += e[1]
718 compresseddeltalen += e[1]
719 if generaldelta:
719 if generaldelta:
720 iterrev = e[3]
720 iterrev = e[3]
721 else:
721 else:
722 iterrev -= 1
722 iterrev -= 1
723 if iterrev in chaininfocache:
723 if iterrev in chaininfocache:
724 t = chaininfocache[iterrev]
724 t = chaininfocache[iterrev]
725 clen += t[0]
725 clen += t[0]
726 compresseddeltalen += t[1]
726 compresseddeltalen += t[1]
727 break
727 break
728 e = index[iterrev]
728 e = index[iterrev]
729 else:
729 else:
730 # Add text length of base since decompressing that also takes
730 # Add text length of base since decompressing that also takes
731 # work. For cache hits the length is already included.
731 # work. For cache hits the length is already included.
732 compresseddeltalen += e[1]
732 compresseddeltalen += e[1]
733 r = (clen, compresseddeltalen)
733 r = (clen, compresseddeltalen)
734 chaininfocache[rev] = r
734 chaininfocache[rev] = r
735 return r
735 return r
736
736
737 def _deltachain(self, rev, stoprev=None):
737 def _deltachain(self, rev, stoprev=None):
738 """Obtain the delta chain for a revision.
738 """Obtain the delta chain for a revision.
739
739
740 ``stoprev`` specifies a revision to stop at. If not specified, we
740 ``stoprev`` specifies a revision to stop at. If not specified, we
741 stop at the base of the chain.
741 stop at the base of the chain.
742
742
743 Returns a 2-tuple of (chain, stopped) where ``chain`` is a list of
743 Returns a 2-tuple of (chain, stopped) where ``chain`` is a list of
744 revs in ascending order and ``stopped`` is a bool indicating whether
744 revs in ascending order and ``stopped`` is a bool indicating whether
745 ``stoprev`` was hit.
745 ``stoprev`` was hit.
746 """
746 """
747 # Try C implementation.
747 # Try C implementation.
748 try:
748 try:
749 return self.index.deltachain(rev, stoprev, self._generaldelta)
749 return self.index.deltachain(rev, stoprev, self._generaldelta)
750 except AttributeError:
750 except AttributeError:
751 pass
751 pass
752
752
753 chain = []
753 chain = []
754
754
755 # Alias to prevent attribute lookup in tight loop.
755 # Alias to prevent attribute lookup in tight loop.
756 index = self.index
756 index = self.index
757 generaldelta = self._generaldelta
757 generaldelta = self._generaldelta
758
758
759 iterrev = rev
759 iterrev = rev
760 e = index[iterrev]
760 e = index[iterrev]
761 while iterrev != e[3] and iterrev != stoprev:
761 while iterrev != e[3] and iterrev != stoprev:
762 chain.append(iterrev)
762 chain.append(iterrev)
763 if generaldelta:
763 if generaldelta:
764 iterrev = e[3]
764 iterrev = e[3]
765 else:
765 else:
766 iterrev -= 1
766 iterrev -= 1
767 e = index[iterrev]
767 e = index[iterrev]
768
768
769 if iterrev == stoprev:
769 if iterrev == stoprev:
770 stopped = True
770 stopped = True
771 else:
771 else:
772 chain.append(iterrev)
772 chain.append(iterrev)
773 stopped = False
773 stopped = False
774
774
775 chain.reverse()
775 chain.reverse()
776 return chain, stopped
776 return chain, stopped
777
777
778 def ancestors(self, revs, stoprev=0, inclusive=False):
778 def ancestors(self, revs, stoprev=0, inclusive=False):
779 """Generate the ancestors of 'revs' in reverse revision order.
779 """Generate the ancestors of 'revs' in reverse revision order.
780 Does not generate revs lower than stoprev.
780 Does not generate revs lower than stoprev.
781
781
782 See the documentation for ancestor.lazyancestors for more details."""
782 See the documentation for ancestor.lazyancestors for more details."""
783
783
784 # first, make sure start revisions aren't filtered
784 # first, make sure start revisions aren't filtered
785 revs = list(revs)
785 revs = list(revs)
786 checkrev = self.node
786 checkrev = self.node
787 for r in revs:
787 for r in revs:
788 checkrev(r)
788 checkrev(r)
789 # and we're sure ancestors aren't filtered as well
789 # and we're sure ancestors aren't filtered as well
790
790
791 if rustancestor is not None:
791 if rustancestor is not None:
792 lazyancestors = rustancestor.LazyAncestors
792 lazyancestors = rustancestor.LazyAncestors
793 arg = self.index
793 arg = self.index
794 elif util.safehasattr(parsers, 'rustlazyancestors'):
794 elif util.safehasattr(parsers, 'rustlazyancestors'):
795 lazyancestors = ancestor.rustlazyancestors
795 lazyancestors = ancestor.rustlazyancestors
796 arg = self.index
796 arg = self.index
797 else:
797 else:
798 lazyancestors = ancestor.lazyancestors
798 lazyancestors = ancestor.lazyancestors
799 arg = self._uncheckedparentrevs
799 arg = self._uncheckedparentrevs
800 return lazyancestors(arg, revs, stoprev=stoprev, inclusive=inclusive)
800 return lazyancestors(arg, revs, stoprev=stoprev, inclusive=inclusive)
801
801
802 def descendants(self, revs):
802 def descendants(self, revs):
803 return dagop.descendantrevs(revs, self.revs, self.parentrevs)
803 return dagop.descendantrevs(revs, self.revs, self.parentrevs)
804
804
805 def findcommonmissing(self, common=None, heads=None):
805 def findcommonmissing(self, common=None, heads=None):
806 """Return a tuple of the ancestors of common and the ancestors of heads
806 """Return a tuple of the ancestors of common and the ancestors of heads
807 that are not ancestors of common. In revset terminology, we return the
807 that are not ancestors of common. In revset terminology, we return the
808 tuple:
808 tuple:
809
809
810 ::common, (::heads) - (::common)
810 ::common, (::heads) - (::common)
811
811
812 The list is sorted by revision number, meaning it is
812 The list is sorted by revision number, meaning it is
813 topologically sorted.
813 topologically sorted.
814
814
815 'heads' and 'common' are both lists of node IDs. If heads is
815 'heads' and 'common' are both lists of node IDs. If heads is
816 not supplied, uses all of the revlog's heads. If common is not
816 not supplied, uses all of the revlog's heads. If common is not
817 supplied, uses nullid."""
817 supplied, uses nullid."""
818 if common is None:
818 if common is None:
819 common = [nullid]
819 common = [nullid]
820 if heads is None:
820 if heads is None:
821 heads = self.heads()
821 heads = self.heads()
822
822
823 common = [self.rev(n) for n in common]
823 common = [self.rev(n) for n in common]
824 heads = [self.rev(n) for n in heads]
824 heads = [self.rev(n) for n in heads]
825
825
826 # we want the ancestors, but inclusive
826 # we want the ancestors, but inclusive
827 class lazyset(object):
827 class lazyset(object):
828 def __init__(self, lazyvalues):
828 def __init__(self, lazyvalues):
829 self.addedvalues = set()
829 self.addedvalues = set()
830 self.lazyvalues = lazyvalues
830 self.lazyvalues = lazyvalues
831
831
832 def __contains__(self, value):
832 def __contains__(self, value):
833 return value in self.addedvalues or value in self.lazyvalues
833 return value in self.addedvalues or value in self.lazyvalues
834
834
835 def __iter__(self):
835 def __iter__(self):
836 added = self.addedvalues
836 added = self.addedvalues
837 for r in added:
837 for r in added:
838 yield r
838 yield r
839 for r in self.lazyvalues:
839 for r in self.lazyvalues:
840 if not r in added:
840 if not r in added:
841 yield r
841 yield r
842
842
843 def add(self, value):
843 def add(self, value):
844 self.addedvalues.add(value)
844 self.addedvalues.add(value)
845
845
846 def update(self, values):
846 def update(self, values):
847 self.addedvalues.update(values)
847 self.addedvalues.update(values)
848
848
849 has = lazyset(self.ancestors(common))
849 has = lazyset(self.ancestors(common))
850 has.add(nullrev)
850 has.add(nullrev)
851 has.update(common)
851 has.update(common)
852
852
853 # take all ancestors from heads that aren't in has
853 # take all ancestors from heads that aren't in has
854 missing = set()
854 missing = set()
855 visit = collections.deque(r for r in heads if r not in has)
855 visit = collections.deque(r for r in heads if r not in has)
856 while visit:
856 while visit:
857 r = visit.popleft()
857 r = visit.popleft()
858 if r in missing:
858 if r in missing:
859 continue
859 continue
860 else:
860 else:
861 missing.add(r)
861 missing.add(r)
862 for p in self.parentrevs(r):
862 for p in self.parentrevs(r):
863 if p not in has:
863 if p not in has:
864 visit.append(p)
864 visit.append(p)
865 missing = list(missing)
865 missing = list(missing)
866 missing.sort()
866 missing.sort()
867 return has, [self.node(miss) for miss in missing]
867 return has, [self.node(miss) for miss in missing]
868
868
869 def incrementalmissingrevs(self, common=None):
869 def incrementalmissingrevs(self, common=None):
870 """Return an object that can be used to incrementally compute the
870 """Return an object that can be used to incrementally compute the
871 revision numbers of the ancestors of arbitrary sets that are not
871 revision numbers of the ancestors of arbitrary sets that are not
872 ancestors of common. This is an ancestor.incrementalmissingancestors
872 ancestors of common. This is an ancestor.incrementalmissingancestors
873 object.
873 object.
874
874
875 'common' is a list of revision numbers. If common is not supplied, uses
875 'common' is a list of revision numbers. If common is not supplied, uses
876 nullrev.
876 nullrev.
877 """
877 """
878 if common is None:
878 if common is None:
879 common = [nullrev]
879 common = [nullrev]
880
880
881 if rustancestor is not None:
881 if rustancestor is not None:
882 return rustancestor.MissingAncestors(self.index, common)
882 return rustancestor.MissingAncestors(self.index, common)
883 return ancestor.incrementalmissingancestors(self.parentrevs, common)
883 return ancestor.incrementalmissingancestors(self.parentrevs, common)
884
884
885 def findmissingrevs(self, common=None, heads=None):
885 def findmissingrevs(self, common=None, heads=None):
886 """Return the revision numbers of the ancestors of heads that
886 """Return the revision numbers of the ancestors of heads that
887 are not ancestors of common.
887 are not ancestors of common.
888
888
889 More specifically, return a list of revision numbers corresponding to
889 More specifically, return a list of revision numbers corresponding to
890 nodes N such that every N satisfies the following constraints:
890 nodes N such that every N satisfies the following constraints:
891
891
892 1. N is an ancestor of some node in 'heads'
892 1. N is an ancestor of some node in 'heads'
893 2. N is not an ancestor of any node in 'common'
893 2. N is not an ancestor of any node in 'common'
894
894
895 The list is sorted by revision number, meaning it is
895 The list is sorted by revision number, meaning it is
896 topologically sorted.
896 topologically sorted.
897
897
898 'heads' and 'common' are both lists of revision numbers. If heads is
898 'heads' and 'common' are both lists of revision numbers. If heads is
899 not supplied, uses all of the revlog's heads. If common is not
899 not supplied, uses all of the revlog's heads. If common is not
900 supplied, uses nullid."""
900 supplied, uses nullid."""
901 if common is None:
901 if common is None:
902 common = [nullrev]
902 common = [nullrev]
903 if heads is None:
903 if heads is None:
904 heads = self.headrevs()
904 heads = self.headrevs()
905
905
906 inc = self.incrementalmissingrevs(common=common)
906 inc = self.incrementalmissingrevs(common=common)
907 return inc.missingancestors(heads)
907 return inc.missingancestors(heads)
908
908
909 def findmissing(self, common=None, heads=None):
909 def findmissing(self, common=None, heads=None):
910 """Return the ancestors of heads that are not ancestors of common.
910 """Return the ancestors of heads that are not ancestors of common.
911
911
912 More specifically, return a list of nodes N such that every N
912 More specifically, return a list of nodes N such that every N
913 satisfies the following constraints:
913 satisfies the following constraints:
914
914
915 1. N is an ancestor of some node in 'heads'
915 1. N is an ancestor of some node in 'heads'
916 2. N is not an ancestor of any node in 'common'
916 2. N is not an ancestor of any node in 'common'
917
917
918 The list is sorted by revision number, meaning it is
918 The list is sorted by revision number, meaning it is
919 topologically sorted.
919 topologically sorted.
920
920
921 'heads' and 'common' are both lists of node IDs. If heads is
921 'heads' and 'common' are both lists of node IDs. If heads is
922 not supplied, uses all of the revlog's heads. If common is not
922 not supplied, uses all of the revlog's heads. If common is not
923 supplied, uses nullid."""
923 supplied, uses nullid."""
924 if common is None:
924 if common is None:
925 common = [nullid]
925 common = [nullid]
926 if heads is None:
926 if heads is None:
927 heads = self.heads()
927 heads = self.heads()
928
928
929 common = [self.rev(n) for n in common]
929 common = [self.rev(n) for n in common]
930 heads = [self.rev(n) for n in heads]
930 heads = [self.rev(n) for n in heads]
931
931
932 inc = self.incrementalmissingrevs(common=common)
932 inc = self.incrementalmissingrevs(common=common)
933 return [self.node(r) for r in inc.missingancestors(heads)]
933 return [self.node(r) for r in inc.missingancestors(heads)]
934
934
935 def nodesbetween(self, roots=None, heads=None):
935 def nodesbetween(self, roots=None, heads=None):
936 """Return a topological path from 'roots' to 'heads'.
936 """Return a topological path from 'roots' to 'heads'.
937
937
938 Return a tuple (nodes, outroots, outheads) where 'nodes' is a
938 Return a tuple (nodes, outroots, outheads) where 'nodes' is a
939 topologically sorted list of all nodes N that satisfy both of
939 topologically sorted list of all nodes N that satisfy both of
940 these constraints:
940 these constraints:
941
941
942 1. N is a descendant of some node in 'roots'
942 1. N is a descendant of some node in 'roots'
943 2. N is an ancestor of some node in 'heads'
943 2. N is an ancestor of some node in 'heads'
944
944
945 Every node is considered to be both a descendant and an ancestor
945 Every node is considered to be both a descendant and an ancestor
946 of itself, so every reachable node in 'roots' and 'heads' will be
946 of itself, so every reachable node in 'roots' and 'heads' will be
947 included in 'nodes'.
947 included in 'nodes'.
948
948
949 'outroots' is the list of reachable nodes in 'roots', i.e., the
949 'outroots' is the list of reachable nodes in 'roots', i.e., the
950 subset of 'roots' that is returned in 'nodes'. Likewise,
950 subset of 'roots' that is returned in 'nodes'. Likewise,
951 'outheads' is the subset of 'heads' that is also in 'nodes'.
951 'outheads' is the subset of 'heads' that is also in 'nodes'.
952
952
953 'roots' and 'heads' are both lists of node IDs. If 'roots' is
953 'roots' and 'heads' are both lists of node IDs. If 'roots' is
954 unspecified, uses nullid as the only root. If 'heads' is
954 unspecified, uses nullid as the only root. If 'heads' is
955 unspecified, uses list of all of the revlog's heads."""
955 unspecified, uses list of all of the revlog's heads."""
956 nonodes = ([], [], [])
956 nonodes = ([], [], [])
957 if roots is not None:
957 if roots is not None:
958 roots = list(roots)
958 roots = list(roots)
959 if not roots:
959 if not roots:
960 return nonodes
960 return nonodes
961 lowestrev = min([self.rev(n) for n in roots])
961 lowestrev = min([self.rev(n) for n in roots])
962 else:
962 else:
963 roots = [nullid] # Everybody's a descendant of nullid
963 roots = [nullid] # Everybody's a descendant of nullid
964 lowestrev = nullrev
964 lowestrev = nullrev
965 if (lowestrev == nullrev) and (heads is None):
965 if (lowestrev == nullrev) and (heads is None):
966 # We want _all_ the nodes!
966 # We want _all_ the nodes!
967 return ([self.node(r) for r in self], [nullid], list(self.heads()))
967 return ([self.node(r) for r in self], [nullid], list(self.heads()))
968 if heads is None:
968 if heads is None:
969 # All nodes are ancestors, so the latest ancestor is the last
969 # All nodes are ancestors, so the latest ancestor is the last
970 # node.
970 # node.
971 highestrev = len(self) - 1
971 highestrev = len(self) - 1
972 # Set ancestors to None to signal that every node is an ancestor.
972 # Set ancestors to None to signal that every node is an ancestor.
973 ancestors = None
973 ancestors = None
974 # Set heads to an empty dictionary for later discovery of heads
974 # Set heads to an empty dictionary for later discovery of heads
975 heads = {}
975 heads = {}
976 else:
976 else:
977 heads = list(heads)
977 heads = list(heads)
978 if not heads:
978 if not heads:
979 return nonodes
979 return nonodes
980 ancestors = set()
980 ancestors = set()
981 # Turn heads into a dictionary so we can remove 'fake' heads.
981 # Turn heads into a dictionary so we can remove 'fake' heads.
982 # Also, later we will be using it to filter out the heads we can't
982 # Also, later we will be using it to filter out the heads we can't
983 # find from roots.
983 # find from roots.
984 heads = dict.fromkeys(heads, False)
984 heads = dict.fromkeys(heads, False)
985 # Start at the top and keep marking parents until we're done.
985 # Start at the top and keep marking parents until we're done.
986 nodestotag = set(heads)
986 nodestotag = set(heads)
987 # Remember where the top was so we can use it as a limit later.
987 # Remember where the top was so we can use it as a limit later.
988 highestrev = max([self.rev(n) for n in nodestotag])
988 highestrev = max([self.rev(n) for n in nodestotag])
989 while nodestotag:
989 while nodestotag:
990 # grab a node to tag
990 # grab a node to tag
991 n = nodestotag.pop()
991 n = nodestotag.pop()
992 # Never tag nullid
992 # Never tag nullid
993 if n == nullid:
993 if n == nullid:
994 continue
994 continue
995 # A node's revision number represents its place in a
995 # A node's revision number represents its place in a
996 # topologically sorted list of nodes.
996 # topologically sorted list of nodes.
997 r = self.rev(n)
997 r = self.rev(n)
998 if r >= lowestrev:
998 if r >= lowestrev:
999 if n not in ancestors:
999 if n not in ancestors:
1000 # If we are possibly a descendant of one of the roots
1000 # If we are possibly a descendant of one of the roots
1001 # and we haven't already been marked as an ancestor
1001 # and we haven't already been marked as an ancestor
1002 ancestors.add(n) # Mark as ancestor
1002 ancestors.add(n) # Mark as ancestor
1003 # Add non-nullid parents to list of nodes to tag.
1003 # Add non-nullid parents to list of nodes to tag.
1004 nodestotag.update([p for p in self.parents(n) if
1004 nodestotag.update([p for p in self.parents(n) if
1005 p != nullid])
1005 p != nullid])
1006 elif n in heads: # We've seen it before, is it a fake head?
1006 elif n in heads: # We've seen it before, is it a fake head?
1007 # So it is, real heads should not be the ancestors of
1007 # So it is, real heads should not be the ancestors of
1008 # any other heads.
1008 # any other heads.
1009 heads.pop(n)
1009 heads.pop(n)
1010 if not ancestors:
1010 if not ancestors:
1011 return nonodes
1011 return nonodes
1012 # Now that we have our set of ancestors, we want to remove any
1012 # Now that we have our set of ancestors, we want to remove any
1013 # roots that are not ancestors.
1013 # roots that are not ancestors.
1014
1014
1015 # If one of the roots was nullid, everything is included anyway.
1015 # If one of the roots was nullid, everything is included anyway.
1016 if lowestrev > nullrev:
1016 if lowestrev > nullrev:
1017 # But, since we weren't, let's recompute the lowest rev to not
1017 # But, since we weren't, let's recompute the lowest rev to not
1018 # include roots that aren't ancestors.
1018 # include roots that aren't ancestors.
1019
1019
1020 # Filter out roots that aren't ancestors of heads
1020 # Filter out roots that aren't ancestors of heads
1021 roots = [root for root in roots if root in ancestors]
1021 roots = [root for root in roots if root in ancestors]
1022 # Recompute the lowest revision
1022 # Recompute the lowest revision
1023 if roots:
1023 if roots:
1024 lowestrev = min([self.rev(root) for root in roots])
1024 lowestrev = min([self.rev(root) for root in roots])
1025 else:
1025 else:
1026 # No more roots? Return empty list
1026 # No more roots? Return empty list
1027 return nonodes
1027 return nonodes
1028 else:
1028 else:
1029 # We are descending from nullid, and don't need to care about
1029 # We are descending from nullid, and don't need to care about
1030 # any other roots.
1030 # any other roots.
1031 lowestrev = nullrev
1031 lowestrev = nullrev
1032 roots = [nullid]
1032 roots = [nullid]
1033 # Transform our roots list into a set.
1033 # Transform our roots list into a set.
1034 descendants = set(roots)
1034 descendants = set(roots)
1035 # Also, keep the original roots so we can filter out roots that aren't
1035 # Also, keep the original roots so we can filter out roots that aren't
1036 # 'real' roots (i.e. are descended from other roots).
1036 # 'real' roots (i.e. are descended from other roots).
1037 roots = descendants.copy()
1037 roots = descendants.copy()
1038 # Our topologically sorted list of output nodes.
1038 # Our topologically sorted list of output nodes.
1039 orderedout = []
1039 orderedout = []
1040 # Don't start at nullid since we don't want nullid in our output list,
1040 # Don't start at nullid since we don't want nullid in our output list,
1041 # and if nullid shows up in descendants, empty parents will look like
1041 # and if nullid shows up in descendants, empty parents will look like
1042 # they're descendants.
1042 # they're descendants.
1043 for r in self.revs(start=max(lowestrev, 0), stop=highestrev + 1):
1043 for r in self.revs(start=max(lowestrev, 0), stop=highestrev + 1):
1044 n = self.node(r)
1044 n = self.node(r)
1045 isdescendant = False
1045 isdescendant = False
1046 if lowestrev == nullrev: # Everybody is a descendant of nullid
1046 if lowestrev == nullrev: # Everybody is a descendant of nullid
1047 isdescendant = True
1047 isdescendant = True
1048 elif n in descendants:
1048 elif n in descendants:
1049 # n is already a descendant
1049 # n is already a descendant
1050 isdescendant = True
1050 isdescendant = True
1051 # This check only needs to be done here because all the roots
1051 # This check only needs to be done here because all the roots
1052 # will start being marked is descendants before the loop.
1052 # will start being marked is descendants before the loop.
1053 if n in roots:
1053 if n in roots:
1054 # If n was a root, check if it's a 'real' root.
1054 # If n was a root, check if it's a 'real' root.
1055 p = tuple(self.parents(n))
1055 p = tuple(self.parents(n))
1056 # If any of its parents are descendants, it's not a root.
1056 # If any of its parents are descendants, it's not a root.
1057 if (p[0] in descendants) or (p[1] in descendants):
1057 if (p[0] in descendants) or (p[1] in descendants):
1058 roots.remove(n)
1058 roots.remove(n)
1059 else:
1059 else:
1060 p = tuple(self.parents(n))
1060 p = tuple(self.parents(n))
1061 # A node is a descendant if either of its parents are
1061 # A node is a descendant if either of its parents are
1062 # descendants. (We seeded the dependents list with the roots
1062 # descendants. (We seeded the dependents list with the roots
1063 # up there, remember?)
1063 # up there, remember?)
1064 if (p[0] in descendants) or (p[1] in descendants):
1064 if (p[0] in descendants) or (p[1] in descendants):
1065 descendants.add(n)
1065 descendants.add(n)
1066 isdescendant = True
1066 isdescendant = True
1067 if isdescendant and ((ancestors is None) or (n in ancestors)):
1067 if isdescendant and ((ancestors is None) or (n in ancestors)):
1068 # Only include nodes that are both descendants and ancestors.
1068 # Only include nodes that are both descendants and ancestors.
1069 orderedout.append(n)
1069 orderedout.append(n)
1070 if (ancestors is not None) and (n in heads):
1070 if (ancestors is not None) and (n in heads):
1071 # We're trying to figure out which heads are reachable
1071 # We're trying to figure out which heads are reachable
1072 # from roots.
1072 # from roots.
1073 # Mark this head as having been reached
1073 # Mark this head as having been reached
1074 heads[n] = True
1074 heads[n] = True
1075 elif ancestors is None:
1075 elif ancestors is None:
1076 # Otherwise, we're trying to discover the heads.
1076 # Otherwise, we're trying to discover the heads.
1077 # Assume this is a head because if it isn't, the next step
1077 # Assume this is a head because if it isn't, the next step
1078 # will eventually remove it.
1078 # will eventually remove it.
1079 heads[n] = True
1079 heads[n] = True
1080 # But, obviously its parents aren't.
1080 # But, obviously its parents aren't.
1081 for p in self.parents(n):
1081 for p in self.parents(n):
1082 heads.pop(p, None)
1082 heads.pop(p, None)
1083 heads = [head for head, flag in heads.iteritems() if flag]
1083 heads = [head for head, flag in heads.iteritems() if flag]
1084 roots = list(roots)
1084 roots = list(roots)
1085 assert orderedout
1085 assert orderedout
1086 assert roots
1086 assert roots
1087 assert heads
1087 assert heads
1088 return (orderedout, roots, heads)
1088 return (orderedout, roots, heads)
1089
1089
1090 def headrevs(self, revs=None):
1090 def headrevs(self, revs=None):
1091 if revs is None:
1091 if revs is None:
1092 try:
1092 try:
1093 return self.index.headrevs()
1093 return self.index.headrevs()
1094 except AttributeError:
1094 except AttributeError:
1095 return self._headrevs()
1095 return self._headrevs()
1096 if rustdagop is not None:
1096 if rustdagop is not None:
1097 return rustdagop.headrevs(self.index, revs)
1097 return rustdagop.headrevs(self.index, revs)
1098 return dagop.headrevs(revs, self._uncheckedparentrevs)
1098 return dagop.headrevs(revs, self._uncheckedparentrevs)
1099
1099
1100 def computephases(self, roots):
1100 def computephases(self, roots):
1101 return self.index.computephasesmapsets(roots)
1101 return self.index.computephasesmapsets(roots)
1102
1102
1103 def _headrevs(self):
1103 def _headrevs(self):
1104 count = len(self)
1104 count = len(self)
1105 if not count:
1105 if not count:
1106 return [nullrev]
1106 return [nullrev]
1107 # we won't iter over filtered rev so nobody is a head at start
1107 # we won't iter over filtered rev so nobody is a head at start
1108 ishead = [0] * (count + 1)
1108 ishead = [0] * (count + 1)
1109 index = self.index
1109 index = self.index
1110 for r in self:
1110 for r in self:
1111 ishead[r] = 1 # I may be an head
1111 ishead[r] = 1 # I may be an head
1112 e = index[r]
1112 e = index[r]
1113 ishead[e[5]] = ishead[e[6]] = 0 # my parent are not
1113 ishead[e[5]] = ishead[e[6]] = 0 # my parent are not
1114 return [r for r, val in enumerate(ishead) if val]
1114 return [r for r, val in enumerate(ishead) if val]
1115
1115
1116 def heads(self, start=None, stop=None):
1116 def heads(self, start=None, stop=None):
1117 """return the list of all nodes that have no children
1117 """return the list of all nodes that have no children
1118
1118
1119 if start is specified, only heads that are descendants of
1119 if start is specified, only heads that are descendants of
1120 start will be returned
1120 start will be returned
1121 if stop is specified, it will consider all the revs from stop
1121 if stop is specified, it will consider all the revs from stop
1122 as if they had no children
1122 as if they had no children
1123 """
1123 """
1124 if start is None and stop is None:
1124 if start is None and stop is None:
1125 if not len(self):
1125 if not len(self):
1126 return [nullid]
1126 return [nullid]
1127 return [self.node(r) for r in self.headrevs()]
1127 return [self.node(r) for r in self.headrevs()]
1128
1128
1129 if start is None:
1129 if start is None:
1130 start = nullrev
1130 start = nullrev
1131 else:
1131 else:
1132 start = self.rev(start)
1132 start = self.rev(start)
1133
1133
1134 stoprevs = set(self.rev(n) for n in stop or [])
1134 stoprevs = set(self.rev(n) for n in stop or [])
1135
1135
1136 revs = dagop.headrevssubset(self.revs, self.parentrevs, startrev=start,
1136 revs = dagop.headrevssubset(self.revs, self.parentrevs, startrev=start,
1137 stoprevs=stoprevs)
1137 stoprevs=stoprevs)
1138
1138
1139 return [self.node(rev) for rev in revs]
1139 return [self.node(rev) for rev in revs]
1140
1140
1141 def children(self, node):
1141 def children(self, node):
1142 """find the children of a given node"""
1142 """find the children of a given node"""
1143 c = []
1143 c = []
1144 p = self.rev(node)
1144 p = self.rev(node)
1145 for r in self.revs(start=p + 1):
1145 for r in self.revs(start=p + 1):
1146 prevs = [pr for pr in self.parentrevs(r) if pr != nullrev]
1146 prevs = [pr for pr in self.parentrevs(r) if pr != nullrev]
1147 if prevs:
1147 if prevs:
1148 for pr in prevs:
1148 for pr in prevs:
1149 if pr == p:
1149 if pr == p:
1150 c.append(self.node(r))
1150 c.append(self.node(r))
1151 elif p == nullrev:
1151 elif p == nullrev:
1152 c.append(self.node(r))
1152 c.append(self.node(r))
1153 return c
1153 return c
1154
1154
1155 def commonancestorsheads(self, a, b):
1155 def commonancestorsheads(self, a, b):
1156 """calculate all the heads of the common ancestors of nodes a and b"""
1156 """calculate all the heads of the common ancestors of nodes a and b"""
1157 a, b = self.rev(a), self.rev(b)
1157 a, b = self.rev(a), self.rev(b)
1158 ancs = self._commonancestorsheads(a, b)
1158 ancs = self._commonancestorsheads(a, b)
1159 return pycompat.maplist(self.node, ancs)
1159 return pycompat.maplist(self.node, ancs)
1160
1160
1161 def _commonancestorsheads(self, *revs):
1161 def _commonancestorsheads(self, *revs):
1162 """calculate all the heads of the common ancestors of revs"""
1162 """calculate all the heads of the common ancestors of revs"""
1163 try:
1163 try:
1164 ancs = self.index.commonancestorsheads(*revs)
1164 ancs = self.index.commonancestorsheads(*revs)
1165 except (AttributeError, OverflowError): # C implementation failed
1165 except (AttributeError, OverflowError): # C implementation failed
1166 ancs = ancestor.commonancestorsheads(self.parentrevs, *revs)
1166 ancs = ancestor.commonancestorsheads(self.parentrevs, *revs)
1167 return ancs
1167 return ancs
1168
1168
1169 def isancestor(self, a, b):
1169 def isancestor(self, a, b):
1170 """return True if node a is an ancestor of node b
1170 """return True if node a is an ancestor of node b
1171
1171
1172 A revision is considered an ancestor of itself."""
1172 A revision is considered an ancestor of itself."""
1173 a, b = self.rev(a), self.rev(b)
1173 a, b = self.rev(a), self.rev(b)
1174 return self.isancestorrev(a, b)
1174 return self.isancestorrev(a, b)
1175
1175
1176 def isancestorrev(self, a, b):
1176 def isancestorrev(self, a, b):
1177 """return True if revision a is an ancestor of revision b
1177 """return True if revision a is an ancestor of revision b
1178
1178
1179 A revision is considered an ancestor of itself.
1179 A revision is considered an ancestor of itself.
1180
1180
1181 The implementation of this is trivial but the use of
1181 The implementation of this is trivial but the use of
1182 reachableroots is not."""
1182 reachableroots is not."""
1183 if a == nullrev:
1183 if a == nullrev:
1184 return True
1184 return True
1185 elif a == b:
1185 elif a == b:
1186 return True
1186 return True
1187 elif a > b:
1187 elif a > b:
1188 return False
1188 return False
1189 return bool(self.reachableroots(a, [b], [a], includepath=False))
1189 return bool(self.reachableroots(a, [b], [a], includepath=False))
1190
1190
1191 def reachableroots(self, minroot, heads, roots, includepath=False):
1191 def reachableroots(self, minroot, heads, roots, includepath=False):
1192 """return (heads(::<roots> and <roots>::<heads>))
1192 """return (heads(::<roots> and <roots>::<heads>))
1193
1193
1194 If includepath is True, return (<roots>::<heads>)."""
1194 If includepath is True, return (<roots>::<heads>)."""
1195 try:
1195 try:
1196 return self.index.reachableroots2(minroot, heads, roots,
1196 return self.index.reachableroots2(minroot, heads, roots,
1197 includepath)
1197 includepath)
1198 except AttributeError:
1198 except AttributeError:
1199 return dagop._reachablerootspure(self.parentrevs,
1199 return dagop._reachablerootspure(self.parentrevs,
1200 minroot, roots, heads, includepath)
1200 minroot, roots, heads, includepath)
1201
1201
1202 def ancestor(self, a, b):
1202 def ancestor(self, a, b):
1203 """calculate the "best" common ancestor of nodes a and b"""
1203 """calculate the "best" common ancestor of nodes a and b"""
1204
1204
1205 a, b = self.rev(a), self.rev(b)
1205 a, b = self.rev(a), self.rev(b)
1206 try:
1206 try:
1207 ancs = self.index.ancestors(a, b)
1207 ancs = self.index.ancestors(a, b)
1208 except (AttributeError, OverflowError):
1208 except (AttributeError, OverflowError):
1209 ancs = ancestor.ancestors(self.parentrevs, a, b)
1209 ancs = ancestor.ancestors(self.parentrevs, a, b)
1210 if ancs:
1210 if ancs:
1211 # choose a consistent winner when there's a tie
1211 # choose a consistent winner when there's a tie
1212 return min(map(self.node, ancs))
1212 return min(map(self.node, ancs))
1213 return nullid
1213 return nullid
1214
1214
1215 def _match(self, id):
1215 def _match(self, id):
1216 if isinstance(id, int):
1216 if isinstance(id, int):
1217 # rev
1217 # rev
1218 return self.node(id)
1218 return self.node(id)
1219 if len(id) == 20:
1219 if len(id) == 20:
1220 # possibly a binary node
1220 # possibly a binary node
1221 # odds of a binary node being all hex in ASCII are 1 in 10**25
1221 # odds of a binary node being all hex in ASCII are 1 in 10**25
1222 try:
1222 try:
1223 node = id
1223 node = id
1224 self.rev(node) # quick search the index
1224 self.rev(node) # quick search the index
1225 return node
1225 return node
1226 except error.LookupError:
1226 except error.LookupError:
1227 pass # may be partial hex id
1227 pass # may be partial hex id
1228 try:
1228 try:
1229 # str(rev)
1229 # str(rev)
1230 rev = int(id)
1230 rev = int(id)
1231 if "%d" % rev != id:
1231 if "%d" % rev != id:
1232 raise ValueError
1232 raise ValueError
1233 if rev < 0:
1233 if rev < 0:
1234 rev = len(self) + rev
1234 rev = len(self) + rev
1235 if rev < 0 or rev >= len(self):
1235 if rev < 0 or rev >= len(self):
1236 raise ValueError
1236 raise ValueError
1237 return self.node(rev)
1237 return self.node(rev)
1238 except (ValueError, OverflowError):
1238 except (ValueError, OverflowError):
1239 pass
1239 pass
1240 if len(id) == 40:
1240 if len(id) == 40:
1241 try:
1241 try:
1242 # a full hex nodeid?
1242 # a full hex nodeid?
1243 node = bin(id)
1243 node = bin(id)
1244 self.rev(node)
1244 self.rev(node)
1245 return node
1245 return node
1246 except (TypeError, error.LookupError):
1246 except (TypeError, error.LookupError):
1247 pass
1247 pass
1248
1248
1249 def _partialmatch(self, id):
1249 def _partialmatch(self, id):
1250 # we don't care wdirfilenodeids as they should be always full hash
1250 # we don't care wdirfilenodeids as they should be always full hash
1251 maybewdir = wdirhex.startswith(id)
1251 maybewdir = wdirhex.startswith(id)
1252 try:
1252 try:
1253 partial = self.index.partialmatch(id)
1253 partial = self.index.partialmatch(id)
1254 if partial and self.hasnode(partial):
1254 if partial and self.hasnode(partial):
1255 if maybewdir:
1255 if maybewdir:
1256 # single 'ff...' match in radix tree, ambiguous with wdir
1256 # single 'ff...' match in radix tree, ambiguous with wdir
1257 raise error.RevlogError
1257 raise error.RevlogError
1258 return partial
1258 return partial
1259 if maybewdir:
1259 if maybewdir:
1260 # no 'ff...' match in radix tree, wdir identified
1260 # no 'ff...' match in radix tree, wdir identified
1261 raise error.WdirUnsupported
1261 raise error.WdirUnsupported
1262 return None
1262 return None
1263 except error.RevlogError:
1263 except error.RevlogError:
1264 # parsers.c radix tree lookup gave multiple matches
1264 # parsers.c radix tree lookup gave multiple matches
1265 # fast path: for unfiltered changelog, radix tree is accurate
1265 # fast path: for unfiltered changelog, radix tree is accurate
1266 if not getattr(self, 'filteredrevs', None):
1266 if not getattr(self, 'filteredrevs', None):
1267 raise error.AmbiguousPrefixLookupError(
1267 raise error.AmbiguousPrefixLookupError(
1268 id, self.indexfile, _('ambiguous identifier'))
1268 id, self.indexfile, _('ambiguous identifier'))
1269 # fall through to slow path that filters hidden revisions
1269 # fall through to slow path that filters hidden revisions
1270 except (AttributeError, ValueError):
1270 except (AttributeError, ValueError):
1271 # we are pure python, or key was too short to search radix tree
1271 # we are pure python, or key was too short to search radix tree
1272 pass
1272 pass
1273
1273
1274 if id in self._pcache:
1274 if id in self._pcache:
1275 return self._pcache[id]
1275 return self._pcache[id]
1276
1276
1277 if len(id) <= 40:
1277 if len(id) <= 40:
1278 try:
1278 try:
1279 # hex(node)[:...]
1279 # hex(node)[:...]
1280 l = len(id) // 2 # grab an even number of digits
1280 l = len(id) // 2 # grab an even number of digits
1281 prefix = bin(id[:l * 2])
1281 prefix = bin(id[:l * 2])
1282 nl = [e[7] for e in self.index if e[7].startswith(prefix)]
1282 nl = [e[7] for e in self.index if e[7].startswith(prefix)]
1283 nl = [n for n in nl if hex(n).startswith(id) and
1283 nl = [n for n in nl if hex(n).startswith(id) and
1284 self.hasnode(n)]
1284 self.hasnode(n)]
1285 if nullhex.startswith(id):
1285 if nullhex.startswith(id):
1286 nl.append(nullid)
1286 nl.append(nullid)
1287 if len(nl) > 0:
1287 if len(nl) > 0:
1288 if len(nl) == 1 and not maybewdir:
1288 if len(nl) == 1 and not maybewdir:
1289 self._pcache[id] = nl[0]
1289 self._pcache[id] = nl[0]
1290 return nl[0]
1290 return nl[0]
1291 raise error.AmbiguousPrefixLookupError(
1291 raise error.AmbiguousPrefixLookupError(
1292 id, self.indexfile, _('ambiguous identifier'))
1292 id, self.indexfile, _('ambiguous identifier'))
1293 if maybewdir:
1293 if maybewdir:
1294 raise error.WdirUnsupported
1294 raise error.WdirUnsupported
1295 return None
1295 return None
1296 except TypeError:
1296 except TypeError:
1297 pass
1297 pass
1298
1298
1299 def lookup(self, id):
1299 def lookup(self, id):
1300 """locate a node based on:
1300 """locate a node based on:
1301 - revision number or str(revision number)
1301 - revision number or str(revision number)
1302 - nodeid or subset of hex nodeid
1302 - nodeid or subset of hex nodeid
1303 """
1303 """
1304 n = self._match(id)
1304 n = self._match(id)
1305 if n is not None:
1305 if n is not None:
1306 return n
1306 return n
1307 n = self._partialmatch(id)
1307 n = self._partialmatch(id)
1308 if n:
1308 if n:
1309 return n
1309 return n
1310
1310
1311 raise error.LookupError(id, self.indexfile, _('no match found'))
1311 raise error.LookupError(id, self.indexfile, _('no match found'))
1312
1312
1313 def shortest(self, node, minlength=1):
1313 def shortest(self, node, minlength=1):
1314 """Find the shortest unambiguous prefix that matches node."""
1314 """Find the shortest unambiguous prefix that matches node."""
1315 def isvalid(prefix):
1315 def isvalid(prefix):
1316 try:
1316 try:
1317 matchednode = self._partialmatch(prefix)
1317 matchednode = self._partialmatch(prefix)
1318 except error.AmbiguousPrefixLookupError:
1318 except error.AmbiguousPrefixLookupError:
1319 return False
1319 return False
1320 except error.WdirUnsupported:
1320 except error.WdirUnsupported:
1321 # single 'ff...' match
1321 # single 'ff...' match
1322 return True
1322 return True
1323 if matchednode is None:
1323 if matchednode is None:
1324 raise error.LookupError(node, self.indexfile, _('no node'))
1324 raise error.LookupError(node, self.indexfile, _('no node'))
1325 return True
1325 return True
1326
1326
1327 def maybewdir(prefix):
1327 def maybewdir(prefix):
1328 return all(c == 'f' for c in pycompat.iterbytestr(prefix))
1328 return all(c == 'f' for c in pycompat.iterbytestr(prefix))
1329
1329
1330 hexnode = hex(node)
1330 hexnode = hex(node)
1331
1331
1332 def disambiguate(hexnode, minlength):
1332 def disambiguate(hexnode, minlength):
1333 """Disambiguate against wdirid."""
1333 """Disambiguate against wdirid."""
1334 for length in range(minlength, 41):
1334 for length in range(minlength, 41):
1335 prefix = hexnode[:length]
1335 prefix = hexnode[:length]
1336 if not maybewdir(prefix):
1336 if not maybewdir(prefix):
1337 return prefix
1337 return prefix
1338
1338
1339 if not getattr(self, 'filteredrevs', None):
1339 if not getattr(self, 'filteredrevs', None):
1340 try:
1340 try:
1341 length = max(self.index.shortest(node), minlength)
1341 length = max(self.index.shortest(node), minlength)
1342 return disambiguate(hexnode, length)
1342 return disambiguate(hexnode, length)
1343 except error.RevlogError:
1343 except error.RevlogError:
1344 if node != wdirid:
1344 if node != wdirid:
1345 raise error.LookupError(node, self.indexfile, _('no node'))
1345 raise error.LookupError(node, self.indexfile, _('no node'))
1346 except AttributeError:
1346 except AttributeError:
1347 # Fall through to pure code
1347 # Fall through to pure code
1348 pass
1348 pass
1349
1349
1350 if node == wdirid:
1350 if node == wdirid:
1351 for length in range(minlength, 41):
1351 for length in range(minlength, 41):
1352 prefix = hexnode[:length]
1352 prefix = hexnode[:length]
1353 if isvalid(prefix):
1353 if isvalid(prefix):
1354 return prefix
1354 return prefix
1355
1355
1356 for length in range(minlength, 41):
1356 for length in range(minlength, 41):
1357 prefix = hexnode[:length]
1357 prefix = hexnode[:length]
1358 if isvalid(prefix):
1358 if isvalid(prefix):
1359 return disambiguate(hexnode, length)
1359 return disambiguate(hexnode, length)
1360
1360
1361 def cmp(self, node, text):
1361 def cmp(self, node, text):
1362 """compare text with a given file revision
1362 """compare text with a given file revision
1363
1363
1364 returns True if text is different than what is stored.
1364 returns True if text is different than what is stored.
1365 """
1365 """
1366 p1, p2 = self.parents(node)
1366 p1, p2 = self.parents(node)
1367 return storageutil.hashrevisionsha1(text, p1, p2) != node
1367 return storageutil.hashrevisionsha1(text, p1, p2) != node
1368
1368
1369 def _cachesegment(self, offset, data):
1369 def _cachesegment(self, offset, data):
1370 """Add a segment to the revlog cache.
1370 """Add a segment to the revlog cache.
1371
1371
1372 Accepts an absolute offset and the data that is at that location.
1372 Accepts an absolute offset and the data that is at that location.
1373 """
1373 """
1374 o, d = self._chunkcache
1374 o, d = self._chunkcache
1375 # try to add to existing cache
1375 # try to add to existing cache
1376 if o + len(d) == offset and len(d) + len(data) < _chunksize:
1376 if o + len(d) == offset and len(d) + len(data) < _chunksize:
1377 self._chunkcache = o, d + data
1377 self._chunkcache = o, d + data
1378 else:
1378 else:
1379 self._chunkcache = offset, data
1379 self._chunkcache = offset, data
1380
1380
1381 def _readsegment(self, offset, length, df=None):
1381 def _readsegment(self, offset, length, df=None):
1382 """Load a segment of raw data from the revlog.
1382 """Load a segment of raw data from the revlog.
1383
1383
1384 Accepts an absolute offset, length to read, and an optional existing
1384 Accepts an absolute offset, length to read, and an optional existing
1385 file handle to read from.
1385 file handle to read from.
1386
1386
1387 If an existing file handle is passed, it will be seeked and the
1387 If an existing file handle is passed, it will be seeked and the
1388 original seek position will NOT be restored.
1388 original seek position will NOT be restored.
1389
1389
1390 Returns a str or buffer of raw byte data.
1390 Returns a str or buffer of raw byte data.
1391
1391
1392 Raises if the requested number of bytes could not be read.
1392 Raises if the requested number of bytes could not be read.
1393 """
1393 """
1394 # Cache data both forward and backward around the requested
1394 # Cache data both forward and backward around the requested
1395 # data, in a fixed size window. This helps speed up operations
1395 # data, in a fixed size window. This helps speed up operations
1396 # involving reading the revlog backwards.
1396 # involving reading the revlog backwards.
1397 cachesize = self._chunkcachesize
1397 cachesize = self._chunkcachesize
1398 realoffset = offset & ~(cachesize - 1)
1398 realoffset = offset & ~(cachesize - 1)
1399 reallength = (((offset + length + cachesize) & ~(cachesize - 1))
1399 reallength = (((offset + length + cachesize) & ~(cachesize - 1))
1400 - realoffset)
1400 - realoffset)
1401 with self._datareadfp(df) as df:
1401 with self._datareadfp(df) as df:
1402 df.seek(realoffset)
1402 df.seek(realoffset)
1403 d = df.read(reallength)
1403 d = df.read(reallength)
1404
1404
1405 self._cachesegment(realoffset, d)
1405 self._cachesegment(realoffset, d)
1406 if offset != realoffset or reallength != length:
1406 if offset != realoffset or reallength != length:
1407 startoffset = offset - realoffset
1407 startoffset = offset - realoffset
1408 if len(d) - startoffset < length:
1408 if len(d) - startoffset < length:
1409 raise error.RevlogError(
1409 raise error.RevlogError(
1410 _('partial read of revlog %s; expected %d bytes from '
1410 _('partial read of revlog %s; expected %d bytes from '
1411 'offset %d, got %d') %
1411 'offset %d, got %d') %
1412 (self.indexfile if self._inline else self.datafile,
1412 (self.indexfile if self._inline else self.datafile,
1413 length, realoffset, len(d) - startoffset))
1413 length, realoffset, len(d) - startoffset))
1414
1414
1415 return util.buffer(d, startoffset, length)
1415 return util.buffer(d, startoffset, length)
1416
1416
1417 if len(d) < length:
1417 if len(d) < length:
1418 raise error.RevlogError(
1418 raise error.RevlogError(
1419 _('partial read of revlog %s; expected %d bytes from offset '
1419 _('partial read of revlog %s; expected %d bytes from offset '
1420 '%d, got %d') %
1420 '%d, got %d') %
1421 (self.indexfile if self._inline else self.datafile,
1421 (self.indexfile if self._inline else self.datafile,
1422 length, offset, len(d)))
1422 length, offset, len(d)))
1423
1423
1424 return d
1424 return d
1425
1425
1426 def _getsegment(self, offset, length, df=None):
1426 def _getsegment(self, offset, length, df=None):
1427 """Obtain a segment of raw data from the revlog.
1427 """Obtain a segment of raw data from the revlog.
1428
1428
1429 Accepts an absolute offset, length of bytes to obtain, and an
1429 Accepts an absolute offset, length of bytes to obtain, and an
1430 optional file handle to the already-opened revlog. If the file
1430 optional file handle to the already-opened revlog. If the file
1431 handle is used, it's original seek position will not be preserved.
1431 handle is used, it's original seek position will not be preserved.
1432
1432
1433 Requests for data may be returned from a cache.
1433 Requests for data may be returned from a cache.
1434
1434
1435 Returns a str or a buffer instance of raw byte data.
1435 Returns a str or a buffer instance of raw byte data.
1436 """
1436 """
1437 o, d = self._chunkcache
1437 o, d = self._chunkcache
1438 l = len(d)
1438 l = len(d)
1439
1439
1440 # is it in the cache?
1440 # is it in the cache?
1441 cachestart = offset - o
1441 cachestart = offset - o
1442 cacheend = cachestart + length
1442 cacheend = cachestart + length
1443 if cachestart >= 0 and cacheend <= l:
1443 if cachestart >= 0 and cacheend <= l:
1444 if cachestart == 0 and cacheend == l:
1444 if cachestart == 0 and cacheend == l:
1445 return d # avoid a copy
1445 return d # avoid a copy
1446 return util.buffer(d, cachestart, cacheend - cachestart)
1446 return util.buffer(d, cachestart, cacheend - cachestart)
1447
1447
1448 return self._readsegment(offset, length, df=df)
1448 return self._readsegment(offset, length, df=df)
1449
1449
1450 def _getsegmentforrevs(self, startrev, endrev, df=None):
1450 def _getsegmentforrevs(self, startrev, endrev, df=None):
1451 """Obtain a segment of raw data corresponding to a range of revisions.
1451 """Obtain a segment of raw data corresponding to a range of revisions.
1452
1452
1453 Accepts the start and end revisions and an optional already-open
1453 Accepts the start and end revisions and an optional already-open
1454 file handle to be used for reading. If the file handle is read, its
1454 file handle to be used for reading. If the file handle is read, its
1455 seek position will not be preserved.
1455 seek position will not be preserved.
1456
1456
1457 Requests for data may be satisfied by a cache.
1457 Requests for data may be satisfied by a cache.
1458
1458
1459 Returns a 2-tuple of (offset, data) for the requested range of
1459 Returns a 2-tuple of (offset, data) for the requested range of
1460 revisions. Offset is the integer offset from the beginning of the
1460 revisions. Offset is the integer offset from the beginning of the
1461 revlog and data is a str or buffer of the raw byte data.
1461 revlog and data is a str or buffer of the raw byte data.
1462
1462
1463 Callers will need to call ``self.start(rev)`` and ``self.length(rev)``
1463 Callers will need to call ``self.start(rev)`` and ``self.length(rev)``
1464 to determine where each revision's data begins and ends.
1464 to determine where each revision's data begins and ends.
1465 """
1465 """
1466 # Inlined self.start(startrev) & self.end(endrev) for perf reasons
1466 # Inlined self.start(startrev) & self.end(endrev) for perf reasons
1467 # (functions are expensive).
1467 # (functions are expensive).
1468 index = self.index
1468 index = self.index
1469 istart = index[startrev]
1469 istart = index[startrev]
1470 start = int(istart[0] >> 16)
1470 start = int(istart[0] >> 16)
1471 if startrev == endrev:
1471 if startrev == endrev:
1472 end = start + istart[1]
1472 end = start + istart[1]
1473 else:
1473 else:
1474 iend = index[endrev]
1474 iend = index[endrev]
1475 end = int(iend[0] >> 16) + iend[1]
1475 end = int(iend[0] >> 16) + iend[1]
1476
1476
1477 if self._inline:
1477 if self._inline:
1478 start += (startrev + 1) * self._io.size
1478 start += (startrev + 1) * self._io.size
1479 end += (endrev + 1) * self._io.size
1479 end += (endrev + 1) * self._io.size
1480 length = end - start
1480 length = end - start
1481
1481
1482 return start, self._getsegment(start, length, df=df)
1482 return start, self._getsegment(start, length, df=df)
1483
1483
1484 def _chunk(self, rev, df=None):
1484 def _chunk(self, rev, df=None):
1485 """Obtain a single decompressed chunk for a revision.
1485 """Obtain a single decompressed chunk for a revision.
1486
1486
1487 Accepts an integer revision and an optional already-open file handle
1487 Accepts an integer revision and an optional already-open file handle
1488 to be used for reading. If used, the seek position of the file will not
1488 to be used for reading. If used, the seek position of the file will not
1489 be preserved.
1489 be preserved.
1490
1490
1491 Returns a str holding uncompressed data for the requested revision.
1491 Returns a str holding uncompressed data for the requested revision.
1492 """
1492 """
1493 return self.decompress(self._getsegmentforrevs(rev, rev, df=df)[1])
1493 return self.decompress(self._getsegmentforrevs(rev, rev, df=df)[1])
1494
1494
1495 def _chunks(self, revs, df=None, targetsize=None):
1495 def _chunks(self, revs, df=None, targetsize=None):
1496 """Obtain decompressed chunks for the specified revisions.
1496 """Obtain decompressed chunks for the specified revisions.
1497
1497
1498 Accepts an iterable of numeric revisions that are assumed to be in
1498 Accepts an iterable of numeric revisions that are assumed to be in
1499 ascending order. Also accepts an optional already-open file handle
1499 ascending order. Also accepts an optional already-open file handle
1500 to be used for reading. If used, the seek position of the file will
1500 to be used for reading. If used, the seek position of the file will
1501 not be preserved.
1501 not be preserved.
1502
1502
1503 This function is similar to calling ``self._chunk()`` multiple times,
1503 This function is similar to calling ``self._chunk()`` multiple times,
1504 but is faster.
1504 but is faster.
1505
1505
1506 Returns a list with decompressed data for each requested revision.
1506 Returns a list with decompressed data for each requested revision.
1507 """
1507 """
1508 if not revs:
1508 if not revs:
1509 return []
1509 return []
1510 start = self.start
1510 start = self.start
1511 length = self.length
1511 length = self.length
1512 inline = self._inline
1512 inline = self._inline
1513 iosize = self._io.size
1513 iosize = self._io.size
1514 buffer = util.buffer
1514 buffer = util.buffer
1515
1515
1516 l = []
1516 l = []
1517 ladd = l.append
1517 ladd = l.append
1518
1518
1519 if not self._withsparseread:
1519 if not self._withsparseread:
1520 slicedchunks = (revs,)
1520 slicedchunks = (revs,)
1521 else:
1521 else:
1522 slicedchunks = deltautil.slicechunk(self, revs,
1522 slicedchunks = deltautil.slicechunk(self, revs,
1523 targetsize=targetsize)
1523 targetsize=targetsize)
1524
1524
1525 for revschunk in slicedchunks:
1525 for revschunk in slicedchunks:
1526 firstrev = revschunk[0]
1526 firstrev = revschunk[0]
1527 # Skip trailing revisions with empty diff
1527 # Skip trailing revisions with empty diff
1528 for lastrev in revschunk[::-1]:
1528 for lastrev in revschunk[::-1]:
1529 if length(lastrev) != 0:
1529 if length(lastrev) != 0:
1530 break
1530 break
1531
1531
1532 try:
1532 try:
1533 offset, data = self._getsegmentforrevs(firstrev, lastrev, df=df)
1533 offset, data = self._getsegmentforrevs(firstrev, lastrev, df=df)
1534 except OverflowError:
1534 except OverflowError:
1535 # issue4215 - we can't cache a run of chunks greater than
1535 # issue4215 - we can't cache a run of chunks greater than
1536 # 2G on Windows
1536 # 2G on Windows
1537 return [self._chunk(rev, df=df) for rev in revschunk]
1537 return [self._chunk(rev, df=df) for rev in revschunk]
1538
1538
1539 decomp = self.decompress
1539 decomp = self.decompress
1540 for rev in revschunk:
1540 for rev in revschunk:
1541 chunkstart = start(rev)
1541 chunkstart = start(rev)
1542 if inline:
1542 if inline:
1543 chunkstart += (rev + 1) * iosize
1543 chunkstart += (rev + 1) * iosize
1544 chunklength = length(rev)
1544 chunklength = length(rev)
1545 ladd(decomp(buffer(data, chunkstart - offset, chunklength)))
1545 ladd(decomp(buffer(data, chunkstart - offset, chunklength)))
1546
1546
1547 return l
1547 return l
1548
1548
1549 def _chunkclear(self):
1549 def _chunkclear(self):
1550 """Clear the raw chunk cache."""
1550 """Clear the raw chunk cache."""
1551 self._chunkcache = (0, '')
1551 self._chunkcache = (0, '')
1552
1552
1553 def deltaparent(self, rev):
1553 def deltaparent(self, rev):
1554 """return deltaparent of the given revision"""
1554 """return deltaparent of the given revision"""
1555 base = self.index[rev][3]
1555 base = self.index[rev][3]
1556 if base == rev:
1556 if base == rev:
1557 return nullrev
1557 return nullrev
1558 elif self._generaldelta:
1558 elif self._generaldelta:
1559 return base
1559 return base
1560 else:
1560 else:
1561 return rev - 1
1561 return rev - 1
1562
1562
1563 def issnapshot(self, rev):
1563 def issnapshot(self, rev):
1564 """tells whether rev is a snapshot
1564 """tells whether rev is a snapshot
1565 """
1565 """
1566 if not self._sparserevlog:
1566 if not self._sparserevlog:
1567 return self.deltaparent(rev) == nullrev
1567 return self.deltaparent(rev) == nullrev
1568 elif util.safehasattr(self.index, 'issnapshot'):
1568 elif util.safehasattr(self.index, 'issnapshot'):
1569 # directly assign the method to cache the testing and access
1569 # directly assign the method to cache the testing and access
1570 self.issnapshot = self.index.issnapshot
1570 self.issnapshot = self.index.issnapshot
1571 return self.issnapshot(rev)
1571 return self.issnapshot(rev)
1572 if rev == nullrev:
1572 if rev == nullrev:
1573 return True
1573 return True
1574 entry = self.index[rev]
1574 entry = self.index[rev]
1575 base = entry[3]
1575 base = entry[3]
1576 if base == rev:
1576 if base == rev:
1577 return True
1577 return True
1578 if base == nullrev:
1578 if base == nullrev:
1579 return True
1579 return True
1580 p1 = entry[5]
1580 p1 = entry[5]
1581 p2 = entry[6]
1581 p2 = entry[6]
1582 if base == p1 or base == p2:
1582 if base == p1 or base == p2:
1583 return False
1583 return False
1584 return self.issnapshot(base)
1584 return self.issnapshot(base)
1585
1585
1586 def snapshotdepth(self, rev):
1586 def snapshotdepth(self, rev):
1587 """number of snapshot in the chain before this one"""
1587 """number of snapshot in the chain before this one"""
1588 if not self.issnapshot(rev):
1588 if not self.issnapshot(rev):
1589 raise error.ProgrammingError('revision %d not a snapshot')
1589 raise error.ProgrammingError('revision %d not a snapshot')
1590 return len(self._deltachain(rev)[0]) - 1
1590 return len(self._deltachain(rev)[0]) - 1
1591
1591
1592 def revdiff(self, rev1, rev2):
1592 def revdiff(self, rev1, rev2):
1593 """return or calculate a delta between two revisions
1593 """return or calculate a delta between two revisions
1594
1594
1595 The delta calculated is in binary form and is intended to be written to
1595 The delta calculated is in binary form and is intended to be written to
1596 revlog data directly. So this function needs raw revision data.
1596 revlog data directly. So this function needs raw revision data.
1597 """
1597 """
1598 if rev1 != nullrev and self.deltaparent(rev2) == rev1:
1598 if rev1 != nullrev and self.deltaparent(rev2) == rev1:
1599 return bytes(self._chunk(rev2))
1599 return bytes(self._chunk(rev2))
1600
1600
1601 return mdiff.textdiff(self.rawdata(rev1),
1601 return mdiff.textdiff(self.rawdata(rev1),
1602 self.rawdata(rev2))
1602 self.rawdata(rev2))
1603
1603
1604 def revision(self, nodeorrev, _df=None, raw=False):
1604 def revision(self, nodeorrev, _df=None, raw=False):
1605 """return an uncompressed revision of a given node or revision
1605 """return an uncompressed revision of a given node or revision
1606 number.
1606 number.
1607
1607
1608 _df - an existing file handle to read from. (internal-only)
1608 _df - an existing file handle to read from. (internal-only)
1609 raw - an optional argument specifying if the revision data is to be
1609 raw - an optional argument specifying if the revision data is to be
1610 treated as raw data when applying flag transforms. 'raw' should be set
1610 treated as raw data when applying flag transforms. 'raw' should be set
1611 to True when generating changegroups or in debug commands.
1611 to True when generating changegroups or in debug commands.
1612 """
1612 """
1613 if raw:
1613 if raw:
1614 msg = ('revlog.revision(..., raw=True) is deprecated, '
1614 msg = ('revlog.revision(..., raw=True) is deprecated, '
1615 'use revlog.rawdata(...)')
1615 'use revlog.rawdata(...)')
1616 util.nouideprecwarn(msg, '5.2', stacklevel=2)
1616 util.nouideprecwarn(msg, '5.2', stacklevel=2)
1617 return self._revisiondata(nodeorrev, _df, raw=raw)[0]
1617 return self._revisiondata(nodeorrev, _df, raw=raw)[0]
1618
1618
1619 def sidedata(self, nodeorrev, _df=None):
1619 def sidedata(self, nodeorrev, _df=None):
1620 """a map of extra data related to the changeset but not part of the hash
1620 """a map of extra data related to the changeset but not part of the hash
1621
1621
1622 This function currently return a dictionary. However, more advanced
1622 This function currently return a dictionary. However, more advanced
1623 mapping object will likely be used in the future for a more
1623 mapping object will likely be used in the future for a more
1624 efficient/lazy code.
1624 efficient/lazy code.
1625 """
1625 """
1626 return self._revisiondata(nodeorrev, _df)[1]
1626 return self._revisiondata(nodeorrev, _df)[1]
1627
1627
1628 def _revisiondata(self, nodeorrev, _df=None, raw=False):
1628 def _revisiondata(self, nodeorrev, _df=None, raw=False):
1629 # deal with <nodeorrev> argument type
1629 # deal with <nodeorrev> argument type
1630 if isinstance(nodeorrev, int):
1630 if isinstance(nodeorrev, int):
1631 rev = nodeorrev
1631 rev = nodeorrev
1632 node = self.node(rev)
1632 node = self.node(rev)
1633 else:
1633 else:
1634 node = nodeorrev
1634 node = nodeorrev
1635 rev = None
1635 rev = None
1636
1636
1637 # fast path the special `nullid` rev
1637 # fast path the special `nullid` rev
1638 if node == nullid:
1638 if node == nullid:
1639 return "", {}
1639 return "", {}
1640
1640
1641 # The text as stored inside the revlog. Might be the revision or might
1641 # The text as stored inside the revlog. Might be the revision or might
1642 # need to be processed to retrieve the revision.
1642 # need to be processed to retrieve the revision.
1643 rawtext = None
1643 rawtext = None
1644
1644
1645 rev, rawtext, validated = self._rawtext(node, rev, _df=_df)
1645 rev, rawtext, validated = self._rawtext(node, rev, _df=_df)
1646
1646
1647 if raw and validated:
1647 if raw and validated:
1648 # if we don't want to process the raw text and that raw
1648 # if we don't want to process the raw text and that raw
1649 # text is cached, we can exit early.
1649 # text is cached, we can exit early.
1650 return rawtext, {}
1650 return rawtext, {}
1651 if rev is None:
1651 if rev is None:
1652 rev = self.rev(node)
1652 rev = self.rev(node)
1653 # the revlog's flag for this revision
1653 # the revlog's flag for this revision
1654 # (usually alter its state or content)
1654 # (usually alter its state or content)
1655 flags = self.flags(rev)
1655 flags = self.flags(rev)
1656
1656
1657 if validated and flags == REVIDX_DEFAULT_FLAGS:
1657 if validated and flags == REVIDX_DEFAULT_FLAGS:
1658 # no extra flags set, no flag processor runs, text = rawtext
1658 # no extra flags set, no flag processor runs, text = rawtext
1659 return rawtext, {}
1659 return rawtext, {}
1660
1660
1661 sidedata = {}
1661 sidedata = {}
1662 if raw:
1662 if raw:
1663 validatehash = self._processflagsraw(rawtext, flags)
1663 validatehash = self._processflagsraw(rawtext, flags)
1664 text = rawtext
1664 text = rawtext
1665 else:
1665 else:
1666 r = self._processflagsread(rawtext, flags)
1666 r = self._processflagsread(rawtext, flags)
1667 text, validatehash, sidedata = r
1667 text, validatehash, sidedata = r
1668 if validatehash:
1668 if validatehash:
1669 self.checkhash(text, node, rev=rev)
1669 self.checkhash(text, node, rev=rev)
1670 if not validated:
1670 if not validated:
1671 self._revisioncache = (node, rev, rawtext)
1671 self._revisioncache = (node, rev, rawtext)
1672
1672
1673 return text, sidedata
1673 return text, sidedata
1674
1674
1675 def _rawtext(self, node, rev, _df=None):
1675 def _rawtext(self, node, rev, _df=None):
1676 """return the possibly unvalidated rawtext for a revision
1676 """return the possibly unvalidated rawtext for a revision
1677
1677
1678 returns (rev, rawtext, validated)
1678 returns (rev, rawtext, validated)
1679 """
1679 """
1680
1680
1681 # revision in the cache (could be useful to apply delta)
1681 # revision in the cache (could be useful to apply delta)
1682 cachedrev = None
1682 cachedrev = None
1683 # An intermediate text to apply deltas to
1683 # An intermediate text to apply deltas to
1684 basetext = None
1684 basetext = None
1685
1685
1686 # Check if we have the entry in cache
1686 # Check if we have the entry in cache
1687 # The cache entry looks like (node, rev, rawtext)
1687 # The cache entry looks like (node, rev, rawtext)
1688 if self._revisioncache:
1688 if self._revisioncache:
1689 if self._revisioncache[0] == node:
1689 if self._revisioncache[0] == node:
1690 return (rev, self._revisioncache[2], True)
1690 return (rev, self._revisioncache[2], True)
1691 cachedrev = self._revisioncache[1]
1691 cachedrev = self._revisioncache[1]
1692
1692
1693 if rev is None:
1693 if rev is None:
1694 rev = self.rev(node)
1694 rev = self.rev(node)
1695
1695
1696 chain, stopped = self._deltachain(rev, stoprev=cachedrev)
1696 chain, stopped = self._deltachain(rev, stoprev=cachedrev)
1697 if stopped:
1697 if stopped:
1698 basetext = self._revisioncache[2]
1698 basetext = self._revisioncache[2]
1699
1699
1700 # drop cache to save memory, the caller is expected to
1700 # drop cache to save memory, the caller is expected to
1701 # update self._revisioncache after validating the text
1701 # update self._revisioncache after validating the text
1702 self._revisioncache = None
1702 self._revisioncache = None
1703
1703
1704 targetsize = None
1704 targetsize = None
1705 rawsize = self.index[rev][2]
1705 rawsize = self.index[rev][2]
1706 if 0 <= rawsize:
1706 if 0 <= rawsize:
1707 targetsize = 4 * rawsize
1707 targetsize = 4 * rawsize
1708
1708
1709 bins = self._chunks(chain, df=_df, targetsize=targetsize)
1709 bins = self._chunks(chain, df=_df, targetsize=targetsize)
1710 if basetext is None:
1710 if basetext is None:
1711 basetext = bytes(bins[0])
1711 basetext = bytes(bins[0])
1712 bins = bins[1:]
1712 bins = bins[1:]
1713
1713
1714 rawtext = mdiff.patches(basetext, bins)
1714 rawtext = mdiff.patches(basetext, bins)
1715 del basetext # let us have a chance to free memory early
1715 del basetext # let us have a chance to free memory early
1716 return (rev, rawtext, False)
1716 return (rev, rawtext, False)
1717
1717
1718 def rawdata(self, nodeorrev, _df=None):
1718 def rawdata(self, nodeorrev, _df=None):
1719 """return an uncompressed raw data of a given node or revision number.
1719 """return an uncompressed raw data of a given node or revision number.
1720
1720
1721 _df - an existing file handle to read from. (internal-only)
1721 _df - an existing file handle to read from. (internal-only)
1722 """
1722 """
1723 return self._revisiondata(nodeorrev, _df, raw=True)[0]
1723 return self._revisiondata(nodeorrev, _df, raw=True)[0]
1724
1724
1725 def hash(self, text, p1, p2):
1725 def hash(self, text, p1, p2):
1726 """Compute a node hash.
1726 """Compute a node hash.
1727
1727
1728 Available as a function so that subclasses can replace the hash
1728 Available as a function so that subclasses can replace the hash
1729 as needed.
1729 as needed.
1730 """
1730 """
1731 return storageutil.hashrevisionsha1(text, p1, p2)
1731 return storageutil.hashrevisionsha1(text, p1, p2)
1732
1732
1733 def checkhash(self, text, node, p1=None, p2=None, rev=None):
1733 def checkhash(self, text, node, p1=None, p2=None, rev=None):
1734 """Check node hash integrity.
1734 """Check node hash integrity.
1735
1735
1736 Available as a function so that subclasses can extend hash mismatch
1736 Available as a function so that subclasses can extend hash mismatch
1737 behaviors as needed.
1737 behaviors as needed.
1738 """
1738 """
1739 try:
1739 try:
1740 if p1 is None and p2 is None:
1740 if p1 is None and p2 is None:
1741 p1, p2 = self.parents(node)
1741 p1, p2 = self.parents(node)
1742 if node != self.hash(text, p1, p2):
1742 if node != self.hash(text, p1, p2):
1743 # Clear the revision cache on hash failure. The revision cache
1743 # Clear the revision cache on hash failure. The revision cache
1744 # only stores the raw revision and clearing the cache does have
1744 # only stores the raw revision and clearing the cache does have
1745 # the side-effect that we won't have a cache hit when the raw
1745 # the side-effect that we won't have a cache hit when the raw
1746 # revision data is accessed. But this case should be rare and
1746 # revision data is accessed. But this case should be rare and
1747 # it is extra work to teach the cache about the hash
1747 # it is extra work to teach the cache about the hash
1748 # verification state.
1748 # verification state.
1749 if self._revisioncache and self._revisioncache[0] == node:
1749 if self._revisioncache and self._revisioncache[0] == node:
1750 self._revisioncache = None
1750 self._revisioncache = None
1751
1751
1752 revornode = rev
1752 revornode = rev
1753 if revornode is None:
1753 if revornode is None:
1754 revornode = templatefilters.short(hex(node))
1754 revornode = templatefilters.short(hex(node))
1755 raise error.RevlogError(_("integrity check failed on %s:%s")
1755 raise error.RevlogError(_("integrity check failed on %s:%s")
1756 % (self.indexfile, pycompat.bytestr(revornode)))
1756 % (self.indexfile, pycompat.bytestr(revornode)))
1757 except error.RevlogError:
1757 except error.RevlogError:
1758 if self._censorable and storageutil.iscensoredtext(text):
1758 if self._censorable and storageutil.iscensoredtext(text):
1759 raise error.CensoredNodeError(self.indexfile, node, text)
1759 raise error.CensoredNodeError(self.indexfile, node, text)
1760 raise
1760 raise
1761
1761
1762 def _enforceinlinesize(self, tr, fp=None):
1762 def _enforceinlinesize(self, tr, fp=None):
1763 """Check if the revlog is too big for inline and convert if so.
1763 """Check if the revlog is too big for inline and convert if so.
1764
1764
1765 This should be called after revisions are added to the revlog. If the
1765 This should be called after revisions are added to the revlog. If the
1766 revlog has grown too large to be an inline revlog, it will convert it
1766 revlog has grown too large to be an inline revlog, it will convert it
1767 to use multiple index and data files.
1767 to use multiple index and data files.
1768 """
1768 """
1769 tiprev = len(self) - 1
1769 tiprev = len(self) - 1
1770 if (not self._inline or
1770 if (not self._inline or
1771 (self.start(tiprev) + self.length(tiprev)) < _maxinline):
1771 (self.start(tiprev) + self.length(tiprev)) < _maxinline):
1772 return
1772 return
1773
1773
1774 trinfo = tr.find(self.indexfile)
1774 trinfo = tr.find(self.indexfile)
1775 if trinfo is None:
1775 if trinfo is None:
1776 raise error.RevlogError(_("%s not found in the transaction")
1776 raise error.RevlogError(_("%s not found in the transaction")
1777 % self.indexfile)
1777 % self.indexfile)
1778
1778
1779 trindex = trinfo[2]
1779 trindex = trinfo[2]
1780 if trindex is not None:
1780 if trindex is not None:
1781 dataoff = self.start(trindex)
1781 dataoff = self.start(trindex)
1782 else:
1782 else:
1783 # revlog was stripped at start of transaction, use all leftover data
1783 # revlog was stripped at start of transaction, use all leftover data
1784 trindex = len(self) - 1
1784 trindex = len(self) - 1
1785 dataoff = self.end(tiprev)
1785 dataoff = self.end(tiprev)
1786
1786
1787 tr.add(self.datafile, dataoff)
1787 tr.add(self.datafile, dataoff)
1788
1788
1789 if fp:
1789 if fp:
1790 fp.flush()
1790 fp.flush()
1791 fp.close()
1791 fp.close()
1792 # We can't use the cached file handle after close(). So prevent
1792 # We can't use the cached file handle after close(). So prevent
1793 # its usage.
1793 # its usage.
1794 self._writinghandles = None
1794 self._writinghandles = None
1795
1795
1796 with self._indexfp('r') as ifh, self._datafp('w') as dfh:
1796 with self._indexfp('r') as ifh, self._datafp('w') as dfh:
1797 for r in self:
1797 for r in self:
1798 dfh.write(self._getsegmentforrevs(r, r, df=ifh)[1])
1798 dfh.write(self._getsegmentforrevs(r, r, df=ifh)[1])
1799
1799
1800 with self._indexfp('w') as fp:
1800 with self._indexfp('w') as fp:
1801 self.version &= ~FLAG_INLINE_DATA
1801 self.version &= ~FLAG_INLINE_DATA
1802 self._inline = False
1802 self._inline = False
1803 io = self._io
1803 io = self._io
1804 for i in self:
1804 for i in self:
1805 e = io.packentry(self.index[i], self.node, self.version, i)
1805 e = io.packentry(self.index[i], self.node, self.version, i)
1806 fp.write(e)
1806 fp.write(e)
1807
1807
1808 # the temp file replace the real index when we exit the context
1808 # the temp file replace the real index when we exit the context
1809 # manager
1809 # manager
1810
1810
1811 tr.replace(self.indexfile, trindex * self._io.size)
1811 tr.replace(self.indexfile, trindex * self._io.size)
1812 self._chunkclear()
1812 self._chunkclear()
1813
1813
1814 def _nodeduplicatecallback(self, transaction, node):
1814 def _nodeduplicatecallback(self, transaction, node):
1815 """called when trying to add a node already stored.
1815 """called when trying to add a node already stored.
1816 """
1816 """
1817
1817
1818 def addrevision(self, text, transaction, link, p1, p2, cachedelta=None,
1818 def addrevision(self, text, transaction, link, p1, p2, cachedelta=None,
1819 node=None, flags=REVIDX_DEFAULT_FLAGS, deltacomputer=None):
1819 node=None, flags=REVIDX_DEFAULT_FLAGS, deltacomputer=None):
1820 """add a revision to the log
1820 """add a revision to the log
1821
1821
1822 text - the revision data to add
1822 text - the revision data to add
1823 transaction - the transaction object used for rollback
1823 transaction - the transaction object used for rollback
1824 link - the linkrev data to add
1824 link - the linkrev data to add
1825 p1, p2 - the parent nodeids of the revision
1825 p1, p2 - the parent nodeids of the revision
1826 cachedelta - an optional precomputed delta
1826 cachedelta - an optional precomputed delta
1827 node - nodeid of revision; typically node is not specified, and it is
1827 node - nodeid of revision; typically node is not specified, and it is
1828 computed by default as hash(text, p1, p2), however subclasses might
1828 computed by default as hash(text, p1, p2), however subclasses might
1829 use different hashing method (and override checkhash() in such case)
1829 use different hashing method (and override checkhash() in such case)
1830 flags - the known flags to set on the revision
1830 flags - the known flags to set on the revision
1831 deltacomputer - an optional deltacomputer instance shared between
1831 deltacomputer - an optional deltacomputer instance shared between
1832 multiple calls
1832 multiple calls
1833 """
1833 """
1834 if link == nullrev:
1834 if link == nullrev:
1835 raise error.RevlogError(_("attempted to add linkrev -1 to %s")
1835 raise error.RevlogError(_("attempted to add linkrev -1 to %s")
1836 % self.indexfile)
1836 % self.indexfile)
1837
1837
1838 if flags:
1838 if flags:
1839 node = node or self.hash(text, p1, p2)
1839 node = node or self.hash(text, p1, p2)
1840
1840
1841 rawtext, validatehash = self._processflagswrite(text, flags)
1841 rawtext, validatehash = self._processflagswrite(text, flags)
1842
1842
1843 # If the flag processor modifies the revision data, ignore any provided
1843 # If the flag processor modifies the revision data, ignore any provided
1844 # cachedelta.
1844 # cachedelta.
1845 if rawtext != text:
1845 if rawtext != text:
1846 cachedelta = None
1846 cachedelta = None
1847
1847
1848 if len(rawtext) > _maxentrysize:
1848 if len(rawtext) > _maxentrysize:
1849 raise error.RevlogError(
1849 raise error.RevlogError(
1850 _("%s: size of %d bytes exceeds maximum revlog storage of 2GiB")
1850 _("%s: size of %d bytes exceeds maximum revlog storage of 2GiB")
1851 % (self.indexfile, len(rawtext)))
1851 % (self.indexfile, len(rawtext)))
1852
1852
1853 node = node or self.hash(rawtext, p1, p2)
1853 node = node or self.hash(rawtext, p1, p2)
1854 if node in self.nodemap:
1854 if node in self.nodemap:
1855 return node
1855 return node
1856
1856
1857 if validatehash:
1857 if validatehash:
1858 self.checkhash(rawtext, node, p1=p1, p2=p2)
1858 self.checkhash(rawtext, node, p1=p1, p2=p2)
1859
1859
1860 return self.addrawrevision(rawtext, transaction, link, p1, p2, node,
1860 return self.addrawrevision(rawtext, transaction, link, p1, p2, node,
1861 flags, cachedelta=cachedelta,
1861 flags, cachedelta=cachedelta,
1862 deltacomputer=deltacomputer)
1862 deltacomputer=deltacomputer)
1863
1863
1864 def addrawrevision(self, rawtext, transaction, link, p1, p2, node, flags,
1864 def addrawrevision(self, rawtext, transaction, link, p1, p2, node, flags,
1865 cachedelta=None, deltacomputer=None):
1865 cachedelta=None, deltacomputer=None):
1866 """add a raw revision with known flags, node and parents
1866 """add a raw revision with known flags, node and parents
1867 useful when reusing a revision not stored in this revlog (ex: received
1867 useful when reusing a revision not stored in this revlog (ex: received
1868 over wire, or read from an external bundle).
1868 over wire, or read from an external bundle).
1869 """
1869 """
1870 dfh = None
1870 dfh = None
1871 if not self._inline:
1871 if not self._inline:
1872 dfh = self._datafp("a+")
1872 dfh = self._datafp("a+")
1873 ifh = self._indexfp("a+")
1873 ifh = self._indexfp("a+")
1874 try:
1874 try:
1875 return self._addrevision(node, rawtext, transaction, link, p1, p2,
1875 return self._addrevision(node, rawtext, transaction, link, p1, p2,
1876 flags, cachedelta, ifh, dfh,
1876 flags, cachedelta, ifh, dfh,
1877 deltacomputer=deltacomputer)
1877 deltacomputer=deltacomputer)
1878 finally:
1878 finally:
1879 if dfh:
1879 if dfh:
1880 dfh.close()
1880 dfh.close()
1881 ifh.close()
1881 ifh.close()
1882
1882
1883 def compress(self, data):
1883 def compress(self, data):
1884 """Generate a possibly-compressed representation of data."""
1884 """Generate a possibly-compressed representation of data."""
1885 if not data:
1885 if not data:
1886 return '', data
1886 return '', data
1887
1887
1888 compressed = self._compressor.compress(data)
1888 compressed = self._compressor.compress(data)
1889
1889
1890 if compressed:
1890 if compressed:
1891 # The revlog compressor added the header in the returned data.
1891 # The revlog compressor added the header in the returned data.
1892 return '', compressed
1892 return '', compressed
1893
1893
1894 if data[0:1] == '\0':
1894 if data[0:1] == '\0':
1895 return '', data
1895 return '', data
1896 return 'u', data
1896 return 'u', data
1897
1897
1898 def decompress(self, data):
1898 def decompress(self, data):
1899 """Decompress a revlog chunk.
1899 """Decompress a revlog chunk.
1900
1900
1901 The chunk is expected to begin with a header identifying the
1901 The chunk is expected to begin with a header identifying the
1902 format type so it can be routed to an appropriate decompressor.
1902 format type so it can be routed to an appropriate decompressor.
1903 """
1903 """
1904 if not data:
1904 if not data:
1905 return data
1905 return data
1906
1906
1907 # Revlogs are read much more frequently than they are written and many
1907 # Revlogs are read much more frequently than they are written and many
1908 # chunks only take microseconds to decompress, so performance is
1908 # chunks only take microseconds to decompress, so performance is
1909 # important here.
1909 # important here.
1910 #
1910 #
1911 # We can make a few assumptions about revlogs:
1911 # We can make a few assumptions about revlogs:
1912 #
1912 #
1913 # 1) the majority of chunks will be compressed (as opposed to inline
1913 # 1) the majority of chunks will be compressed (as opposed to inline
1914 # raw data).
1914 # raw data).
1915 # 2) decompressing *any* data will likely by at least 10x slower than
1915 # 2) decompressing *any* data will likely by at least 10x slower than
1916 # returning raw inline data.
1916 # returning raw inline data.
1917 # 3) we want to prioritize common and officially supported compression
1917 # 3) we want to prioritize common and officially supported compression
1918 # engines
1918 # engines
1919 #
1919 #
1920 # It follows that we want to optimize for "decompress compressed data
1920 # It follows that we want to optimize for "decompress compressed data
1921 # when encoded with common and officially supported compression engines"
1921 # when encoded with common and officially supported compression engines"
1922 # case over "raw data" and "data encoded by less common or non-official
1922 # case over "raw data" and "data encoded by less common or non-official
1923 # compression engines." That is why we have the inline lookup first
1923 # compression engines." That is why we have the inline lookup first
1924 # followed by the compengines lookup.
1924 # followed by the compengines lookup.
1925 #
1925 #
1926 # According to `hg perfrevlogchunks`, this is ~0.5% faster for zlib
1926 # According to `hg perfrevlogchunks`, this is ~0.5% faster for zlib
1927 # compressed chunks. And this matters for changelog and manifest reads.
1927 # compressed chunks. And this matters for changelog and manifest reads.
1928 t = data[0:1]
1928 t = data[0:1]
1929
1929
1930 if t == 'x':
1930 if t == 'x':
1931 try:
1931 try:
1932 return _zlibdecompress(data)
1932 return _zlibdecompress(data)
1933 except zlib.error as e:
1933 except zlib.error as e:
1934 raise error.RevlogError(_('revlog decompress error: %s') %
1934 raise error.RevlogError(_('revlog decompress error: %s') %
1935 stringutil.forcebytestr(e))
1935 stringutil.forcebytestr(e))
1936 # '\0' is more common than 'u' so it goes first.
1936 # '\0' is more common than 'u' so it goes first.
1937 elif t == '\0':
1937 elif t == '\0':
1938 return data
1938 return data
1939 elif t == 'u':
1939 elif t == 'u':
1940 return util.buffer(data, 1)
1940 return util.buffer(data, 1)
1941
1941
1942 try:
1942 try:
1943 compressor = self._decompressors[t]
1943 compressor = self._decompressors[t]
1944 except KeyError:
1944 except KeyError:
1945 try:
1945 try:
1946 engine = util.compengines.forrevlogheader(t)
1946 engine = util.compengines.forrevlogheader(t)
1947 compressor = engine.revlogcompressor(self._compengineopts)
1947 compressor = engine.revlogcompressor(self._compengineopts)
1948 self._decompressors[t] = compressor
1948 self._decompressors[t] = compressor
1949 except KeyError:
1949 except KeyError:
1950 raise error.RevlogError(_('unknown compression type %r') % t)
1950 raise error.RevlogError(_('unknown compression type %r') % t)
1951
1951
1952 return compressor.decompress(data)
1952 return compressor.decompress(data)
1953
1953
1954 def _addrevision(self, node, rawtext, transaction, link, p1, p2, flags,
1954 def _addrevision(self, node, rawtext, transaction, link, p1, p2, flags,
1955 cachedelta, ifh, dfh, alwayscache=False,
1955 cachedelta, ifh, dfh, alwayscache=False,
1956 deltacomputer=None):
1956 deltacomputer=None):
1957 """internal function to add revisions to the log
1957 """internal function to add revisions to the log
1958
1958
1959 see addrevision for argument descriptions.
1959 see addrevision for argument descriptions.
1960
1960
1961 note: "addrevision" takes non-raw text, "_addrevision" takes raw text.
1961 note: "addrevision" takes non-raw text, "_addrevision" takes raw text.
1962
1962
1963 if "deltacomputer" is not provided or None, a defaultdeltacomputer will
1963 if "deltacomputer" is not provided or None, a defaultdeltacomputer will
1964 be used.
1964 be used.
1965
1965
1966 invariants:
1966 invariants:
1967 - rawtext is optional (can be None); if not set, cachedelta must be set.
1967 - rawtext is optional (can be None); if not set, cachedelta must be set.
1968 if both are set, they must correspond to each other.
1968 if both are set, they must correspond to each other.
1969 """
1969 """
1970 if node == nullid:
1970 if node == nullid:
1971 raise error.RevlogError(_("%s: attempt to add null revision") %
1971 raise error.RevlogError(_("%s: attempt to add null revision") %
1972 self.indexfile)
1972 self.indexfile)
1973 if node == wdirid or node in wdirfilenodeids:
1973 if node == wdirid or node in wdirfilenodeids:
1974 raise error.RevlogError(_("%s: attempt to add wdir revision") %
1974 raise error.RevlogError(_("%s: attempt to add wdir revision") %
1975 self.indexfile)
1975 self.indexfile)
1976
1976
1977 if self._inline:
1977 if self._inline:
1978 fh = ifh
1978 fh = ifh
1979 else:
1979 else:
1980 fh = dfh
1980 fh = dfh
1981
1981
1982 btext = [rawtext]
1982 btext = [rawtext]
1983
1983
1984 curr = len(self)
1984 curr = len(self)
1985 prev = curr - 1
1985 prev = curr - 1
1986 offset = self.end(prev)
1986 offset = self.end(prev)
1987 p1r, p2r = self.rev(p1), self.rev(p2)
1987 p1r, p2r = self.rev(p1), self.rev(p2)
1988
1988
1989 # full versions are inserted when the needed deltas
1989 # full versions are inserted when the needed deltas
1990 # become comparable to the uncompressed text
1990 # become comparable to the uncompressed text
1991 if rawtext is None:
1991 if rawtext is None:
1992 # need rawtext size, before changed by flag processors, which is
1992 # need rawtext size, before changed by flag processors, which is
1993 # the non-raw size. use revlog explicitly to avoid filelog's extra
1993 # the non-raw size. use revlog explicitly to avoid filelog's extra
1994 # logic that might remove metadata size.
1994 # logic that might remove metadata size.
1995 textlen = mdiff.patchedsize(revlog.size(self, cachedelta[0]),
1995 textlen = mdiff.patchedsize(revlog.size(self, cachedelta[0]),
1996 cachedelta[1])
1996 cachedelta[1])
1997 else:
1997 else:
1998 textlen = len(rawtext)
1998 textlen = len(rawtext)
1999
1999
2000 if deltacomputer is None:
2000 if deltacomputer is None:
2001 deltacomputer = deltautil.deltacomputer(self)
2001 deltacomputer = deltautil.deltacomputer(self)
2002
2002
2003 revinfo = _revisioninfo(node, p1, p2, btext, textlen, cachedelta, flags)
2003 revinfo = _revisioninfo(node, p1, p2, btext, textlen, cachedelta, flags)
2004
2004
2005 deltainfo = deltacomputer.finddeltainfo(revinfo, fh)
2005 deltainfo = deltacomputer.finddeltainfo(revinfo, fh)
2006
2006
2007 e = (offset_type(offset, flags), deltainfo.deltalen, textlen,
2007 e = (offset_type(offset, flags), deltainfo.deltalen, textlen,
2008 deltainfo.base, link, p1r, p2r, node)
2008 deltainfo.base, link, p1r, p2r, node)
2009 self.index.append(e)
2009 self.index.append(e)
2010 self.nodemap[node] = curr
2010 self.nodemap[node] = curr
2011
2011
2012 # Reset the pure node cache start lookup offset to account for new
2012 # Reset the pure node cache start lookup offset to account for new
2013 # revision.
2013 # revision.
2014 if self._nodepos is not None:
2014 if self._nodepos is not None:
2015 self._nodepos = curr
2015 self._nodepos = curr
2016
2016
2017 entry = self._io.packentry(e, self.node, self.version, curr)
2017 entry = self._io.packentry(e, self.node, self.version, curr)
2018 self._writeentry(transaction, ifh, dfh, entry, deltainfo.data,
2018 self._writeentry(transaction, ifh, dfh, entry, deltainfo.data,
2019 link, offset)
2019 link, offset)
2020
2020
2021 rawtext = btext[0]
2021 rawtext = btext[0]
2022
2022
2023 if alwayscache and rawtext is None:
2023 if alwayscache and rawtext is None:
2024 rawtext = deltacomputer.buildtext(revinfo, fh)
2024 rawtext = deltacomputer.buildtext(revinfo, fh)
2025
2025
2026 if type(rawtext) == bytes: # only accept immutable objects
2026 if type(rawtext) == bytes: # only accept immutable objects
2027 self._revisioncache = (node, curr, rawtext)
2027 self._revisioncache = (node, curr, rawtext)
2028 self._chainbasecache[curr] = deltainfo.chainbase
2028 self._chainbasecache[curr] = deltainfo.chainbase
2029 return node
2029 return node
2030
2030
2031 def _writeentry(self, transaction, ifh, dfh, entry, data, link, offset):
2031 def _writeentry(self, transaction, ifh, dfh, entry, data, link, offset):
2032 # Files opened in a+ mode have inconsistent behavior on various
2032 # Files opened in a+ mode have inconsistent behavior on various
2033 # platforms. Windows requires that a file positioning call be made
2033 # platforms. Windows requires that a file positioning call be made
2034 # when the file handle transitions between reads and writes. See
2034 # when the file handle transitions between reads and writes. See
2035 # 3686fa2b8eee and the mixedfilemodewrapper in windows.py. On other
2035 # 3686fa2b8eee and the mixedfilemodewrapper in windows.py. On other
2036 # platforms, Python or the platform itself can be buggy. Some versions
2036 # platforms, Python or the platform itself can be buggy. Some versions
2037 # of Solaris have been observed to not append at the end of the file
2037 # of Solaris have been observed to not append at the end of the file
2038 # if the file was seeked to before the end. See issue4943 for more.
2038 # if the file was seeked to before the end. See issue4943 for more.
2039 #
2039 #
2040 # We work around this issue by inserting a seek() before writing.
2040 # We work around this issue by inserting a seek() before writing.
2041 # Note: This is likely not necessary on Python 3. However, because
2041 # Note: This is likely not necessary on Python 3. However, because
2042 # the file handle is reused for reads and may be seeked there, we need
2042 # the file handle is reused for reads and may be seeked there, we need
2043 # to be careful before changing this.
2043 # to be careful before changing this.
2044 ifh.seek(0, os.SEEK_END)
2044 ifh.seek(0, os.SEEK_END)
2045 if dfh:
2045 if dfh:
2046 dfh.seek(0, os.SEEK_END)
2046 dfh.seek(0, os.SEEK_END)
2047
2047
2048 curr = len(self) - 1
2048 curr = len(self) - 1
2049 if not self._inline:
2049 if not self._inline:
2050 transaction.add(self.datafile, offset)
2050 transaction.add(self.datafile, offset)
2051 transaction.add(self.indexfile, curr * len(entry))
2051 transaction.add(self.indexfile, curr * len(entry))
2052 if data[0]:
2052 if data[0]:
2053 dfh.write(data[0])
2053 dfh.write(data[0])
2054 dfh.write(data[1])
2054 dfh.write(data[1])
2055 ifh.write(entry)
2055 ifh.write(entry)
2056 else:
2056 else:
2057 offset += curr * self._io.size
2057 offset += curr * self._io.size
2058 transaction.add(self.indexfile, offset, curr)
2058 transaction.add(self.indexfile, offset, curr)
2059 ifh.write(entry)
2059 ifh.write(entry)
2060 ifh.write(data[0])
2060 ifh.write(data[0])
2061 ifh.write(data[1])
2061 ifh.write(data[1])
2062 self._enforceinlinesize(transaction, ifh)
2062 self._enforceinlinesize(transaction, ifh)
2063
2063
2064 def addgroup(self, deltas, linkmapper, transaction, addrevisioncb=None):
2064 def addgroup(self, deltas, linkmapper, transaction, addrevisioncb=None):
2065 """
2065 """
2066 add a delta group
2066 add a delta group
2067
2067
2068 given a set of deltas, add them to the revision log. the
2068 given a set of deltas, add them to the revision log. the
2069 first delta is against its parent, which should be in our
2069 first delta is against its parent, which should be in our
2070 log, the rest are against the previous delta.
2070 log, the rest are against the previous delta.
2071
2071
2072 If ``addrevisioncb`` is defined, it will be called with arguments of
2072 If ``addrevisioncb`` is defined, it will be called with arguments of
2073 this revlog and the node that was added.
2073 this revlog and the node that was added.
2074 """
2074 """
2075
2075
2076 if self._writinghandles:
2076 if self._writinghandles:
2077 raise error.ProgrammingError('cannot nest addgroup() calls')
2077 raise error.ProgrammingError('cannot nest addgroup() calls')
2078
2078
2079 nodes = []
2079 nodes = []
2080
2080
2081 r = len(self)
2081 r = len(self)
2082 end = 0
2082 end = 0
2083 if r:
2083 if r:
2084 end = self.end(r - 1)
2084 end = self.end(r - 1)
2085 ifh = self._indexfp("a+")
2085 ifh = self._indexfp("a+")
2086 isize = r * self._io.size
2086 isize = r * self._io.size
2087 if self._inline:
2087 if self._inline:
2088 transaction.add(self.indexfile, end + isize, r)
2088 transaction.add(self.indexfile, end + isize, r)
2089 dfh = None
2089 dfh = None
2090 else:
2090 else:
2091 transaction.add(self.indexfile, isize, r)
2091 transaction.add(self.indexfile, isize, r)
2092 transaction.add(self.datafile, end)
2092 transaction.add(self.datafile, end)
2093 dfh = self._datafp("a+")
2093 dfh = self._datafp("a+")
2094 def flush():
2094 def flush():
2095 if dfh:
2095 if dfh:
2096 dfh.flush()
2096 dfh.flush()
2097 ifh.flush()
2097 ifh.flush()
2098
2098
2099 self._writinghandles = (ifh, dfh)
2099 self._writinghandles = (ifh, dfh)
2100
2100
2101 try:
2101 try:
2102 deltacomputer = deltautil.deltacomputer(self)
2102 deltacomputer = deltautil.deltacomputer(self)
2103 # loop through our set of deltas
2103 # loop through our set of deltas
2104 for data in deltas:
2104 for data in deltas:
2105 node, p1, p2, linknode, deltabase, delta, flags = data
2105 node, p1, p2, linknode, deltabase, delta, flags = data
2106 link = linkmapper(linknode)
2106 link = linkmapper(linknode)
2107 flags = flags or REVIDX_DEFAULT_FLAGS
2107 flags = flags or REVIDX_DEFAULT_FLAGS
2108
2108
2109 nodes.append(node)
2109 nodes.append(node)
2110
2110
2111 if node in self.nodemap:
2111 if node in self.nodemap:
2112 self._nodeduplicatecallback(transaction, node)
2112 self._nodeduplicatecallback(transaction, node)
2113 # this can happen if two branches make the same change
2113 # this can happen if two branches make the same change
2114 continue
2114 continue
2115
2115
2116 for p in (p1, p2):
2116 for p in (p1, p2):
2117 if p not in self.nodemap:
2117 if p not in self.nodemap:
2118 raise error.LookupError(p, self.indexfile,
2118 raise error.LookupError(p, self.indexfile,
2119 _('unknown parent'))
2119 _('unknown parent'))
2120
2120
2121 if deltabase not in self.nodemap:
2121 if deltabase not in self.nodemap:
2122 raise error.LookupError(deltabase, self.indexfile,
2122 raise error.LookupError(deltabase, self.indexfile,
2123 _('unknown delta base'))
2123 _('unknown delta base'))
2124
2124
2125 baserev = self.rev(deltabase)
2125 baserev = self.rev(deltabase)
2126
2126
2127 if baserev != nullrev and self.iscensored(baserev):
2127 if baserev != nullrev and self.iscensored(baserev):
2128 # if base is censored, delta must be full replacement in a
2128 # if base is censored, delta must be full replacement in a
2129 # single patch operation
2129 # single patch operation
2130 hlen = struct.calcsize(">lll")
2130 hlen = struct.calcsize(">lll")
2131 oldlen = self.rawsize(baserev)
2131 oldlen = self.rawsize(baserev)
2132 newlen = len(delta) - hlen
2132 newlen = len(delta) - hlen
2133 if delta[:hlen] != mdiff.replacediffheader(oldlen, newlen):
2133 if delta[:hlen] != mdiff.replacediffheader(oldlen, newlen):
2134 raise error.CensoredBaseError(self.indexfile,
2134 raise error.CensoredBaseError(self.indexfile,
2135 self.node(baserev))
2135 self.node(baserev))
2136
2136
2137 if not flags and self._peek_iscensored(baserev, delta, flush):
2137 if not flags and self._peek_iscensored(baserev, delta, flush):
2138 flags |= REVIDX_ISCENSORED
2138 flags |= REVIDX_ISCENSORED
2139
2139
2140 # We assume consumers of addrevisioncb will want to retrieve
2140 # We assume consumers of addrevisioncb will want to retrieve
2141 # the added revision, which will require a call to
2141 # the added revision, which will require a call to
2142 # revision(). revision() will fast path if there is a cache
2142 # revision(). revision() will fast path if there is a cache
2143 # hit. So, we tell _addrevision() to always cache in this case.
2143 # hit. So, we tell _addrevision() to always cache in this case.
2144 # We're only using addgroup() in the context of changegroup
2144 # We're only using addgroup() in the context of changegroup
2145 # generation so the revision data can always be handled as raw
2145 # generation so the revision data can always be handled as raw
2146 # by the flagprocessor.
2146 # by the flagprocessor.
2147 self._addrevision(node, None, transaction, link,
2147 self._addrevision(node, None, transaction, link,
2148 p1, p2, flags, (baserev, delta),
2148 p1, p2, flags, (baserev, delta),
2149 ifh, dfh,
2149 ifh, dfh,
2150 alwayscache=bool(addrevisioncb),
2150 alwayscache=bool(addrevisioncb),
2151 deltacomputer=deltacomputer)
2151 deltacomputer=deltacomputer)
2152
2152
2153 if addrevisioncb:
2153 if addrevisioncb:
2154 addrevisioncb(self, node)
2154 addrevisioncb(self, node)
2155
2155
2156 if not dfh and not self._inline:
2156 if not dfh and not self._inline:
2157 # addrevision switched from inline to conventional
2157 # addrevision switched from inline to conventional
2158 # reopen the index
2158 # reopen the index
2159 ifh.close()
2159 ifh.close()
2160 dfh = self._datafp("a+")
2160 dfh = self._datafp("a+")
2161 ifh = self._indexfp("a+")
2161 ifh = self._indexfp("a+")
2162 self._writinghandles = (ifh, dfh)
2162 self._writinghandles = (ifh, dfh)
2163 finally:
2163 finally:
2164 self._writinghandles = None
2164 self._writinghandles = None
2165
2165
2166 if dfh:
2166 if dfh:
2167 dfh.close()
2167 dfh.close()
2168 ifh.close()
2168 ifh.close()
2169
2169
2170 return nodes
2170 return nodes
2171
2171
2172 def iscensored(self, rev):
2172 def iscensored(self, rev):
2173 """Check if a file revision is censored."""
2173 """Check if a file revision is censored."""
2174 if not self._censorable:
2174 if not self._censorable:
2175 return False
2175 return False
2176
2176
2177 return self.flags(rev) & REVIDX_ISCENSORED
2177 return self.flags(rev) & REVIDX_ISCENSORED
2178
2178
2179 def _peek_iscensored(self, baserev, delta, flush):
2179 def _peek_iscensored(self, baserev, delta, flush):
2180 """Quickly check if a delta produces a censored revision."""
2180 """Quickly check if a delta produces a censored revision."""
2181 if not self._censorable:
2181 if not self._censorable:
2182 return False
2182 return False
2183
2183
2184 return storageutil.deltaiscensored(delta, baserev, self.rawsize)
2184 return storageutil.deltaiscensored(delta, baserev, self.rawsize)
2185
2185
2186 def getstrippoint(self, minlink):
2186 def getstrippoint(self, minlink):
2187 """find the minimum rev that must be stripped to strip the linkrev
2187 """find the minimum rev that must be stripped to strip the linkrev
2188
2188
2189 Returns a tuple containing the minimum rev and a set of all revs that
2189 Returns a tuple containing the minimum rev and a set of all revs that
2190 have linkrevs that will be broken by this strip.
2190 have linkrevs that will be broken by this strip.
2191 """
2191 """
2192 return storageutil.resolvestripinfo(minlink, len(self) - 1,
2192 return storageutil.resolvestripinfo(minlink, len(self) - 1,
2193 self.headrevs(),
2193 self.headrevs(),
2194 self.linkrev, self.parentrevs)
2194 self.linkrev, self.parentrevs)
2195
2195
2196 def strip(self, minlink, transaction):
2196 def strip(self, minlink, transaction):
2197 """truncate the revlog on the first revision with a linkrev >= minlink
2197 """truncate the revlog on the first revision with a linkrev >= minlink
2198
2198
2199 This function is called when we're stripping revision minlink and
2199 This function is called when we're stripping revision minlink and
2200 its descendants from the repository.
2200 its descendants from the repository.
2201
2201
2202 We have to remove all revisions with linkrev >= minlink, because
2202 We have to remove all revisions with linkrev >= minlink, because
2203 the equivalent changelog revisions will be renumbered after the
2203 the equivalent changelog revisions will be renumbered after the
2204 strip.
2204 strip.
2205
2205
2206 So we truncate the revlog on the first of these revisions, and
2206 So we truncate the revlog on the first of these revisions, and
2207 trust that the caller has saved the revisions that shouldn't be
2207 trust that the caller has saved the revisions that shouldn't be
2208 removed and that it'll re-add them after this truncation.
2208 removed and that it'll re-add them after this truncation.
2209 """
2209 """
2210 if len(self) == 0:
2210 if len(self) == 0:
2211 return
2211 return
2212
2212
2213 rev, _ = self.getstrippoint(minlink)
2213 rev, _ = self.getstrippoint(minlink)
2214 if rev == len(self):
2214 if rev == len(self):
2215 return
2215 return
2216
2216
2217 # first truncate the files on disk
2217 # first truncate the files on disk
2218 end = self.start(rev)
2218 end = self.start(rev)
2219 if not self._inline:
2219 if not self._inline:
2220 transaction.add(self.datafile, end)
2220 transaction.add(self.datafile, end)
2221 end = rev * self._io.size
2221 end = rev * self._io.size
2222 else:
2222 else:
2223 end += rev * self._io.size
2223 end += rev * self._io.size
2224
2224
2225 transaction.add(self.indexfile, end)
2225 transaction.add(self.indexfile, end)
2226
2226
2227 # then reset internal state in memory to forget those revisions
2227 # then reset internal state in memory to forget those revisions
2228 self._revisioncache = None
2228 self._revisioncache = None
2229 self._chaininfocache = {}
2229 self._chaininfocache = {}
2230 self._chunkclear()
2230 self._chunkclear()
2231 for x in pycompat.xrange(rev, len(self)):
2231 for x in pycompat.xrange(rev, len(self)):
2232 del self.nodemap[self.node(x)]
2232 del self.nodemap[self.node(x)]
2233
2233
2234 del self.index[rev:-1]
2234 del self.index[rev:-1]
2235 self._nodepos = None
2235 self._nodepos = None
2236
2236
2237 def checksize(self):
2237 def checksize(self):
2238 """Check size of index and data files
2238 """Check size of index and data files
2239
2239
2240 return a (dd, di) tuple.
2240 return a (dd, di) tuple.
2241 - dd: extra bytes for the "data" file
2241 - dd: extra bytes for the "data" file
2242 - di: extra bytes for the "index" file
2242 - di: extra bytes for the "index" file
2243
2243
2244 A healthy revlog will return (0, 0).
2244 A healthy revlog will return (0, 0).
2245 """
2245 """
2246 expected = 0
2246 expected = 0
2247 if len(self):
2247 if len(self):
2248 expected = max(0, self.end(len(self) - 1))
2248 expected = max(0, self.end(len(self) - 1))
2249
2249
2250 try:
2250 try:
2251 with self._datafp() as f:
2251 with self._datafp() as f:
2252 f.seek(0, io.SEEK_END)
2252 f.seek(0, io.SEEK_END)
2253 actual = f.tell()
2253 actual = f.tell()
2254 dd = actual - expected
2254 dd = actual - expected
2255 except IOError as inst:
2255 except IOError as inst:
2256 if inst.errno != errno.ENOENT:
2256 if inst.errno != errno.ENOENT:
2257 raise
2257 raise
2258 dd = 0
2258 dd = 0
2259
2259
2260 try:
2260 try:
2261 f = self.opener(self.indexfile)
2261 f = self.opener(self.indexfile)
2262 f.seek(0, io.SEEK_END)
2262 f.seek(0, io.SEEK_END)
2263 actual = f.tell()
2263 actual = f.tell()
2264 f.close()
2264 f.close()
2265 s = self._io.size
2265 s = self._io.size
2266 i = max(0, actual // s)
2266 i = max(0, actual // s)
2267 di = actual - (i * s)
2267 di = actual - (i * s)
2268 if self._inline:
2268 if self._inline:
2269 databytes = 0
2269 databytes = 0
2270 for r in self:
2270 for r in self:
2271 databytes += max(0, self.length(r))
2271 databytes += max(0, self.length(r))
2272 dd = 0
2272 dd = 0
2273 di = actual - len(self) * s - databytes
2273 di = actual - len(self) * s - databytes
2274 except IOError as inst:
2274 except IOError as inst:
2275 if inst.errno != errno.ENOENT:
2275 if inst.errno != errno.ENOENT:
2276 raise
2276 raise
2277 di = 0
2277 di = 0
2278
2278
2279 return (dd, di)
2279 return (dd, di)
2280
2280
2281 def files(self):
2281 def files(self):
2282 res = [self.indexfile]
2282 res = [self.indexfile]
2283 if not self._inline:
2283 if not self._inline:
2284 res.append(self.datafile)
2284 res.append(self.datafile)
2285 return res
2285 return res
2286
2286
2287 def emitrevisions(self, nodes, nodesorder=None, revisiondata=False,
2287 def emitrevisions(self, nodes, nodesorder=None, revisiondata=False,
2288 assumehaveparentrevisions=False,
2288 assumehaveparentrevisions=False,
2289 deltamode=repository.CG_DELTAMODE_STD):
2289 deltamode=repository.CG_DELTAMODE_STD):
2290 if nodesorder not in ('nodes', 'storage', 'linear', None):
2290 if nodesorder not in ('nodes', 'storage', 'linear', None):
2291 raise error.ProgrammingError('unhandled value for nodesorder: %s' %
2291 raise error.ProgrammingError('unhandled value for nodesorder: %s' %
2292 nodesorder)
2292 nodesorder)
2293
2293
2294 if nodesorder is None and not self._generaldelta:
2294 if nodesorder is None and not self._generaldelta:
2295 nodesorder = 'storage'
2295 nodesorder = 'storage'
2296
2296
2297 if (not self._storedeltachains and
2297 if (not self._storedeltachains and
2298 deltamode != repository.CG_DELTAMODE_PREV):
2298 deltamode != repository.CG_DELTAMODE_PREV):
2299 deltamode = repository.CG_DELTAMODE_FULL
2299 deltamode = repository.CG_DELTAMODE_FULL
2300
2300
2301 return storageutil.emitrevisions(
2301 return storageutil.emitrevisions(
2302 self, nodes, nodesorder, revlogrevisiondelta,
2302 self, nodes, nodesorder, revlogrevisiondelta,
2303 deltaparentfn=self.deltaparent,
2303 deltaparentfn=self.deltaparent,
2304 candeltafn=self.candelta,
2304 candeltafn=self.candelta,
2305 rawsizefn=self.rawsize,
2305 rawsizefn=self.rawsize,
2306 revdifffn=self.revdiff,
2306 revdifffn=self.revdiff,
2307 flagsfn=self.flags,
2307 flagsfn=self.flags,
2308 deltamode=deltamode,
2308 deltamode=deltamode,
2309 revisiondata=revisiondata,
2309 revisiondata=revisiondata,
2310 assumehaveparentrevisions=assumehaveparentrevisions)
2310 assumehaveparentrevisions=assumehaveparentrevisions)
2311
2311
2312 DELTAREUSEALWAYS = 'always'
2312 DELTAREUSEALWAYS = 'always'
2313 DELTAREUSESAMEREVS = 'samerevs'
2313 DELTAREUSESAMEREVS = 'samerevs'
2314 DELTAREUSENEVER = 'never'
2314 DELTAREUSENEVER = 'never'
2315
2315
2316 DELTAREUSEFULLADD = 'fulladd'
2316 DELTAREUSEFULLADD = 'fulladd'
2317
2317
2318 DELTAREUSEALL = {'always', 'samerevs', 'never', 'fulladd'}
2318 DELTAREUSEALL = {'always', 'samerevs', 'never', 'fulladd'}
2319
2319
2320 def clone(self, tr, destrevlog, addrevisioncb=None,
2320 def clone(self, tr, destrevlog, addrevisioncb=None,
2321 deltareuse=DELTAREUSESAMEREVS, forcedeltabothparents=None):
2321 deltareuse=DELTAREUSESAMEREVS, forcedeltabothparents=None):
2322 """Copy this revlog to another, possibly with format changes.
2322 """Copy this revlog to another, possibly with format changes.
2323
2323
2324 The destination revlog will contain the same revisions and nodes.
2324 The destination revlog will contain the same revisions and nodes.
2325 However, it may not be bit-for-bit identical due to e.g. delta encoding
2325 However, it may not be bit-for-bit identical due to e.g. delta encoding
2326 differences.
2326 differences.
2327
2327
2328 The ``deltareuse`` argument control how deltas from the existing revlog
2328 The ``deltareuse`` argument control how deltas from the existing revlog
2329 are preserved in the destination revlog. The argument can have the
2329 are preserved in the destination revlog. The argument can have the
2330 following values:
2330 following values:
2331
2331
2332 DELTAREUSEALWAYS
2332 DELTAREUSEALWAYS
2333 Deltas will always be reused (if possible), even if the destination
2333 Deltas will always be reused (if possible), even if the destination
2334 revlog would not select the same revisions for the delta. This is the
2334 revlog would not select the same revisions for the delta. This is the
2335 fastest mode of operation.
2335 fastest mode of operation.
2336 DELTAREUSESAMEREVS
2336 DELTAREUSESAMEREVS
2337 Deltas will be reused if the destination revlog would pick the same
2337 Deltas will be reused if the destination revlog would pick the same
2338 revisions for the delta. This mode strikes a balance between speed
2338 revisions for the delta. This mode strikes a balance between speed
2339 and optimization.
2339 and optimization.
2340 DELTAREUSENEVER
2340 DELTAREUSENEVER
2341 Deltas will never be reused. This is the slowest mode of execution.
2341 Deltas will never be reused. This is the slowest mode of execution.
2342 This mode can be used to recompute deltas (e.g. if the diff/delta
2342 This mode can be used to recompute deltas (e.g. if the diff/delta
2343 algorithm changes).
2343 algorithm changes).
2344
2344
2345 Delta computation can be slow, so the choice of delta reuse policy can
2345 Delta computation can be slow, so the choice of delta reuse policy can
2346 significantly affect run time.
2346 significantly affect run time.
2347
2347
2348 The default policy (``DELTAREUSESAMEREVS``) strikes a balance between
2348 The default policy (``DELTAREUSESAMEREVS``) strikes a balance between
2349 two extremes. Deltas will be reused if they are appropriate. But if the
2349 two extremes. Deltas will be reused if they are appropriate. But if the
2350 delta could choose a better revision, it will do so. This means if you
2350 delta could choose a better revision, it will do so. This means if you
2351 are converting a non-generaldelta revlog to a generaldelta revlog,
2351 are converting a non-generaldelta revlog to a generaldelta revlog,
2352 deltas will be recomputed if the delta's parent isn't a parent of the
2352 deltas will be recomputed if the delta's parent isn't a parent of the
2353 revision.
2353 revision.
2354
2354
2355 In addition to the delta policy, the ``forcedeltabothparents``
2355 In addition to the delta policy, the ``forcedeltabothparents``
2356 argument controls whether to force compute deltas against both parents
2356 argument controls whether to force compute deltas against both parents
2357 for merges. By default, the current default is used.
2357 for merges. By default, the current default is used.
2358 """
2358 """
2359 if deltareuse not in self.DELTAREUSEALL:
2359 if deltareuse not in self.DELTAREUSEALL:
2360 raise ValueError(_('value for deltareuse invalid: %s') % deltareuse)
2360 raise ValueError(_('value for deltareuse invalid: %s') % deltareuse)
2361
2361
2362 if len(destrevlog):
2362 if len(destrevlog):
2363 raise ValueError(_('destination revlog is not empty'))
2363 raise ValueError(_('destination revlog is not empty'))
2364
2364
2365 if getattr(self, 'filteredrevs', None):
2365 if getattr(self, 'filteredrevs', None):
2366 raise ValueError(_('source revlog has filtered revisions'))
2366 raise ValueError(_('source revlog has filtered revisions'))
2367 if getattr(destrevlog, 'filteredrevs', None):
2367 if getattr(destrevlog, 'filteredrevs', None):
2368 raise ValueError(_('destination revlog has filtered revisions'))
2368 raise ValueError(_('destination revlog has filtered revisions'))
2369
2369
2370 # lazydelta and lazydeltabase controls whether to reuse a cached delta,
2370 # lazydelta and lazydeltabase controls whether to reuse a cached delta,
2371 # if possible.
2371 # if possible.
2372 oldlazydelta = destrevlog._lazydelta
2372 oldlazydelta = destrevlog._lazydelta
2373 oldlazydeltabase = destrevlog._lazydeltabase
2373 oldlazydeltabase = destrevlog._lazydeltabase
2374 oldamd = destrevlog._deltabothparents
2374 oldamd = destrevlog._deltabothparents
2375
2375
2376 try:
2376 try:
2377 if deltareuse == self.DELTAREUSEALWAYS:
2377 if deltareuse == self.DELTAREUSEALWAYS:
2378 destrevlog._lazydeltabase = True
2378 destrevlog._lazydeltabase = True
2379 destrevlog._lazydelta = True
2379 destrevlog._lazydelta = True
2380 elif deltareuse == self.DELTAREUSESAMEREVS:
2380 elif deltareuse == self.DELTAREUSESAMEREVS:
2381 destrevlog._lazydeltabase = False
2381 destrevlog._lazydeltabase = False
2382 destrevlog._lazydelta = True
2382 destrevlog._lazydelta = True
2383 elif deltareuse == self.DELTAREUSENEVER:
2383 elif deltareuse == self.DELTAREUSENEVER:
2384 destrevlog._lazydeltabase = False
2384 destrevlog._lazydeltabase = False
2385 destrevlog._lazydelta = False
2385 destrevlog._lazydelta = False
2386
2386
2387 destrevlog._deltabothparents = forcedeltabothparents or oldamd
2387 destrevlog._deltabothparents = forcedeltabothparents or oldamd
2388
2388
2389 deltacomputer = deltautil.deltacomputer(destrevlog)
2389 deltacomputer = deltautil.deltacomputer(destrevlog)
2390 index = self.index
2390 index = self.index
2391 for rev in self:
2391 for rev in self:
2392 entry = index[rev]
2392 entry = index[rev]
2393
2393
2394 # Some classes override linkrev to take filtered revs into
2394 # Some classes override linkrev to take filtered revs into
2395 # account. Use raw entry from index.
2395 # account. Use raw entry from index.
2396 flags = entry[0] & 0xffff
2396 flags = entry[0] & 0xffff
2397 linkrev = entry[4]
2397 linkrev = entry[4]
2398 p1 = index[entry[5]][7]
2398 p1 = index[entry[5]][7]
2399 p2 = index[entry[6]][7]
2399 p2 = index[entry[6]][7]
2400 node = entry[7]
2400 node = entry[7]
2401
2401
2402 # (Possibly) reuse the delta from the revlog if allowed and
2402 # (Possibly) reuse the delta from the revlog if allowed and
2403 # the revlog chunk is a delta.
2403 # the revlog chunk is a delta.
2404 cachedelta = None
2404 cachedelta = None
2405 rawtext = None
2405 rawtext = None
2406 if (deltareuse != self.DELTAREUSEFULLADD
2406 if (deltareuse != self.DELTAREUSEFULLADD
2407 and destrevlog._lazydelta):
2407 and destrevlog._lazydelta):
2408 dp = self.deltaparent(rev)
2408 dp = self.deltaparent(rev)
2409 if dp != nullrev:
2409 if dp != nullrev:
2410 cachedelta = (dp, bytes(self._chunk(rev)))
2410 cachedelta = (dp, bytes(self._chunk(rev)))
2411
2411
2412 if not cachedelta:
2412 if not cachedelta:
2413 rawtext = self.rawdata(rev)
2413 rawtext = self.rawdata(rev)
2414
2414
2415
2415
2416 if deltareuse == self.DELTAREUSEFULLADD:
2416 if deltareuse == self.DELTAREUSEFULLADD:
2417 destrevlog.addrevision(rawtext, tr, linkrev, p1, p2,
2417 destrevlog.addrevision(rawtext, tr, linkrev, p1, p2,
2418 cachedelta=cachedelta,
2418 cachedelta=cachedelta,
2419 node=node, flags=flags,
2419 node=node, flags=flags,
2420 deltacomputer=deltacomputer)
2420 deltacomputer=deltacomputer)
2421 else:
2421 else:
2422 ifh = destrevlog.opener(destrevlog.indexfile, 'a+',
2422 ifh = destrevlog.opener(destrevlog.indexfile, 'a+',
2423 checkambig=False)
2423 checkambig=False)
2424 dfh = None
2424 dfh = None
2425 if not destrevlog._inline:
2425 if not destrevlog._inline:
2426 dfh = destrevlog.opener(destrevlog.datafile, 'a+')
2426 dfh = destrevlog.opener(destrevlog.datafile, 'a+')
2427 try:
2427 try:
2428 destrevlog._addrevision(node, rawtext, tr, linkrev, p1,
2428 destrevlog._addrevision(node, rawtext, tr, linkrev, p1,
2429 p2, flags, cachedelta, ifh, dfh,
2429 p2, flags, cachedelta, ifh, dfh,
2430 deltacomputer=deltacomputer)
2430 deltacomputer=deltacomputer)
2431 finally:
2431 finally:
2432 if dfh:
2432 if dfh:
2433 dfh.close()
2433 dfh.close()
2434 ifh.close()
2434 ifh.close()
2435
2435
2436 if addrevisioncb:
2436 if addrevisioncb:
2437 addrevisioncb(self, rev, node)
2437 addrevisioncb(self, rev, node)
2438 finally:
2438 finally:
2439 destrevlog._lazydelta = oldlazydelta
2439 destrevlog._lazydelta = oldlazydelta
2440 destrevlog._lazydeltabase = oldlazydeltabase
2440 destrevlog._lazydeltabase = oldlazydeltabase
2441 destrevlog._deltabothparents = oldamd
2441 destrevlog._deltabothparents = oldamd
2442
2442
2443 def censorrevision(self, tr, censornode, tombstone=b''):
2443 def censorrevision(self, tr, censornode, tombstone=b''):
2444 if (self.version & 0xFFFF) == REVLOGV0:
2444 if (self.version & 0xFFFF) == REVLOGV0:
2445 raise error.RevlogError(_('cannot censor with version %d revlogs') %
2445 raise error.RevlogError(_('cannot censor with version %d revlogs') %
2446 self.version)
2446 self.version)
2447
2447
2448 censorrev = self.rev(censornode)
2448 censorrev = self.rev(censornode)
2449 tombstone = storageutil.packmeta({b'censored': tombstone}, b'')
2449 tombstone = storageutil.packmeta({b'censored': tombstone}, b'')
2450
2450
2451 if len(tombstone) > self.rawsize(censorrev):
2451 if len(tombstone) > self.rawsize(censorrev):
2452 raise error.Abort(_('censor tombstone must be no longer than '
2452 raise error.Abort(_('censor tombstone must be no longer than '
2453 'censored data'))
2453 'censored data'))
2454
2454
2455 # Rewriting the revlog in place is hard. Our strategy for censoring is
2455 # Rewriting the revlog in place is hard. Our strategy for censoring is
2456 # to create a new revlog, copy all revisions to it, then replace the
2456 # to create a new revlog, copy all revisions to it, then replace the
2457 # revlogs on transaction close.
2457 # revlogs on transaction close.
2458
2458
2459 newindexfile = self.indexfile + b'.tmpcensored'
2459 newindexfile = self.indexfile + b'.tmpcensored'
2460 newdatafile = self.datafile + b'.tmpcensored'
2460 newdatafile = self.datafile + b'.tmpcensored'
2461
2461
2462 # This is a bit dangerous. We could easily have a mismatch of state.
2462 # This is a bit dangerous. We could easily have a mismatch of state.
2463 newrl = revlog(self.opener, newindexfile, newdatafile,
2463 newrl = revlog(self.opener, newindexfile, newdatafile,
2464 censorable=True)
2464 censorable=True)
2465 newrl.version = self.version
2465 newrl.version = self.version
2466 newrl._generaldelta = self._generaldelta
2466 newrl._generaldelta = self._generaldelta
2467 newrl._io = self._io
2467 newrl._io = self._io
2468
2468
2469 for rev in self.revs():
2469 for rev in self.revs():
2470 node = self.node(rev)
2470 node = self.node(rev)
2471 p1, p2 = self.parents(node)
2471 p1, p2 = self.parents(node)
2472
2472
2473 if rev == censorrev:
2473 if rev == censorrev:
2474 newrl.addrawrevision(tombstone, tr, self.linkrev(censorrev),
2474 newrl.addrawrevision(tombstone, tr, self.linkrev(censorrev),
2475 p1, p2, censornode, REVIDX_ISCENSORED)
2475 p1, p2, censornode, REVIDX_ISCENSORED)
2476
2476
2477 if newrl.deltaparent(rev) != nullrev:
2477 if newrl.deltaparent(rev) != nullrev:
2478 raise error.Abort(_('censored revision stored as delta; '
2478 raise error.Abort(_('censored revision stored as delta; '
2479 'cannot censor'),
2479 'cannot censor'),
2480 hint=_('censoring of revlogs is not '
2480 hint=_('censoring of revlogs is not '
2481 'fully implemented; please report '
2481 'fully implemented; please report '
2482 'this bug'))
2482 'this bug'))
2483 continue
2483 continue
2484
2484
2485 if self.iscensored(rev):
2485 if self.iscensored(rev):
2486 if self.deltaparent(rev) != nullrev:
2486 if self.deltaparent(rev) != nullrev:
2487 raise error.Abort(_('cannot censor due to censored '
2487 raise error.Abort(_('cannot censor due to censored '
2488 'revision having delta stored'))
2488 'revision having delta stored'))
2489 rawtext = self._chunk(rev)
2489 rawtext = self._chunk(rev)
2490 else:
2490 else:
2491 rawtext = self.rawdata(rev)
2491 rawtext = self.rawdata(rev)
2492
2492
2493 newrl.addrawrevision(rawtext, tr, self.linkrev(rev), p1, p2, node,
2493 newrl.addrawrevision(rawtext, tr, self.linkrev(rev), p1, p2, node,
2494 self.flags(rev))
2494 self.flags(rev))
2495
2495
2496 tr.addbackup(self.indexfile, location='store')
2496 tr.addbackup(self.indexfile, location='store')
2497 if not self._inline:
2497 if not self._inline:
2498 tr.addbackup(self.datafile, location='store')
2498 tr.addbackup(self.datafile, location='store')
2499
2499
2500 self.opener.rename(newrl.indexfile, self.indexfile)
2500 self.opener.rename(newrl.indexfile, self.indexfile)
2501 if not self._inline:
2501 if not self._inline:
2502 self.opener.rename(newrl.datafile, self.datafile)
2502 self.opener.rename(newrl.datafile, self.datafile)
2503
2503
2504 self.clearcaches()
2504 self.clearcaches()
2505 self._loadindex()
2505 self._loadindex()
2506
2506
2507 def verifyintegrity(self, state):
2507 def verifyintegrity(self, state):
2508 """Verifies the integrity of the revlog.
2508 """Verifies the integrity of the revlog.
2509
2509
2510 Yields ``revlogproblem`` instances describing problems that are
2510 Yields ``revlogproblem`` instances describing problems that are
2511 found.
2511 found.
2512 """
2512 """
2513 dd, di = self.checksize()
2513 dd, di = self.checksize()
2514 if dd:
2514 if dd:
2515 yield revlogproblem(error=_('data length off by %d bytes') % dd)
2515 yield revlogproblem(error=_('data length off by %d bytes') % dd)
2516 if di:
2516 if di:
2517 yield revlogproblem(error=_('index contains %d extra bytes') % di)
2517 yield revlogproblem(error=_('index contains %d extra bytes') % di)
2518
2518
2519 version = self.version & 0xFFFF
2519 version = self.version & 0xFFFF
2520
2520
2521 # The verifier tells us what version revlog we should be.
2521 # The verifier tells us what version revlog we should be.
2522 if version != state['expectedversion']:
2522 if version != state['expectedversion']:
2523 yield revlogproblem(
2523 yield revlogproblem(
2524 warning=_("warning: '%s' uses revlog format %d; expected %d") %
2524 warning=_("warning: '%s' uses revlog format %d; expected %d") %
2525 (self.indexfile, version, state['expectedversion']))
2525 (self.indexfile, version, state['expectedversion']))
2526
2526
2527 state['skipread'] = set()
2527 state['skipread'] = set()
2528
2528
2529 for rev in self:
2529 for rev in self:
2530 node = self.node(rev)
2530 node = self.node(rev)
2531
2531
2532 # Verify contents. 4 cases to care about:
2532 # Verify contents. 4 cases to care about:
2533 #
2533 #
2534 # common: the most common case
2534 # common: the most common case
2535 # rename: with a rename
2535 # rename: with a rename
2536 # meta: file content starts with b'\1\n', the metadata
2536 # meta: file content starts with b'\1\n', the metadata
2537 # header defined in filelog.py, but without a rename
2537 # header defined in filelog.py, but without a rename
2538 # ext: content stored externally
2538 # ext: content stored externally
2539 #
2539 #
2540 # More formally, their differences are shown below:
2540 # More formally, their differences are shown below:
2541 #
2541 #
2542 # | common | rename | meta | ext
2542 # | common | rename | meta | ext
2543 # -------------------------------------------------------
2543 # -------------------------------------------------------
2544 # flags() | 0 | 0 | 0 | not 0
2544 # flags() | 0 | 0 | 0 | not 0
2545 # renamed() | False | True | False | ?
2545 # renamed() | False | True | False | ?
2546 # rawtext[0:2]=='\1\n'| False | True | True | ?
2546 # rawtext[0:2]=='\1\n'| False | True | True | ?
2547 #
2547 #
2548 # "rawtext" means the raw text stored in revlog data, which
2548 # "rawtext" means the raw text stored in revlog data, which
2549 # could be retrieved by "rawdata(rev)". "text"
2549 # could be retrieved by "rawdata(rev)". "text"
2550 # mentioned below is "revision(rev)".
2550 # mentioned below is "revision(rev)".
2551 #
2551 #
2552 # There are 3 different lengths stored physically:
2552 # There are 3 different lengths stored physically:
2553 # 1. L1: rawsize, stored in revlog index
2553 # 1. L1: rawsize, stored in revlog index
2554 # 2. L2: len(rawtext), stored in revlog data
2554 # 2. L2: len(rawtext), stored in revlog data
2555 # 3. L3: len(text), stored in revlog data if flags==0, or
2555 # 3. L3: len(text), stored in revlog data if flags==0, or
2556 # possibly somewhere else if flags!=0
2556 # possibly somewhere else if flags!=0
2557 #
2557 #
2558 # L1 should be equal to L2. L3 could be different from them.
2558 # L1 should be equal to L2. L3 could be different from them.
2559 # "text" may or may not affect commit hash depending on flag
2559 # "text" may or may not affect commit hash depending on flag
2560 # processors (see flagutil.addflagprocessor).
2560 # processors (see flagutil.addflagprocessor).
2561 #
2561 #
2562 # | common | rename | meta | ext
2562 # | common | rename | meta | ext
2563 # -------------------------------------------------
2563 # -------------------------------------------------
2564 # rawsize() | L1 | L1 | L1 | L1
2564 # rawsize() | L1 | L1 | L1 | L1
2565 # size() | L1 | L2-LM | L1(*) | L1 (?)
2565 # size() | L1 | L2-LM | L1(*) | L1 (?)
2566 # len(rawtext) | L2 | L2 | L2 | L2
2566 # len(rawtext) | L2 | L2 | L2 | L2
2567 # len(text) | L2 | L2 | L2 | L3
2567 # len(text) | L2 | L2 | L2 | L3
2568 # len(read()) | L2 | L2-LM | L2-LM | L3 (?)
2568 # len(read()) | L2 | L2-LM | L2-LM | L3 (?)
2569 #
2569 #
2570 # LM: length of metadata, depending on rawtext
2570 # LM: length of metadata, depending on rawtext
2571 # (*): not ideal, see comment in filelog.size
2571 # (*): not ideal, see comment in filelog.size
2572 # (?): could be "- len(meta)" if the resolved content has
2572 # (?): could be "- len(meta)" if the resolved content has
2573 # rename metadata
2573 # rename metadata
2574 #
2574 #
2575 # Checks needed to be done:
2575 # Checks needed to be done:
2576 # 1. length check: L1 == L2, in all cases.
2576 # 1. length check: L1 == L2, in all cases.
2577 # 2. hash check: depending on flag processor, we may need to
2577 # 2. hash check: depending on flag processor, we may need to
2578 # use either "text" (external), or "rawtext" (in revlog).
2578 # use either "text" (external), or "rawtext" (in revlog).
2579
2579
2580 try:
2580 try:
2581 skipflags = state.get('skipflags', 0)
2581 skipflags = state.get('skipflags', 0)
2582 if skipflags:
2582 if skipflags:
2583 skipflags &= self.flags(rev)
2583 skipflags &= self.flags(rev)
2584
2584
2585 if skipflags:
2585 if skipflags:
2586 state['skipread'].add(node)
2586 state['skipread'].add(node)
2587 else:
2587 else:
2588 # Side-effect: read content and verify hash.
2588 # Side-effect: read content and verify hash.
2589 self.revision(node)
2589 self.revision(node)
2590
2590
2591 l1 = self.rawsize(rev)
2591 l1 = self.rawsize(rev)
2592 l2 = len(self.rawdata(node))
2592 l2 = len(self.rawdata(node))
2593
2593
2594 if l1 != l2:
2594 if l1 != l2:
2595 yield revlogproblem(
2595 yield revlogproblem(
2596 error=_('unpacked size is %d, %d expected') % (l2, l1),
2596 error=_('unpacked size is %d, %d expected') % (l2, l1),
2597 node=node)
2597 node=node)
2598
2598
2599 except error.CensoredNodeError:
2599 except error.CensoredNodeError:
2600 if state['erroroncensored']:
2600 if state['erroroncensored']:
2601 yield revlogproblem(error=_('censored file data'),
2601 yield revlogproblem(error=_('censored file data'),
2602 node=node)
2602 node=node)
2603 state['skipread'].add(node)
2603 state['skipread'].add(node)
2604 except Exception as e:
2604 except Exception as e:
2605 yield revlogproblem(
2605 yield revlogproblem(
2606 error=_('unpacking %s: %s') % (short(node),
2606 error=_('unpacking %s: %s') % (short(node),
2607 stringutil.forcebytestr(e)),
2607 stringutil.forcebytestr(e)),
2608 node=node)
2608 node=node)
2609 state['skipread'].add(node)
2609 state['skipread'].add(node)
2610
2610
2611 def storageinfo(self, exclusivefiles=False, sharedfiles=False,
2611 def storageinfo(self, exclusivefiles=False, sharedfiles=False,
2612 revisionscount=False, trackedsize=False,
2612 revisionscount=False, trackedsize=False,
2613 storedsize=False):
2613 storedsize=False):
2614 d = {}
2614 d = {}
2615
2615
2616 if exclusivefiles:
2616 if exclusivefiles:
2617 d['exclusivefiles'] = [(self.opener, self.indexfile)]
2617 d['exclusivefiles'] = [(self.opener, self.indexfile)]
2618 if not self._inline:
2618 if not self._inline:
2619 d['exclusivefiles'].append((self.opener, self.datafile))
2619 d['exclusivefiles'].append((self.opener, self.datafile))
2620
2620
2621 if sharedfiles:
2621 if sharedfiles:
2622 d['sharedfiles'] = []
2622 d['sharedfiles'] = []
2623
2623
2624 if revisionscount:
2624 if revisionscount:
2625 d['revisionscount'] = len(self)
2625 d['revisionscount'] = len(self)
2626
2626
2627 if trackedsize:
2627 if trackedsize:
2628 d['trackedsize'] = sum(map(self.rawsize, iter(self)))
2628 d['trackedsize'] = sum(map(self.rawsize, iter(self)))
2629
2629
2630 if storedsize:
2630 if storedsize:
2631 d['storedsize'] = sum(self.opener.stat(path).st_size
2631 d['storedsize'] = sum(self.opener.stat(path).st_size
2632 for path in self.files())
2632 for path in self.files())
2633
2633
2634 return d
2634 return d
@@ -1,200 +1,201 b''
1 # flagutils.py - code to deal with revlog flags and their processors
1 # flagutils.py - code to deal with revlog flags and their processors
2 #
2 #
3 # Copyright 2016 Remi Chaintron <remi@fb.com>
3 # Copyright 2016 Remi Chaintron <remi@fb.com>
4 # Copyright 2016-2019 Pierre-Yves David <pierre-yves.david@ens-lyon.org>
4 # Copyright 2016-2019 Pierre-Yves David <pierre-yves.david@ens-lyon.org>
5 #
5 #
6 # This software may be used and distributed according to the terms of the
6 # This software may be used and distributed according to the terms of the
7 # GNU General Public License version 2 or any later version.
7 # GNU General Public License version 2 or any later version.
8
8
9 from __future__ import absolute_import
9 from __future__ import absolute_import
10
10
11 from ..i18n import _
11 from ..i18n import _
12
12
13 from .constants import (
13 from .constants import (
14 REVIDX_DEFAULT_FLAGS,
14 REVIDX_DEFAULT_FLAGS,
15 REVIDX_ELLIPSIS,
15 REVIDX_ELLIPSIS,
16 REVIDX_EXTSTORED,
16 REVIDX_EXTSTORED,
17 REVIDX_FLAGS_ORDER,
17 REVIDX_FLAGS_ORDER,
18 REVIDX_ISCENSORED,
18 REVIDX_ISCENSORED,
19 REVIDX_RAWTEXT_CHANGING_FLAGS,
19 REVIDX_RAWTEXT_CHANGING_FLAGS,
20 )
20 )
21
21
22 from .. import (
22 from .. import (
23 error,
23 error,
24 util
24 util
25 )
25 )
26
26
27 # blanked usage of all the name to prevent pyflakes constraints
27 # blanked usage of all the name to prevent pyflakes constraints
28 # We need these name available in the module for extensions.
28 # We need these name available in the module for extensions.
29 REVIDX_ISCENSORED
29 REVIDX_ISCENSORED
30 REVIDX_ELLIPSIS
30 REVIDX_ELLIPSIS
31 REVIDX_EXTSTORED
31 REVIDX_EXTSTORED
32 REVIDX_DEFAULT_FLAGS
32 REVIDX_DEFAULT_FLAGS
33 REVIDX_FLAGS_ORDER
33 REVIDX_FLAGS_ORDER
34 REVIDX_RAWTEXT_CHANGING_FLAGS
34 REVIDX_RAWTEXT_CHANGING_FLAGS
35
35
36 REVIDX_KNOWN_FLAGS = util.bitsfrom(REVIDX_FLAGS_ORDER)
36 REVIDX_KNOWN_FLAGS = util.bitsfrom(REVIDX_FLAGS_ORDER)
37
37
38 # Store flag processors (cf. 'addflagprocessor()' to register)
38 # Store flag processors (cf. 'addflagprocessor()' to register)
39 flagprocessors = {
39 flagprocessors = {
40 REVIDX_ISCENSORED: None,
40 REVIDX_ISCENSORED: None,
41 }
41 }
42
42
43 def addflagprocessor(flag, processor):
43 def addflagprocessor(flag, processor):
44 """Register a flag processor on a revision data flag.
44 """Register a flag processor on a revision data flag.
45
45
46 Invariant:
46 Invariant:
47 - Flags need to be defined in REVIDX_KNOWN_FLAGS and REVIDX_FLAGS_ORDER,
47 - Flags need to be defined in REVIDX_KNOWN_FLAGS and REVIDX_FLAGS_ORDER,
48 and REVIDX_RAWTEXT_CHANGING_FLAGS if they can alter rawtext.
48 and REVIDX_RAWTEXT_CHANGING_FLAGS if they can alter rawtext.
49 - Only one flag processor can be registered on a specific flag.
49 - Only one flag processor can be registered on a specific flag.
50 - flagprocessors must be 3-tuples of functions (read, write, raw) with the
50 - flagprocessors must be 3-tuples of functions (read, write, raw) with the
51 following signatures:
51 following signatures:
52 - (read) f(self, rawtext) -> text, bool
52 - (read) f(self, rawtext) -> text, bool
53 - (write) f(self, text) -> rawtext, bool
53 - (write) f(self, text) -> rawtext, bool
54 - (raw) f(self, rawtext) -> bool
54 - (raw) f(self, rawtext) -> bool
55 "text" is presented to the user. "rawtext" is stored in revlog data, not
55 "text" is presented to the user. "rawtext" is stored in revlog data, not
56 directly visible to the user.
56 directly visible to the user.
57 The boolean returned by these transforms is used to determine whether
57 The boolean returned by these transforms is used to determine whether
58 the returned text can be used for hash integrity checking. For example,
58 the returned text can be used for hash integrity checking. For example,
59 if "write" returns False, then "text" is used to generate hash. If
59 if "write" returns False, then "text" is used to generate hash. If
60 "write" returns True, that basically means "rawtext" returned by "write"
60 "write" returns True, that basically means "rawtext" returned by "write"
61 should be used to generate hash. Usually, "write" and "read" return
61 should be used to generate hash. Usually, "write" and "read" return
62 different booleans. And "raw" returns a same boolean as "write".
62 different booleans. And "raw" returns a same boolean as "write".
63
63
64 Note: The 'raw' transform is used for changegroup generation and in some
64 Note: The 'raw' transform is used for changegroup generation and in some
65 debug commands. In this case the transform only indicates whether the
65 debug commands. In this case the transform only indicates whether the
66 contents can be used for hash integrity checks.
66 contents can be used for hash integrity checks.
67 """
67 """
68 insertflagprocessor(flag, processor, flagprocessors)
68 insertflagprocessor(flag, processor, flagprocessors)
69
69
70 def insertflagprocessor(flag, processor, flagprocessors):
70 def insertflagprocessor(flag, processor, flagprocessors):
71 if not flag & REVIDX_KNOWN_FLAGS:
71 if not flag & REVIDX_KNOWN_FLAGS:
72 msg = _("cannot register processor on unknown flag '%#x'.") % (flag)
72 msg = _("cannot register processor on unknown flag '%#x'.") % (flag)
73 raise error.ProgrammingError(msg)
73 raise error.ProgrammingError(msg)
74 if flag not in REVIDX_FLAGS_ORDER:
74 if flag not in REVIDX_FLAGS_ORDER:
75 msg = _("flag '%#x' undefined in REVIDX_FLAGS_ORDER.") % (flag)
75 msg = _("flag '%#x' undefined in REVIDX_FLAGS_ORDER.") % (flag)
76 raise error.ProgrammingError(msg)
76 raise error.ProgrammingError(msg)
77 if flag in flagprocessors:
77 if flag in flagprocessors:
78 msg = _("cannot register multiple processors on flag '%#x'.") % (flag)
78 msg = _("cannot register multiple processors on flag '%#x'.") % (flag)
79 raise error.Abort(msg)
79 raise error.Abort(msg)
80 flagprocessors[flag] = processor
80 flagprocessors[flag] = processor
81
81
82 class flagprocessorsmixin(object):
82 class flagprocessorsmixin(object):
83 """basic mixin to support revlog flag processing
83 """basic mixin to support revlog flag processing
84
84
85 Make sure the `_flagprocessors` attribute is set at ``__init__`` time.
85 Make sure the `_flagprocessors` attribute is set at ``__init__`` time.
86
86
87 See the documentation of the ``_processflags`` method for details.
87 See the documentation of the ``_processflags`` method for details.
88 """
88 """
89
89
90 _flagserrorclass = error.RevlogError
90 _flagserrorclass = error.RevlogError
91
91
92 def _processflags(self, text, flags, operation, raw=False):
92 def _processflags(self, text, flags, operation, raw=False):
93 """deprecated entry point to access flag processors"""
93 """deprecated entry point to access flag processors"""
94 msg = ('_processflag(...) use the specialized variant')
94 msg = ('_processflag(...) use the specialized variant')
95 util.nouideprecwarn(msg, '5.2', stacklevel=2)
95 util.nouideprecwarn(msg, '5.2', stacklevel=2)
96 if raw:
96 if raw:
97 return text, self._processflagsraw(text, flags)
97 return text, self._processflagsraw(text, flags)
98 elif operation == 'read':
98 elif operation == 'read':
99 return self._processflagsread(text, flags)
99 return self._processflagsread(text, flags)
100 else: # write operation
100 else: # write operation
101 return self._processflagswrite(text, flags)
101 return self._processflagswrite(text, flags)
102
102
103 def _processflagsread(self, text, flags):
103 def _processflagsread(self, text, flags):
104 """Inspect revision data flags and applies read transformations defined
104 """Inspect revision data flags and applies read transformations defined
105 by registered flag processors.
105 by registered flag processors.
106
106
107 ``text`` - the revision data to process
107 ``text`` - the revision data to process
108 ``flags`` - the revision flags
108 ``flags`` - the revision flags
109 ``raw`` - an optional argument describing if the raw transform should be
109 ``raw`` - an optional argument describing if the raw transform should be
110 applied.
110 applied.
111
111
112 This method processes the flags in the order (or reverse order if
112 This method processes the flags in the order (or reverse order if
113 ``operation`` is 'write') defined by REVIDX_FLAGS_ORDER, applying the
113 ``operation`` is 'write') defined by REVIDX_FLAGS_ORDER, applying the
114 flag processors registered for present flags. The order of flags defined
114 flag processors registered for present flags. The order of flags defined
115 in REVIDX_FLAGS_ORDER needs to be stable to allow non-commutativity.
115 in REVIDX_FLAGS_ORDER needs to be stable to allow non-commutativity.
116
116
117 Returns a 2-tuple of ``(text, validatehash)`` where ``text`` is the
117 Returns a 2-tuple of ``(text, validatehash)`` where ``text`` is the
118 processed text and ``validatehash`` is a bool indicating whether the
118 processed text and ``validatehash`` is a bool indicating whether the
119 returned text should be checked for hash integrity.
119 returned text should be checked for hash integrity.
120 """
120 """
121 return self._processflagsfunc(text, flags, 'read')
121 return self._processflagsfunc(text, flags, 'read')
122
122
123 def _processflagswrite(self, text, flags):
123 def _processflagswrite(self, text, flags):
124 """Inspect revision data flags and applies write transformations defined
124 """Inspect revision data flags and applies write transformations defined
125 by registered flag processors.
125 by registered flag processors.
126
126
127 ``text`` - the revision data to process
127 ``text`` - the revision data to process
128 ``flags`` - the revision flags
128 ``flags`` - the revision flags
129
129
130 This method processes the flags in the order (or reverse order if
130 This method processes the flags in the order (or reverse order if
131 ``operation`` is 'write') defined by REVIDX_FLAGS_ORDER, applying the
131 ``operation`` is 'write') defined by REVIDX_FLAGS_ORDER, applying the
132 flag processors registered for present flags. The order of flags defined
132 flag processors registered for present flags. The order of flags defined
133 in REVIDX_FLAGS_ORDER needs to be stable to allow non-commutativity.
133 in REVIDX_FLAGS_ORDER needs to be stable to allow non-commutativity.
134
134
135 Returns a 2-tuple of ``(text, validatehash)`` where ``text`` is the
135 Returns a 2-tuple of ``(text, validatehash)`` where ``text`` is the
136 processed text and ``validatehash`` is a bool indicating whether the
136 processed text and ``validatehash`` is a bool indicating whether the
137 returned text should be checked for hash integrity.
137 returned text should be checked for hash integrity.
138 """
138 """
139 return self._processflagsfunc(text, flags, 'write')[:2]
139 return self._processflagsfunc(text, flags, 'write')[:2]
140
140
141 def _processflagsraw(self, text, flags):
141 def _processflagsraw(self, text, flags):
142 """Inspect revision data flags to check is the content hash should be
142 """Inspect revision data flags to check is the content hash should be
143 validated.
143 validated.
144
144
145 ``text`` - the revision data to process
145 ``text`` - the revision data to process
146 ``flags`` - the revision flags
146 ``flags`` - the revision flags
147
147
148 This method processes the flags in the order (or reverse order if
148 This method processes the flags in the order (or reverse order if
149 ``operation`` is 'write') defined by REVIDX_FLAGS_ORDER, applying the
149 ``operation`` is 'write') defined by REVIDX_FLAGS_ORDER, applying the
150 flag processors registered for present flags. The order of flags defined
150 flag processors registered for present flags. The order of flags defined
151 in REVIDX_FLAGS_ORDER needs to be stable to allow non-commutativity.
151 in REVIDX_FLAGS_ORDER needs to be stable to allow non-commutativity.
152
152
153 Returns a 2-tuple of ``(text, validatehash)`` where ``text`` is the
153 Returns a 2-tuple of ``(text, validatehash)`` where ``text`` is the
154 processed text and ``validatehash`` is a bool indicating whether the
154 processed text and ``validatehash`` is a bool indicating whether the
155 returned text should be checked for hash integrity.
155 returned text should be checked for hash integrity.
156 """
156 """
157 return self._processflagsfunc(text, flags, 'raw')[1]
157 return self._processflagsfunc(text, flags, 'raw')[1]
158
158
159 def _processflagsfunc(self, text, flags, operation):
159 def _processflagsfunc(self, text, flags, operation):
160 # fast path: no flag processors will run
160 # fast path: no flag processors will run
161 if flags == 0:
161 if flags == 0:
162 return text, True, {}
162 return text, True, {}
163 if operation not in ('read', 'write', 'raw'):
163 if operation not in ('read', 'write', 'raw'):
164 raise error.ProgrammingError(_("invalid '%s' operation") %
164 raise error.ProgrammingError(_("invalid '%s' operation") %
165 operation)
165 operation)
166 # Check all flags are known.
166 # Check all flags are known.
167 if flags & ~REVIDX_KNOWN_FLAGS:
167 if flags & ~REVIDX_KNOWN_FLAGS:
168 raise self._flagserrorclass(_("incompatible revision flag '%#x'") %
168 raise self._flagserrorclass(_("incompatible revision flag '%#x'") %
169 (flags & ~REVIDX_KNOWN_FLAGS))
169 (flags & ~REVIDX_KNOWN_FLAGS))
170 validatehash = True
170 validatehash = True
171 # Depending on the operation (read or write), the order might be
171 # Depending on the operation (read or write), the order might be
172 # reversed due to non-commutative transforms.
172 # reversed due to non-commutative transforms.
173 orderedflags = REVIDX_FLAGS_ORDER
173 orderedflags = REVIDX_FLAGS_ORDER
174 if operation == 'write':
174 if operation == 'write':
175 orderedflags = reversed(orderedflags)
175 orderedflags = reversed(orderedflags)
176
176
177 outsidedata = {}
177 outsidedata = {}
178 for flag in orderedflags:
178 for flag in orderedflags:
179 # If a flagprocessor has been registered for a known flag, apply the
179 # If a flagprocessor has been registered for a known flag, apply the
180 # related operation transform and update result tuple.
180 # related operation transform and update result tuple.
181 if flag & flags:
181 if flag & flags:
182 vhash = True
182 vhash = True
183
183
184 if flag not in self._flagprocessors:
184 if flag not in self._flagprocessors:
185 message = _("missing processor for flag '%#x'") % (flag)
185 message = _("missing processor for flag '%#x'") % (flag)
186 raise self._flagserrorclass(message)
186 raise self._flagserrorclass(message)
187
187
188 processor = self._flagprocessors[flag]
188 processor = self._flagprocessors[flag]
189 if processor is not None:
189 if processor is not None:
190 readtransform, writetransform, rawtransform = processor
190 readtransform, writetransform, rawtransform = processor
191
191
192 if operation == 'raw':
192 if operation == 'raw':
193 vhash = rawtransform(self, text)
193 vhash = rawtransform(self, text)
194 elif operation == 'read':
194 elif operation == 'read':
195 text, vhash = readtransform(self, text)
195 text, vhash, s = readtransform(self, text)
196 outsidedata.update(s)
196 else: # write operation
197 else: # write operation
197 text, vhash = writetransform(self, text)
198 text, vhash = writetransform(self, text)
198 validatehash = validatehash and vhash
199 validatehash = validatehash and vhash
199
200
200 return text, validatehash, outsidedata
201 return text, validatehash, outsidedata
@@ -1,139 +1,142 b''
1 # coding=UTF-8
1 # coding=UTF-8
2
2
3 from __future__ import absolute_import
3 from __future__ import absolute_import
4
4
5 import base64
5 import base64
6 import zlib
6 import zlib
7
7
8 from mercurial import (
8 from mercurial import (
9 changegroup,
9 changegroup,
10 exchange,
10 exchange,
11 extensions,
11 extensions,
12 revlog,
12 revlog,
13 util,
13 util,
14 )
14 )
15 from mercurial.revlogutils import (
15 from mercurial.revlogutils import (
16 flagutil,
16 flagutil,
17 )
17 )
18
18
19 # Test only: These flags are defined here only in the context of testing the
19 # Test only: These flags are defined here only in the context of testing the
20 # behavior of the flag processor. The canonical way to add flags is to get in
20 # behavior of the flag processor. The canonical way to add flags is to get in
21 # touch with the community and make them known in revlog.
21 # touch with the community and make them known in revlog.
22 REVIDX_NOOP = (1 << 3)
22 REVIDX_NOOP = (1 << 3)
23 REVIDX_BASE64 = (1 << 2)
23 REVIDX_BASE64 = (1 << 2)
24 REVIDX_GZIP = (1 << 1)
24 REVIDX_GZIP = (1 << 1)
25 REVIDX_FAIL = 1
25 REVIDX_FAIL = 1
26
26
27 def validatehash(self, text):
27 def validatehash(self, text):
28 return True
28 return True
29
29
30 def bypass(self, text):
30 def bypass(self, text):
31 return False
31 return False
32
32
33 def noopdonothing(self, text):
33 def noopdonothing(self, text):
34 return (text, True)
34 return (text, True)
35
35
36 def noopdonothingread(self, text):
37 return (text, True, {})
38
36 def b64encode(self, text):
39 def b64encode(self, text):
37 return (base64.b64encode(text), False)
40 return (base64.b64encode(text), False)
38
41
39 def b64decode(self, text):
42 def b64decode(self, text):
40 return (base64.b64decode(text), True)
43 return (base64.b64decode(text), True, {})
41
44
42 def gzipcompress(self, text):
45 def gzipcompress(self, text):
43 return (zlib.compress(text), False)
46 return (zlib.compress(text), False)
44
47
45 def gzipdecompress(self, text):
48 def gzipdecompress(self, text):
46 return (zlib.decompress(text), True)
49 return (zlib.decompress(text), True, {})
47
50
48 def supportedoutgoingversions(orig, repo):
51 def supportedoutgoingversions(orig, repo):
49 versions = orig(repo)
52 versions = orig(repo)
50 versions.discard(b'01')
53 versions.discard(b'01')
51 versions.discard(b'02')
54 versions.discard(b'02')
52 versions.add(b'03')
55 versions.add(b'03')
53 return versions
56 return versions
54
57
55 def allsupportedversions(orig, ui):
58 def allsupportedversions(orig, ui):
56 versions = orig(ui)
59 versions = orig(ui)
57 versions.add(b'03')
60 versions.add(b'03')
58 return versions
61 return versions
59
62
60 def makewrappedfile(obj):
63 def makewrappedfile(obj):
61 class wrappedfile(obj.__class__):
64 class wrappedfile(obj.__class__):
62 def addrevision(self, text, transaction, link, p1, p2,
65 def addrevision(self, text, transaction, link, p1, p2,
63 cachedelta=None, node=None,
66 cachedelta=None, node=None,
64 flags=flagutil.REVIDX_DEFAULT_FLAGS):
67 flags=flagutil.REVIDX_DEFAULT_FLAGS):
65 if b'[NOOP]' in text:
68 if b'[NOOP]' in text:
66 flags |= REVIDX_NOOP
69 flags |= REVIDX_NOOP
67
70
68 if b'[BASE64]' in text:
71 if b'[BASE64]' in text:
69 flags |= REVIDX_BASE64
72 flags |= REVIDX_BASE64
70
73
71 if b'[GZIP]' in text:
74 if b'[GZIP]' in text:
72 flags |= REVIDX_GZIP
75 flags |= REVIDX_GZIP
73
76
74 # This addrevision wrapper is meant to add a flag we will not have
77 # This addrevision wrapper is meant to add a flag we will not have
75 # transforms registered for, ensuring we handle this error case.
78 # transforms registered for, ensuring we handle this error case.
76 if b'[FAIL]' in text:
79 if b'[FAIL]' in text:
77 flags |= REVIDX_FAIL
80 flags |= REVIDX_FAIL
78
81
79 return super(wrappedfile, self).addrevision(text, transaction, link,
82 return super(wrappedfile, self).addrevision(text, transaction, link,
80 p1, p2,
83 p1, p2,
81 cachedelta=cachedelta,
84 cachedelta=cachedelta,
82 node=node,
85 node=node,
83 flags=flags)
86 flags=flags)
84
87
85 obj.__class__ = wrappedfile
88 obj.__class__ = wrappedfile
86
89
87 def reposetup(ui, repo):
90 def reposetup(ui, repo):
88 class wrappingflagprocessorrepo(repo.__class__):
91 class wrappingflagprocessorrepo(repo.__class__):
89 def file(self, f):
92 def file(self, f):
90 orig = super(wrappingflagprocessorrepo, self).file(f)
93 orig = super(wrappingflagprocessorrepo, self).file(f)
91 makewrappedfile(orig)
94 makewrappedfile(orig)
92 return orig
95 return orig
93
96
94 repo.__class__ = wrappingflagprocessorrepo
97 repo.__class__ = wrappingflagprocessorrepo
95
98
96 def extsetup(ui):
99 def extsetup(ui):
97 # Enable changegroup3 for flags to be sent over the wire
100 # Enable changegroup3 for flags to be sent over the wire
98 wrapfunction = extensions.wrapfunction
101 wrapfunction = extensions.wrapfunction
99 wrapfunction(changegroup,
102 wrapfunction(changegroup,
100 'supportedoutgoingversions',
103 'supportedoutgoingversions',
101 supportedoutgoingversions)
104 supportedoutgoingversions)
102 wrapfunction(changegroup,
105 wrapfunction(changegroup,
103 'allsupportedversions',
106 'allsupportedversions',
104 allsupportedversions)
107 allsupportedversions)
105
108
106 # Teach revlog about our test flags
109 # Teach revlog about our test flags
107 flags = [REVIDX_NOOP, REVIDX_BASE64, REVIDX_GZIP, REVIDX_FAIL]
110 flags = [REVIDX_NOOP, REVIDX_BASE64, REVIDX_GZIP, REVIDX_FAIL]
108 flagutil.REVIDX_KNOWN_FLAGS |= util.bitsfrom(flags)
111 flagutil.REVIDX_KNOWN_FLAGS |= util.bitsfrom(flags)
109 revlog.REVIDX_FLAGS_ORDER.extend(flags)
112 revlog.REVIDX_FLAGS_ORDER.extend(flags)
110
113
111 # Teach exchange to use changegroup 3
114 # Teach exchange to use changegroup 3
112 for k in exchange._bundlespeccontentopts.keys():
115 for k in exchange._bundlespeccontentopts.keys():
113 exchange._bundlespeccontentopts[k][b"cg.version"] = b"03"
116 exchange._bundlespeccontentopts[k][b"cg.version"] = b"03"
114
117
115 # Register flag processors for each extension
118 # Register flag processors for each extension
116 flagutil.addflagprocessor(
119 flagutil.addflagprocessor(
117 REVIDX_NOOP,
120 REVIDX_NOOP,
118 (
121 (
119 noopdonothing,
122 noopdonothingread,
120 noopdonothing,
123 noopdonothing,
121 validatehash,
124 validatehash,
122 )
125 )
123 )
126 )
124 flagutil.addflagprocessor(
127 flagutil.addflagprocessor(
125 REVIDX_BASE64,
128 REVIDX_BASE64,
126 (
129 (
127 b64decode,
130 b64decode,
128 b64encode,
131 b64encode,
129 bypass,
132 bypass,
130 ),
133 ),
131 )
134 )
132 flagutil.addflagprocessor(
135 flagutil.addflagprocessor(
133 REVIDX_GZIP,
136 REVIDX_GZIP,
134 (
137 (
135 gzipdecompress,
138 gzipdecompress,
136 gzipcompress,
139 gzipcompress,
137 bypass
140 bypass
138 )
141 )
139 )
142 )
@@ -1,455 +1,455 b''
1 # test revlog interaction about raw data (flagprocessor)
1 # test revlog interaction about raw data (flagprocessor)
2
2
3 from __future__ import absolute_import, print_function
3 from __future__ import absolute_import, print_function
4
4
5 import collections
5 import collections
6 import hashlib
6 import hashlib
7 import sys
7 import sys
8
8
9 from mercurial import (
9 from mercurial import (
10 encoding,
10 encoding,
11 node,
11 node,
12 revlog,
12 revlog,
13 transaction,
13 transaction,
14 vfs,
14 vfs,
15 )
15 )
16
16
17 from mercurial.revlogutils import (
17 from mercurial.revlogutils import (
18 deltas,
18 deltas,
19 flagutil,
19 flagutil,
20 )
20 )
21
21
22 # TESTTMP is optional. This makes it convenient to run without run-tests.py
22 # TESTTMP is optional. This makes it convenient to run without run-tests.py
23 tvfs = vfs.vfs(encoding.environ.get(b'TESTTMP', b'/tmp'))
23 tvfs = vfs.vfs(encoding.environ.get(b'TESTTMP', b'/tmp'))
24
24
25 # Enable generaldelta otherwise revlog won't use delta as expected by the test
25 # Enable generaldelta otherwise revlog won't use delta as expected by the test
26 tvfs.options = {b'generaldelta': True, b'revlogv1': True,
26 tvfs.options = {b'generaldelta': True, b'revlogv1': True,
27 b'sparse-revlog': True}
27 b'sparse-revlog': True}
28
28
29 # The test wants to control whether to use delta explicitly, based on
29 # The test wants to control whether to use delta explicitly, based on
30 # "storedeltachains".
30 # "storedeltachains".
31 revlog.revlog._isgooddeltainfo = lambda self, d, textlen: self._storedeltachains
31 revlog.revlog._isgooddeltainfo = lambda self, d, textlen: self._storedeltachains
32
32
33 def abort(msg):
33 def abort(msg):
34 print('abort: %s' % msg)
34 print('abort: %s' % msg)
35 # Return 0 so run-tests.py could compare the output.
35 # Return 0 so run-tests.py could compare the output.
36 sys.exit()
36 sys.exit()
37
37
38 # Register a revlog processor for flag EXTSTORED.
38 # Register a revlog processor for flag EXTSTORED.
39 #
39 #
40 # It simply prepends a fixed header, and replaces '1' to 'i'. So it has
40 # It simply prepends a fixed header, and replaces '1' to 'i'. So it has
41 # insertion and replacement, and may be interesting to test revlog's line-based
41 # insertion and replacement, and may be interesting to test revlog's line-based
42 # deltas.
42 # deltas.
43 _extheader = b'E\n'
43 _extheader = b'E\n'
44
44
45 def readprocessor(self, rawtext):
45 def readprocessor(self, rawtext):
46 # True: the returned text could be used to verify hash
46 # True: the returned text could be used to verify hash
47 text = rawtext[len(_extheader):].replace(b'i', b'1')
47 text = rawtext[len(_extheader):].replace(b'i', b'1')
48 return text, True
48 return text, True, {}
49
49
50 def writeprocessor(self, text):
50 def writeprocessor(self, text):
51 # False: the returned rawtext shouldn't be used to verify hash
51 # False: the returned rawtext shouldn't be used to verify hash
52 rawtext = _extheader + text.replace(b'1', b'i')
52 rawtext = _extheader + text.replace(b'1', b'i')
53 return rawtext, False
53 return rawtext, False
54
54
55 def rawprocessor(self, rawtext):
55 def rawprocessor(self, rawtext):
56 # False: do not verify hash. Only the content returned by "readprocessor"
56 # False: do not verify hash. Only the content returned by "readprocessor"
57 # can be used to verify hash.
57 # can be used to verify hash.
58 return False
58 return False
59
59
60 flagutil.addflagprocessor(revlog.REVIDX_EXTSTORED,
60 flagutil.addflagprocessor(revlog.REVIDX_EXTSTORED,
61 (readprocessor, writeprocessor, rawprocessor))
61 (readprocessor, writeprocessor, rawprocessor))
62
62
63 # Utilities about reading and appending revlog
63 # Utilities about reading and appending revlog
64
64
65 def newtransaction():
65 def newtransaction():
66 # A transaction is required to write revlogs
66 # A transaction is required to write revlogs
67 report = lambda msg: None
67 report = lambda msg: None
68 return transaction.transaction(report, tvfs, {'plain': tvfs}, b'journal')
68 return transaction.transaction(report, tvfs, {'plain': tvfs}, b'journal')
69
69
70 def newrevlog(name=b'_testrevlog.i', recreate=False):
70 def newrevlog(name=b'_testrevlog.i', recreate=False):
71 if recreate:
71 if recreate:
72 tvfs.tryunlink(name)
72 tvfs.tryunlink(name)
73 rlog = revlog.revlog(tvfs, name)
73 rlog = revlog.revlog(tvfs, name)
74 return rlog
74 return rlog
75
75
76 def appendrev(rlog, text, tr, isext=False, isdelta=True):
76 def appendrev(rlog, text, tr, isext=False, isdelta=True):
77 '''Append a revision. If isext is True, set the EXTSTORED flag so flag
77 '''Append a revision. If isext is True, set the EXTSTORED flag so flag
78 processor will be used (and rawtext is different from text). If isdelta is
78 processor will be used (and rawtext is different from text). If isdelta is
79 True, force the revision to be a delta, otherwise it's full text.
79 True, force the revision to be a delta, otherwise it's full text.
80 '''
80 '''
81 nextrev = len(rlog)
81 nextrev = len(rlog)
82 p1 = rlog.node(nextrev - 1)
82 p1 = rlog.node(nextrev - 1)
83 p2 = node.nullid
83 p2 = node.nullid
84 if isext:
84 if isext:
85 flags = revlog.REVIDX_EXTSTORED
85 flags = revlog.REVIDX_EXTSTORED
86 else:
86 else:
87 flags = revlog.REVIDX_DEFAULT_FLAGS
87 flags = revlog.REVIDX_DEFAULT_FLAGS
88 # Change storedeltachains temporarily, to override revlog's delta decision
88 # Change storedeltachains temporarily, to override revlog's delta decision
89 rlog._storedeltachains = isdelta
89 rlog._storedeltachains = isdelta
90 try:
90 try:
91 rlog.addrevision(text, tr, nextrev, p1, p2, flags=flags)
91 rlog.addrevision(text, tr, nextrev, p1, p2, flags=flags)
92 return nextrev
92 return nextrev
93 except Exception as ex:
93 except Exception as ex:
94 abort('rev %d: failed to append: %s' % (nextrev, ex))
94 abort('rev %d: failed to append: %s' % (nextrev, ex))
95 finally:
95 finally:
96 # Restore storedeltachains. It is always True, see revlog.__init__
96 # Restore storedeltachains. It is always True, see revlog.__init__
97 rlog._storedeltachains = True
97 rlog._storedeltachains = True
98
98
99 def addgroupcopy(rlog, tr, destname=b'_destrevlog.i', optimaldelta=True):
99 def addgroupcopy(rlog, tr, destname=b'_destrevlog.i', optimaldelta=True):
100 '''Copy revlog to destname using revlog.addgroup. Return the copied revlog.
100 '''Copy revlog to destname using revlog.addgroup. Return the copied revlog.
101
101
102 This emulates push or pull. They use changegroup. Changegroup requires
102 This emulates push or pull. They use changegroup. Changegroup requires
103 repo to work. We don't have a repo, so a dummy changegroup is used.
103 repo to work. We don't have a repo, so a dummy changegroup is used.
104
104
105 If optimaldelta is True, use optimized delta parent, so the destination
105 If optimaldelta is True, use optimized delta parent, so the destination
106 revlog could probably reuse it. Otherwise it builds sub-optimal delta, and
106 revlog could probably reuse it. Otherwise it builds sub-optimal delta, and
107 the destination revlog needs more work to use it.
107 the destination revlog needs more work to use it.
108
108
109 This exercises some revlog.addgroup (and revlog._addrevision(text=None))
109 This exercises some revlog.addgroup (and revlog._addrevision(text=None))
110 code path, which is not covered by "appendrev" alone.
110 code path, which is not covered by "appendrev" alone.
111 '''
111 '''
112 class dummychangegroup(object):
112 class dummychangegroup(object):
113 @staticmethod
113 @staticmethod
114 def deltachunk(pnode):
114 def deltachunk(pnode):
115 pnode = pnode or node.nullid
115 pnode = pnode or node.nullid
116 parentrev = rlog.rev(pnode)
116 parentrev = rlog.rev(pnode)
117 r = parentrev + 1
117 r = parentrev + 1
118 if r >= len(rlog):
118 if r >= len(rlog):
119 return {}
119 return {}
120 if optimaldelta:
120 if optimaldelta:
121 deltaparent = parentrev
121 deltaparent = parentrev
122 else:
122 else:
123 # suboptimal deltaparent
123 # suboptimal deltaparent
124 deltaparent = min(0, parentrev)
124 deltaparent = min(0, parentrev)
125 if not rlog.candelta(deltaparent, r):
125 if not rlog.candelta(deltaparent, r):
126 deltaparent = -1
126 deltaparent = -1
127 return {b'node': rlog.node(r), b'p1': pnode, b'p2': node.nullid,
127 return {b'node': rlog.node(r), b'p1': pnode, b'p2': node.nullid,
128 b'cs': rlog.node(rlog.linkrev(r)), b'flags': rlog.flags(r),
128 b'cs': rlog.node(rlog.linkrev(r)), b'flags': rlog.flags(r),
129 b'deltabase': rlog.node(deltaparent),
129 b'deltabase': rlog.node(deltaparent),
130 b'delta': rlog.revdiff(deltaparent, r)}
130 b'delta': rlog.revdiff(deltaparent, r)}
131
131
132 def deltaiter(self):
132 def deltaiter(self):
133 chain = None
133 chain = None
134 for chunkdata in iter(lambda: self.deltachunk(chain), {}):
134 for chunkdata in iter(lambda: self.deltachunk(chain), {}):
135 node = chunkdata[b'node']
135 node = chunkdata[b'node']
136 p1 = chunkdata[b'p1']
136 p1 = chunkdata[b'p1']
137 p2 = chunkdata[b'p2']
137 p2 = chunkdata[b'p2']
138 cs = chunkdata[b'cs']
138 cs = chunkdata[b'cs']
139 deltabase = chunkdata[b'deltabase']
139 deltabase = chunkdata[b'deltabase']
140 delta = chunkdata[b'delta']
140 delta = chunkdata[b'delta']
141 flags = chunkdata[b'flags']
141 flags = chunkdata[b'flags']
142
142
143 chain = node
143 chain = node
144
144
145 yield (node, p1, p2, cs, deltabase, delta, flags)
145 yield (node, p1, p2, cs, deltabase, delta, flags)
146
146
147 def linkmap(lnode):
147 def linkmap(lnode):
148 return rlog.rev(lnode)
148 return rlog.rev(lnode)
149
149
150 dlog = newrevlog(destname, recreate=True)
150 dlog = newrevlog(destname, recreate=True)
151 dummydeltas = dummychangegroup().deltaiter()
151 dummydeltas = dummychangegroup().deltaiter()
152 dlog.addgroup(dummydeltas, linkmap, tr)
152 dlog.addgroup(dummydeltas, linkmap, tr)
153 return dlog
153 return dlog
154
154
155 def lowlevelcopy(rlog, tr, destname=b'_destrevlog.i'):
155 def lowlevelcopy(rlog, tr, destname=b'_destrevlog.i'):
156 '''Like addgroupcopy, but use the low level revlog._addrevision directly.
156 '''Like addgroupcopy, but use the low level revlog._addrevision directly.
157
157
158 It exercises some code paths that are hard to reach easily otherwise.
158 It exercises some code paths that are hard to reach easily otherwise.
159 '''
159 '''
160 dlog = newrevlog(destname, recreate=True)
160 dlog = newrevlog(destname, recreate=True)
161 for r in rlog:
161 for r in rlog:
162 p1 = rlog.node(r - 1)
162 p1 = rlog.node(r - 1)
163 p2 = node.nullid
163 p2 = node.nullid
164 if r == 0 or (rlog.flags(r) & revlog.REVIDX_EXTSTORED):
164 if r == 0 or (rlog.flags(r) & revlog.REVIDX_EXTSTORED):
165 text = rlog.rawdata(r)
165 text = rlog.rawdata(r)
166 cachedelta = None
166 cachedelta = None
167 else:
167 else:
168 # deltaparent cannot have EXTSTORED flag.
168 # deltaparent cannot have EXTSTORED flag.
169 deltaparent = max([-1] +
169 deltaparent = max([-1] +
170 [p for p in range(r)
170 [p for p in range(r)
171 if rlog.flags(p) & revlog.REVIDX_EXTSTORED == 0])
171 if rlog.flags(p) & revlog.REVIDX_EXTSTORED == 0])
172 text = None
172 text = None
173 cachedelta = (deltaparent, rlog.revdiff(deltaparent, r))
173 cachedelta = (deltaparent, rlog.revdiff(deltaparent, r))
174 flags = rlog.flags(r)
174 flags = rlog.flags(r)
175 ifh = dfh = None
175 ifh = dfh = None
176 try:
176 try:
177 ifh = dlog.opener(dlog.indexfile, b'a+')
177 ifh = dlog.opener(dlog.indexfile, b'a+')
178 if not dlog._inline:
178 if not dlog._inline:
179 dfh = dlog.opener(dlog.datafile, b'a+')
179 dfh = dlog.opener(dlog.datafile, b'a+')
180 dlog._addrevision(rlog.node(r), text, tr, r, p1, p2, flags,
180 dlog._addrevision(rlog.node(r), text, tr, r, p1, p2, flags,
181 cachedelta, ifh, dfh)
181 cachedelta, ifh, dfh)
182 finally:
182 finally:
183 if dfh is not None:
183 if dfh is not None:
184 dfh.close()
184 dfh.close()
185 if ifh is not None:
185 if ifh is not None:
186 ifh.close()
186 ifh.close()
187 return dlog
187 return dlog
188
188
189 # Utilities to generate revisions for testing
189 # Utilities to generate revisions for testing
190
190
191 def genbits(n):
191 def genbits(n):
192 '''Given a number n, generate (2 ** (n * 2) + 1) numbers in range(2 ** n).
192 '''Given a number n, generate (2 ** (n * 2) + 1) numbers in range(2 ** n).
193 i.e. the generated numbers have a width of n bits.
193 i.e. the generated numbers have a width of n bits.
194
194
195 The combination of two adjacent numbers will cover all possible cases.
195 The combination of two adjacent numbers will cover all possible cases.
196 That is to say, given any x, y where both x, and y are in range(2 ** n),
196 That is to say, given any x, y where both x, and y are in range(2 ** n),
197 there is an x followed immediately by y in the generated sequence.
197 there is an x followed immediately by y in the generated sequence.
198 '''
198 '''
199 m = 2 ** n
199 m = 2 ** n
200
200
201 # Gray Code. See https://en.wikipedia.org/wiki/Gray_code
201 # Gray Code. See https://en.wikipedia.org/wiki/Gray_code
202 gray = lambda x: x ^ (x >> 1)
202 gray = lambda x: x ^ (x >> 1)
203 reversegray = dict((gray(i), i) for i in range(m))
203 reversegray = dict((gray(i), i) for i in range(m))
204
204
205 # Generate (n * 2) bit gray code, yield lower n bits as X, and look for
205 # Generate (n * 2) bit gray code, yield lower n bits as X, and look for
206 # the next unused gray code where higher n bits equal to X.
206 # the next unused gray code where higher n bits equal to X.
207
207
208 # For gray codes whose higher bits are X, a[X] of them have been used.
208 # For gray codes whose higher bits are X, a[X] of them have been used.
209 a = [0] * m
209 a = [0] * m
210
210
211 # Iterate from 0.
211 # Iterate from 0.
212 x = 0
212 x = 0
213 yield x
213 yield x
214 for i in range(m * m):
214 for i in range(m * m):
215 x = reversegray[x]
215 x = reversegray[x]
216 y = gray(a[x] + x * m) & (m - 1)
216 y = gray(a[x] + x * m) & (m - 1)
217 assert a[x] < m
217 assert a[x] < m
218 a[x] += 1
218 a[x] += 1
219 x = y
219 x = y
220 yield x
220 yield x
221
221
222 def gentext(rev):
222 def gentext(rev):
223 '''Given a revision number, generate dummy text'''
223 '''Given a revision number, generate dummy text'''
224 return b''.join(b'%d\n' % j for j in range(-1, rev % 5))
224 return b''.join(b'%d\n' % j for j in range(-1, rev % 5))
225
225
226 def writecases(rlog, tr):
226 def writecases(rlog, tr):
227 '''Write some revisions interested to the test.
227 '''Write some revisions interested to the test.
228
228
229 The test is interested in 3 properties of a revision:
229 The test is interested in 3 properties of a revision:
230
230
231 - Is it a delta or a full text? (isdelta)
231 - Is it a delta or a full text? (isdelta)
232 This is to catch some delta application issues.
232 This is to catch some delta application issues.
233 - Does it have a flag of EXTSTORED? (isext)
233 - Does it have a flag of EXTSTORED? (isext)
234 This is to catch some flag processor issues. Especially when
234 This is to catch some flag processor issues. Especially when
235 interacted with revlog deltas.
235 interacted with revlog deltas.
236 - Is its text empty? (isempty)
236 - Is its text empty? (isempty)
237 This is less important. It is intended to try to catch some careless
237 This is less important. It is intended to try to catch some careless
238 checks like "if text" instead of "if text is None". Note: if flag
238 checks like "if text" instead of "if text is None". Note: if flag
239 processor is involved, raw text may be not empty.
239 processor is involved, raw text may be not empty.
240
240
241 Write 65 revisions. So that all combinations of the above flags for
241 Write 65 revisions. So that all combinations of the above flags for
242 adjacent revisions are covered. That is to say,
242 adjacent revisions are covered. That is to say,
243
243
244 len(set(
244 len(set(
245 (r.delta, r.ext, r.empty, (r+1).delta, (r+1).ext, (r+1).empty)
245 (r.delta, r.ext, r.empty, (r+1).delta, (r+1).ext, (r+1).empty)
246 for r in range(len(rlog) - 1)
246 for r in range(len(rlog) - 1)
247 )) is 64.
247 )) is 64.
248
248
249 Where "r.delta", "r.ext", and "r.empty" are booleans matching properties
249 Where "r.delta", "r.ext", and "r.empty" are booleans matching properties
250 mentioned above.
250 mentioned above.
251
251
252 Return expected [(text, rawtext)].
252 Return expected [(text, rawtext)].
253 '''
253 '''
254 result = []
254 result = []
255 for i, x in enumerate(genbits(3)):
255 for i, x in enumerate(genbits(3)):
256 isdelta, isext, isempty = bool(x & 1), bool(x & 2), bool(x & 4)
256 isdelta, isext, isempty = bool(x & 1), bool(x & 2), bool(x & 4)
257 if isempty:
257 if isempty:
258 text = b''
258 text = b''
259 else:
259 else:
260 text = gentext(i)
260 text = gentext(i)
261 rev = appendrev(rlog, text, tr, isext=isext, isdelta=isdelta)
261 rev = appendrev(rlog, text, tr, isext=isext, isdelta=isdelta)
262
262
263 # Verify text, rawtext, and rawsize
263 # Verify text, rawtext, and rawsize
264 if isext:
264 if isext:
265 rawtext = writeprocessor(None, text)[0]
265 rawtext = writeprocessor(None, text)[0]
266 else:
266 else:
267 rawtext = text
267 rawtext = text
268 if rlog.rawsize(rev) != len(rawtext):
268 if rlog.rawsize(rev) != len(rawtext):
269 abort('rev %d: wrong rawsize' % rev)
269 abort('rev %d: wrong rawsize' % rev)
270 if rlog.revision(rev, raw=False) != text:
270 if rlog.revision(rev, raw=False) != text:
271 abort('rev %d: wrong text' % rev)
271 abort('rev %d: wrong text' % rev)
272 if rlog.rawdata(rev) != rawtext:
272 if rlog.rawdata(rev) != rawtext:
273 abort('rev %d: wrong rawtext' % rev)
273 abort('rev %d: wrong rawtext' % rev)
274 result.append((text, rawtext))
274 result.append((text, rawtext))
275
275
276 # Verify flags like isdelta, isext work as expected
276 # Verify flags like isdelta, isext work as expected
277 # isdelta can be overridden to False if this or p1 has isext set
277 # isdelta can be overridden to False if this or p1 has isext set
278 if bool(rlog.deltaparent(rev) > -1) and not isdelta:
278 if bool(rlog.deltaparent(rev) > -1) and not isdelta:
279 abort('rev %d: isdelta is unexpected' % rev)
279 abort('rev %d: isdelta is unexpected' % rev)
280 if bool(rlog.flags(rev)) != isext:
280 if bool(rlog.flags(rev)) != isext:
281 abort('rev %d: isext is ineffective' % rev)
281 abort('rev %d: isext is ineffective' % rev)
282 return result
282 return result
283
283
284 # Main test and checking
284 # Main test and checking
285
285
286 def checkrevlog(rlog, expected):
286 def checkrevlog(rlog, expected):
287 '''Check if revlog has expected contents. expected is [(text, rawtext)]'''
287 '''Check if revlog has expected contents. expected is [(text, rawtext)]'''
288 # Test using different access orders. This could expose some issues
288 # Test using different access orders. This could expose some issues
289 # depending on revlog caching (see revlog._cache).
289 # depending on revlog caching (see revlog._cache).
290 for r0 in range(len(rlog) - 1):
290 for r0 in range(len(rlog) - 1):
291 r1 = r0 + 1
291 r1 = r0 + 1
292 for revorder in [[r0, r1], [r1, r0]]:
292 for revorder in [[r0, r1], [r1, r0]]:
293 for raworder in [[True], [False], [True, False], [False, True]]:
293 for raworder in [[True], [False], [True, False], [False, True]]:
294 nlog = newrevlog()
294 nlog = newrevlog()
295 for rev in revorder:
295 for rev in revorder:
296 for raw in raworder:
296 for raw in raworder:
297 if raw:
297 if raw:
298 t = nlog.rawdata(rev)
298 t = nlog.rawdata(rev)
299 else:
299 else:
300 t = nlog.revision(rev)
300 t = nlog.revision(rev)
301 if t != expected[rev][int(raw)]:
301 if t != expected[rev][int(raw)]:
302 abort('rev %d: corrupted %stext'
302 abort('rev %d: corrupted %stext'
303 % (rev, raw and 'raw' or ''))
303 % (rev, raw and 'raw' or ''))
304
304
305 slicingdata = [
305 slicingdata = [
306 ([0, 1, 2, 3, 55, 56, 58, 59, 60],
306 ([0, 1, 2, 3, 55, 56, 58, 59, 60],
307 [[0, 1], [2], [58], [59, 60]],
307 [[0, 1], [2], [58], [59, 60]],
308 10),
308 10),
309 ([0, 1, 2, 3, 55, 56, 58, 59, 60],
309 ([0, 1, 2, 3, 55, 56, 58, 59, 60],
310 [[0, 1], [2], [58], [59, 60]],
310 [[0, 1], [2], [58], [59, 60]],
311 10),
311 10),
312 ([-1, 0, 1, 2, 3, 55, 56, 58, 59, 60],
312 ([-1, 0, 1, 2, 3, 55, 56, 58, 59, 60],
313 [[-1, 0, 1], [2], [58], [59, 60]],
313 [[-1, 0, 1], [2], [58], [59, 60]],
314 10),
314 10),
315 ]
315 ]
316
316
317 def slicingtest(rlog):
317 def slicingtest(rlog):
318 oldmin = rlog._srmingapsize
318 oldmin = rlog._srmingapsize
319 try:
319 try:
320 # the test revlog is small, we remove the floor under which we
320 # the test revlog is small, we remove the floor under which we
321 # slicing is diregarded.
321 # slicing is diregarded.
322 rlog._srmingapsize = 0
322 rlog._srmingapsize = 0
323 for item in slicingdata:
323 for item in slicingdata:
324 chain, expected, target = item
324 chain, expected, target = item
325 result = deltas.slicechunk(rlog, chain, targetsize=target)
325 result = deltas.slicechunk(rlog, chain, targetsize=target)
326 result = list(result)
326 result = list(result)
327 if result != expected:
327 if result != expected:
328 print('slicing differ:')
328 print('slicing differ:')
329 print(' chain: %s' % chain)
329 print(' chain: %s' % chain)
330 print(' target: %s' % target)
330 print(' target: %s' % target)
331 print(' expected: %s' % expected)
331 print(' expected: %s' % expected)
332 print(' result: %s' % result)
332 print(' result: %s' % result)
333 finally:
333 finally:
334 rlog._srmingapsize = oldmin
334 rlog._srmingapsize = oldmin
335
335
336 def md5sum(s):
336 def md5sum(s):
337 return hashlib.md5(s).digest()
337 return hashlib.md5(s).digest()
338
338
339 def _maketext(*coord):
339 def _maketext(*coord):
340 """create piece of text according to range of integers
340 """create piece of text according to range of integers
341
341
342 The test returned use a md5sum of the integer to make it less
342 The test returned use a md5sum of the integer to make it less
343 compressible"""
343 compressible"""
344 pieces = []
344 pieces = []
345 for start, size in coord:
345 for start, size in coord:
346 num = range(start, start + size)
346 num = range(start, start + size)
347 p = [md5sum(b'%d' % r) for r in num]
347 p = [md5sum(b'%d' % r) for r in num]
348 pieces.append(b'\n'.join(p))
348 pieces.append(b'\n'.join(p))
349 return b'\n'.join(pieces) + b'\n'
349 return b'\n'.join(pieces) + b'\n'
350
350
351 data = [
351 data = [
352 _maketext((0, 120), (456, 60)),
352 _maketext((0, 120), (456, 60)),
353 _maketext((0, 120), (345, 60)),
353 _maketext((0, 120), (345, 60)),
354 _maketext((0, 120), (734, 60)),
354 _maketext((0, 120), (734, 60)),
355 _maketext((0, 120), (734, 60), (923, 45)),
355 _maketext((0, 120), (734, 60), (923, 45)),
356 _maketext((0, 120), (734, 60), (234, 45)),
356 _maketext((0, 120), (734, 60), (234, 45)),
357 _maketext((0, 120), (734, 60), (564, 45)),
357 _maketext((0, 120), (734, 60), (564, 45)),
358 _maketext((0, 120), (734, 60), (361, 45)),
358 _maketext((0, 120), (734, 60), (361, 45)),
359 _maketext((0, 120), (734, 60), (489, 45)),
359 _maketext((0, 120), (734, 60), (489, 45)),
360 _maketext((0, 120), (123, 60)),
360 _maketext((0, 120), (123, 60)),
361 _maketext((0, 120), (145, 60)),
361 _maketext((0, 120), (145, 60)),
362 _maketext((0, 120), (104, 60)),
362 _maketext((0, 120), (104, 60)),
363 _maketext((0, 120), (430, 60)),
363 _maketext((0, 120), (430, 60)),
364 _maketext((0, 120), (430, 60), (923, 45)),
364 _maketext((0, 120), (430, 60), (923, 45)),
365 _maketext((0, 120), (430, 60), (234, 45)),
365 _maketext((0, 120), (430, 60), (234, 45)),
366 _maketext((0, 120), (430, 60), (564, 45)),
366 _maketext((0, 120), (430, 60), (564, 45)),
367 _maketext((0, 120), (430, 60), (361, 45)),
367 _maketext((0, 120), (430, 60), (361, 45)),
368 _maketext((0, 120), (430, 60), (489, 45)),
368 _maketext((0, 120), (430, 60), (489, 45)),
369 _maketext((0, 120), (249, 60)),
369 _maketext((0, 120), (249, 60)),
370 _maketext((0, 120), (832, 60)),
370 _maketext((0, 120), (832, 60)),
371 _maketext((0, 120), (891, 60)),
371 _maketext((0, 120), (891, 60)),
372 _maketext((0, 120), (543, 60)),
372 _maketext((0, 120), (543, 60)),
373 _maketext((0, 120), (120, 60)),
373 _maketext((0, 120), (120, 60)),
374 _maketext((0, 120), (60, 60), (768, 30)),
374 _maketext((0, 120), (60, 60), (768, 30)),
375 _maketext((0, 120), (60, 60), (260, 30)),
375 _maketext((0, 120), (60, 60), (260, 30)),
376 _maketext((0, 120), (60, 60), (450, 30)),
376 _maketext((0, 120), (60, 60), (450, 30)),
377 _maketext((0, 120), (60, 60), (361, 30)),
377 _maketext((0, 120), (60, 60), (361, 30)),
378 _maketext((0, 120), (60, 60), (886, 30)),
378 _maketext((0, 120), (60, 60), (886, 30)),
379 _maketext((0, 120), (60, 60), (116, 30)),
379 _maketext((0, 120), (60, 60), (116, 30)),
380 _maketext((0, 120), (60, 60), (567, 30), (629, 40)),
380 _maketext((0, 120), (60, 60), (567, 30), (629, 40)),
381 _maketext((0, 120), (60, 60), (569, 30), (745, 40)),
381 _maketext((0, 120), (60, 60), (569, 30), (745, 40)),
382 _maketext((0, 120), (60, 60), (777, 30), (700, 40)),
382 _maketext((0, 120), (60, 60), (777, 30), (700, 40)),
383 _maketext((0, 120), (60, 60), (618, 30), (398, 40), (158, 10)),
383 _maketext((0, 120), (60, 60), (618, 30), (398, 40), (158, 10)),
384 ]
384 ]
385
385
386 def makesnapshot(tr):
386 def makesnapshot(tr):
387 rl = newrevlog(name=b'_snaprevlog3.i', recreate=True)
387 rl = newrevlog(name=b'_snaprevlog3.i', recreate=True)
388 for i in data:
388 for i in data:
389 appendrev(rl, i, tr)
389 appendrev(rl, i, tr)
390 return rl
390 return rl
391
391
392 snapshots = [-1, 0, 6, 8, 11, 17, 19, 21, 25, 30]
392 snapshots = [-1, 0, 6, 8, 11, 17, 19, 21, 25, 30]
393 def issnapshottest(rlog):
393 def issnapshottest(rlog):
394 result = []
394 result = []
395 if rlog.issnapshot(-1):
395 if rlog.issnapshot(-1):
396 result.append(-1)
396 result.append(-1)
397 for rev in rlog:
397 for rev in rlog:
398 if rlog.issnapshot(rev):
398 if rlog.issnapshot(rev):
399 result.append(rev)
399 result.append(rev)
400 if snapshots != result:
400 if snapshots != result:
401 print('snapshot differ:')
401 print('snapshot differ:')
402 print(' expected: %s' % snapshots)
402 print(' expected: %s' % snapshots)
403 print(' got: %s' % result)
403 print(' got: %s' % result)
404
404
405 snapshotmapall = {0: [6, 8, 11, 17, 19, 25], 8: [21], -1: [0, 30]}
405 snapshotmapall = {0: [6, 8, 11, 17, 19, 25], 8: [21], -1: [0, 30]}
406 snapshotmap15 = {0: [17, 19, 25], 8: [21], -1: [30]}
406 snapshotmap15 = {0: [17, 19, 25], 8: [21], -1: [30]}
407 def findsnapshottest(rlog):
407 def findsnapshottest(rlog):
408 resultall = collections.defaultdict(list)
408 resultall = collections.defaultdict(list)
409 deltas._findsnapshots(rlog, resultall, 0)
409 deltas._findsnapshots(rlog, resultall, 0)
410 resultall = dict(resultall.items())
410 resultall = dict(resultall.items())
411 if resultall != snapshotmapall:
411 if resultall != snapshotmapall:
412 print('snapshot map differ:')
412 print('snapshot map differ:')
413 print(' expected: %s' % snapshotmapall)
413 print(' expected: %s' % snapshotmapall)
414 print(' got: %s' % resultall)
414 print(' got: %s' % resultall)
415 result15 = collections.defaultdict(list)
415 result15 = collections.defaultdict(list)
416 deltas._findsnapshots(rlog, result15, 15)
416 deltas._findsnapshots(rlog, result15, 15)
417 result15 = dict(result15.items())
417 result15 = dict(result15.items())
418 if result15 != snapshotmap15:
418 if result15 != snapshotmap15:
419 print('snapshot map differ:')
419 print('snapshot map differ:')
420 print(' expected: %s' % snapshotmap15)
420 print(' expected: %s' % snapshotmap15)
421 print(' got: %s' % result15)
421 print(' got: %s' % result15)
422
422
423 def maintest():
423 def maintest():
424 with newtransaction() as tr:
424 with newtransaction() as tr:
425 rl = newrevlog(recreate=True)
425 rl = newrevlog(recreate=True)
426 expected = writecases(rl, tr)
426 expected = writecases(rl, tr)
427 checkrevlog(rl, expected)
427 checkrevlog(rl, expected)
428 print('local test passed')
428 print('local test passed')
429 # Copy via revlog.addgroup
429 # Copy via revlog.addgroup
430 rl1 = addgroupcopy(rl, tr)
430 rl1 = addgroupcopy(rl, tr)
431 checkrevlog(rl1, expected)
431 checkrevlog(rl1, expected)
432 rl2 = addgroupcopy(rl, tr, optimaldelta=False)
432 rl2 = addgroupcopy(rl, tr, optimaldelta=False)
433 checkrevlog(rl2, expected)
433 checkrevlog(rl2, expected)
434 print('addgroupcopy test passed')
434 print('addgroupcopy test passed')
435 # Copy via revlog.clone
435 # Copy via revlog.clone
436 rl3 = newrevlog(name=b'_destrevlog3.i', recreate=True)
436 rl3 = newrevlog(name=b'_destrevlog3.i', recreate=True)
437 rl.clone(tr, rl3)
437 rl.clone(tr, rl3)
438 checkrevlog(rl3, expected)
438 checkrevlog(rl3, expected)
439 print('clone test passed')
439 print('clone test passed')
440 # Copy via low-level revlog._addrevision
440 # Copy via low-level revlog._addrevision
441 rl4 = lowlevelcopy(rl, tr)
441 rl4 = lowlevelcopy(rl, tr)
442 checkrevlog(rl4, expected)
442 checkrevlog(rl4, expected)
443 print('lowlevelcopy test passed')
443 print('lowlevelcopy test passed')
444 slicingtest(rl)
444 slicingtest(rl)
445 print('slicing test passed')
445 print('slicing test passed')
446 rl5 = makesnapshot(tr)
446 rl5 = makesnapshot(tr)
447 issnapshottest(rl5)
447 issnapshottest(rl5)
448 print('issnapshot test passed')
448 print('issnapshot test passed')
449 findsnapshottest(rl5)
449 findsnapshottest(rl5)
450 print('findsnapshot test passed')
450 print('findsnapshot test passed')
451
451
452 try:
452 try:
453 maintest()
453 maintest()
454 except Exception as ex:
454 except Exception as ex:
455 abort('crashed: %s' % ex)
455 abort('crashed: %s' % ex)
General Comments 0
You need to be logged in to leave comments. Login now