##// END OF EJS Templates
changegroup: declare shallow flag in constructor...
Gregory Szorc -
r38940:cdb9bc21 default
parent child Browse files
Show More
@@ -1,1402 +1,1408 b''
1 # changegroup.py - Mercurial changegroup manipulation functions
1 # changegroup.py - Mercurial changegroup manipulation functions
2 #
2 #
3 # Copyright 2006 Matt Mackall <mpm@selenic.com>
3 # Copyright 2006 Matt Mackall <mpm@selenic.com>
4 #
4 #
5 # This software may be used and distributed according to the terms of the
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
6 # GNU General Public License version 2 or any later version.
7
7
8 from __future__ import absolute_import
8 from __future__ import absolute_import
9
9
10 import os
10 import os
11 import struct
11 import struct
12 import weakref
12 import weakref
13
13
14 from .i18n import _
14 from .i18n import _
15 from .node import (
15 from .node import (
16 hex,
16 hex,
17 nullid,
17 nullid,
18 nullrev,
18 nullrev,
19 short,
19 short,
20 )
20 )
21
21
22 from .thirdparty import (
22 from .thirdparty import (
23 attr,
23 attr,
24 )
24 )
25
25
26 from . import (
26 from . import (
27 dagutil,
27 dagutil,
28 error,
28 error,
29 manifest,
29 manifest,
30 match as matchmod,
30 match as matchmod,
31 mdiff,
31 mdiff,
32 phases,
32 phases,
33 pycompat,
33 pycompat,
34 repository,
34 repository,
35 revlog,
35 revlog,
36 util,
36 util,
37 )
37 )
38
38
39 from .utils import (
39 from .utils import (
40 stringutil,
40 stringutil,
41 )
41 )
42
42
43 _CHANGEGROUPV1_DELTA_HEADER = struct.Struct("20s20s20s20s")
43 _CHANGEGROUPV1_DELTA_HEADER = struct.Struct("20s20s20s20s")
44 _CHANGEGROUPV2_DELTA_HEADER = struct.Struct("20s20s20s20s20s")
44 _CHANGEGROUPV2_DELTA_HEADER = struct.Struct("20s20s20s20s20s")
45 _CHANGEGROUPV3_DELTA_HEADER = struct.Struct(">20s20s20s20s20sH")
45 _CHANGEGROUPV3_DELTA_HEADER = struct.Struct(">20s20s20s20s20sH")
46
46
47 LFS_REQUIREMENT = 'lfs'
47 LFS_REQUIREMENT = 'lfs'
48
48
49 readexactly = util.readexactly
49 readexactly = util.readexactly
50
50
51 def getchunk(stream):
51 def getchunk(stream):
52 """return the next chunk from stream as a string"""
52 """return the next chunk from stream as a string"""
53 d = readexactly(stream, 4)
53 d = readexactly(stream, 4)
54 l = struct.unpack(">l", d)[0]
54 l = struct.unpack(">l", d)[0]
55 if l <= 4:
55 if l <= 4:
56 if l:
56 if l:
57 raise error.Abort(_("invalid chunk length %d") % l)
57 raise error.Abort(_("invalid chunk length %d") % l)
58 return ""
58 return ""
59 return readexactly(stream, l - 4)
59 return readexactly(stream, l - 4)
60
60
61 def chunkheader(length):
61 def chunkheader(length):
62 """return a changegroup chunk header (string)"""
62 """return a changegroup chunk header (string)"""
63 return struct.pack(">l", length + 4)
63 return struct.pack(">l", length + 4)
64
64
65 def closechunk():
65 def closechunk():
66 """return a changegroup chunk header (string) for a zero-length chunk"""
66 """return a changegroup chunk header (string) for a zero-length chunk"""
67 return struct.pack(">l", 0)
67 return struct.pack(">l", 0)
68
68
69 def writechunks(ui, chunks, filename, vfs=None):
69 def writechunks(ui, chunks, filename, vfs=None):
70 """Write chunks to a file and return its filename.
70 """Write chunks to a file and return its filename.
71
71
72 The stream is assumed to be a bundle file.
72 The stream is assumed to be a bundle file.
73 Existing files will not be overwritten.
73 Existing files will not be overwritten.
74 If no filename is specified, a temporary file is created.
74 If no filename is specified, a temporary file is created.
75 """
75 """
76 fh = None
76 fh = None
77 cleanup = None
77 cleanup = None
78 try:
78 try:
79 if filename:
79 if filename:
80 if vfs:
80 if vfs:
81 fh = vfs.open(filename, "wb")
81 fh = vfs.open(filename, "wb")
82 else:
82 else:
83 # Increase default buffer size because default is usually
83 # Increase default buffer size because default is usually
84 # small (4k is common on Linux).
84 # small (4k is common on Linux).
85 fh = open(filename, "wb", 131072)
85 fh = open(filename, "wb", 131072)
86 else:
86 else:
87 fd, filename = pycompat.mkstemp(prefix="hg-bundle-", suffix=".hg")
87 fd, filename = pycompat.mkstemp(prefix="hg-bundle-", suffix=".hg")
88 fh = os.fdopen(fd, r"wb")
88 fh = os.fdopen(fd, r"wb")
89 cleanup = filename
89 cleanup = filename
90 for c in chunks:
90 for c in chunks:
91 fh.write(c)
91 fh.write(c)
92 cleanup = None
92 cleanup = None
93 return filename
93 return filename
94 finally:
94 finally:
95 if fh is not None:
95 if fh is not None:
96 fh.close()
96 fh.close()
97 if cleanup is not None:
97 if cleanup is not None:
98 if filename and vfs:
98 if filename and vfs:
99 vfs.unlink(cleanup)
99 vfs.unlink(cleanup)
100 else:
100 else:
101 os.unlink(cleanup)
101 os.unlink(cleanup)
102
102
103 class cg1unpacker(object):
103 class cg1unpacker(object):
104 """Unpacker for cg1 changegroup streams.
104 """Unpacker for cg1 changegroup streams.
105
105
106 A changegroup unpacker handles the framing of the revision data in
106 A changegroup unpacker handles the framing of the revision data in
107 the wire format. Most consumers will want to use the apply()
107 the wire format. Most consumers will want to use the apply()
108 method to add the changes from the changegroup to a repository.
108 method to add the changes from the changegroup to a repository.
109
109
110 If you're forwarding a changegroup unmodified to another consumer,
110 If you're forwarding a changegroup unmodified to another consumer,
111 use getchunks(), which returns an iterator of changegroup
111 use getchunks(), which returns an iterator of changegroup
112 chunks. This is mostly useful for cases where you need to know the
112 chunks. This is mostly useful for cases where you need to know the
113 data stream has ended by observing the end of the changegroup.
113 data stream has ended by observing the end of the changegroup.
114
114
115 deltachunk() is useful only if you're applying delta data. Most
115 deltachunk() is useful only if you're applying delta data. Most
116 consumers should prefer apply() instead.
116 consumers should prefer apply() instead.
117
117
118 A few other public methods exist. Those are used only for
118 A few other public methods exist. Those are used only for
119 bundlerepo and some debug commands - their use is discouraged.
119 bundlerepo and some debug commands - their use is discouraged.
120 """
120 """
121 deltaheader = _CHANGEGROUPV1_DELTA_HEADER
121 deltaheader = _CHANGEGROUPV1_DELTA_HEADER
122 deltaheadersize = deltaheader.size
122 deltaheadersize = deltaheader.size
123 version = '01'
123 version = '01'
124 _grouplistcount = 1 # One list of files after the manifests
124 _grouplistcount = 1 # One list of files after the manifests
125
125
126 def __init__(self, fh, alg, extras=None):
126 def __init__(self, fh, alg, extras=None):
127 if alg is None:
127 if alg is None:
128 alg = 'UN'
128 alg = 'UN'
129 if alg not in util.compengines.supportedbundletypes:
129 if alg not in util.compengines.supportedbundletypes:
130 raise error.Abort(_('unknown stream compression type: %s')
130 raise error.Abort(_('unknown stream compression type: %s')
131 % alg)
131 % alg)
132 if alg == 'BZ':
132 if alg == 'BZ':
133 alg = '_truncatedBZ'
133 alg = '_truncatedBZ'
134
134
135 compengine = util.compengines.forbundletype(alg)
135 compengine = util.compengines.forbundletype(alg)
136 self._stream = compengine.decompressorreader(fh)
136 self._stream = compengine.decompressorreader(fh)
137 self._type = alg
137 self._type = alg
138 self.extras = extras or {}
138 self.extras = extras or {}
139 self.callback = None
139 self.callback = None
140
140
141 # These methods (compressed, read, seek, tell) all appear to only
141 # These methods (compressed, read, seek, tell) all appear to only
142 # be used by bundlerepo, but it's a little hard to tell.
142 # be used by bundlerepo, but it's a little hard to tell.
143 def compressed(self):
143 def compressed(self):
144 return self._type is not None and self._type != 'UN'
144 return self._type is not None and self._type != 'UN'
145 def read(self, l):
145 def read(self, l):
146 return self._stream.read(l)
146 return self._stream.read(l)
147 def seek(self, pos):
147 def seek(self, pos):
148 return self._stream.seek(pos)
148 return self._stream.seek(pos)
149 def tell(self):
149 def tell(self):
150 return self._stream.tell()
150 return self._stream.tell()
151 def close(self):
151 def close(self):
152 return self._stream.close()
152 return self._stream.close()
153
153
154 def _chunklength(self):
154 def _chunklength(self):
155 d = readexactly(self._stream, 4)
155 d = readexactly(self._stream, 4)
156 l = struct.unpack(">l", d)[0]
156 l = struct.unpack(">l", d)[0]
157 if l <= 4:
157 if l <= 4:
158 if l:
158 if l:
159 raise error.Abort(_("invalid chunk length %d") % l)
159 raise error.Abort(_("invalid chunk length %d") % l)
160 return 0
160 return 0
161 if self.callback:
161 if self.callback:
162 self.callback()
162 self.callback()
163 return l - 4
163 return l - 4
164
164
165 def changelogheader(self):
165 def changelogheader(self):
166 """v10 does not have a changelog header chunk"""
166 """v10 does not have a changelog header chunk"""
167 return {}
167 return {}
168
168
169 def manifestheader(self):
169 def manifestheader(self):
170 """v10 does not have a manifest header chunk"""
170 """v10 does not have a manifest header chunk"""
171 return {}
171 return {}
172
172
173 def filelogheader(self):
173 def filelogheader(self):
174 """return the header of the filelogs chunk, v10 only has the filename"""
174 """return the header of the filelogs chunk, v10 only has the filename"""
175 l = self._chunklength()
175 l = self._chunklength()
176 if not l:
176 if not l:
177 return {}
177 return {}
178 fname = readexactly(self._stream, l)
178 fname = readexactly(self._stream, l)
179 return {'filename': fname}
179 return {'filename': fname}
180
180
181 def _deltaheader(self, headertuple, prevnode):
181 def _deltaheader(self, headertuple, prevnode):
182 node, p1, p2, cs = headertuple
182 node, p1, p2, cs = headertuple
183 if prevnode is None:
183 if prevnode is None:
184 deltabase = p1
184 deltabase = p1
185 else:
185 else:
186 deltabase = prevnode
186 deltabase = prevnode
187 flags = 0
187 flags = 0
188 return node, p1, p2, deltabase, cs, flags
188 return node, p1, p2, deltabase, cs, flags
189
189
190 def deltachunk(self, prevnode):
190 def deltachunk(self, prevnode):
191 l = self._chunklength()
191 l = self._chunklength()
192 if not l:
192 if not l:
193 return {}
193 return {}
194 headerdata = readexactly(self._stream, self.deltaheadersize)
194 headerdata = readexactly(self._stream, self.deltaheadersize)
195 header = self.deltaheader.unpack(headerdata)
195 header = self.deltaheader.unpack(headerdata)
196 delta = readexactly(self._stream, l - self.deltaheadersize)
196 delta = readexactly(self._stream, l - self.deltaheadersize)
197 node, p1, p2, deltabase, cs, flags = self._deltaheader(header, prevnode)
197 node, p1, p2, deltabase, cs, flags = self._deltaheader(header, prevnode)
198 return (node, p1, p2, cs, deltabase, delta, flags)
198 return (node, p1, p2, cs, deltabase, delta, flags)
199
199
200 def getchunks(self):
200 def getchunks(self):
201 """returns all the chunks contains in the bundle
201 """returns all the chunks contains in the bundle
202
202
203 Used when you need to forward the binary stream to a file or another
203 Used when you need to forward the binary stream to a file or another
204 network API. To do so, it parse the changegroup data, otherwise it will
204 network API. To do so, it parse the changegroup data, otherwise it will
205 block in case of sshrepo because it don't know the end of the stream.
205 block in case of sshrepo because it don't know the end of the stream.
206 """
206 """
207 # For changegroup 1 and 2, we expect 3 parts: changelog, manifestlog,
207 # For changegroup 1 and 2, we expect 3 parts: changelog, manifestlog,
208 # and a list of filelogs. For changegroup 3, we expect 4 parts:
208 # and a list of filelogs. For changegroup 3, we expect 4 parts:
209 # changelog, manifestlog, a list of tree manifestlogs, and a list of
209 # changelog, manifestlog, a list of tree manifestlogs, and a list of
210 # filelogs.
210 # filelogs.
211 #
211 #
212 # Changelog and manifestlog parts are terminated with empty chunks. The
212 # Changelog and manifestlog parts are terminated with empty chunks. The
213 # tree and file parts are a list of entry sections. Each entry section
213 # tree and file parts are a list of entry sections. Each entry section
214 # is a series of chunks terminating in an empty chunk. The list of these
214 # is a series of chunks terminating in an empty chunk. The list of these
215 # entry sections is terminated in yet another empty chunk, so we know
215 # entry sections is terminated in yet another empty chunk, so we know
216 # we've reached the end of the tree/file list when we reach an empty
216 # we've reached the end of the tree/file list when we reach an empty
217 # chunk that was proceeded by no non-empty chunks.
217 # chunk that was proceeded by no non-empty chunks.
218
218
219 parts = 0
219 parts = 0
220 while parts < 2 + self._grouplistcount:
220 while parts < 2 + self._grouplistcount:
221 noentries = True
221 noentries = True
222 while True:
222 while True:
223 chunk = getchunk(self)
223 chunk = getchunk(self)
224 if not chunk:
224 if not chunk:
225 # The first two empty chunks represent the end of the
225 # The first two empty chunks represent the end of the
226 # changelog and the manifestlog portions. The remaining
226 # changelog and the manifestlog portions. The remaining
227 # empty chunks represent either A) the end of individual
227 # empty chunks represent either A) the end of individual
228 # tree or file entries in the file list, or B) the end of
228 # tree or file entries in the file list, or B) the end of
229 # the entire list. It's the end of the entire list if there
229 # the entire list. It's the end of the entire list if there
230 # were no entries (i.e. noentries is True).
230 # were no entries (i.e. noentries is True).
231 if parts < 2:
231 if parts < 2:
232 parts += 1
232 parts += 1
233 elif noentries:
233 elif noentries:
234 parts += 1
234 parts += 1
235 break
235 break
236 noentries = False
236 noentries = False
237 yield chunkheader(len(chunk))
237 yield chunkheader(len(chunk))
238 pos = 0
238 pos = 0
239 while pos < len(chunk):
239 while pos < len(chunk):
240 next = pos + 2**20
240 next = pos + 2**20
241 yield chunk[pos:next]
241 yield chunk[pos:next]
242 pos = next
242 pos = next
243 yield closechunk()
243 yield closechunk()
244
244
245 def _unpackmanifests(self, repo, revmap, trp, prog):
245 def _unpackmanifests(self, repo, revmap, trp, prog):
246 self.callback = prog.increment
246 self.callback = prog.increment
247 # no need to check for empty manifest group here:
247 # no need to check for empty manifest group here:
248 # if the result of the merge of 1 and 2 is the same in 3 and 4,
248 # if the result of the merge of 1 and 2 is the same in 3 and 4,
249 # no new manifest will be created and the manifest group will
249 # no new manifest will be created and the manifest group will
250 # be empty during the pull
250 # be empty during the pull
251 self.manifestheader()
251 self.manifestheader()
252 deltas = self.deltaiter()
252 deltas = self.deltaiter()
253 repo.manifestlog.addgroup(deltas, revmap, trp)
253 repo.manifestlog.addgroup(deltas, revmap, trp)
254 prog.complete()
254 prog.complete()
255 self.callback = None
255 self.callback = None
256
256
257 def apply(self, repo, tr, srctype, url, targetphase=phases.draft,
257 def apply(self, repo, tr, srctype, url, targetphase=phases.draft,
258 expectedtotal=None):
258 expectedtotal=None):
259 """Add the changegroup returned by source.read() to this repo.
259 """Add the changegroup returned by source.read() to this repo.
260 srctype is a string like 'push', 'pull', or 'unbundle'. url is
260 srctype is a string like 'push', 'pull', or 'unbundle'. url is
261 the URL of the repo where this changegroup is coming from.
261 the URL of the repo where this changegroup is coming from.
262
262
263 Return an integer summarizing the change to this repo:
263 Return an integer summarizing the change to this repo:
264 - nothing changed or no source: 0
264 - nothing changed or no source: 0
265 - more heads than before: 1+added heads (2..n)
265 - more heads than before: 1+added heads (2..n)
266 - fewer heads than before: -1-removed heads (-2..-n)
266 - fewer heads than before: -1-removed heads (-2..-n)
267 - number of heads stays the same: 1
267 - number of heads stays the same: 1
268 """
268 """
269 repo = repo.unfiltered()
269 repo = repo.unfiltered()
270 def csmap(x):
270 def csmap(x):
271 repo.ui.debug("add changeset %s\n" % short(x))
271 repo.ui.debug("add changeset %s\n" % short(x))
272 return len(cl)
272 return len(cl)
273
273
274 def revmap(x):
274 def revmap(x):
275 return cl.rev(x)
275 return cl.rev(x)
276
276
277 changesets = files = revisions = 0
277 changesets = files = revisions = 0
278
278
279 try:
279 try:
280 # The transaction may already carry source information. In this
280 # The transaction may already carry source information. In this
281 # case we use the top level data. We overwrite the argument
281 # case we use the top level data. We overwrite the argument
282 # because we need to use the top level value (if they exist)
282 # because we need to use the top level value (if they exist)
283 # in this function.
283 # in this function.
284 srctype = tr.hookargs.setdefault('source', srctype)
284 srctype = tr.hookargs.setdefault('source', srctype)
285 url = tr.hookargs.setdefault('url', url)
285 url = tr.hookargs.setdefault('url', url)
286 repo.hook('prechangegroup',
286 repo.hook('prechangegroup',
287 throw=True, **pycompat.strkwargs(tr.hookargs))
287 throw=True, **pycompat.strkwargs(tr.hookargs))
288
288
289 # write changelog data to temp files so concurrent readers
289 # write changelog data to temp files so concurrent readers
290 # will not see an inconsistent view
290 # will not see an inconsistent view
291 cl = repo.changelog
291 cl = repo.changelog
292 cl.delayupdate(tr)
292 cl.delayupdate(tr)
293 oldheads = set(cl.heads())
293 oldheads = set(cl.heads())
294
294
295 trp = weakref.proxy(tr)
295 trp = weakref.proxy(tr)
296 # pull off the changeset group
296 # pull off the changeset group
297 repo.ui.status(_("adding changesets\n"))
297 repo.ui.status(_("adding changesets\n"))
298 clstart = len(cl)
298 clstart = len(cl)
299 progress = repo.ui.makeprogress(_('changesets'), unit=_('chunks'),
299 progress = repo.ui.makeprogress(_('changesets'), unit=_('chunks'),
300 total=expectedtotal)
300 total=expectedtotal)
301 self.callback = progress.increment
301 self.callback = progress.increment
302
302
303 efiles = set()
303 efiles = set()
304 def onchangelog(cl, node):
304 def onchangelog(cl, node):
305 efiles.update(cl.readfiles(node))
305 efiles.update(cl.readfiles(node))
306
306
307 self.changelogheader()
307 self.changelogheader()
308 deltas = self.deltaiter()
308 deltas = self.deltaiter()
309 cgnodes = cl.addgroup(deltas, csmap, trp, addrevisioncb=onchangelog)
309 cgnodes = cl.addgroup(deltas, csmap, trp, addrevisioncb=onchangelog)
310 efiles = len(efiles)
310 efiles = len(efiles)
311
311
312 if not cgnodes:
312 if not cgnodes:
313 repo.ui.develwarn('applied empty changegroup',
313 repo.ui.develwarn('applied empty changegroup',
314 config='warn-empty-changegroup')
314 config='warn-empty-changegroup')
315 clend = len(cl)
315 clend = len(cl)
316 changesets = clend - clstart
316 changesets = clend - clstart
317 progress.complete()
317 progress.complete()
318 self.callback = None
318 self.callback = None
319
319
320 # pull off the manifest group
320 # pull off the manifest group
321 repo.ui.status(_("adding manifests\n"))
321 repo.ui.status(_("adding manifests\n"))
322 # We know that we'll never have more manifests than we had
322 # We know that we'll never have more manifests than we had
323 # changesets.
323 # changesets.
324 progress = repo.ui.makeprogress(_('manifests'), unit=_('chunks'),
324 progress = repo.ui.makeprogress(_('manifests'), unit=_('chunks'),
325 total=changesets)
325 total=changesets)
326 self._unpackmanifests(repo, revmap, trp, progress)
326 self._unpackmanifests(repo, revmap, trp, progress)
327
327
328 needfiles = {}
328 needfiles = {}
329 if repo.ui.configbool('server', 'validate'):
329 if repo.ui.configbool('server', 'validate'):
330 cl = repo.changelog
330 cl = repo.changelog
331 ml = repo.manifestlog
331 ml = repo.manifestlog
332 # validate incoming csets have their manifests
332 # validate incoming csets have their manifests
333 for cset in pycompat.xrange(clstart, clend):
333 for cset in pycompat.xrange(clstart, clend):
334 mfnode = cl.changelogrevision(cset).manifest
334 mfnode = cl.changelogrevision(cset).manifest
335 mfest = ml[mfnode].readdelta()
335 mfest = ml[mfnode].readdelta()
336 # store file cgnodes we must see
336 # store file cgnodes we must see
337 for f, n in mfest.iteritems():
337 for f, n in mfest.iteritems():
338 needfiles.setdefault(f, set()).add(n)
338 needfiles.setdefault(f, set()).add(n)
339
339
340 # process the files
340 # process the files
341 repo.ui.status(_("adding file changes\n"))
341 repo.ui.status(_("adding file changes\n"))
342 newrevs, newfiles = _addchangegroupfiles(
342 newrevs, newfiles = _addchangegroupfiles(
343 repo, self, revmap, trp, efiles, needfiles)
343 repo, self, revmap, trp, efiles, needfiles)
344 revisions += newrevs
344 revisions += newrevs
345 files += newfiles
345 files += newfiles
346
346
347 deltaheads = 0
347 deltaheads = 0
348 if oldheads:
348 if oldheads:
349 heads = cl.heads()
349 heads = cl.heads()
350 deltaheads = len(heads) - len(oldheads)
350 deltaheads = len(heads) - len(oldheads)
351 for h in heads:
351 for h in heads:
352 if h not in oldheads and repo[h].closesbranch():
352 if h not in oldheads and repo[h].closesbranch():
353 deltaheads -= 1
353 deltaheads -= 1
354 htext = ""
354 htext = ""
355 if deltaheads:
355 if deltaheads:
356 htext = _(" (%+d heads)") % deltaheads
356 htext = _(" (%+d heads)") % deltaheads
357
357
358 repo.ui.status(_("added %d changesets"
358 repo.ui.status(_("added %d changesets"
359 " with %d changes to %d files%s\n")
359 " with %d changes to %d files%s\n")
360 % (changesets, revisions, files, htext))
360 % (changesets, revisions, files, htext))
361 repo.invalidatevolatilesets()
361 repo.invalidatevolatilesets()
362
362
363 if changesets > 0:
363 if changesets > 0:
364 if 'node' not in tr.hookargs:
364 if 'node' not in tr.hookargs:
365 tr.hookargs['node'] = hex(cl.node(clstart))
365 tr.hookargs['node'] = hex(cl.node(clstart))
366 tr.hookargs['node_last'] = hex(cl.node(clend - 1))
366 tr.hookargs['node_last'] = hex(cl.node(clend - 1))
367 hookargs = dict(tr.hookargs)
367 hookargs = dict(tr.hookargs)
368 else:
368 else:
369 hookargs = dict(tr.hookargs)
369 hookargs = dict(tr.hookargs)
370 hookargs['node'] = hex(cl.node(clstart))
370 hookargs['node'] = hex(cl.node(clstart))
371 hookargs['node_last'] = hex(cl.node(clend - 1))
371 hookargs['node_last'] = hex(cl.node(clend - 1))
372 repo.hook('pretxnchangegroup',
372 repo.hook('pretxnchangegroup',
373 throw=True, **pycompat.strkwargs(hookargs))
373 throw=True, **pycompat.strkwargs(hookargs))
374
374
375 added = [cl.node(r) for r in pycompat.xrange(clstart, clend)]
375 added = [cl.node(r) for r in pycompat.xrange(clstart, clend)]
376 phaseall = None
376 phaseall = None
377 if srctype in ('push', 'serve'):
377 if srctype in ('push', 'serve'):
378 # Old servers can not push the boundary themselves.
378 # Old servers can not push the boundary themselves.
379 # New servers won't push the boundary if changeset already
379 # New servers won't push the boundary if changeset already
380 # exists locally as secret
380 # exists locally as secret
381 #
381 #
382 # We should not use added here but the list of all change in
382 # We should not use added here but the list of all change in
383 # the bundle
383 # the bundle
384 if repo.publishing():
384 if repo.publishing():
385 targetphase = phaseall = phases.public
385 targetphase = phaseall = phases.public
386 else:
386 else:
387 # closer target phase computation
387 # closer target phase computation
388
388
389 # Those changesets have been pushed from the
389 # Those changesets have been pushed from the
390 # outside, their phases are going to be pushed
390 # outside, their phases are going to be pushed
391 # alongside. Therefor `targetphase` is
391 # alongside. Therefor `targetphase` is
392 # ignored.
392 # ignored.
393 targetphase = phaseall = phases.draft
393 targetphase = phaseall = phases.draft
394 if added:
394 if added:
395 phases.registernew(repo, tr, targetphase, added)
395 phases.registernew(repo, tr, targetphase, added)
396 if phaseall is not None:
396 if phaseall is not None:
397 phases.advanceboundary(repo, tr, phaseall, cgnodes)
397 phases.advanceboundary(repo, tr, phaseall, cgnodes)
398
398
399 if changesets > 0:
399 if changesets > 0:
400
400
401 def runhooks():
401 def runhooks():
402 # These hooks run when the lock releases, not when the
402 # These hooks run when the lock releases, not when the
403 # transaction closes. So it's possible for the changelog
403 # transaction closes. So it's possible for the changelog
404 # to have changed since we last saw it.
404 # to have changed since we last saw it.
405 if clstart >= len(repo):
405 if clstart >= len(repo):
406 return
406 return
407
407
408 repo.hook("changegroup", **pycompat.strkwargs(hookargs))
408 repo.hook("changegroup", **pycompat.strkwargs(hookargs))
409
409
410 for n in added:
410 for n in added:
411 args = hookargs.copy()
411 args = hookargs.copy()
412 args['node'] = hex(n)
412 args['node'] = hex(n)
413 del args['node_last']
413 del args['node_last']
414 repo.hook("incoming", **pycompat.strkwargs(args))
414 repo.hook("incoming", **pycompat.strkwargs(args))
415
415
416 newheads = [h for h in repo.heads()
416 newheads = [h for h in repo.heads()
417 if h not in oldheads]
417 if h not in oldheads]
418 repo.ui.log("incoming",
418 repo.ui.log("incoming",
419 "%d incoming changes - new heads: %s\n",
419 "%d incoming changes - new heads: %s\n",
420 len(added),
420 len(added),
421 ', '.join([hex(c[:6]) for c in newheads]))
421 ', '.join([hex(c[:6]) for c in newheads]))
422
422
423 tr.addpostclose('changegroup-runhooks-%020i' % clstart,
423 tr.addpostclose('changegroup-runhooks-%020i' % clstart,
424 lambda tr: repo._afterlock(runhooks))
424 lambda tr: repo._afterlock(runhooks))
425 finally:
425 finally:
426 repo.ui.flush()
426 repo.ui.flush()
427 # never return 0 here:
427 # never return 0 here:
428 if deltaheads < 0:
428 if deltaheads < 0:
429 ret = deltaheads - 1
429 ret = deltaheads - 1
430 else:
430 else:
431 ret = deltaheads + 1
431 ret = deltaheads + 1
432 return ret
432 return ret
433
433
434 def deltaiter(self):
434 def deltaiter(self):
435 """
435 """
436 returns an iterator of the deltas in this changegroup
436 returns an iterator of the deltas in this changegroup
437
437
438 Useful for passing to the underlying storage system to be stored.
438 Useful for passing to the underlying storage system to be stored.
439 """
439 """
440 chain = None
440 chain = None
441 for chunkdata in iter(lambda: self.deltachunk(chain), {}):
441 for chunkdata in iter(lambda: self.deltachunk(chain), {}):
442 # Chunkdata: (node, p1, p2, cs, deltabase, delta, flags)
442 # Chunkdata: (node, p1, p2, cs, deltabase, delta, flags)
443 yield chunkdata
443 yield chunkdata
444 chain = chunkdata[0]
444 chain = chunkdata[0]
445
445
446 class cg2unpacker(cg1unpacker):
446 class cg2unpacker(cg1unpacker):
447 """Unpacker for cg2 streams.
447 """Unpacker for cg2 streams.
448
448
449 cg2 streams add support for generaldelta, so the delta header
449 cg2 streams add support for generaldelta, so the delta header
450 format is slightly different. All other features about the data
450 format is slightly different. All other features about the data
451 remain the same.
451 remain the same.
452 """
452 """
453 deltaheader = _CHANGEGROUPV2_DELTA_HEADER
453 deltaheader = _CHANGEGROUPV2_DELTA_HEADER
454 deltaheadersize = deltaheader.size
454 deltaheadersize = deltaheader.size
455 version = '02'
455 version = '02'
456
456
457 def _deltaheader(self, headertuple, prevnode):
457 def _deltaheader(self, headertuple, prevnode):
458 node, p1, p2, deltabase, cs = headertuple
458 node, p1, p2, deltabase, cs = headertuple
459 flags = 0
459 flags = 0
460 return node, p1, p2, deltabase, cs, flags
460 return node, p1, p2, deltabase, cs, flags
461
461
462 class cg3unpacker(cg2unpacker):
462 class cg3unpacker(cg2unpacker):
463 """Unpacker for cg3 streams.
463 """Unpacker for cg3 streams.
464
464
465 cg3 streams add support for exchanging treemanifests and revlog
465 cg3 streams add support for exchanging treemanifests and revlog
466 flags. It adds the revlog flags to the delta header and an empty chunk
466 flags. It adds the revlog flags to the delta header and an empty chunk
467 separating manifests and files.
467 separating manifests and files.
468 """
468 """
469 deltaheader = _CHANGEGROUPV3_DELTA_HEADER
469 deltaheader = _CHANGEGROUPV3_DELTA_HEADER
470 deltaheadersize = deltaheader.size
470 deltaheadersize = deltaheader.size
471 version = '03'
471 version = '03'
472 _grouplistcount = 2 # One list of manifests and one list of files
472 _grouplistcount = 2 # One list of manifests and one list of files
473
473
474 def _deltaheader(self, headertuple, prevnode):
474 def _deltaheader(self, headertuple, prevnode):
475 node, p1, p2, deltabase, cs, flags = headertuple
475 node, p1, p2, deltabase, cs, flags = headertuple
476 return node, p1, p2, deltabase, cs, flags
476 return node, p1, p2, deltabase, cs, flags
477
477
478 def _unpackmanifests(self, repo, revmap, trp, prog):
478 def _unpackmanifests(self, repo, revmap, trp, prog):
479 super(cg3unpacker, self)._unpackmanifests(repo, revmap, trp, prog)
479 super(cg3unpacker, self)._unpackmanifests(repo, revmap, trp, prog)
480 for chunkdata in iter(self.filelogheader, {}):
480 for chunkdata in iter(self.filelogheader, {}):
481 # If we get here, there are directory manifests in the changegroup
481 # If we get here, there are directory manifests in the changegroup
482 d = chunkdata["filename"]
482 d = chunkdata["filename"]
483 repo.ui.debug("adding %s revisions\n" % d)
483 repo.ui.debug("adding %s revisions\n" % d)
484 dirlog = repo.manifestlog._revlog.dirlog(d)
484 dirlog = repo.manifestlog._revlog.dirlog(d)
485 deltas = self.deltaiter()
485 deltas = self.deltaiter()
486 if not dirlog.addgroup(deltas, revmap, trp):
486 if not dirlog.addgroup(deltas, revmap, trp):
487 raise error.Abort(_("received dir revlog group is empty"))
487 raise error.Abort(_("received dir revlog group is empty"))
488
488
489 class headerlessfixup(object):
489 class headerlessfixup(object):
490 def __init__(self, fh, h):
490 def __init__(self, fh, h):
491 self._h = h
491 self._h = h
492 self._fh = fh
492 self._fh = fh
493 def read(self, n):
493 def read(self, n):
494 if self._h:
494 if self._h:
495 d, self._h = self._h[:n], self._h[n:]
495 d, self._h = self._h[:n], self._h[n:]
496 if len(d) < n:
496 if len(d) < n:
497 d += readexactly(self._fh, n - len(d))
497 d += readexactly(self._fh, n - len(d))
498 return d
498 return d
499 return readexactly(self._fh, n)
499 return readexactly(self._fh, n)
500
500
501 @attr.s(slots=True, frozen=True)
501 @attr.s(slots=True, frozen=True)
502 class revisiondelta(object):
502 class revisiondelta(object):
503 """Describes a delta entry in a changegroup.
503 """Describes a delta entry in a changegroup.
504
504
505 Captured data is sufficient to serialize the delta into multiple
505 Captured data is sufficient to serialize the delta into multiple
506 formats.
506 formats.
507 """
507 """
508 # 20 byte node of this revision.
508 # 20 byte node of this revision.
509 node = attr.ib()
509 node = attr.ib()
510 # 20 byte nodes of parent revisions.
510 # 20 byte nodes of parent revisions.
511 p1node = attr.ib()
511 p1node = attr.ib()
512 p2node = attr.ib()
512 p2node = attr.ib()
513 # 20 byte node of node this delta is against.
513 # 20 byte node of node this delta is against.
514 basenode = attr.ib()
514 basenode = attr.ib()
515 # 20 byte node of changeset revision this delta is associated with.
515 # 20 byte node of changeset revision this delta is associated with.
516 linknode = attr.ib()
516 linknode = attr.ib()
517 # 2 bytes of flags to apply to revision data.
517 # 2 bytes of flags to apply to revision data.
518 flags = attr.ib()
518 flags = attr.ib()
519 # Iterable of chunks holding raw delta data.
519 # Iterable of chunks holding raw delta data.
520 deltachunks = attr.ib()
520 deltachunks = attr.ib()
521
521
522 class cgpacker(object):
522 class cgpacker(object):
523 def __init__(self, repo, filematcher, version, allowreorder,
523 def __init__(self, repo, filematcher, version, allowreorder,
524 useprevdelta, builddeltaheader, manifestsend,
524 useprevdelta, builddeltaheader, manifestsend,
525 sendtreemanifests, bundlecaps=None):
525 sendtreemanifests, bundlecaps=None, shallow=False):
526 """Given a source repo, construct a bundler.
526 """Given a source repo, construct a bundler.
527
527
528 filematcher is a matcher that matches on files to include in the
528 filematcher is a matcher that matches on files to include in the
529 changegroup. Used to facilitate sparse changegroups.
529 changegroup. Used to facilitate sparse changegroups.
530
530
531 allowreorder controls whether reordering of revisions is allowed.
531 allowreorder controls whether reordering of revisions is allowed.
532 This value is used when ``bundle.reorder`` is ``auto`` or isn't
532 This value is used when ``bundle.reorder`` is ``auto`` or isn't
533 set.
533 set.
534
534
535 useprevdelta controls whether revisions should always delta against
535 useprevdelta controls whether revisions should always delta against
536 the previous revision in the changegroup.
536 the previous revision in the changegroup.
537
537
538 builddeltaheader is a callable that constructs the header for a group
538 builddeltaheader is a callable that constructs the header for a group
539 delta.
539 delta.
540
540
541 manifestsend is a chunk to send after manifests have been fully emitted.
541 manifestsend is a chunk to send after manifests have been fully emitted.
542
542
543 sendtreemanifests indicates whether tree manifests should be emitted.
543 sendtreemanifests indicates whether tree manifests should be emitted.
544
544
545 bundlecaps is optional and can be used to specify the set of
545 bundlecaps is optional and can be used to specify the set of
546 capabilities which can be used to build the bundle. While bundlecaps is
546 capabilities which can be used to build the bundle. While bundlecaps is
547 unused in core Mercurial, extensions rely on this feature to communicate
547 unused in core Mercurial, extensions rely on this feature to communicate
548 capabilities to customize the changegroup packer.
548 capabilities to customize the changegroup packer.
549
550 shallow indicates whether shallow data might be sent. The packer may
551 need to pack file contents not introduced by the changes being packed.
549 """
552 """
550 assert filematcher
553 assert filematcher
551 self._filematcher = filematcher
554 self._filematcher = filematcher
552
555
553 self.version = version
556 self.version = version
554 self._useprevdelta = useprevdelta
557 self._useprevdelta = useprevdelta
555 self._builddeltaheader = builddeltaheader
558 self._builddeltaheader = builddeltaheader
556 self._manifestsend = manifestsend
559 self._manifestsend = manifestsend
557 self._sendtreemanifests = sendtreemanifests
560 self._sendtreemanifests = sendtreemanifests
558
561
559 # Set of capabilities we can use to build the bundle.
562 # Set of capabilities we can use to build the bundle.
560 if bundlecaps is None:
563 if bundlecaps is None:
561 bundlecaps = set()
564 bundlecaps = set()
562 self._bundlecaps = bundlecaps
565 self._bundlecaps = bundlecaps
566 self._isshallow = shallow
563
567
564 # experimental config: bundle.reorder
568 # experimental config: bundle.reorder
565 reorder = repo.ui.config('bundle', 'reorder')
569 reorder = repo.ui.config('bundle', 'reorder')
566 if reorder == 'auto':
570 if reorder == 'auto':
567 self._reorder = allowreorder
571 self._reorder = allowreorder
568 else:
572 else:
569 self._reorder = stringutil.parsebool(reorder)
573 self._reorder = stringutil.parsebool(reorder)
570
574
571 self._repo = repo
575 self._repo = repo
572
576
573 if self._repo.ui.verbose and not self._repo.ui.debugflag:
577 if self._repo.ui.verbose and not self._repo.ui.debugflag:
574 self._verbosenote = self._repo.ui.note
578 self._verbosenote = self._repo.ui.note
575 else:
579 else:
576 self._verbosenote = lambda s: None
580 self._verbosenote = lambda s: None
577
581
578 def _close(self):
582 def _close(self):
579 # Ellipses serving mode.
583 # Ellipses serving mode.
580 getattr(self, '_clrev_to_localrev', {}).clear()
584 getattr(self, '_clrev_to_localrev', {}).clear()
581 if getattr(self, '_next_clrev_to_localrev', {}):
585 if getattr(self, '_next_clrev_to_localrev', {}):
582 self._clrev_to_localrev = self._next_clrev_to_localrev
586 self._clrev_to_localrev = self._next_clrev_to_localrev
583 del self._next_clrev_to_localrev
587 del self._next_clrev_to_localrev
584 self._changelog_done = True
588 self._changelog_done = True
585
589
586 return closechunk()
590 return closechunk()
587
591
588 def _fileheader(self, fname):
592 def _fileheader(self, fname):
589 return chunkheader(len(fname)) + fname
593 return chunkheader(len(fname)) + fname
590
594
591 # Extracted both for clarity and for overriding in extensions.
595 # Extracted both for clarity and for overriding in extensions.
592 def _sortgroup(self, store, nodelist, lookup):
596 def _sortgroup(self, store, nodelist, lookup):
593 """Sort nodes for change group and turn them into revnums."""
597 """Sort nodes for change group and turn them into revnums."""
594 # Ellipses serving mode.
598 # Ellipses serving mode.
595 #
599 #
596 # In a perfect world, we'd generate better ellipsis-ified graphs
600 # In a perfect world, we'd generate better ellipsis-ified graphs
597 # for non-changelog revlogs. In practice, we haven't started doing
601 # for non-changelog revlogs. In practice, we haven't started doing
598 # that yet, so the resulting DAGs for the manifestlog and filelogs
602 # that yet, so the resulting DAGs for the manifestlog and filelogs
599 # are actually full of bogus parentage on all the ellipsis
603 # are actually full of bogus parentage on all the ellipsis
600 # nodes. This has the side effect that, while the contents are
604 # nodes. This has the side effect that, while the contents are
601 # correct, the individual DAGs might be completely out of whack in
605 # correct, the individual DAGs might be completely out of whack in
602 # a case like 882681bc3166 and its ancestors (back about 10
606 # a case like 882681bc3166 and its ancestors (back about 10
603 # revisions or so) in the main hg repo.
607 # revisions or so) in the main hg repo.
604 #
608 #
605 # The one invariant we *know* holds is that the new (potentially
609 # The one invariant we *know* holds is that the new (potentially
606 # bogus) DAG shape will be valid if we order the nodes in the
610 # bogus) DAG shape will be valid if we order the nodes in the
607 # order that they're introduced in dramatis personae by the
611 # order that they're introduced in dramatis personae by the
608 # changelog, so what we do is we sort the non-changelog histories
612 # changelog, so what we do is we sort the non-changelog histories
609 # by the order in which they are used by the changelog.
613 # by the order in which they are used by the changelog.
610 if util.safehasattr(self, '_full_nodes') and self._clnode_to_rev:
614 if util.safehasattr(self, '_full_nodes') and self._clnode_to_rev:
611 key = lambda n: self._clnode_to_rev[lookup(n)]
615 key = lambda n: self._clnode_to_rev[lookup(n)]
612 return [store.rev(n) for n in sorted(nodelist, key=key)]
616 return [store.rev(n) for n in sorted(nodelist, key=key)]
613
617
614 # for generaldelta revlogs, we linearize the revs; this will both be
618 # for generaldelta revlogs, we linearize the revs; this will both be
615 # much quicker and generate a much smaller bundle
619 # much quicker and generate a much smaller bundle
616 if (store._generaldelta and self._reorder is None) or self._reorder:
620 if (store._generaldelta and self._reorder is None) or self._reorder:
617 dag = dagutil.revlogdag(store)
621 dag = dagutil.revlogdag(store)
618 return dag.linearize(set(store.rev(n) for n in nodelist))
622 return dag.linearize(set(store.rev(n) for n in nodelist))
619 else:
623 else:
620 return sorted([store.rev(n) for n in nodelist])
624 return sorted([store.rev(n) for n in nodelist])
621
625
622 def group(self, nodelist, store, lookup, units=None):
626 def group(self, nodelist, store, lookup, units=None):
623 """Calculate a delta group, yielding a sequence of changegroup chunks
627 """Calculate a delta group, yielding a sequence of changegroup chunks
624 (strings).
628 (strings).
625
629
626 Given a list of changeset revs, return a set of deltas and
630 Given a list of changeset revs, return a set of deltas and
627 metadata corresponding to nodes. The first delta is
631 metadata corresponding to nodes. The first delta is
628 first parent(nodelist[0]) -> nodelist[0], the receiver is
632 first parent(nodelist[0]) -> nodelist[0], the receiver is
629 guaranteed to have this parent as it has all history before
633 guaranteed to have this parent as it has all history before
630 these changesets. In the case firstparent is nullrev the
634 these changesets. In the case firstparent is nullrev the
631 changegroup starts with a full revision.
635 changegroup starts with a full revision.
632
636
633 If units is not None, progress detail will be generated, units specifies
637 If units is not None, progress detail will be generated, units specifies
634 the type of revlog that is touched (changelog, manifest, etc.).
638 the type of revlog that is touched (changelog, manifest, etc.).
635 """
639 """
636 # if we don't have any revisions touched by these changesets, bail
640 # if we don't have any revisions touched by these changesets, bail
637 if len(nodelist) == 0:
641 if len(nodelist) == 0:
638 yield self._close()
642 yield self._close()
639 return
643 return
640
644
641 revs = self._sortgroup(store, nodelist, lookup)
645 revs = self._sortgroup(store, nodelist, lookup)
642
646
643 # add the parent of the first rev
647 # add the parent of the first rev
644 p = store.parentrevs(revs[0])[0]
648 p = store.parentrevs(revs[0])[0]
645 revs.insert(0, p)
649 revs.insert(0, p)
646
650
647 # build deltas
651 # build deltas
648 progress = None
652 progress = None
649 if units is not None:
653 if units is not None:
650 progress = self._repo.ui.makeprogress(_('bundling'), unit=units,
654 progress = self._repo.ui.makeprogress(_('bundling'), unit=units,
651 total=(len(revs) - 1))
655 total=(len(revs) - 1))
652 for r in pycompat.xrange(len(revs) - 1):
656 for r in pycompat.xrange(len(revs) - 1):
653 if progress:
657 if progress:
654 progress.update(r + 1)
658 progress.update(r + 1)
655 prev, curr = revs[r], revs[r + 1]
659 prev, curr = revs[r], revs[r + 1]
656 linknode = lookup(store.node(curr))
660 linknode = lookup(store.node(curr))
657 for c in self._revchunk(store, curr, prev, linknode):
661 for c in self._revchunk(store, curr, prev, linknode):
658 yield c
662 yield c
659
663
660 if progress:
664 if progress:
661 progress.complete()
665 progress.complete()
662 yield self._close()
666 yield self._close()
663
667
664 # filter any nodes that claim to be part of the known set
668 # filter any nodes that claim to be part of the known set
665 def _prune(self, store, missing, commonrevs):
669 def _prune(self, store, missing, commonrevs):
666 # TODO this violates storage abstraction for manifests.
670 # TODO this violates storage abstraction for manifests.
667 if isinstance(store, manifest.manifestrevlog):
671 if isinstance(store, manifest.manifestrevlog):
668 if not self._filematcher.visitdir(store._dir[:-1] or '.'):
672 if not self._filematcher.visitdir(store._dir[:-1] or '.'):
669 return []
673 return []
670
674
671 rr, rl = store.rev, store.linkrev
675 rr, rl = store.rev, store.linkrev
672 return [n for n in missing if rl(rr(n)) not in commonrevs]
676 return [n for n in missing if rl(rr(n)) not in commonrevs]
673
677
674 def _packmanifests(self, dir, mfnodes, lookuplinknode):
678 def _packmanifests(self, dir, mfnodes, lookuplinknode):
675 """Pack flat manifests into a changegroup stream."""
679 """Pack flat manifests into a changegroup stream."""
676 assert not dir
680 assert not dir
677 for chunk in self.group(mfnodes, self._repo.manifestlog._revlog,
681 for chunk in self.group(mfnodes, self._repo.manifestlog._revlog,
678 lookuplinknode, units=_('manifests')):
682 lookuplinknode, units=_('manifests')):
679 yield chunk
683 yield chunk
680
684
681 def _packtreemanifests(self, dir, mfnodes, lookuplinknode):
685 def _packtreemanifests(self, dir, mfnodes, lookuplinknode):
682 """Version of _packmanifests that operates on directory manifests.
686 """Version of _packmanifests that operates on directory manifests.
683
687
684 Encodes the directory name in the output so multiple manifests
688 Encodes the directory name in the output so multiple manifests
685 can be sent.
689 can be sent.
686 """
690 """
687 assert self.version == b'03'
691 assert self.version == b'03'
688
692
689 if dir:
693 if dir:
690 yield self._fileheader(dir)
694 yield self._fileheader(dir)
691
695
692 # TODO violates storage abstractions by assuming revlogs.
696 # TODO violates storage abstractions by assuming revlogs.
693 dirlog = self._repo.manifestlog._revlog.dirlog(dir)
697 dirlog = self._repo.manifestlog._revlog.dirlog(dir)
694 for chunk in self.group(mfnodes, dirlog, lookuplinknode,
698 for chunk in self.group(mfnodes, dirlog, lookuplinknode,
695 units=_('manifests')):
699 units=_('manifests')):
696 yield chunk
700 yield chunk
697
701
698 def generate(self, commonrevs, clnodes, fastpathlinkrev, source):
702 def generate(self, commonrevs, clnodes, fastpathlinkrev, source):
699 '''yield a sequence of changegroup chunks (strings)'''
703 '''yield a sequence of changegroup chunks (strings)'''
700 repo = self._repo
704 repo = self._repo
701 cl = repo.changelog
705 cl = repo.changelog
702
706
703 clrevorder = {}
707 clrevorder = {}
704 mfs = {} # needed manifests
708 mfs = {} # needed manifests
705 fnodes = {} # needed file nodes
709 fnodes = {} # needed file nodes
706 mfl = repo.manifestlog
710 mfl = repo.manifestlog
707 # TODO violates storage abstraction.
711 # TODO violates storage abstraction.
708 mfrevlog = mfl._revlog
712 mfrevlog = mfl._revlog
709 changedfiles = set()
713 changedfiles = set()
710
714
711 ellipsesmode = util.safehasattr(self, '_full_nodes')
715 ellipsesmode = util.safehasattr(self, '_full_nodes')
712
716
713 # Callback for the changelog, used to collect changed files and
717 # Callback for the changelog, used to collect changed files and
714 # manifest nodes.
718 # manifest nodes.
715 # Returns the linkrev node (identity in the changelog case).
719 # Returns the linkrev node (identity in the changelog case).
716 def lookupcl(x):
720 def lookupcl(x):
717 c = cl.read(x)
721 c = cl.read(x)
718 clrevorder[x] = len(clrevorder)
722 clrevorder[x] = len(clrevorder)
719
723
720 if ellipsesmode:
724 if ellipsesmode:
721 # Only update mfs if x is going to be sent. Otherwise we
725 # Only update mfs if x is going to be sent. Otherwise we
722 # end up with bogus linkrevs specified for manifests and
726 # end up with bogus linkrevs specified for manifests and
723 # we skip some manifest nodes that we should otherwise
727 # we skip some manifest nodes that we should otherwise
724 # have sent.
728 # have sent.
725 if (x in self._full_nodes
729 if (x in self._full_nodes
726 or cl.rev(x) in self._precomputed_ellipsis):
730 or cl.rev(x) in self._precomputed_ellipsis):
727 n = c[0]
731 n = c[0]
728 # Record the first changeset introducing this manifest
732 # Record the first changeset introducing this manifest
729 # version.
733 # version.
730 mfs.setdefault(n, x)
734 mfs.setdefault(n, x)
731 # Set this narrow-specific dict so we have the lowest
735 # Set this narrow-specific dict so we have the lowest
732 # manifest revnum to look up for this cl revnum. (Part of
736 # manifest revnum to look up for this cl revnum. (Part of
733 # mapping changelog ellipsis parents to manifest ellipsis
737 # mapping changelog ellipsis parents to manifest ellipsis
734 # parents)
738 # parents)
735 self._next_clrev_to_localrev.setdefault(cl.rev(x),
739 self._next_clrev_to_localrev.setdefault(cl.rev(x),
736 mfrevlog.rev(n))
740 mfrevlog.rev(n))
737 # We can't trust the changed files list in the changeset if the
741 # We can't trust the changed files list in the changeset if the
738 # client requested a shallow clone.
742 # client requested a shallow clone.
739 if self._is_shallow:
743 if self._isshallow:
740 changedfiles.update(mfl[c[0]].read().keys())
744 changedfiles.update(mfl[c[0]].read().keys())
741 else:
745 else:
742 changedfiles.update(c[3])
746 changedfiles.update(c[3])
743 else:
747 else:
744
748
745 n = c[0]
749 n = c[0]
746 # record the first changeset introducing this manifest version
750 # record the first changeset introducing this manifest version
747 mfs.setdefault(n, x)
751 mfs.setdefault(n, x)
748 # Record a complete list of potentially-changed files in
752 # Record a complete list of potentially-changed files in
749 # this manifest.
753 # this manifest.
750 changedfiles.update(c[3])
754 changedfiles.update(c[3])
751
755
752 return x
756 return x
753
757
754 self._verbosenote(_('uncompressed size of bundle content:\n'))
758 self._verbosenote(_('uncompressed size of bundle content:\n'))
755 size = 0
759 size = 0
756 for chunk in self.group(clnodes, cl, lookupcl, units=_('changesets')):
760 for chunk in self.group(clnodes, cl, lookupcl, units=_('changesets')):
757 size += len(chunk)
761 size += len(chunk)
758 yield chunk
762 yield chunk
759 self._verbosenote(_('%8.i (changelog)\n') % size)
763 self._verbosenote(_('%8.i (changelog)\n') % size)
760
764
761 # We need to make sure that the linkrev in the changegroup refers to
765 # We need to make sure that the linkrev in the changegroup refers to
762 # the first changeset that introduced the manifest or file revision.
766 # the first changeset that introduced the manifest or file revision.
763 # The fastpath is usually safer than the slowpath, because the filelogs
767 # The fastpath is usually safer than the slowpath, because the filelogs
764 # are walked in revlog order.
768 # are walked in revlog order.
765 #
769 #
766 # When taking the slowpath with reorder=None and the manifest revlog
770 # When taking the slowpath with reorder=None and the manifest revlog
767 # uses generaldelta, the manifest may be walked in the "wrong" order.
771 # uses generaldelta, the manifest may be walked in the "wrong" order.
768 # Without 'clrevorder', we would get an incorrect linkrev (see fix in
772 # Without 'clrevorder', we would get an incorrect linkrev (see fix in
769 # cc0ff93d0c0c).
773 # cc0ff93d0c0c).
770 #
774 #
771 # When taking the fastpath, we are only vulnerable to reordering
775 # When taking the fastpath, we are only vulnerable to reordering
772 # of the changelog itself. The changelog never uses generaldelta, so
776 # of the changelog itself. The changelog never uses generaldelta, so
773 # it is only reordered when reorder=True. To handle this case, we
777 # it is only reordered when reorder=True. To handle this case, we
774 # simply take the slowpath, which already has the 'clrevorder' logic.
778 # simply take the slowpath, which already has the 'clrevorder' logic.
775 # This was also fixed in cc0ff93d0c0c.
779 # This was also fixed in cc0ff93d0c0c.
776 fastpathlinkrev = fastpathlinkrev and not self._reorder
780 fastpathlinkrev = fastpathlinkrev and not self._reorder
777 # Treemanifests don't work correctly with fastpathlinkrev
781 # Treemanifests don't work correctly with fastpathlinkrev
778 # either, because we don't discover which directory nodes to
782 # either, because we don't discover which directory nodes to
779 # send along with files. This could probably be fixed.
783 # send along with files. This could probably be fixed.
780 fastpathlinkrev = fastpathlinkrev and (
784 fastpathlinkrev = fastpathlinkrev and (
781 'treemanifest' not in repo.requirements)
785 'treemanifest' not in repo.requirements)
782
786
783 for chunk in self.generatemanifests(commonrevs, clrevorder,
787 for chunk in self.generatemanifests(commonrevs, clrevorder,
784 fastpathlinkrev, mfs, fnodes, source):
788 fastpathlinkrev, mfs, fnodes, source):
785 yield chunk
789 yield chunk
786
790
787 if ellipsesmode:
791 if ellipsesmode:
788 mfdicts = None
792 mfdicts = None
789 if self._is_shallow:
793 if self._isshallow:
790 mfdicts = [(self._repo.manifestlog[n].read(), lr)
794 mfdicts = [(self._repo.manifestlog[n].read(), lr)
791 for (n, lr) in mfs.iteritems()]
795 for (n, lr) in mfs.iteritems()]
792
796
793 mfs.clear()
797 mfs.clear()
794 clrevs = set(cl.rev(x) for x in clnodes)
798 clrevs = set(cl.rev(x) for x in clnodes)
795
799
796 if not fastpathlinkrev:
800 if not fastpathlinkrev:
797 def linknodes(unused, fname):
801 def linknodes(unused, fname):
798 return fnodes.get(fname, {})
802 return fnodes.get(fname, {})
799 else:
803 else:
800 cln = cl.node
804 cln = cl.node
801 def linknodes(filerevlog, fname):
805 def linknodes(filerevlog, fname):
802 llr = filerevlog.linkrev
806 llr = filerevlog.linkrev
803 fln = filerevlog.node
807 fln = filerevlog.node
804 revs = ((r, llr(r)) for r in filerevlog)
808 revs = ((r, llr(r)) for r in filerevlog)
805 return dict((fln(r), cln(lr)) for r, lr in revs if lr in clrevs)
809 return dict((fln(r), cln(lr)) for r, lr in revs if lr in clrevs)
806
810
807 if ellipsesmode:
811 if ellipsesmode:
808 # We need to pass the mfdicts variable down into
812 # We need to pass the mfdicts variable down into
809 # generatefiles(), but more than one command might have
813 # generatefiles(), but more than one command might have
810 # wrapped generatefiles so we can't modify the function
814 # wrapped generatefiles so we can't modify the function
811 # signature. Instead, we pass the data to ourselves using an
815 # signature. Instead, we pass the data to ourselves using an
812 # instance attribute. I'm sorry.
816 # instance attribute. I'm sorry.
813 self._mfdicts = mfdicts
817 self._mfdicts = mfdicts
814
818
815 for chunk in self.generatefiles(changedfiles, linknodes, commonrevs,
819 for chunk in self.generatefiles(changedfiles, linknodes, commonrevs,
816 source):
820 source):
817 yield chunk
821 yield chunk
818
822
819 yield self._close()
823 yield self._close()
820
824
821 if clnodes:
825 if clnodes:
822 repo.hook('outgoing', node=hex(clnodes[0]), source=source)
826 repo.hook('outgoing', node=hex(clnodes[0]), source=source)
823
827
824 def generatemanifests(self, commonrevs, clrevorder, fastpathlinkrev, mfs,
828 def generatemanifests(self, commonrevs, clrevorder, fastpathlinkrev, mfs,
825 fnodes, source):
829 fnodes, source):
826 """Returns an iterator of changegroup chunks containing manifests.
830 """Returns an iterator of changegroup chunks containing manifests.
827
831
828 `source` is unused here, but is used by extensions like remotefilelog to
832 `source` is unused here, but is used by extensions like remotefilelog to
829 change what is sent based in pulls vs pushes, etc.
833 change what is sent based in pulls vs pushes, etc.
830 """
834 """
831 repo = self._repo
835 repo = self._repo
832 mfl = repo.manifestlog
836 mfl = repo.manifestlog
833 dirlog = mfl._revlog.dirlog
837 dirlog = mfl._revlog.dirlog
834 tmfnodes = {'': mfs}
838 tmfnodes = {'': mfs}
835
839
836 # Callback for the manifest, used to collect linkrevs for filelog
840 # Callback for the manifest, used to collect linkrevs for filelog
837 # revisions.
841 # revisions.
838 # Returns the linkrev node (collected in lookupcl).
842 # Returns the linkrev node (collected in lookupcl).
839 def makelookupmflinknode(dir, nodes):
843 def makelookupmflinknode(dir, nodes):
840 if fastpathlinkrev:
844 if fastpathlinkrev:
841 assert not dir
845 assert not dir
842 return mfs.__getitem__
846 return mfs.__getitem__
843
847
844 def lookupmflinknode(x):
848 def lookupmflinknode(x):
845 """Callback for looking up the linknode for manifests.
849 """Callback for looking up the linknode for manifests.
846
850
847 Returns the linkrev node for the specified manifest.
851 Returns the linkrev node for the specified manifest.
848
852
849 SIDE EFFECT:
853 SIDE EFFECT:
850
854
851 1) fclnodes gets populated with the list of relevant
855 1) fclnodes gets populated with the list of relevant
852 file nodes if we're not using fastpathlinkrev
856 file nodes if we're not using fastpathlinkrev
853 2) When treemanifests are in use, collects treemanifest nodes
857 2) When treemanifests are in use, collects treemanifest nodes
854 to send
858 to send
855
859
856 Note that this means manifests must be completely sent to
860 Note that this means manifests must be completely sent to
857 the client before you can trust the list of files and
861 the client before you can trust the list of files and
858 treemanifests to send.
862 treemanifests to send.
859 """
863 """
860 clnode = nodes[x]
864 clnode = nodes[x]
861 mdata = mfl.get(dir, x).readfast(shallow=True)
865 mdata = mfl.get(dir, x).readfast(shallow=True)
862 for p, n, fl in mdata.iterentries():
866 for p, n, fl in mdata.iterentries():
863 if fl == 't': # subdirectory manifest
867 if fl == 't': # subdirectory manifest
864 subdir = dir + p + '/'
868 subdir = dir + p + '/'
865 tmfclnodes = tmfnodes.setdefault(subdir, {})
869 tmfclnodes = tmfnodes.setdefault(subdir, {})
866 tmfclnode = tmfclnodes.setdefault(n, clnode)
870 tmfclnode = tmfclnodes.setdefault(n, clnode)
867 if clrevorder[clnode] < clrevorder[tmfclnode]:
871 if clrevorder[clnode] < clrevorder[tmfclnode]:
868 tmfclnodes[n] = clnode
872 tmfclnodes[n] = clnode
869 else:
873 else:
870 f = dir + p
874 f = dir + p
871 fclnodes = fnodes.setdefault(f, {})
875 fclnodes = fnodes.setdefault(f, {})
872 fclnode = fclnodes.setdefault(n, clnode)
876 fclnode = fclnodes.setdefault(n, clnode)
873 if clrevorder[clnode] < clrevorder[fclnode]:
877 if clrevorder[clnode] < clrevorder[fclnode]:
874 fclnodes[n] = clnode
878 fclnodes[n] = clnode
875 return clnode
879 return clnode
876 return lookupmflinknode
880 return lookupmflinknode
877
881
878 fn = (self._packtreemanifests if self._sendtreemanifests
882 fn = (self._packtreemanifests if self._sendtreemanifests
879 else self._packmanifests)
883 else self._packmanifests)
880 size = 0
884 size = 0
881 while tmfnodes:
885 while tmfnodes:
882 dir, nodes = tmfnodes.popitem()
886 dir, nodes = tmfnodes.popitem()
883 prunednodes = self._prune(dirlog(dir), nodes, commonrevs)
887 prunednodes = self._prune(dirlog(dir), nodes, commonrevs)
884 if not dir or prunednodes:
888 if not dir or prunednodes:
885 for x in fn(dir, prunednodes, makelookupmflinknode(dir, nodes)):
889 for x in fn(dir, prunednodes, makelookupmflinknode(dir, nodes)):
886 size += len(x)
890 size += len(x)
887 yield x
891 yield x
888 self._verbosenote(_('%8.i (manifests)\n') % size)
892 self._verbosenote(_('%8.i (manifests)\n') % size)
889 yield self._manifestsend
893 yield self._manifestsend
890
894
891 # The 'source' parameter is useful for extensions
895 # The 'source' parameter is useful for extensions
892 def generatefiles(self, changedfiles, linknodes, commonrevs, source):
896 def generatefiles(self, changedfiles, linknodes, commonrevs, source):
893 changedfiles = list(filter(self._filematcher, changedfiles))
897 changedfiles = list(filter(self._filematcher, changedfiles))
894
898
895 if getattr(self, '_is_shallow', False):
899 if self._isshallow:
896 # See comment in generate() for why this sadness is a thing.
900 # See comment in generate() for why this sadness is a thing.
897 mfdicts = self._mfdicts
901 mfdicts = self._mfdicts
898 del self._mfdicts
902 del self._mfdicts
899 # In a shallow clone, the linknodes callback needs to also include
903 # In a shallow clone, the linknodes callback needs to also include
900 # those file nodes that are in the manifests we sent but weren't
904 # those file nodes that are in the manifests we sent but weren't
901 # introduced by those manifests.
905 # introduced by those manifests.
902 commonctxs = [self._repo[c] for c in commonrevs]
906 commonctxs = [self._repo[c] for c in commonrevs]
903 oldlinknodes = linknodes
907 oldlinknodes = linknodes
904 clrev = self._repo.changelog.rev
908 clrev = self._repo.changelog.rev
905
909
906 # Defining this function has a side-effect of overriding the
910 # Defining this function has a side-effect of overriding the
907 # function of the same name that was passed in as an argument.
911 # function of the same name that was passed in as an argument.
908 # TODO have caller pass in appropriate function.
912 # TODO have caller pass in appropriate function.
909 def linknodes(flog, fname):
913 def linknodes(flog, fname):
910 for c in commonctxs:
914 for c in commonctxs:
911 try:
915 try:
912 fnode = c.filenode(fname)
916 fnode = c.filenode(fname)
913 self._clrev_to_localrev[c.rev()] = flog.rev(fnode)
917 self._clrev_to_localrev[c.rev()] = flog.rev(fnode)
914 except error.ManifestLookupError:
918 except error.ManifestLookupError:
915 pass
919 pass
916 links = oldlinknodes(flog, fname)
920 links = oldlinknodes(flog, fname)
917 if len(links) != len(mfdicts):
921 if len(links) != len(mfdicts):
918 for mf, lr in mfdicts:
922 for mf, lr in mfdicts:
919 fnode = mf.get(fname, None)
923 fnode = mf.get(fname, None)
920 if fnode in links:
924 if fnode in links:
921 links[fnode] = min(links[fnode], lr, key=clrev)
925 links[fnode] = min(links[fnode], lr, key=clrev)
922 elif fnode:
926 elif fnode:
923 links[fnode] = lr
927 links[fnode] = lr
924 return links
928 return links
925
929
926 return self._generatefiles(changedfiles, linknodes, commonrevs, source)
930 return self._generatefiles(changedfiles, linknodes, commonrevs, source)
927
931
928 def _generatefiles(self, changedfiles, linknodes, commonrevs, source):
932 def _generatefiles(self, changedfiles, linknodes, commonrevs, source):
929 repo = self._repo
933 repo = self._repo
930 progress = repo.ui.makeprogress(_('bundling'), unit=_('files'),
934 progress = repo.ui.makeprogress(_('bundling'), unit=_('files'),
931 total=len(changedfiles))
935 total=len(changedfiles))
932 for i, fname in enumerate(sorted(changedfiles)):
936 for i, fname in enumerate(sorted(changedfiles)):
933 filerevlog = repo.file(fname)
937 filerevlog = repo.file(fname)
934 if not filerevlog:
938 if not filerevlog:
935 raise error.Abort(_("empty or missing file data for %s") %
939 raise error.Abort(_("empty or missing file data for %s") %
936 fname)
940 fname)
937
941
938 linkrevnodes = linknodes(filerevlog, fname)
942 linkrevnodes = linknodes(filerevlog, fname)
939 # Lookup for filenodes, we collected the linkrev nodes above in the
943 # Lookup for filenodes, we collected the linkrev nodes above in the
940 # fastpath case and with lookupmf in the slowpath case.
944 # fastpath case and with lookupmf in the slowpath case.
941 def lookupfilelog(x):
945 def lookupfilelog(x):
942 return linkrevnodes[x]
946 return linkrevnodes[x]
943
947
944 filenodes = self._prune(filerevlog, linkrevnodes, commonrevs)
948 filenodes = self._prune(filerevlog, linkrevnodes, commonrevs)
945 if filenodes:
949 if filenodes:
946 progress.update(i + 1, item=fname)
950 progress.update(i + 1, item=fname)
947 h = self._fileheader(fname)
951 h = self._fileheader(fname)
948 size = len(h)
952 size = len(h)
949 yield h
953 yield h
950 for chunk in self.group(filenodes, filerevlog, lookupfilelog):
954 for chunk in self.group(filenodes, filerevlog, lookupfilelog):
951 size += len(chunk)
955 size += len(chunk)
952 yield chunk
956 yield chunk
953 self._verbosenote(_('%8.i %s\n') % (size, fname))
957 self._verbosenote(_('%8.i %s\n') % (size, fname))
954 progress.complete()
958 progress.complete()
955
959
956 def _deltaparent(self, store, rev, p1, p2, prev):
960 def _deltaparent(self, store, rev, p1, p2, prev):
957 if self._useprevdelta:
961 if self._useprevdelta:
958 if not store.candelta(prev, rev):
962 if not store.candelta(prev, rev):
959 raise error.ProgrammingError(
963 raise error.ProgrammingError(
960 'cg1 should not be used in this case')
964 'cg1 should not be used in this case')
961 return prev
965 return prev
962
966
963 # Narrow ellipses mode.
967 # Narrow ellipses mode.
964 if util.safehasattr(self, '_full_nodes'):
968 if util.safehasattr(self, '_full_nodes'):
965 # TODO: send better deltas when in narrow mode.
969 # TODO: send better deltas when in narrow mode.
966 #
970 #
967 # changegroup.group() loops over revisions to send,
971 # changegroup.group() loops over revisions to send,
968 # including revisions we'll skip. What this means is that
972 # including revisions we'll skip. What this means is that
969 # `prev` will be a potentially useless delta base for all
973 # `prev` will be a potentially useless delta base for all
970 # ellipsis nodes, as the client likely won't have it. In
974 # ellipsis nodes, as the client likely won't have it. In
971 # the future we should do bookkeeping about which nodes
975 # the future we should do bookkeeping about which nodes
972 # have been sent to the client, and try to be
976 # have been sent to the client, and try to be
973 # significantly smarter about delta bases. This is
977 # significantly smarter about delta bases. This is
974 # slightly tricky because this same code has to work for
978 # slightly tricky because this same code has to work for
975 # all revlogs, and we don't have the linkrev/linknode here.
979 # all revlogs, and we don't have the linkrev/linknode here.
976 return p1
980 return p1
977
981
978 dp = store.deltaparent(rev)
982 dp = store.deltaparent(rev)
979 if dp == nullrev and store.storedeltachains:
983 if dp == nullrev and store.storedeltachains:
980 # Avoid sending full revisions when delta parent is null. Pick prev
984 # Avoid sending full revisions when delta parent is null. Pick prev
981 # in that case. It's tempting to pick p1 in this case, as p1 will
985 # in that case. It's tempting to pick p1 in this case, as p1 will
982 # be smaller in the common case. However, computing a delta against
986 # be smaller in the common case. However, computing a delta against
983 # p1 may require resolving the raw text of p1, which could be
987 # p1 may require resolving the raw text of p1, which could be
984 # expensive. The revlog caches should have prev cached, meaning
988 # expensive. The revlog caches should have prev cached, meaning
985 # less CPU for changegroup generation. There is likely room to add
989 # less CPU for changegroup generation. There is likely room to add
986 # a flag and/or config option to control this behavior.
990 # a flag and/or config option to control this behavior.
987 base = prev
991 base = prev
988 elif dp == nullrev:
992 elif dp == nullrev:
989 # revlog is configured to use full snapshot for a reason,
993 # revlog is configured to use full snapshot for a reason,
990 # stick to full snapshot.
994 # stick to full snapshot.
991 base = nullrev
995 base = nullrev
992 elif dp not in (p1, p2, prev):
996 elif dp not in (p1, p2, prev):
993 # Pick prev when we can't be sure remote has the base revision.
997 # Pick prev when we can't be sure remote has the base revision.
994 return prev
998 return prev
995 else:
999 else:
996 base = dp
1000 base = dp
997
1001
998 if base != nullrev and not store.candelta(base, rev):
1002 if base != nullrev and not store.candelta(base, rev):
999 base = nullrev
1003 base = nullrev
1000
1004
1001 return base
1005 return base
1002
1006
1003 def _revchunk(self, store, rev, prev, linknode):
1007 def _revchunk(self, store, rev, prev, linknode):
1004 if util.safehasattr(self, '_full_nodes'):
1008 if util.safehasattr(self, '_full_nodes'):
1005 fn = self._revisiondeltanarrow
1009 fn = self._revisiondeltanarrow
1006 else:
1010 else:
1007 fn = self._revisiondeltanormal
1011 fn = self._revisiondeltanormal
1008
1012
1009 delta = fn(store, rev, prev, linknode)
1013 delta = fn(store, rev, prev, linknode)
1010 if not delta:
1014 if not delta:
1011 return
1015 return
1012
1016
1013 meta = self._builddeltaheader(delta)
1017 meta = self._builddeltaheader(delta)
1014 l = len(meta) + sum(len(x) for x in delta.deltachunks)
1018 l = len(meta) + sum(len(x) for x in delta.deltachunks)
1015
1019
1016 yield chunkheader(l)
1020 yield chunkheader(l)
1017 yield meta
1021 yield meta
1018 for x in delta.deltachunks:
1022 for x in delta.deltachunks:
1019 yield x
1023 yield x
1020
1024
1021 def _revisiondeltanormal(self, store, rev, prev, linknode):
1025 def _revisiondeltanormal(self, store, rev, prev, linknode):
1022 node = store.node(rev)
1026 node = store.node(rev)
1023 p1, p2 = store.parentrevs(rev)
1027 p1, p2 = store.parentrevs(rev)
1024 base = self._deltaparent(store, rev, p1, p2, prev)
1028 base = self._deltaparent(store, rev, p1, p2, prev)
1025
1029
1026 prefix = ''
1030 prefix = ''
1027 if store.iscensored(base) or store.iscensored(rev):
1031 if store.iscensored(base) or store.iscensored(rev):
1028 try:
1032 try:
1029 delta = store.revision(node, raw=True)
1033 delta = store.revision(node, raw=True)
1030 except error.CensoredNodeError as e:
1034 except error.CensoredNodeError as e:
1031 delta = e.tombstone
1035 delta = e.tombstone
1032 if base == nullrev:
1036 if base == nullrev:
1033 prefix = mdiff.trivialdiffheader(len(delta))
1037 prefix = mdiff.trivialdiffheader(len(delta))
1034 else:
1038 else:
1035 baselen = store.rawsize(base)
1039 baselen = store.rawsize(base)
1036 prefix = mdiff.replacediffheader(baselen, len(delta))
1040 prefix = mdiff.replacediffheader(baselen, len(delta))
1037 elif base == nullrev:
1041 elif base == nullrev:
1038 delta = store.revision(node, raw=True)
1042 delta = store.revision(node, raw=True)
1039 prefix = mdiff.trivialdiffheader(len(delta))
1043 prefix = mdiff.trivialdiffheader(len(delta))
1040 else:
1044 else:
1041 delta = store.revdiff(base, rev)
1045 delta = store.revdiff(base, rev)
1042 p1n, p2n = store.parents(node)
1046 p1n, p2n = store.parents(node)
1043
1047
1044 return revisiondelta(
1048 return revisiondelta(
1045 node=node,
1049 node=node,
1046 p1node=p1n,
1050 p1node=p1n,
1047 p2node=p2n,
1051 p2node=p2n,
1048 basenode=store.node(base),
1052 basenode=store.node(base),
1049 linknode=linknode,
1053 linknode=linknode,
1050 flags=store.flags(rev),
1054 flags=store.flags(rev),
1051 deltachunks=(prefix, delta),
1055 deltachunks=(prefix, delta),
1052 )
1056 )
1053
1057
1054 def _revisiondeltanarrow(self, store, rev, prev, linknode):
1058 def _revisiondeltanarrow(self, store, rev, prev, linknode):
1055 # build up some mapping information that's useful later. See
1059 # build up some mapping information that's useful later. See
1056 # the local() nested function below.
1060 # the local() nested function below.
1057 if not self._changelog_done:
1061 if not self._changelog_done:
1058 self._clnode_to_rev[linknode] = rev
1062 self._clnode_to_rev[linknode] = rev
1059 linkrev = rev
1063 linkrev = rev
1060 self._clrev_to_localrev[linkrev] = rev
1064 self._clrev_to_localrev[linkrev] = rev
1061 else:
1065 else:
1062 linkrev = self._clnode_to_rev[linknode]
1066 linkrev = self._clnode_to_rev[linknode]
1063 self._clrev_to_localrev[linkrev] = rev
1067 self._clrev_to_localrev[linkrev] = rev
1064
1068
1065 # This is a node to send in full, because the changeset it
1069 # This is a node to send in full, because the changeset it
1066 # corresponds to was a full changeset.
1070 # corresponds to was a full changeset.
1067 if linknode in self._full_nodes:
1071 if linknode in self._full_nodes:
1068 return self._revisiondeltanormal(store, rev, prev, linknode)
1072 return self._revisiondeltanormal(store, rev, prev, linknode)
1069
1073
1070 # At this point, a node can either be one we should skip or an
1074 # At this point, a node can either be one we should skip or an
1071 # ellipsis. If it's not an ellipsis, bail immediately.
1075 # ellipsis. If it's not an ellipsis, bail immediately.
1072 if linkrev not in self._precomputed_ellipsis:
1076 if linkrev not in self._precomputed_ellipsis:
1073 return
1077 return
1074
1078
1075 linkparents = self._precomputed_ellipsis[linkrev]
1079 linkparents = self._precomputed_ellipsis[linkrev]
1076 def local(clrev):
1080 def local(clrev):
1077 """Turn a changelog revnum into a local revnum.
1081 """Turn a changelog revnum into a local revnum.
1078
1082
1079 The ellipsis dag is stored as revnums on the changelog,
1083 The ellipsis dag is stored as revnums on the changelog,
1080 but when we're producing ellipsis entries for
1084 but when we're producing ellipsis entries for
1081 non-changelog revlogs, we need to turn those numbers into
1085 non-changelog revlogs, we need to turn those numbers into
1082 something local. This does that for us, and during the
1086 something local. This does that for us, and during the
1083 changelog sending phase will also expand the stored
1087 changelog sending phase will also expand the stored
1084 mappings as needed.
1088 mappings as needed.
1085 """
1089 """
1086 if clrev == nullrev:
1090 if clrev == nullrev:
1087 return nullrev
1091 return nullrev
1088
1092
1089 if not self._changelog_done:
1093 if not self._changelog_done:
1090 # If we're doing the changelog, it's possible that we
1094 # If we're doing the changelog, it's possible that we
1091 # have a parent that is already on the client, and we
1095 # have a parent that is already on the client, and we
1092 # need to store some extra mapping information so that
1096 # need to store some extra mapping information so that
1093 # our contained ellipsis nodes will be able to resolve
1097 # our contained ellipsis nodes will be able to resolve
1094 # their parents.
1098 # their parents.
1095 if clrev not in self._clrev_to_localrev:
1099 if clrev not in self._clrev_to_localrev:
1096 clnode = store.node(clrev)
1100 clnode = store.node(clrev)
1097 self._clnode_to_rev[clnode] = clrev
1101 self._clnode_to_rev[clnode] = clrev
1098 return clrev
1102 return clrev
1099
1103
1100 # Walk the ellipsis-ized changelog breadth-first looking for a
1104 # Walk the ellipsis-ized changelog breadth-first looking for a
1101 # change that has been linked from the current revlog.
1105 # change that has been linked from the current revlog.
1102 #
1106 #
1103 # For a flat manifest revlog only a single step should be necessary
1107 # For a flat manifest revlog only a single step should be necessary
1104 # as all relevant changelog entries are relevant to the flat
1108 # as all relevant changelog entries are relevant to the flat
1105 # manifest.
1109 # manifest.
1106 #
1110 #
1107 # For a filelog or tree manifest dirlog however not every changelog
1111 # For a filelog or tree manifest dirlog however not every changelog
1108 # entry will have been relevant, so we need to skip some changelog
1112 # entry will have been relevant, so we need to skip some changelog
1109 # nodes even after ellipsis-izing.
1113 # nodes even after ellipsis-izing.
1110 walk = [clrev]
1114 walk = [clrev]
1111 while walk:
1115 while walk:
1112 p = walk[0]
1116 p = walk[0]
1113 walk = walk[1:]
1117 walk = walk[1:]
1114 if p in self._clrev_to_localrev:
1118 if p in self._clrev_to_localrev:
1115 return self._clrev_to_localrev[p]
1119 return self._clrev_to_localrev[p]
1116 elif p in self._full_nodes:
1120 elif p in self._full_nodes:
1117 walk.extend([pp for pp in self._repo.changelog.parentrevs(p)
1121 walk.extend([pp for pp in self._repo.changelog.parentrevs(p)
1118 if pp != nullrev])
1122 if pp != nullrev])
1119 elif p in self._precomputed_ellipsis:
1123 elif p in self._precomputed_ellipsis:
1120 walk.extend([pp for pp in self._precomputed_ellipsis[p]
1124 walk.extend([pp for pp in self._precomputed_ellipsis[p]
1121 if pp != nullrev])
1125 if pp != nullrev])
1122 else:
1126 else:
1123 # In this case, we've got an ellipsis with parents
1127 # In this case, we've got an ellipsis with parents
1124 # outside the current bundle (likely an
1128 # outside the current bundle (likely an
1125 # incremental pull). We "know" that we can use the
1129 # incremental pull). We "know" that we can use the
1126 # value of this same revlog at whatever revision
1130 # value of this same revlog at whatever revision
1127 # is pointed to by linknode. "Know" is in scare
1131 # is pointed to by linknode. "Know" is in scare
1128 # quotes because I haven't done enough examination
1132 # quotes because I haven't done enough examination
1129 # of edge cases to convince myself this is really
1133 # of edge cases to convince myself this is really
1130 # a fact - it works for all the (admittedly
1134 # a fact - it works for all the (admittedly
1131 # thorough) cases in our testsuite, but I would be
1135 # thorough) cases in our testsuite, but I would be
1132 # somewhat unsurprised to find a case in the wild
1136 # somewhat unsurprised to find a case in the wild
1133 # where this breaks down a bit. That said, I don't
1137 # where this breaks down a bit. That said, I don't
1134 # know if it would hurt anything.
1138 # know if it would hurt anything.
1135 for i in pycompat.xrange(rev, 0, -1):
1139 for i in pycompat.xrange(rev, 0, -1):
1136 if store.linkrev(i) == clrev:
1140 if store.linkrev(i) == clrev:
1137 return i
1141 return i
1138 # We failed to resolve a parent for this node, so
1142 # We failed to resolve a parent for this node, so
1139 # we crash the changegroup construction.
1143 # we crash the changegroup construction.
1140 raise error.Abort(
1144 raise error.Abort(
1141 'unable to resolve parent while packing %r %r'
1145 'unable to resolve parent while packing %r %r'
1142 ' for changeset %r' % (store.indexfile, rev, clrev))
1146 ' for changeset %r' % (store.indexfile, rev, clrev))
1143
1147
1144 return nullrev
1148 return nullrev
1145
1149
1146 if not linkparents or (
1150 if not linkparents or (
1147 store.parentrevs(rev) == (nullrev, nullrev)):
1151 store.parentrevs(rev) == (nullrev, nullrev)):
1148 p1, p2 = nullrev, nullrev
1152 p1, p2 = nullrev, nullrev
1149 elif len(linkparents) == 1:
1153 elif len(linkparents) == 1:
1150 p1, = sorted(local(p) for p in linkparents)
1154 p1, = sorted(local(p) for p in linkparents)
1151 p2 = nullrev
1155 p2 = nullrev
1152 else:
1156 else:
1153 p1, p2 = sorted(local(p) for p in linkparents)
1157 p1, p2 = sorted(local(p) for p in linkparents)
1154
1158
1155 n = store.node(rev)
1159 n = store.node(rev)
1156 p1n, p2n = store.node(p1), store.node(p2)
1160 p1n, p2n = store.node(p1), store.node(p2)
1157 flags = store.flags(rev)
1161 flags = store.flags(rev)
1158 flags |= revlog.REVIDX_ELLIPSIS
1162 flags |= revlog.REVIDX_ELLIPSIS
1159
1163
1160 # TODO: try and actually send deltas for ellipsis data blocks
1164 # TODO: try and actually send deltas for ellipsis data blocks
1161 data = store.revision(n)
1165 data = store.revision(n)
1162 diffheader = mdiff.trivialdiffheader(len(data))
1166 diffheader = mdiff.trivialdiffheader(len(data))
1163
1167
1164 return revisiondelta(
1168 return revisiondelta(
1165 node=n,
1169 node=n,
1166 p1node=p1n,
1170 p1node=p1n,
1167 p2node=p2n,
1171 p2node=p2n,
1168 basenode=nullid,
1172 basenode=nullid,
1169 linknode=linknode,
1173 linknode=linknode,
1170 flags=flags,
1174 flags=flags,
1171 deltachunks=(diffheader, data),
1175 deltachunks=(diffheader, data),
1172 )
1176 )
1173
1177
1174 def _makecg1packer(repo, filematcher, bundlecaps):
1178 def _makecg1packer(repo, filematcher, bundlecaps, shallow=False):
1175 builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack(
1179 builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack(
1176 d.node, d.p1node, d.p2node, d.linknode)
1180 d.node, d.p1node, d.p2node, d.linknode)
1177
1181
1178 return cgpacker(repo, filematcher, b'01',
1182 return cgpacker(repo, filematcher, b'01',
1179 useprevdelta=True,
1183 useprevdelta=True,
1180 allowreorder=None,
1184 allowreorder=None,
1181 builddeltaheader=builddeltaheader,
1185 builddeltaheader=builddeltaheader,
1182 manifestsend=b'',
1186 manifestsend=b'',
1183 sendtreemanifests=False,
1187 sendtreemanifests=False,
1184 bundlecaps=bundlecaps)
1188 bundlecaps=bundlecaps,
1189 shallow=shallow)
1185
1190
1186 def _makecg2packer(repo, filematcher, bundlecaps):
1191 def _makecg2packer(repo, filematcher, bundlecaps, shallow=False):
1187 builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack(
1192 builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack(
1188 d.node, d.p1node, d.p2node, d.basenode, d.linknode)
1193 d.node, d.p1node, d.p2node, d.basenode, d.linknode)
1189
1194
1190 # Since generaldelta is directly supported by cg2, reordering
1195 # Since generaldelta is directly supported by cg2, reordering
1191 # generally doesn't help, so we disable it by default (treating
1196 # generally doesn't help, so we disable it by default (treating
1192 # bundle.reorder=auto just like bundle.reorder=False).
1197 # bundle.reorder=auto just like bundle.reorder=False).
1193 return cgpacker(repo, filematcher, b'02',
1198 return cgpacker(repo, filematcher, b'02',
1194 useprevdelta=False,
1199 useprevdelta=False,
1195 allowreorder=False,
1200 allowreorder=False,
1196 builddeltaheader=builddeltaheader,
1201 builddeltaheader=builddeltaheader,
1197 manifestsend=b'',
1202 manifestsend=b'',
1198 sendtreemanifests=False,
1203 sendtreemanifests=False,
1199 bundlecaps=bundlecaps)
1204 bundlecaps=bundlecaps,
1205 shallow=shallow)
1200
1206
1201 def _makecg3packer(repo, filematcher, bundlecaps):
1207 def _makecg3packer(repo, filematcher, bundlecaps, shallow=False):
1202 builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack(
1208 builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack(
1203 d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags)
1209 d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags)
1204
1210
1205 return cgpacker(repo, filematcher, b'03',
1211 return cgpacker(repo, filematcher, b'03',
1206 useprevdelta=False,
1212 useprevdelta=False,
1207 allowreorder=False,
1213 allowreorder=False,
1208 builddeltaheader=builddeltaheader,
1214 builddeltaheader=builddeltaheader,
1209 manifestsend=closechunk(),
1215 manifestsend=closechunk(),
1210 sendtreemanifests=True,
1216 sendtreemanifests=True,
1211 bundlecaps=bundlecaps)
1217 bundlecaps=bundlecaps,
1218 shallow=shallow)
1212
1219
1213 _packermap = {'01': (_makecg1packer, cg1unpacker),
1220 _packermap = {'01': (_makecg1packer, cg1unpacker),
1214 # cg2 adds support for exchanging generaldelta
1221 # cg2 adds support for exchanging generaldelta
1215 '02': (_makecg2packer, cg2unpacker),
1222 '02': (_makecg2packer, cg2unpacker),
1216 # cg3 adds support for exchanging revlog flags and treemanifests
1223 # cg3 adds support for exchanging revlog flags and treemanifests
1217 '03': (_makecg3packer, cg3unpacker),
1224 '03': (_makecg3packer, cg3unpacker),
1218 }
1225 }
1219
1226
1220 def allsupportedversions(repo):
1227 def allsupportedversions(repo):
1221 versions = set(_packermap.keys())
1228 versions = set(_packermap.keys())
1222 if not (repo.ui.configbool('experimental', 'changegroup3') or
1229 if not (repo.ui.configbool('experimental', 'changegroup3') or
1223 repo.ui.configbool('experimental', 'treemanifest') or
1230 repo.ui.configbool('experimental', 'treemanifest') or
1224 'treemanifest' in repo.requirements):
1231 'treemanifest' in repo.requirements):
1225 versions.discard('03')
1232 versions.discard('03')
1226 return versions
1233 return versions
1227
1234
1228 # Changegroup versions that can be applied to the repo
1235 # Changegroup versions that can be applied to the repo
1229 def supportedincomingversions(repo):
1236 def supportedincomingversions(repo):
1230 return allsupportedversions(repo)
1237 return allsupportedversions(repo)
1231
1238
1232 # Changegroup versions that can be created from the repo
1239 # Changegroup versions that can be created from the repo
1233 def supportedoutgoingversions(repo):
1240 def supportedoutgoingversions(repo):
1234 versions = allsupportedversions(repo)
1241 versions = allsupportedversions(repo)
1235 if 'treemanifest' in repo.requirements:
1242 if 'treemanifest' in repo.requirements:
1236 # Versions 01 and 02 support only flat manifests and it's just too
1243 # Versions 01 and 02 support only flat manifests and it's just too
1237 # expensive to convert between the flat manifest and tree manifest on
1244 # expensive to convert between the flat manifest and tree manifest on
1238 # the fly. Since tree manifests are hashed differently, all of history
1245 # the fly. Since tree manifests are hashed differently, all of history
1239 # would have to be converted. Instead, we simply don't even pretend to
1246 # would have to be converted. Instead, we simply don't even pretend to
1240 # support versions 01 and 02.
1247 # support versions 01 and 02.
1241 versions.discard('01')
1248 versions.discard('01')
1242 versions.discard('02')
1249 versions.discard('02')
1243 if repository.NARROW_REQUIREMENT in repo.requirements:
1250 if repository.NARROW_REQUIREMENT in repo.requirements:
1244 # Versions 01 and 02 don't support revlog flags, and we need to
1251 # Versions 01 and 02 don't support revlog flags, and we need to
1245 # support that for stripping and unbundling to work.
1252 # support that for stripping and unbundling to work.
1246 versions.discard('01')
1253 versions.discard('01')
1247 versions.discard('02')
1254 versions.discard('02')
1248 if LFS_REQUIREMENT in repo.requirements:
1255 if LFS_REQUIREMENT in repo.requirements:
1249 # Versions 01 and 02 don't support revlog flags, and we need to
1256 # Versions 01 and 02 don't support revlog flags, and we need to
1250 # mark LFS entries with REVIDX_EXTSTORED.
1257 # mark LFS entries with REVIDX_EXTSTORED.
1251 versions.discard('01')
1258 versions.discard('01')
1252 versions.discard('02')
1259 versions.discard('02')
1253
1260
1254 return versions
1261 return versions
1255
1262
1256 def localversion(repo):
1263 def localversion(repo):
1257 # Finds the best version to use for bundles that are meant to be used
1264 # Finds the best version to use for bundles that are meant to be used
1258 # locally, such as those from strip and shelve, and temporary bundles.
1265 # locally, such as those from strip and shelve, and temporary bundles.
1259 return max(supportedoutgoingversions(repo))
1266 return max(supportedoutgoingversions(repo))
1260
1267
1261 def safeversion(repo):
1268 def safeversion(repo):
1262 # Finds the smallest version that it's safe to assume clients of the repo
1269 # Finds the smallest version that it's safe to assume clients of the repo
1263 # will support. For example, all hg versions that support generaldelta also
1270 # will support. For example, all hg versions that support generaldelta also
1264 # support changegroup 02.
1271 # support changegroup 02.
1265 versions = supportedoutgoingversions(repo)
1272 versions = supportedoutgoingversions(repo)
1266 if 'generaldelta' in repo.requirements:
1273 if 'generaldelta' in repo.requirements:
1267 versions.discard('01')
1274 versions.discard('01')
1268 assert versions
1275 assert versions
1269 return min(versions)
1276 return min(versions)
1270
1277
1271 def getbundler(version, repo, bundlecaps=None, filematcher=None):
1278 def getbundler(version, repo, bundlecaps=None, filematcher=None,
1279 shallow=False):
1272 assert version in supportedoutgoingversions(repo)
1280 assert version in supportedoutgoingversions(repo)
1273
1281
1274 if filematcher is None:
1282 if filematcher is None:
1275 filematcher = matchmod.alwaysmatcher(repo.root, '')
1283 filematcher = matchmod.alwaysmatcher(repo.root, '')
1276
1284
1277 if version == '01' and not filematcher.always():
1285 if version == '01' and not filematcher.always():
1278 raise error.ProgrammingError('version 01 changegroups do not support '
1286 raise error.ProgrammingError('version 01 changegroups do not support '
1279 'sparse file matchers')
1287 'sparse file matchers')
1280
1288
1281 # Requested files could include files not in the local store. So
1289 # Requested files could include files not in the local store. So
1282 # filter those out.
1290 # filter those out.
1283 filematcher = matchmod.intersectmatchers(repo.narrowmatch(),
1291 filematcher = matchmod.intersectmatchers(repo.narrowmatch(),
1284 filematcher)
1292 filematcher)
1285
1293
1286 fn = _packermap[version][0]
1294 fn = _packermap[version][0]
1287 return fn(repo, filematcher, bundlecaps)
1295 return fn(repo, filematcher, bundlecaps, shallow=shallow)
1288
1296
1289 def getunbundler(version, fh, alg, extras=None):
1297 def getunbundler(version, fh, alg, extras=None):
1290 return _packermap[version][1](fh, alg, extras=extras)
1298 return _packermap[version][1](fh, alg, extras=extras)
1291
1299
1292 def _changegroupinfo(repo, nodes, source):
1300 def _changegroupinfo(repo, nodes, source):
1293 if repo.ui.verbose or source == 'bundle':
1301 if repo.ui.verbose or source == 'bundle':
1294 repo.ui.status(_("%d changesets found\n") % len(nodes))
1302 repo.ui.status(_("%d changesets found\n") % len(nodes))
1295 if repo.ui.debugflag:
1303 if repo.ui.debugflag:
1296 repo.ui.debug("list of changesets:\n")
1304 repo.ui.debug("list of changesets:\n")
1297 for node in nodes:
1305 for node in nodes:
1298 repo.ui.debug("%s\n" % hex(node))
1306 repo.ui.debug("%s\n" % hex(node))
1299
1307
1300 def makechangegroup(repo, outgoing, version, source, fastpath=False,
1308 def makechangegroup(repo, outgoing, version, source, fastpath=False,
1301 bundlecaps=None):
1309 bundlecaps=None):
1302 cgstream = makestream(repo, outgoing, version, source,
1310 cgstream = makestream(repo, outgoing, version, source,
1303 fastpath=fastpath, bundlecaps=bundlecaps)
1311 fastpath=fastpath, bundlecaps=bundlecaps)
1304 return getunbundler(version, util.chunkbuffer(cgstream), None,
1312 return getunbundler(version, util.chunkbuffer(cgstream), None,
1305 {'clcount': len(outgoing.missing) })
1313 {'clcount': len(outgoing.missing) })
1306
1314
1307 def makestream(repo, outgoing, version, source, fastpath=False,
1315 def makestream(repo, outgoing, version, source, fastpath=False,
1308 bundlecaps=None, filematcher=None):
1316 bundlecaps=None, filematcher=None):
1309 bundler = getbundler(version, repo, bundlecaps=bundlecaps,
1317 bundler = getbundler(version, repo, bundlecaps=bundlecaps,
1310 filematcher=filematcher)
1318 filematcher=filematcher)
1311
1319
1312 repo = repo.unfiltered()
1320 repo = repo.unfiltered()
1313 commonrevs = outgoing.common
1321 commonrevs = outgoing.common
1314 csets = outgoing.missing
1322 csets = outgoing.missing
1315 heads = outgoing.missingheads
1323 heads = outgoing.missingheads
1316 # We go through the fast path if we get told to, or if all (unfiltered
1324 # We go through the fast path if we get told to, or if all (unfiltered
1317 # heads have been requested (since we then know there all linkrevs will
1325 # heads have been requested (since we then know there all linkrevs will
1318 # be pulled by the client).
1326 # be pulled by the client).
1319 heads.sort()
1327 heads.sort()
1320 fastpathlinkrev = fastpath or (
1328 fastpathlinkrev = fastpath or (
1321 repo.filtername is None and heads == sorted(repo.heads()))
1329 repo.filtername is None and heads == sorted(repo.heads()))
1322
1330
1323 repo.hook('preoutgoing', throw=True, source=source)
1331 repo.hook('preoutgoing', throw=True, source=source)
1324 _changegroupinfo(repo, csets, source)
1332 _changegroupinfo(repo, csets, source)
1325 return bundler.generate(commonrevs, csets, fastpathlinkrev, source)
1333 return bundler.generate(commonrevs, csets, fastpathlinkrev, source)
1326
1334
1327 def _addchangegroupfiles(repo, source, revmap, trp, expectedfiles, needfiles):
1335 def _addchangegroupfiles(repo, source, revmap, trp, expectedfiles, needfiles):
1328 revisions = 0
1336 revisions = 0
1329 files = 0
1337 files = 0
1330 progress = repo.ui.makeprogress(_('files'), unit=_('files'),
1338 progress = repo.ui.makeprogress(_('files'), unit=_('files'),
1331 total=expectedfiles)
1339 total=expectedfiles)
1332 for chunkdata in iter(source.filelogheader, {}):
1340 for chunkdata in iter(source.filelogheader, {}):
1333 files += 1
1341 files += 1
1334 f = chunkdata["filename"]
1342 f = chunkdata["filename"]
1335 repo.ui.debug("adding %s revisions\n" % f)
1343 repo.ui.debug("adding %s revisions\n" % f)
1336 progress.increment()
1344 progress.increment()
1337 fl = repo.file(f)
1345 fl = repo.file(f)
1338 o = len(fl)
1346 o = len(fl)
1339 try:
1347 try:
1340 deltas = source.deltaiter()
1348 deltas = source.deltaiter()
1341 if not fl.addgroup(deltas, revmap, trp):
1349 if not fl.addgroup(deltas, revmap, trp):
1342 raise error.Abort(_("received file revlog group is empty"))
1350 raise error.Abort(_("received file revlog group is empty"))
1343 except error.CensoredBaseError as e:
1351 except error.CensoredBaseError as e:
1344 raise error.Abort(_("received delta base is censored: %s") % e)
1352 raise error.Abort(_("received delta base is censored: %s") % e)
1345 revisions += len(fl) - o
1353 revisions += len(fl) - o
1346 if f in needfiles:
1354 if f in needfiles:
1347 needs = needfiles[f]
1355 needs = needfiles[f]
1348 for new in pycompat.xrange(o, len(fl)):
1356 for new in pycompat.xrange(o, len(fl)):
1349 n = fl.node(new)
1357 n = fl.node(new)
1350 if n in needs:
1358 if n in needs:
1351 needs.remove(n)
1359 needs.remove(n)
1352 else:
1360 else:
1353 raise error.Abort(
1361 raise error.Abort(
1354 _("received spurious file revlog entry"))
1362 _("received spurious file revlog entry"))
1355 if not needs:
1363 if not needs:
1356 del needfiles[f]
1364 del needfiles[f]
1357 progress.complete()
1365 progress.complete()
1358
1366
1359 for f, needs in needfiles.iteritems():
1367 for f, needs in needfiles.iteritems():
1360 fl = repo.file(f)
1368 fl = repo.file(f)
1361 for n in needs:
1369 for n in needs:
1362 try:
1370 try:
1363 fl.rev(n)
1371 fl.rev(n)
1364 except error.LookupError:
1372 except error.LookupError:
1365 raise error.Abort(
1373 raise error.Abort(
1366 _('missing file data for %s:%s - run hg verify') %
1374 _('missing file data for %s:%s - run hg verify') %
1367 (f, hex(n)))
1375 (f, hex(n)))
1368
1376
1369 return revisions, files
1377 return revisions, files
1370
1378
1371 def _packellipsischangegroup(repo, common, match, relevant_nodes,
1379 def _packellipsischangegroup(repo, common, match, relevant_nodes,
1372 ellipsisroots, visitnodes, depth, source, version):
1380 ellipsisroots, visitnodes, depth, source, version):
1373 if version in ('01', '02'):
1381 if version in ('01', '02'):
1374 raise error.Abort(
1382 raise error.Abort(
1375 'ellipsis nodes require at least cg3 on client and server, '
1383 'ellipsis nodes require at least cg3 on client and server, '
1376 'but negotiated version %s' % version)
1384 'but negotiated version %s' % version)
1377 # We wrap cg1packer.revchunk, using a side channel to pass
1385 # We wrap cg1packer.revchunk, using a side channel to pass
1378 # relevant_nodes into that area. Then if linknode isn't in the
1386 # relevant_nodes into that area. Then if linknode isn't in the
1379 # set, we know we have an ellipsis node and we should defer
1387 # set, we know we have an ellipsis node and we should defer
1380 # sending that node's data. We override close() to detect
1388 # sending that node's data. We override close() to detect
1381 # pending ellipsis nodes and flush them.
1389 # pending ellipsis nodes and flush them.
1382 packer = getbundler(version, repo, filematcher=match)
1390 packer = getbundler(version, repo, filematcher=match,
1391 shallow=depth is not None)
1383 # Give the packer the list of nodes which should not be
1392 # Give the packer the list of nodes which should not be
1384 # ellipsis nodes. We store this rather than the set of nodes
1393 # ellipsis nodes. We store this rather than the set of nodes
1385 # that should be an ellipsis because for very large histories
1394 # that should be an ellipsis because for very large histories
1386 # we expect this to be significantly smaller.
1395 # we expect this to be significantly smaller.
1387 packer._full_nodes = relevant_nodes
1396 packer._full_nodes = relevant_nodes
1388 # Maps ellipsis revs to their roots at the changelog level.
1397 # Maps ellipsis revs to their roots at the changelog level.
1389 packer._precomputed_ellipsis = ellipsisroots
1398 packer._precomputed_ellipsis = ellipsisroots
1390 # Maps CL revs to per-revlog revisions. Cleared in close() at
1399 # Maps CL revs to per-revlog revisions. Cleared in close() at
1391 # the end of each group.
1400 # the end of each group.
1392 packer._clrev_to_localrev = {}
1401 packer._clrev_to_localrev = {}
1393 packer._next_clrev_to_localrev = {}
1402 packer._next_clrev_to_localrev = {}
1394 # Maps changelog nodes to changelog revs. Filled in once
1403 # Maps changelog nodes to changelog revs. Filled in once
1395 # during changelog stage and then left unmodified.
1404 # during changelog stage and then left unmodified.
1396 packer._clnode_to_rev = {}
1405 packer._clnode_to_rev = {}
1397 packer._changelog_done = False
1406 packer._changelog_done = False
1398 # If true, informs the packer that it is serving shallow content and might
1399 # need to pack file contents not introduced by the changes being packed.
1400 packer._is_shallow = depth is not None
1401
1407
1402 return packer.generate(common, visitnodes, False, source)
1408 return packer.generate(common, visitnodes, False, source)
General Comments 0
You need to be logged in to leave comments. Login now