##// END OF EJS Templates
changegroup: extract cgpacker.group() to standalone function...
Gregory Szorc -
r39045:9e8eb2b4 default
parent child Browse files
Show More
@@ -1,1438 +1,1439 b''
1 # changegroup.py - Mercurial changegroup manipulation functions
1 # changegroup.py - Mercurial changegroup manipulation functions
2 #
2 #
3 # Copyright 2006 Matt Mackall <mpm@selenic.com>
3 # Copyright 2006 Matt Mackall <mpm@selenic.com>
4 #
4 #
5 # This software may be used and distributed according to the terms of the
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
6 # GNU General Public License version 2 or any later version.
7
7
8 from __future__ import absolute_import
8 from __future__ import absolute_import
9
9
10 import os
10 import os
11 import struct
11 import struct
12 import weakref
12 import weakref
13
13
14 from .i18n import _
14 from .i18n import _
15 from .node import (
15 from .node import (
16 hex,
16 hex,
17 nullid,
17 nullid,
18 nullrev,
18 nullrev,
19 short,
19 short,
20 )
20 )
21
21
22 from .thirdparty import (
22 from .thirdparty import (
23 attr,
23 attr,
24 )
24 )
25
25
26 from . import (
26 from . import (
27 dagutil,
27 dagutil,
28 error,
28 error,
29 match as matchmod,
29 match as matchmod,
30 mdiff,
30 mdiff,
31 phases,
31 phases,
32 pycompat,
32 pycompat,
33 repository,
33 repository,
34 revlog,
34 revlog,
35 util,
35 util,
36 )
36 )
37
37
38 from .utils import (
38 from .utils import (
39 stringutil,
39 stringutil,
40 )
40 )
41
41
42 _CHANGEGROUPV1_DELTA_HEADER = struct.Struct("20s20s20s20s")
42 _CHANGEGROUPV1_DELTA_HEADER = struct.Struct("20s20s20s20s")
43 _CHANGEGROUPV2_DELTA_HEADER = struct.Struct("20s20s20s20s20s")
43 _CHANGEGROUPV2_DELTA_HEADER = struct.Struct("20s20s20s20s20s")
44 _CHANGEGROUPV3_DELTA_HEADER = struct.Struct(">20s20s20s20s20sH")
44 _CHANGEGROUPV3_DELTA_HEADER = struct.Struct(">20s20s20s20s20sH")
45
45
46 LFS_REQUIREMENT = 'lfs'
46 LFS_REQUIREMENT = 'lfs'
47
47
48 readexactly = util.readexactly
48 readexactly = util.readexactly
49
49
50 def getchunk(stream):
50 def getchunk(stream):
51 """return the next chunk from stream as a string"""
51 """return the next chunk from stream as a string"""
52 d = readexactly(stream, 4)
52 d = readexactly(stream, 4)
53 l = struct.unpack(">l", d)[0]
53 l = struct.unpack(">l", d)[0]
54 if l <= 4:
54 if l <= 4:
55 if l:
55 if l:
56 raise error.Abort(_("invalid chunk length %d") % l)
56 raise error.Abort(_("invalid chunk length %d") % l)
57 return ""
57 return ""
58 return readexactly(stream, l - 4)
58 return readexactly(stream, l - 4)
59
59
60 def chunkheader(length):
60 def chunkheader(length):
61 """return a changegroup chunk header (string)"""
61 """return a changegroup chunk header (string)"""
62 return struct.pack(">l", length + 4)
62 return struct.pack(">l", length + 4)
63
63
64 def closechunk():
64 def closechunk():
65 """return a changegroup chunk header (string) for a zero-length chunk"""
65 """return a changegroup chunk header (string) for a zero-length chunk"""
66 return struct.pack(">l", 0)
66 return struct.pack(">l", 0)
67
67
68 def _fileheader(path):
68 def _fileheader(path):
69 """Obtain a changegroup chunk header for a named path."""
69 """Obtain a changegroup chunk header for a named path."""
70 return chunkheader(len(path)) + path
70 return chunkheader(len(path)) + path
71
71
72 def writechunks(ui, chunks, filename, vfs=None):
72 def writechunks(ui, chunks, filename, vfs=None):
73 """Write chunks to a file and return its filename.
73 """Write chunks to a file and return its filename.
74
74
75 The stream is assumed to be a bundle file.
75 The stream is assumed to be a bundle file.
76 Existing files will not be overwritten.
76 Existing files will not be overwritten.
77 If no filename is specified, a temporary file is created.
77 If no filename is specified, a temporary file is created.
78 """
78 """
79 fh = None
79 fh = None
80 cleanup = None
80 cleanup = None
81 try:
81 try:
82 if filename:
82 if filename:
83 if vfs:
83 if vfs:
84 fh = vfs.open(filename, "wb")
84 fh = vfs.open(filename, "wb")
85 else:
85 else:
86 # Increase default buffer size because default is usually
86 # Increase default buffer size because default is usually
87 # small (4k is common on Linux).
87 # small (4k is common on Linux).
88 fh = open(filename, "wb", 131072)
88 fh = open(filename, "wb", 131072)
89 else:
89 else:
90 fd, filename = pycompat.mkstemp(prefix="hg-bundle-", suffix=".hg")
90 fd, filename = pycompat.mkstemp(prefix="hg-bundle-", suffix=".hg")
91 fh = os.fdopen(fd, r"wb")
91 fh = os.fdopen(fd, r"wb")
92 cleanup = filename
92 cleanup = filename
93 for c in chunks:
93 for c in chunks:
94 fh.write(c)
94 fh.write(c)
95 cleanup = None
95 cleanup = None
96 return filename
96 return filename
97 finally:
97 finally:
98 if fh is not None:
98 if fh is not None:
99 fh.close()
99 fh.close()
100 if cleanup is not None:
100 if cleanup is not None:
101 if filename and vfs:
101 if filename and vfs:
102 vfs.unlink(cleanup)
102 vfs.unlink(cleanup)
103 else:
103 else:
104 os.unlink(cleanup)
104 os.unlink(cleanup)
105
105
106 class cg1unpacker(object):
106 class cg1unpacker(object):
107 """Unpacker for cg1 changegroup streams.
107 """Unpacker for cg1 changegroup streams.
108
108
109 A changegroup unpacker handles the framing of the revision data in
109 A changegroup unpacker handles the framing of the revision data in
110 the wire format. Most consumers will want to use the apply()
110 the wire format. Most consumers will want to use the apply()
111 method to add the changes from the changegroup to a repository.
111 method to add the changes from the changegroup to a repository.
112
112
113 If you're forwarding a changegroup unmodified to another consumer,
113 If you're forwarding a changegroup unmodified to another consumer,
114 use getchunks(), which returns an iterator of changegroup
114 use getchunks(), which returns an iterator of changegroup
115 chunks. This is mostly useful for cases where you need to know the
115 chunks. This is mostly useful for cases where you need to know the
116 data stream has ended by observing the end of the changegroup.
116 data stream has ended by observing the end of the changegroup.
117
117
118 deltachunk() is useful only if you're applying delta data. Most
118 deltachunk() is useful only if you're applying delta data. Most
119 consumers should prefer apply() instead.
119 consumers should prefer apply() instead.
120
120
121 A few other public methods exist. Those are used only for
121 A few other public methods exist. Those are used only for
122 bundlerepo and some debug commands - their use is discouraged.
122 bundlerepo and some debug commands - their use is discouraged.
123 """
123 """
124 deltaheader = _CHANGEGROUPV1_DELTA_HEADER
124 deltaheader = _CHANGEGROUPV1_DELTA_HEADER
125 deltaheadersize = deltaheader.size
125 deltaheadersize = deltaheader.size
126 version = '01'
126 version = '01'
127 _grouplistcount = 1 # One list of files after the manifests
127 _grouplistcount = 1 # One list of files after the manifests
128
128
129 def __init__(self, fh, alg, extras=None):
129 def __init__(self, fh, alg, extras=None):
130 if alg is None:
130 if alg is None:
131 alg = 'UN'
131 alg = 'UN'
132 if alg not in util.compengines.supportedbundletypes:
132 if alg not in util.compengines.supportedbundletypes:
133 raise error.Abort(_('unknown stream compression type: %s')
133 raise error.Abort(_('unknown stream compression type: %s')
134 % alg)
134 % alg)
135 if alg == 'BZ':
135 if alg == 'BZ':
136 alg = '_truncatedBZ'
136 alg = '_truncatedBZ'
137
137
138 compengine = util.compengines.forbundletype(alg)
138 compengine = util.compengines.forbundletype(alg)
139 self._stream = compengine.decompressorreader(fh)
139 self._stream = compengine.decompressorreader(fh)
140 self._type = alg
140 self._type = alg
141 self.extras = extras or {}
141 self.extras = extras or {}
142 self.callback = None
142 self.callback = None
143
143
144 # These methods (compressed, read, seek, tell) all appear to only
144 # These methods (compressed, read, seek, tell) all appear to only
145 # be used by bundlerepo, but it's a little hard to tell.
145 # be used by bundlerepo, but it's a little hard to tell.
146 def compressed(self):
146 def compressed(self):
147 return self._type is not None and self._type != 'UN'
147 return self._type is not None and self._type != 'UN'
148 def read(self, l):
148 def read(self, l):
149 return self._stream.read(l)
149 return self._stream.read(l)
150 def seek(self, pos):
150 def seek(self, pos):
151 return self._stream.seek(pos)
151 return self._stream.seek(pos)
152 def tell(self):
152 def tell(self):
153 return self._stream.tell()
153 return self._stream.tell()
154 def close(self):
154 def close(self):
155 return self._stream.close()
155 return self._stream.close()
156
156
157 def _chunklength(self):
157 def _chunklength(self):
158 d = readexactly(self._stream, 4)
158 d = readexactly(self._stream, 4)
159 l = struct.unpack(">l", d)[0]
159 l = struct.unpack(">l", d)[0]
160 if l <= 4:
160 if l <= 4:
161 if l:
161 if l:
162 raise error.Abort(_("invalid chunk length %d") % l)
162 raise error.Abort(_("invalid chunk length %d") % l)
163 return 0
163 return 0
164 if self.callback:
164 if self.callback:
165 self.callback()
165 self.callback()
166 return l - 4
166 return l - 4
167
167
168 def changelogheader(self):
168 def changelogheader(self):
169 """v10 does not have a changelog header chunk"""
169 """v10 does not have a changelog header chunk"""
170 return {}
170 return {}
171
171
172 def manifestheader(self):
172 def manifestheader(self):
173 """v10 does not have a manifest header chunk"""
173 """v10 does not have a manifest header chunk"""
174 return {}
174 return {}
175
175
176 def filelogheader(self):
176 def filelogheader(self):
177 """return the header of the filelogs chunk, v10 only has the filename"""
177 """return the header of the filelogs chunk, v10 only has the filename"""
178 l = self._chunklength()
178 l = self._chunklength()
179 if not l:
179 if not l:
180 return {}
180 return {}
181 fname = readexactly(self._stream, l)
181 fname = readexactly(self._stream, l)
182 return {'filename': fname}
182 return {'filename': fname}
183
183
184 def _deltaheader(self, headertuple, prevnode):
184 def _deltaheader(self, headertuple, prevnode):
185 node, p1, p2, cs = headertuple
185 node, p1, p2, cs = headertuple
186 if prevnode is None:
186 if prevnode is None:
187 deltabase = p1
187 deltabase = p1
188 else:
188 else:
189 deltabase = prevnode
189 deltabase = prevnode
190 flags = 0
190 flags = 0
191 return node, p1, p2, deltabase, cs, flags
191 return node, p1, p2, deltabase, cs, flags
192
192
193 def deltachunk(self, prevnode):
193 def deltachunk(self, prevnode):
194 l = self._chunklength()
194 l = self._chunklength()
195 if not l:
195 if not l:
196 return {}
196 return {}
197 headerdata = readexactly(self._stream, self.deltaheadersize)
197 headerdata = readexactly(self._stream, self.deltaheadersize)
198 header = self.deltaheader.unpack(headerdata)
198 header = self.deltaheader.unpack(headerdata)
199 delta = readexactly(self._stream, l - self.deltaheadersize)
199 delta = readexactly(self._stream, l - self.deltaheadersize)
200 node, p1, p2, deltabase, cs, flags = self._deltaheader(header, prevnode)
200 node, p1, p2, deltabase, cs, flags = self._deltaheader(header, prevnode)
201 return (node, p1, p2, cs, deltabase, delta, flags)
201 return (node, p1, p2, cs, deltabase, delta, flags)
202
202
203 def getchunks(self):
203 def getchunks(self):
204 """returns all the chunks contains in the bundle
204 """returns all the chunks contains in the bundle
205
205
206 Used when you need to forward the binary stream to a file or another
206 Used when you need to forward the binary stream to a file or another
207 network API. To do so, it parse the changegroup data, otherwise it will
207 network API. To do so, it parse the changegroup data, otherwise it will
208 block in case of sshrepo because it don't know the end of the stream.
208 block in case of sshrepo because it don't know the end of the stream.
209 """
209 """
210 # For changegroup 1 and 2, we expect 3 parts: changelog, manifestlog,
210 # For changegroup 1 and 2, we expect 3 parts: changelog, manifestlog,
211 # and a list of filelogs. For changegroup 3, we expect 4 parts:
211 # and a list of filelogs. For changegroup 3, we expect 4 parts:
212 # changelog, manifestlog, a list of tree manifestlogs, and a list of
212 # changelog, manifestlog, a list of tree manifestlogs, and a list of
213 # filelogs.
213 # filelogs.
214 #
214 #
215 # Changelog and manifestlog parts are terminated with empty chunks. The
215 # Changelog and manifestlog parts are terminated with empty chunks. The
216 # tree and file parts are a list of entry sections. Each entry section
216 # tree and file parts are a list of entry sections. Each entry section
217 # is a series of chunks terminating in an empty chunk. The list of these
217 # is a series of chunks terminating in an empty chunk. The list of these
218 # entry sections is terminated in yet another empty chunk, so we know
218 # entry sections is terminated in yet another empty chunk, so we know
219 # we've reached the end of the tree/file list when we reach an empty
219 # we've reached the end of the tree/file list when we reach an empty
220 # chunk that was proceeded by no non-empty chunks.
220 # chunk that was proceeded by no non-empty chunks.
221
221
222 parts = 0
222 parts = 0
223 while parts < 2 + self._grouplistcount:
223 while parts < 2 + self._grouplistcount:
224 noentries = True
224 noentries = True
225 while True:
225 while True:
226 chunk = getchunk(self)
226 chunk = getchunk(self)
227 if not chunk:
227 if not chunk:
228 # The first two empty chunks represent the end of the
228 # The first two empty chunks represent the end of the
229 # changelog and the manifestlog portions. The remaining
229 # changelog and the manifestlog portions. The remaining
230 # empty chunks represent either A) the end of individual
230 # empty chunks represent either A) the end of individual
231 # tree or file entries in the file list, or B) the end of
231 # tree or file entries in the file list, or B) the end of
232 # the entire list. It's the end of the entire list if there
232 # the entire list. It's the end of the entire list if there
233 # were no entries (i.e. noentries is True).
233 # were no entries (i.e. noentries is True).
234 if parts < 2:
234 if parts < 2:
235 parts += 1
235 parts += 1
236 elif noentries:
236 elif noentries:
237 parts += 1
237 parts += 1
238 break
238 break
239 noentries = False
239 noentries = False
240 yield chunkheader(len(chunk))
240 yield chunkheader(len(chunk))
241 pos = 0
241 pos = 0
242 while pos < len(chunk):
242 while pos < len(chunk):
243 next = pos + 2**20
243 next = pos + 2**20
244 yield chunk[pos:next]
244 yield chunk[pos:next]
245 pos = next
245 pos = next
246 yield closechunk()
246 yield closechunk()
247
247
248 def _unpackmanifests(self, repo, revmap, trp, prog):
248 def _unpackmanifests(self, repo, revmap, trp, prog):
249 self.callback = prog.increment
249 self.callback = prog.increment
250 # no need to check for empty manifest group here:
250 # no need to check for empty manifest group here:
251 # if the result of the merge of 1 and 2 is the same in 3 and 4,
251 # if the result of the merge of 1 and 2 is the same in 3 and 4,
252 # no new manifest will be created and the manifest group will
252 # no new manifest will be created and the manifest group will
253 # be empty during the pull
253 # be empty during the pull
254 self.manifestheader()
254 self.manifestheader()
255 deltas = self.deltaiter()
255 deltas = self.deltaiter()
256 repo.manifestlog.addgroup(deltas, revmap, trp)
256 repo.manifestlog.addgroup(deltas, revmap, trp)
257 prog.complete()
257 prog.complete()
258 self.callback = None
258 self.callback = None
259
259
260 def apply(self, repo, tr, srctype, url, targetphase=phases.draft,
260 def apply(self, repo, tr, srctype, url, targetphase=phases.draft,
261 expectedtotal=None):
261 expectedtotal=None):
262 """Add the changegroup returned by source.read() to this repo.
262 """Add the changegroup returned by source.read() to this repo.
263 srctype is a string like 'push', 'pull', or 'unbundle'. url is
263 srctype is a string like 'push', 'pull', or 'unbundle'. url is
264 the URL of the repo where this changegroup is coming from.
264 the URL of the repo where this changegroup is coming from.
265
265
266 Return an integer summarizing the change to this repo:
266 Return an integer summarizing the change to this repo:
267 - nothing changed or no source: 0
267 - nothing changed or no source: 0
268 - more heads than before: 1+added heads (2..n)
268 - more heads than before: 1+added heads (2..n)
269 - fewer heads than before: -1-removed heads (-2..-n)
269 - fewer heads than before: -1-removed heads (-2..-n)
270 - number of heads stays the same: 1
270 - number of heads stays the same: 1
271 """
271 """
272 repo = repo.unfiltered()
272 repo = repo.unfiltered()
273 def csmap(x):
273 def csmap(x):
274 repo.ui.debug("add changeset %s\n" % short(x))
274 repo.ui.debug("add changeset %s\n" % short(x))
275 return len(cl)
275 return len(cl)
276
276
277 def revmap(x):
277 def revmap(x):
278 return cl.rev(x)
278 return cl.rev(x)
279
279
280 changesets = files = revisions = 0
280 changesets = files = revisions = 0
281
281
282 try:
282 try:
283 # The transaction may already carry source information. In this
283 # The transaction may already carry source information. In this
284 # case we use the top level data. We overwrite the argument
284 # case we use the top level data. We overwrite the argument
285 # because we need to use the top level value (if they exist)
285 # because we need to use the top level value (if they exist)
286 # in this function.
286 # in this function.
287 srctype = tr.hookargs.setdefault('source', srctype)
287 srctype = tr.hookargs.setdefault('source', srctype)
288 url = tr.hookargs.setdefault('url', url)
288 url = tr.hookargs.setdefault('url', url)
289 repo.hook('prechangegroup',
289 repo.hook('prechangegroup',
290 throw=True, **pycompat.strkwargs(tr.hookargs))
290 throw=True, **pycompat.strkwargs(tr.hookargs))
291
291
292 # write changelog data to temp files so concurrent readers
292 # write changelog data to temp files so concurrent readers
293 # will not see an inconsistent view
293 # will not see an inconsistent view
294 cl = repo.changelog
294 cl = repo.changelog
295 cl.delayupdate(tr)
295 cl.delayupdate(tr)
296 oldheads = set(cl.heads())
296 oldheads = set(cl.heads())
297
297
298 trp = weakref.proxy(tr)
298 trp = weakref.proxy(tr)
299 # pull off the changeset group
299 # pull off the changeset group
300 repo.ui.status(_("adding changesets\n"))
300 repo.ui.status(_("adding changesets\n"))
301 clstart = len(cl)
301 clstart = len(cl)
302 progress = repo.ui.makeprogress(_('changesets'), unit=_('chunks'),
302 progress = repo.ui.makeprogress(_('changesets'), unit=_('chunks'),
303 total=expectedtotal)
303 total=expectedtotal)
304 self.callback = progress.increment
304 self.callback = progress.increment
305
305
306 efiles = set()
306 efiles = set()
307 def onchangelog(cl, node):
307 def onchangelog(cl, node):
308 efiles.update(cl.readfiles(node))
308 efiles.update(cl.readfiles(node))
309
309
310 self.changelogheader()
310 self.changelogheader()
311 deltas = self.deltaiter()
311 deltas = self.deltaiter()
312 cgnodes = cl.addgroup(deltas, csmap, trp, addrevisioncb=onchangelog)
312 cgnodes = cl.addgroup(deltas, csmap, trp, addrevisioncb=onchangelog)
313 efiles = len(efiles)
313 efiles = len(efiles)
314
314
315 if not cgnodes:
315 if not cgnodes:
316 repo.ui.develwarn('applied empty changegroup',
316 repo.ui.develwarn('applied empty changegroup',
317 config='warn-empty-changegroup')
317 config='warn-empty-changegroup')
318 clend = len(cl)
318 clend = len(cl)
319 changesets = clend - clstart
319 changesets = clend - clstart
320 progress.complete()
320 progress.complete()
321 self.callback = None
321 self.callback = None
322
322
323 # pull off the manifest group
323 # pull off the manifest group
324 repo.ui.status(_("adding manifests\n"))
324 repo.ui.status(_("adding manifests\n"))
325 # We know that we'll never have more manifests than we had
325 # We know that we'll never have more manifests than we had
326 # changesets.
326 # changesets.
327 progress = repo.ui.makeprogress(_('manifests'), unit=_('chunks'),
327 progress = repo.ui.makeprogress(_('manifests'), unit=_('chunks'),
328 total=changesets)
328 total=changesets)
329 self._unpackmanifests(repo, revmap, trp, progress)
329 self._unpackmanifests(repo, revmap, trp, progress)
330
330
331 needfiles = {}
331 needfiles = {}
332 if repo.ui.configbool('server', 'validate'):
332 if repo.ui.configbool('server', 'validate'):
333 cl = repo.changelog
333 cl = repo.changelog
334 ml = repo.manifestlog
334 ml = repo.manifestlog
335 # validate incoming csets have their manifests
335 # validate incoming csets have their manifests
336 for cset in pycompat.xrange(clstart, clend):
336 for cset in pycompat.xrange(clstart, clend):
337 mfnode = cl.changelogrevision(cset).manifest
337 mfnode = cl.changelogrevision(cset).manifest
338 mfest = ml[mfnode].readdelta()
338 mfest = ml[mfnode].readdelta()
339 # store file cgnodes we must see
339 # store file cgnodes we must see
340 for f, n in mfest.iteritems():
340 for f, n in mfest.iteritems():
341 needfiles.setdefault(f, set()).add(n)
341 needfiles.setdefault(f, set()).add(n)
342
342
343 # process the files
343 # process the files
344 repo.ui.status(_("adding file changes\n"))
344 repo.ui.status(_("adding file changes\n"))
345 newrevs, newfiles = _addchangegroupfiles(
345 newrevs, newfiles = _addchangegroupfiles(
346 repo, self, revmap, trp, efiles, needfiles)
346 repo, self, revmap, trp, efiles, needfiles)
347 revisions += newrevs
347 revisions += newrevs
348 files += newfiles
348 files += newfiles
349
349
350 deltaheads = 0
350 deltaheads = 0
351 if oldheads:
351 if oldheads:
352 heads = cl.heads()
352 heads = cl.heads()
353 deltaheads = len(heads) - len(oldheads)
353 deltaheads = len(heads) - len(oldheads)
354 for h in heads:
354 for h in heads:
355 if h not in oldheads and repo[h].closesbranch():
355 if h not in oldheads and repo[h].closesbranch():
356 deltaheads -= 1
356 deltaheads -= 1
357 htext = ""
357 htext = ""
358 if deltaheads:
358 if deltaheads:
359 htext = _(" (%+d heads)") % deltaheads
359 htext = _(" (%+d heads)") % deltaheads
360
360
361 repo.ui.status(_("added %d changesets"
361 repo.ui.status(_("added %d changesets"
362 " with %d changes to %d files%s\n")
362 " with %d changes to %d files%s\n")
363 % (changesets, revisions, files, htext))
363 % (changesets, revisions, files, htext))
364 repo.invalidatevolatilesets()
364 repo.invalidatevolatilesets()
365
365
366 if changesets > 0:
366 if changesets > 0:
367 if 'node' not in tr.hookargs:
367 if 'node' not in tr.hookargs:
368 tr.hookargs['node'] = hex(cl.node(clstart))
368 tr.hookargs['node'] = hex(cl.node(clstart))
369 tr.hookargs['node_last'] = hex(cl.node(clend - 1))
369 tr.hookargs['node_last'] = hex(cl.node(clend - 1))
370 hookargs = dict(tr.hookargs)
370 hookargs = dict(tr.hookargs)
371 else:
371 else:
372 hookargs = dict(tr.hookargs)
372 hookargs = dict(tr.hookargs)
373 hookargs['node'] = hex(cl.node(clstart))
373 hookargs['node'] = hex(cl.node(clstart))
374 hookargs['node_last'] = hex(cl.node(clend - 1))
374 hookargs['node_last'] = hex(cl.node(clend - 1))
375 repo.hook('pretxnchangegroup',
375 repo.hook('pretxnchangegroup',
376 throw=True, **pycompat.strkwargs(hookargs))
376 throw=True, **pycompat.strkwargs(hookargs))
377
377
378 added = [cl.node(r) for r in pycompat.xrange(clstart, clend)]
378 added = [cl.node(r) for r in pycompat.xrange(clstart, clend)]
379 phaseall = None
379 phaseall = None
380 if srctype in ('push', 'serve'):
380 if srctype in ('push', 'serve'):
381 # Old servers can not push the boundary themselves.
381 # Old servers can not push the boundary themselves.
382 # New servers won't push the boundary if changeset already
382 # New servers won't push the boundary if changeset already
383 # exists locally as secret
383 # exists locally as secret
384 #
384 #
385 # We should not use added here but the list of all change in
385 # We should not use added here but the list of all change in
386 # the bundle
386 # the bundle
387 if repo.publishing():
387 if repo.publishing():
388 targetphase = phaseall = phases.public
388 targetphase = phaseall = phases.public
389 else:
389 else:
390 # closer target phase computation
390 # closer target phase computation
391
391
392 # Those changesets have been pushed from the
392 # Those changesets have been pushed from the
393 # outside, their phases are going to be pushed
393 # outside, their phases are going to be pushed
394 # alongside. Therefor `targetphase` is
394 # alongside. Therefor `targetphase` is
395 # ignored.
395 # ignored.
396 targetphase = phaseall = phases.draft
396 targetphase = phaseall = phases.draft
397 if added:
397 if added:
398 phases.registernew(repo, tr, targetphase, added)
398 phases.registernew(repo, tr, targetphase, added)
399 if phaseall is not None:
399 if phaseall is not None:
400 phases.advanceboundary(repo, tr, phaseall, cgnodes)
400 phases.advanceboundary(repo, tr, phaseall, cgnodes)
401
401
402 if changesets > 0:
402 if changesets > 0:
403
403
404 def runhooks():
404 def runhooks():
405 # These hooks run when the lock releases, not when the
405 # These hooks run when the lock releases, not when the
406 # transaction closes. So it's possible for the changelog
406 # transaction closes. So it's possible for the changelog
407 # to have changed since we last saw it.
407 # to have changed since we last saw it.
408 if clstart >= len(repo):
408 if clstart >= len(repo):
409 return
409 return
410
410
411 repo.hook("changegroup", **pycompat.strkwargs(hookargs))
411 repo.hook("changegroup", **pycompat.strkwargs(hookargs))
412
412
413 for n in added:
413 for n in added:
414 args = hookargs.copy()
414 args = hookargs.copy()
415 args['node'] = hex(n)
415 args['node'] = hex(n)
416 del args['node_last']
416 del args['node_last']
417 repo.hook("incoming", **pycompat.strkwargs(args))
417 repo.hook("incoming", **pycompat.strkwargs(args))
418
418
419 newheads = [h for h in repo.heads()
419 newheads = [h for h in repo.heads()
420 if h not in oldheads]
420 if h not in oldheads]
421 repo.ui.log("incoming",
421 repo.ui.log("incoming",
422 "%d incoming changes - new heads: %s\n",
422 "%d incoming changes - new heads: %s\n",
423 len(added),
423 len(added),
424 ', '.join([hex(c[:6]) for c in newheads]))
424 ', '.join([hex(c[:6]) for c in newheads]))
425
425
426 tr.addpostclose('changegroup-runhooks-%020i' % clstart,
426 tr.addpostclose('changegroup-runhooks-%020i' % clstart,
427 lambda tr: repo._afterlock(runhooks))
427 lambda tr: repo._afterlock(runhooks))
428 finally:
428 finally:
429 repo.ui.flush()
429 repo.ui.flush()
430 # never return 0 here:
430 # never return 0 here:
431 if deltaheads < 0:
431 if deltaheads < 0:
432 ret = deltaheads - 1
432 ret = deltaheads - 1
433 else:
433 else:
434 ret = deltaheads + 1
434 ret = deltaheads + 1
435 return ret
435 return ret
436
436
437 def deltaiter(self):
437 def deltaiter(self):
438 """
438 """
439 returns an iterator of the deltas in this changegroup
439 returns an iterator of the deltas in this changegroup
440
440
441 Useful for passing to the underlying storage system to be stored.
441 Useful for passing to the underlying storage system to be stored.
442 """
442 """
443 chain = None
443 chain = None
444 for chunkdata in iter(lambda: self.deltachunk(chain), {}):
444 for chunkdata in iter(lambda: self.deltachunk(chain), {}):
445 # Chunkdata: (node, p1, p2, cs, deltabase, delta, flags)
445 # Chunkdata: (node, p1, p2, cs, deltabase, delta, flags)
446 yield chunkdata
446 yield chunkdata
447 chain = chunkdata[0]
447 chain = chunkdata[0]
448
448
449 class cg2unpacker(cg1unpacker):
449 class cg2unpacker(cg1unpacker):
450 """Unpacker for cg2 streams.
450 """Unpacker for cg2 streams.
451
451
452 cg2 streams add support for generaldelta, so the delta header
452 cg2 streams add support for generaldelta, so the delta header
453 format is slightly different. All other features about the data
453 format is slightly different. All other features about the data
454 remain the same.
454 remain the same.
455 """
455 """
456 deltaheader = _CHANGEGROUPV2_DELTA_HEADER
456 deltaheader = _CHANGEGROUPV2_DELTA_HEADER
457 deltaheadersize = deltaheader.size
457 deltaheadersize = deltaheader.size
458 version = '02'
458 version = '02'
459
459
460 def _deltaheader(self, headertuple, prevnode):
460 def _deltaheader(self, headertuple, prevnode):
461 node, p1, p2, deltabase, cs = headertuple
461 node, p1, p2, deltabase, cs = headertuple
462 flags = 0
462 flags = 0
463 return node, p1, p2, deltabase, cs, flags
463 return node, p1, p2, deltabase, cs, flags
464
464
465 class cg3unpacker(cg2unpacker):
465 class cg3unpacker(cg2unpacker):
466 """Unpacker for cg3 streams.
466 """Unpacker for cg3 streams.
467
467
468 cg3 streams add support for exchanging treemanifests and revlog
468 cg3 streams add support for exchanging treemanifests and revlog
469 flags. It adds the revlog flags to the delta header and an empty chunk
469 flags. It adds the revlog flags to the delta header and an empty chunk
470 separating manifests and files.
470 separating manifests and files.
471 """
471 """
472 deltaheader = _CHANGEGROUPV3_DELTA_HEADER
472 deltaheader = _CHANGEGROUPV3_DELTA_HEADER
473 deltaheadersize = deltaheader.size
473 deltaheadersize = deltaheader.size
474 version = '03'
474 version = '03'
475 _grouplistcount = 2 # One list of manifests and one list of files
475 _grouplistcount = 2 # One list of manifests and one list of files
476
476
477 def _deltaheader(self, headertuple, prevnode):
477 def _deltaheader(self, headertuple, prevnode):
478 node, p1, p2, deltabase, cs, flags = headertuple
478 node, p1, p2, deltabase, cs, flags = headertuple
479 return node, p1, p2, deltabase, cs, flags
479 return node, p1, p2, deltabase, cs, flags
480
480
481 def _unpackmanifests(self, repo, revmap, trp, prog):
481 def _unpackmanifests(self, repo, revmap, trp, prog):
482 super(cg3unpacker, self)._unpackmanifests(repo, revmap, trp, prog)
482 super(cg3unpacker, self)._unpackmanifests(repo, revmap, trp, prog)
483 for chunkdata in iter(self.filelogheader, {}):
483 for chunkdata in iter(self.filelogheader, {}):
484 # If we get here, there are directory manifests in the changegroup
484 # If we get here, there are directory manifests in the changegroup
485 d = chunkdata["filename"]
485 d = chunkdata["filename"]
486 repo.ui.debug("adding %s revisions\n" % d)
486 repo.ui.debug("adding %s revisions\n" % d)
487 dirlog = repo.manifestlog._revlog.dirlog(d)
487 dirlog = repo.manifestlog._revlog.dirlog(d)
488 deltas = self.deltaiter()
488 deltas = self.deltaiter()
489 if not dirlog.addgroup(deltas, revmap, trp):
489 if not dirlog.addgroup(deltas, revmap, trp):
490 raise error.Abort(_("received dir revlog group is empty"))
490 raise error.Abort(_("received dir revlog group is empty"))
491
491
492 class headerlessfixup(object):
492 class headerlessfixup(object):
493 def __init__(self, fh, h):
493 def __init__(self, fh, h):
494 self._h = h
494 self._h = h
495 self._fh = fh
495 self._fh = fh
496 def read(self, n):
496 def read(self, n):
497 if self._h:
497 if self._h:
498 d, self._h = self._h[:n], self._h[n:]
498 d, self._h = self._h[:n], self._h[n:]
499 if len(d) < n:
499 if len(d) < n:
500 d += readexactly(self._fh, n - len(d))
500 d += readexactly(self._fh, n - len(d))
501 return d
501 return d
502 return readexactly(self._fh, n)
502 return readexactly(self._fh, n)
503
503
504 @attr.s(slots=True, frozen=True)
504 @attr.s(slots=True, frozen=True)
505 class revisiondelta(object):
505 class revisiondelta(object):
506 """Describes a delta entry in a changegroup.
506 """Describes a delta entry in a changegroup.
507
507
508 Captured data is sufficient to serialize the delta into multiple
508 Captured data is sufficient to serialize the delta into multiple
509 formats.
509 formats.
510 """
510 """
511 # 20 byte node of this revision.
511 # 20 byte node of this revision.
512 node = attr.ib()
512 node = attr.ib()
513 # 20 byte nodes of parent revisions.
513 # 20 byte nodes of parent revisions.
514 p1node = attr.ib()
514 p1node = attr.ib()
515 p2node = attr.ib()
515 p2node = attr.ib()
516 # 20 byte node of node this delta is against.
516 # 20 byte node of node this delta is against.
517 basenode = attr.ib()
517 basenode = attr.ib()
518 # 20 byte node of changeset revision this delta is associated with.
518 # 20 byte node of changeset revision this delta is associated with.
519 linknode = attr.ib()
519 linknode = attr.ib()
520 # 2 bytes of flags to apply to revision data.
520 # 2 bytes of flags to apply to revision data.
521 flags = attr.ib()
521 flags = attr.ib()
522 # Iterable of chunks holding raw delta data.
522 # Iterable of chunks holding raw delta data.
523 deltachunks = attr.ib()
523 deltachunks = attr.ib()
524
524
525 def _sortnodesnormal(store, nodes, reorder):
525 def _sortnodesnormal(store, nodes, reorder):
526 """Sort nodes for changegroup generation and turn into revnums."""
526 """Sort nodes for changegroup generation and turn into revnums."""
527 # for generaldelta revlogs, we linearize the revs; this will both be
527 # for generaldelta revlogs, we linearize the revs; this will both be
528 # much quicker and generate a much smaller bundle
528 # much quicker and generate a much smaller bundle
529 if (store._generaldelta and reorder is None) or reorder:
529 if (store._generaldelta and reorder is None) or reorder:
530 dag = dagutil.revlogdag(store)
530 dag = dagutil.revlogdag(store)
531 return dag.linearize(set(store.rev(n) for n in nodes))
531 return dag.linearize(set(store.rev(n) for n in nodes))
532 else:
532 else:
533 return sorted([store.rev(n) for n in nodes])
533 return sorted([store.rev(n) for n in nodes])
534
534
535 def _sortnodesellipsis(store, nodes, cl, lookup):
535 def _sortnodesellipsis(store, nodes, cl, lookup):
536 """Sort nodes for changegroup generation and turn into revnums."""
536 """Sort nodes for changegroup generation and turn into revnums."""
537 # Ellipses serving mode.
537 # Ellipses serving mode.
538 #
538 #
539 # In a perfect world, we'd generate better ellipsis-ified graphs
539 # In a perfect world, we'd generate better ellipsis-ified graphs
540 # for non-changelog revlogs. In practice, we haven't started doing
540 # for non-changelog revlogs. In practice, we haven't started doing
541 # that yet, so the resulting DAGs for the manifestlog and filelogs
541 # that yet, so the resulting DAGs for the manifestlog and filelogs
542 # are actually full of bogus parentage on all the ellipsis
542 # are actually full of bogus parentage on all the ellipsis
543 # nodes. This has the side effect that, while the contents are
543 # nodes. This has the side effect that, while the contents are
544 # correct, the individual DAGs might be completely out of whack in
544 # correct, the individual DAGs might be completely out of whack in
545 # a case like 882681bc3166 and its ancestors (back about 10
545 # a case like 882681bc3166 and its ancestors (back about 10
546 # revisions or so) in the main hg repo.
546 # revisions or so) in the main hg repo.
547 #
547 #
548 # The one invariant we *know* holds is that the new (potentially
548 # The one invariant we *know* holds is that the new (potentially
549 # bogus) DAG shape will be valid if we order the nodes in the
549 # bogus) DAG shape will be valid if we order the nodes in the
550 # order that they're introduced in dramatis personae by the
550 # order that they're introduced in dramatis personae by the
551 # changelog, so what we do is we sort the non-changelog histories
551 # changelog, so what we do is we sort the non-changelog histories
552 # by the order in which they are used by the changelog.
552 # by the order in which they are used by the changelog.
553 key = lambda n: cl.rev(lookup(n))
553 key = lambda n: cl.rev(lookup(n))
554 return [store.rev(n) for n in sorted(nodes, key=key)]
554 return [store.rev(n) for n in sorted(nodes, key=key)]
555
555
556 def _revisiondeltanormal(store, rev, prev, linknode, deltaparentfn):
556 def _revisiondeltanormal(store, rev, prev, linknode, deltaparentfn):
557 """Construct a revision delta for non-ellipses changegroup generation."""
557 """Construct a revision delta for non-ellipses changegroup generation."""
558 node = store.node(rev)
558 node = store.node(rev)
559 p1, p2 = store.parentrevs(rev)
559 p1, p2 = store.parentrevs(rev)
560 base = deltaparentfn(store, rev, p1, p2, prev)
560 base = deltaparentfn(store, rev, p1, p2, prev)
561
561
562 prefix = ''
562 prefix = ''
563 if store.iscensored(base) or store.iscensored(rev):
563 if store.iscensored(base) or store.iscensored(rev):
564 try:
564 try:
565 delta = store.revision(node, raw=True)
565 delta = store.revision(node, raw=True)
566 except error.CensoredNodeError as e:
566 except error.CensoredNodeError as e:
567 delta = e.tombstone
567 delta = e.tombstone
568 if base == nullrev:
568 if base == nullrev:
569 prefix = mdiff.trivialdiffheader(len(delta))
569 prefix = mdiff.trivialdiffheader(len(delta))
570 else:
570 else:
571 baselen = store.rawsize(base)
571 baselen = store.rawsize(base)
572 prefix = mdiff.replacediffheader(baselen, len(delta))
572 prefix = mdiff.replacediffheader(baselen, len(delta))
573 elif base == nullrev:
573 elif base == nullrev:
574 delta = store.revision(node, raw=True)
574 delta = store.revision(node, raw=True)
575 prefix = mdiff.trivialdiffheader(len(delta))
575 prefix = mdiff.trivialdiffheader(len(delta))
576 else:
576 else:
577 delta = store.revdiff(base, rev)
577 delta = store.revdiff(base, rev)
578 p1n, p2n = store.parents(node)
578 p1n, p2n = store.parents(node)
579
579
580 return revisiondelta(
580 return revisiondelta(
581 node=node,
581 node=node,
582 p1node=p1n,
582 p1node=p1n,
583 p2node=p2n,
583 p2node=p2n,
584 basenode=store.node(base),
584 basenode=store.node(base),
585 linknode=linknode,
585 linknode=linknode,
586 flags=store.flags(rev),
586 flags=store.flags(rev),
587 deltachunks=(prefix, delta),
587 deltachunks=(prefix, delta),
588 )
588 )
589
589
590 def _revisiondeltanarrow(cl, store, ischangelog, rev, linkrev,
590 def _revisiondeltanarrow(cl, store, ischangelog, rev, linkrev,
591 linknode, clrevtolocalrev, fullclnodes,
591 linknode, clrevtolocalrev, fullclnodes,
592 precomputedellipsis):
592 precomputedellipsis):
593 linkparents = precomputedellipsis[linkrev]
593 linkparents = precomputedellipsis[linkrev]
594 def local(clrev):
594 def local(clrev):
595 """Turn a changelog revnum into a local revnum.
595 """Turn a changelog revnum into a local revnum.
596
596
597 The ellipsis dag is stored as revnums on the changelog,
597 The ellipsis dag is stored as revnums on the changelog,
598 but when we're producing ellipsis entries for
598 but when we're producing ellipsis entries for
599 non-changelog revlogs, we need to turn those numbers into
599 non-changelog revlogs, we need to turn those numbers into
600 something local. This does that for us, and during the
600 something local. This does that for us, and during the
601 changelog sending phase will also expand the stored
601 changelog sending phase will also expand the stored
602 mappings as needed.
602 mappings as needed.
603 """
603 """
604 if clrev == nullrev:
604 if clrev == nullrev:
605 return nullrev
605 return nullrev
606
606
607 if ischangelog:
607 if ischangelog:
608 return clrev
608 return clrev
609
609
610 # Walk the ellipsis-ized changelog breadth-first looking for a
610 # Walk the ellipsis-ized changelog breadth-first looking for a
611 # change that has been linked from the current revlog.
611 # change that has been linked from the current revlog.
612 #
612 #
613 # For a flat manifest revlog only a single step should be necessary
613 # For a flat manifest revlog only a single step should be necessary
614 # as all relevant changelog entries are relevant to the flat
614 # as all relevant changelog entries are relevant to the flat
615 # manifest.
615 # manifest.
616 #
616 #
617 # For a filelog or tree manifest dirlog however not every changelog
617 # For a filelog or tree manifest dirlog however not every changelog
618 # entry will have been relevant, so we need to skip some changelog
618 # entry will have been relevant, so we need to skip some changelog
619 # nodes even after ellipsis-izing.
619 # nodes even after ellipsis-izing.
620 walk = [clrev]
620 walk = [clrev]
621 while walk:
621 while walk:
622 p = walk[0]
622 p = walk[0]
623 walk = walk[1:]
623 walk = walk[1:]
624 if p in clrevtolocalrev:
624 if p in clrevtolocalrev:
625 return clrevtolocalrev[p]
625 return clrevtolocalrev[p]
626 elif p in fullclnodes:
626 elif p in fullclnodes:
627 walk.extend([pp for pp in cl.parentrevs(p)
627 walk.extend([pp for pp in cl.parentrevs(p)
628 if pp != nullrev])
628 if pp != nullrev])
629 elif p in precomputedellipsis:
629 elif p in precomputedellipsis:
630 walk.extend([pp for pp in precomputedellipsis[p]
630 walk.extend([pp for pp in precomputedellipsis[p]
631 if pp != nullrev])
631 if pp != nullrev])
632 else:
632 else:
633 # In this case, we've got an ellipsis with parents
633 # In this case, we've got an ellipsis with parents
634 # outside the current bundle (likely an
634 # outside the current bundle (likely an
635 # incremental pull). We "know" that we can use the
635 # incremental pull). We "know" that we can use the
636 # value of this same revlog at whatever revision
636 # value of this same revlog at whatever revision
637 # is pointed to by linknode. "Know" is in scare
637 # is pointed to by linknode. "Know" is in scare
638 # quotes because I haven't done enough examination
638 # quotes because I haven't done enough examination
639 # of edge cases to convince myself this is really
639 # of edge cases to convince myself this is really
640 # a fact - it works for all the (admittedly
640 # a fact - it works for all the (admittedly
641 # thorough) cases in our testsuite, but I would be
641 # thorough) cases in our testsuite, but I would be
642 # somewhat unsurprised to find a case in the wild
642 # somewhat unsurprised to find a case in the wild
643 # where this breaks down a bit. That said, I don't
643 # where this breaks down a bit. That said, I don't
644 # know if it would hurt anything.
644 # know if it would hurt anything.
645 for i in pycompat.xrange(rev, 0, -1):
645 for i in pycompat.xrange(rev, 0, -1):
646 if store.linkrev(i) == clrev:
646 if store.linkrev(i) == clrev:
647 return i
647 return i
648 # We failed to resolve a parent for this node, so
648 # We failed to resolve a parent for this node, so
649 # we crash the changegroup construction.
649 # we crash the changegroup construction.
650 raise error.Abort(
650 raise error.Abort(
651 'unable to resolve parent while packing %r %r'
651 'unable to resolve parent while packing %r %r'
652 ' for changeset %r' % (store.indexfile, rev, clrev))
652 ' for changeset %r' % (store.indexfile, rev, clrev))
653
653
654 return nullrev
654 return nullrev
655
655
656 if not linkparents or (
656 if not linkparents or (
657 store.parentrevs(rev) == (nullrev, nullrev)):
657 store.parentrevs(rev) == (nullrev, nullrev)):
658 p1, p2 = nullrev, nullrev
658 p1, p2 = nullrev, nullrev
659 elif len(linkparents) == 1:
659 elif len(linkparents) == 1:
660 p1, = sorted(local(p) for p in linkparents)
660 p1, = sorted(local(p) for p in linkparents)
661 p2 = nullrev
661 p2 = nullrev
662 else:
662 else:
663 p1, p2 = sorted(local(p) for p in linkparents)
663 p1, p2 = sorted(local(p) for p in linkparents)
664
664
665 n = store.node(rev)
665 n = store.node(rev)
666 p1n, p2n = store.node(p1), store.node(p2)
666 p1n, p2n = store.node(p1), store.node(p2)
667 flags = store.flags(rev)
667 flags = store.flags(rev)
668 flags |= revlog.REVIDX_ELLIPSIS
668 flags |= revlog.REVIDX_ELLIPSIS
669
669
670 # TODO: try and actually send deltas for ellipsis data blocks
670 # TODO: try and actually send deltas for ellipsis data blocks
671 data = store.revision(n)
671 data = store.revision(n)
672 diffheader = mdiff.trivialdiffheader(len(data))
672 diffheader = mdiff.trivialdiffheader(len(data))
673
673
674 return revisiondelta(
674 return revisiondelta(
675 node=n,
675 node=n,
676 p1node=p1n,
676 p1node=p1n,
677 p2node=p2n,
677 p2node=p2n,
678 basenode=nullid,
678 basenode=nullid,
679 linknode=linknode,
679 linknode=linknode,
680 flags=flags,
680 flags=flags,
681 deltachunks=(diffheader, data),
681 deltachunks=(diffheader, data),
682 )
682 )
683
683
684 def deltagroup(repo, revs, store, ischangelog, lookup, deltaparentfn,
685 deltaheaderfn, units=None,
686 ellipses=False, clrevtolocalrev=None, fullclnodes=None,
687 precomputedellipsis=None):
688 """Calculate a delta group, yielding a sequence of changegroup chunks
689 (strings).
690
691 Given a list of changeset revs, return a set of deltas and
692 metadata corresponding to nodes. The first delta is
693 first parent(nodelist[0]) -> nodelist[0], the receiver is
694 guaranteed to have this parent as it has all history before
695 these changesets. In the case firstparent is nullrev the
696 changegroup starts with a full revision.
697
698 If units is not None, progress detail will be generated, units specifies
699 the type of revlog that is touched (changelog, manifest, etc.).
700 """
701 # if we don't have any revisions touched by these changesets, bail
702 if len(revs) == 0:
703 yield closechunk()
704 return
705
706 cl = repo.changelog
707
708 # add the parent of the first rev
709 p = store.parentrevs(revs[0])[0]
710 revs.insert(0, p)
711
712 # build deltas
713 progress = None
714 if units is not None:
715 progress = repo.ui.makeprogress(_('bundling'), unit=units,
716 total=(len(revs) - 1))
717 for r in pycompat.xrange(len(revs) - 1):
718 if progress:
719 progress.update(r + 1)
720 prev, curr = revs[r], revs[r + 1]
721 linknode = lookup(store.node(curr))
722
723 if ellipses:
724 linkrev = cl.rev(linknode)
725 clrevtolocalrev[linkrev] = curr
726
727 # This is a node to send in full, because the changeset it
728 # corresponds to was a full changeset.
729 if linknode in fullclnodes:
730 delta = _revisiondeltanormal(store, curr, prev, linknode,
731 deltaparentfn)
732 elif linkrev not in precomputedellipsis:
733 delta = None
734 else:
735 delta = _revisiondeltanarrow(
736 cl, store, ischangelog, curr, linkrev, linknode,
737 clrevtolocalrev, fullclnodes,
738 precomputedellipsis)
739 else:
740 delta = _revisiondeltanormal(store, curr, prev, linknode,
741 deltaparentfn)
742
743 if not delta:
744 continue
745
746 meta = deltaheaderfn(delta)
747 l = len(meta) + sum(len(x) for x in delta.deltachunks)
748 yield chunkheader(l)
749 yield meta
750 for x in delta.deltachunks:
751 yield x
752
753 if progress:
754 progress.complete()
755
756 yield closechunk()
757
684 class cgpacker(object):
758 class cgpacker(object):
685 def __init__(self, repo, filematcher, version, allowreorder,
759 def __init__(self, repo, filematcher, version, allowreorder,
686 deltaparentfn, builddeltaheader, manifestsend,
760 deltaparentfn, builddeltaheader, manifestsend,
687 bundlecaps=None, ellipses=False,
761 bundlecaps=None, ellipses=False,
688 shallow=False, ellipsisroots=None, fullnodes=None):
762 shallow=False, ellipsisroots=None, fullnodes=None):
689 """Given a source repo, construct a bundler.
763 """Given a source repo, construct a bundler.
690
764
691 filematcher is a matcher that matches on files to include in the
765 filematcher is a matcher that matches on files to include in the
692 changegroup. Used to facilitate sparse changegroups.
766 changegroup. Used to facilitate sparse changegroups.
693
767
694 allowreorder controls whether reordering of revisions is allowed.
768 allowreorder controls whether reordering of revisions is allowed.
695 This value is used when ``bundle.reorder`` is ``auto`` or isn't
769 This value is used when ``bundle.reorder`` is ``auto`` or isn't
696 set.
770 set.
697
771
698 deltaparentfn is a callable that resolves the delta parent for
772 deltaparentfn is a callable that resolves the delta parent for
699 a specific revision.
773 a specific revision.
700
774
701 builddeltaheader is a callable that constructs the header for a group
775 builddeltaheader is a callable that constructs the header for a group
702 delta.
776 delta.
703
777
704 manifestsend is a chunk to send after manifests have been fully emitted.
778 manifestsend is a chunk to send after manifests have been fully emitted.
705
779
706 ellipses indicates whether ellipsis serving mode is enabled.
780 ellipses indicates whether ellipsis serving mode is enabled.
707
781
708 bundlecaps is optional and can be used to specify the set of
782 bundlecaps is optional and can be used to specify the set of
709 capabilities which can be used to build the bundle. While bundlecaps is
783 capabilities which can be used to build the bundle. While bundlecaps is
710 unused in core Mercurial, extensions rely on this feature to communicate
784 unused in core Mercurial, extensions rely on this feature to communicate
711 capabilities to customize the changegroup packer.
785 capabilities to customize the changegroup packer.
712
786
713 shallow indicates whether shallow data might be sent. The packer may
787 shallow indicates whether shallow data might be sent. The packer may
714 need to pack file contents not introduced by the changes being packed.
788 need to pack file contents not introduced by the changes being packed.
715
789
716 fullnodes is the set of changelog nodes which should not be ellipsis
790 fullnodes is the set of changelog nodes which should not be ellipsis
717 nodes. We store this rather than the set of nodes that should be
791 nodes. We store this rather than the set of nodes that should be
718 ellipsis because for very large histories we expect this to be
792 ellipsis because for very large histories we expect this to be
719 significantly smaller.
793 significantly smaller.
720 """
794 """
721 assert filematcher
795 assert filematcher
722 self._filematcher = filematcher
796 self._filematcher = filematcher
723
797
724 self.version = version
798 self.version = version
725 self._deltaparentfn = deltaparentfn
799 self._deltaparentfn = deltaparentfn
726 self._builddeltaheader = builddeltaheader
800 self._builddeltaheader = builddeltaheader
727 self._manifestsend = manifestsend
801 self._manifestsend = manifestsend
728 self._ellipses = ellipses
802 self._ellipses = ellipses
729
803
730 # Set of capabilities we can use to build the bundle.
804 # Set of capabilities we can use to build the bundle.
731 if bundlecaps is None:
805 if bundlecaps is None:
732 bundlecaps = set()
806 bundlecaps = set()
733 self._bundlecaps = bundlecaps
807 self._bundlecaps = bundlecaps
734 self._isshallow = shallow
808 self._isshallow = shallow
735 self._fullclnodes = fullnodes
809 self._fullclnodes = fullnodes
736
810
737 # Maps ellipsis revs to their roots at the changelog level.
811 # Maps ellipsis revs to their roots at the changelog level.
738 self._precomputedellipsis = ellipsisroots
812 self._precomputedellipsis = ellipsisroots
739
813
740 # experimental config: bundle.reorder
814 # experimental config: bundle.reorder
741 reorder = repo.ui.config('bundle', 'reorder')
815 reorder = repo.ui.config('bundle', 'reorder')
742 if reorder == 'auto':
816 if reorder == 'auto':
743 self._reorder = allowreorder
817 self._reorder = allowreorder
744 else:
818 else:
745 self._reorder = stringutil.parsebool(reorder)
819 self._reorder = stringutil.parsebool(reorder)
746
820
747 self._repo = repo
821 self._repo = repo
748
822
749 if self._repo.ui.verbose and not self._repo.ui.debugflag:
823 if self._repo.ui.verbose and not self._repo.ui.debugflag:
750 self._verbosenote = self._repo.ui.note
824 self._verbosenote = self._repo.ui.note
751 else:
825 else:
752 self._verbosenote = lambda s: None
826 self._verbosenote = lambda s: None
753
827
754 def group(self, repo, revs, store, ischangelog, lookup, deltaparentfn,
755 deltaheaderfn, units=None,
756 ellipses=False, clrevtolocalrev=None, fullclnodes=None,
757 precomputedellipsis=None):
758 """Calculate a delta group, yielding a sequence of changegroup chunks
759 (strings).
760
761 Given a list of changeset revs, return a set of deltas and
762 metadata corresponding to nodes. The first delta is
763 first parent(nodelist[0]) -> nodelist[0], the receiver is
764 guaranteed to have this parent as it has all history before
765 these changesets. In the case firstparent is nullrev the
766 changegroup starts with a full revision.
767
768 If units is not None, progress detail will be generated, units specifies
769 the type of revlog that is touched (changelog, manifest, etc.).
770 """
771 # if we don't have any revisions touched by these changesets, bail
772 if len(revs) == 0:
773 yield closechunk()
774 return
775
776 cl = repo.changelog
777
778 # add the parent of the first rev
779 p = store.parentrevs(revs[0])[0]
780 revs.insert(0, p)
781
782 # build deltas
783 progress = None
784 if units is not None:
785 progress = repo.ui.makeprogress(_('bundling'), unit=units,
786 total=(len(revs) - 1))
787 for r in pycompat.xrange(len(revs) - 1):
788 if progress:
789 progress.update(r + 1)
790 prev, curr = revs[r], revs[r + 1]
791 linknode = lookup(store.node(curr))
792
793 if ellipses:
794 linkrev = cl.rev(linknode)
795 clrevtolocalrev[linkrev] = curr
796
797 # This is a node to send in full, because the changeset it
798 # corresponds to was a full changeset.
799 if linknode in fullclnodes:
800 delta = _revisiondeltanormal(store, curr, prev, linknode,
801 deltaparentfn)
802 elif linkrev not in precomputedellipsis:
803 delta = None
804 else:
805 delta = _revisiondeltanarrow(
806 cl, store, ischangelog, curr, linkrev, linknode,
807 clrevtolocalrev, fullclnodes,
808 precomputedellipsis)
809 else:
810 delta = _revisiondeltanormal(store, curr, prev, linknode,
811 deltaparentfn)
812
813 if not delta:
814 continue
815
816 meta = deltaheaderfn(delta)
817 l = len(meta) + sum(len(x) for x in delta.deltachunks)
818 yield chunkheader(l)
819 yield meta
820 for x in delta.deltachunks:
821 yield x
822
823 if progress:
824 progress.complete()
825
826 yield closechunk()
827
828 def generate(self, commonrevs, clnodes, fastpathlinkrev, source):
828 def generate(self, commonrevs, clnodes, fastpathlinkrev, source):
829 """Yield a sequence of changegroup byte chunks."""
829 """Yield a sequence of changegroup byte chunks."""
830
830
831 repo = self._repo
831 repo = self._repo
832 cl = repo.changelog
832 cl = repo.changelog
833
833
834 self._verbosenote(_('uncompressed size of bundle content:\n'))
834 self._verbosenote(_('uncompressed size of bundle content:\n'))
835 size = 0
835 size = 0
836
836
837 clstate, chunks = self._generatechangelog(cl, clnodes)
837 clstate, chunks = self._generatechangelog(cl, clnodes)
838 for chunk in chunks:
838 for chunk in chunks:
839 size += len(chunk)
839 size += len(chunk)
840 yield chunk
840 yield chunk
841
841
842 self._verbosenote(_('%8.i (changelog)\n') % size)
842 self._verbosenote(_('%8.i (changelog)\n') % size)
843
843
844 clrevorder = clstate['clrevorder']
844 clrevorder = clstate['clrevorder']
845 mfs = clstate['mfs']
845 mfs = clstate['mfs']
846 changedfiles = clstate['changedfiles']
846 changedfiles = clstate['changedfiles']
847
847
848 # We need to make sure that the linkrev in the changegroup refers to
848 # We need to make sure that the linkrev in the changegroup refers to
849 # the first changeset that introduced the manifest or file revision.
849 # the first changeset that introduced the manifest or file revision.
850 # The fastpath is usually safer than the slowpath, because the filelogs
850 # The fastpath is usually safer than the slowpath, because the filelogs
851 # are walked in revlog order.
851 # are walked in revlog order.
852 #
852 #
853 # When taking the slowpath with reorder=None and the manifest revlog
853 # When taking the slowpath with reorder=None and the manifest revlog
854 # uses generaldelta, the manifest may be walked in the "wrong" order.
854 # uses generaldelta, the manifest may be walked in the "wrong" order.
855 # Without 'clrevorder', we would get an incorrect linkrev (see fix in
855 # Without 'clrevorder', we would get an incorrect linkrev (see fix in
856 # cc0ff93d0c0c).
856 # cc0ff93d0c0c).
857 #
857 #
858 # When taking the fastpath, we are only vulnerable to reordering
858 # When taking the fastpath, we are only vulnerable to reordering
859 # of the changelog itself. The changelog never uses generaldelta, so
859 # of the changelog itself. The changelog never uses generaldelta, so
860 # it is only reordered when reorder=True. To handle this case, we
860 # it is only reordered when reorder=True. To handle this case, we
861 # simply take the slowpath, which already has the 'clrevorder' logic.
861 # simply take the slowpath, which already has the 'clrevorder' logic.
862 # This was also fixed in cc0ff93d0c0c.
862 # This was also fixed in cc0ff93d0c0c.
863 fastpathlinkrev = fastpathlinkrev and not self._reorder
863 fastpathlinkrev = fastpathlinkrev and not self._reorder
864 # Treemanifests don't work correctly with fastpathlinkrev
864 # Treemanifests don't work correctly with fastpathlinkrev
865 # either, because we don't discover which directory nodes to
865 # either, because we don't discover which directory nodes to
866 # send along with files. This could probably be fixed.
866 # send along with files. This could probably be fixed.
867 fastpathlinkrev = fastpathlinkrev and (
867 fastpathlinkrev = fastpathlinkrev and (
868 'treemanifest' not in repo.requirements)
868 'treemanifest' not in repo.requirements)
869
869
870 fnodes = {} # needed file nodes
870 fnodes = {} # needed file nodes
871
871
872 for chunk in self.generatemanifests(commonrevs, clrevorder,
872 for chunk in self.generatemanifests(commonrevs, clrevorder,
873 fastpathlinkrev, mfs, fnodes, source,
873 fastpathlinkrev, mfs, fnodes, source,
874 clstate['clrevtomanifestrev']):
874 clstate['clrevtomanifestrev']):
875 yield chunk
875 yield chunk
876
876
877 mfdicts = None
877 mfdicts = None
878 if self._ellipses and self._isshallow:
878 if self._ellipses and self._isshallow:
879 mfdicts = [(self._repo.manifestlog[n].read(), lr)
879 mfdicts = [(self._repo.manifestlog[n].read(), lr)
880 for (n, lr) in mfs.iteritems()]
880 for (n, lr) in mfs.iteritems()]
881
881
882 mfs.clear()
882 mfs.clear()
883 clrevs = set(cl.rev(x) for x in clnodes)
883 clrevs = set(cl.rev(x) for x in clnodes)
884
884
885 for chunk in self.generatefiles(changedfiles, commonrevs,
885 for chunk in self.generatefiles(changedfiles, commonrevs,
886 source, mfdicts, fastpathlinkrev,
886 source, mfdicts, fastpathlinkrev,
887 fnodes, clrevs):
887 fnodes, clrevs):
888 yield chunk
888 yield chunk
889
889
890 yield closechunk()
890 yield closechunk()
891
891
892 if clnodes:
892 if clnodes:
893 repo.hook('outgoing', node=hex(clnodes[0]), source=source)
893 repo.hook('outgoing', node=hex(clnodes[0]), source=source)
894
894
895 def _generatechangelog(self, cl, nodes):
895 def _generatechangelog(self, cl, nodes):
896 """Generate data for changelog chunks.
896 """Generate data for changelog chunks.
897
897
898 Returns a 2-tuple of a dict containing state and an iterable of
898 Returns a 2-tuple of a dict containing state and an iterable of
899 byte chunks. The state will not be fully populated until the
899 byte chunks. The state will not be fully populated until the
900 chunk stream has been fully consumed.
900 chunk stream has been fully consumed.
901 """
901 """
902 clrevorder = {}
902 clrevorder = {}
903 mfs = {} # needed manifests
903 mfs = {} # needed manifests
904 mfl = self._repo.manifestlog
904 mfl = self._repo.manifestlog
905 # TODO violates storage abstraction.
905 # TODO violates storage abstraction.
906 mfrevlog = mfl._revlog
906 mfrevlog = mfl._revlog
907 changedfiles = set()
907 changedfiles = set()
908 clrevtomanifestrev = {}
908 clrevtomanifestrev = {}
909
909
910 # Callback for the changelog, used to collect changed files and
910 # Callback for the changelog, used to collect changed files and
911 # manifest nodes.
911 # manifest nodes.
912 # Returns the linkrev node (identity in the changelog case).
912 # Returns the linkrev node (identity in the changelog case).
913 def lookupcl(x):
913 def lookupcl(x):
914 c = cl.read(x)
914 c = cl.read(x)
915 clrevorder[x] = len(clrevorder)
915 clrevorder[x] = len(clrevorder)
916
916
917 if self._ellipses:
917 if self._ellipses:
918 # Only update mfs if x is going to be sent. Otherwise we
918 # Only update mfs if x is going to be sent. Otherwise we
919 # end up with bogus linkrevs specified for manifests and
919 # end up with bogus linkrevs specified for manifests and
920 # we skip some manifest nodes that we should otherwise
920 # we skip some manifest nodes that we should otherwise
921 # have sent.
921 # have sent.
922 if (x in self._fullclnodes
922 if (x in self._fullclnodes
923 or cl.rev(x) in self._precomputedellipsis):
923 or cl.rev(x) in self._precomputedellipsis):
924 n = c[0]
924 n = c[0]
925 # Record the first changeset introducing this manifest
925 # Record the first changeset introducing this manifest
926 # version.
926 # version.
927 mfs.setdefault(n, x)
927 mfs.setdefault(n, x)
928 # Set this narrow-specific dict so we have the lowest
928 # Set this narrow-specific dict so we have the lowest
929 # manifest revnum to look up for this cl revnum. (Part of
929 # manifest revnum to look up for this cl revnum. (Part of
930 # mapping changelog ellipsis parents to manifest ellipsis
930 # mapping changelog ellipsis parents to manifest ellipsis
931 # parents)
931 # parents)
932 clrevtomanifestrev.setdefault(cl.rev(x), mfrevlog.rev(n))
932 clrevtomanifestrev.setdefault(cl.rev(x), mfrevlog.rev(n))
933 # We can't trust the changed files list in the changeset if the
933 # We can't trust the changed files list in the changeset if the
934 # client requested a shallow clone.
934 # client requested a shallow clone.
935 if self._isshallow:
935 if self._isshallow:
936 changedfiles.update(mfl[c[0]].read().keys())
936 changedfiles.update(mfl[c[0]].read().keys())
937 else:
937 else:
938 changedfiles.update(c[3])
938 changedfiles.update(c[3])
939 else:
939 else:
940
940
941 n = c[0]
941 n = c[0]
942 # record the first changeset introducing this manifest version
942 # record the first changeset introducing this manifest version
943 mfs.setdefault(n, x)
943 mfs.setdefault(n, x)
944 # Record a complete list of potentially-changed files in
944 # Record a complete list of potentially-changed files in
945 # this manifest.
945 # this manifest.
946 changedfiles.update(c[3])
946 changedfiles.update(c[3])
947
947
948 return x
948 return x
949
949
950 # Changelog doesn't benefit from reordering revisions. So send out
950 # Changelog doesn't benefit from reordering revisions. So send out
951 # revisions in store order.
951 # revisions in store order.
952 revs = sorted(cl.rev(n) for n in nodes)
952 revs = sorted(cl.rev(n) for n in nodes)
953
953
954 state = {
954 state = {
955 'clrevorder': clrevorder,
955 'clrevorder': clrevorder,
956 'mfs': mfs,
956 'mfs': mfs,
957 'changedfiles': changedfiles,
957 'changedfiles': changedfiles,
958 'clrevtomanifestrev': clrevtomanifestrev,
958 'clrevtomanifestrev': clrevtomanifestrev,
959 }
959 }
960
960
961 gen = self.group(self._repo, revs, cl, True, lookupcl,
961 gen = deltagroup(
962 self._deltaparentfn, self._builddeltaheader,
962 self._repo, revs, cl, True, lookupcl,
963 ellipses=self._ellipses,
963 self._deltaparentfn, self._builddeltaheader,
964 units=_('changesets'),
964 ellipses=self._ellipses,
965 clrevtolocalrev={},
965 units=_('changesets'),
966 fullclnodes=self._fullclnodes,
966 clrevtolocalrev={},
967 precomputedellipsis=self._precomputedellipsis)
967 fullclnodes=self._fullclnodes,
968 precomputedellipsis=self._precomputedellipsis)
968
969
969 return state, gen
970 return state, gen
970
971
971 def generatemanifests(self, commonrevs, clrevorder, fastpathlinkrev, mfs,
972 def generatemanifests(self, commonrevs, clrevorder, fastpathlinkrev, mfs,
972 fnodes, source, clrevtolocalrev):
973 fnodes, source, clrevtolocalrev):
973 """Returns an iterator of changegroup chunks containing manifests.
974 """Returns an iterator of changegroup chunks containing manifests.
974
975
975 `source` is unused here, but is used by extensions like remotefilelog to
976 `source` is unused here, but is used by extensions like remotefilelog to
976 change what is sent based in pulls vs pushes, etc.
977 change what is sent based in pulls vs pushes, etc.
977 """
978 """
978 repo = self._repo
979 repo = self._repo
979 cl = repo.changelog
980 cl = repo.changelog
980 mfl = repo.manifestlog
981 mfl = repo.manifestlog
981 dirlog = mfl._revlog.dirlog
982 dirlog = mfl._revlog.dirlog
982 tmfnodes = {'': mfs}
983 tmfnodes = {'': mfs}
983
984
984 # Callback for the manifest, used to collect linkrevs for filelog
985 # Callback for the manifest, used to collect linkrevs for filelog
985 # revisions.
986 # revisions.
986 # Returns the linkrev node (collected in lookupcl).
987 # Returns the linkrev node (collected in lookupcl).
987 def makelookupmflinknode(dir, nodes):
988 def makelookupmflinknode(dir, nodes):
988 if fastpathlinkrev:
989 if fastpathlinkrev:
989 assert not dir
990 assert not dir
990 return mfs.__getitem__
991 return mfs.__getitem__
991
992
992 def lookupmflinknode(x):
993 def lookupmflinknode(x):
993 """Callback for looking up the linknode for manifests.
994 """Callback for looking up the linknode for manifests.
994
995
995 Returns the linkrev node for the specified manifest.
996 Returns the linkrev node for the specified manifest.
996
997
997 SIDE EFFECT:
998 SIDE EFFECT:
998
999
999 1) fclnodes gets populated with the list of relevant
1000 1) fclnodes gets populated with the list of relevant
1000 file nodes if we're not using fastpathlinkrev
1001 file nodes if we're not using fastpathlinkrev
1001 2) When treemanifests are in use, collects treemanifest nodes
1002 2) When treemanifests are in use, collects treemanifest nodes
1002 to send
1003 to send
1003
1004
1004 Note that this means manifests must be completely sent to
1005 Note that this means manifests must be completely sent to
1005 the client before you can trust the list of files and
1006 the client before you can trust the list of files and
1006 treemanifests to send.
1007 treemanifests to send.
1007 """
1008 """
1008 clnode = nodes[x]
1009 clnode = nodes[x]
1009 mdata = mfl.get(dir, x).readfast(shallow=True)
1010 mdata = mfl.get(dir, x).readfast(shallow=True)
1010 for p, n, fl in mdata.iterentries():
1011 for p, n, fl in mdata.iterentries():
1011 if fl == 't': # subdirectory manifest
1012 if fl == 't': # subdirectory manifest
1012 subdir = dir + p + '/'
1013 subdir = dir + p + '/'
1013 tmfclnodes = tmfnodes.setdefault(subdir, {})
1014 tmfclnodes = tmfnodes.setdefault(subdir, {})
1014 tmfclnode = tmfclnodes.setdefault(n, clnode)
1015 tmfclnode = tmfclnodes.setdefault(n, clnode)
1015 if clrevorder[clnode] < clrevorder[tmfclnode]:
1016 if clrevorder[clnode] < clrevorder[tmfclnode]:
1016 tmfclnodes[n] = clnode
1017 tmfclnodes[n] = clnode
1017 else:
1018 else:
1018 f = dir + p
1019 f = dir + p
1019 fclnodes = fnodes.setdefault(f, {})
1020 fclnodes = fnodes.setdefault(f, {})
1020 fclnode = fclnodes.setdefault(n, clnode)
1021 fclnode = fclnodes.setdefault(n, clnode)
1021 if clrevorder[clnode] < clrevorder[fclnode]:
1022 if clrevorder[clnode] < clrevorder[fclnode]:
1022 fclnodes[n] = clnode
1023 fclnodes[n] = clnode
1023 return clnode
1024 return clnode
1024 return lookupmflinknode
1025 return lookupmflinknode
1025
1026
1026 size = 0
1027 size = 0
1027 while tmfnodes:
1028 while tmfnodes:
1028 dir, nodes = tmfnodes.popitem()
1029 dir, nodes = tmfnodes.popitem()
1029 store = dirlog(dir)
1030 store = dirlog(dir)
1030
1031
1031 if not self._filematcher.visitdir(store._dir[:-1] or '.'):
1032 if not self._filematcher.visitdir(store._dir[:-1] or '.'):
1032 prunednodes = []
1033 prunednodes = []
1033 else:
1034 else:
1034 frev, flr = store.rev, store.linkrev
1035 frev, flr = store.rev, store.linkrev
1035 prunednodes = [n for n in nodes
1036 prunednodes = [n for n in nodes
1036 if flr(frev(n)) not in commonrevs]
1037 if flr(frev(n)) not in commonrevs]
1037
1038
1038 if dir and not prunednodes:
1039 if dir and not prunednodes:
1039 continue
1040 continue
1040
1041
1041 lookupfn = makelookupmflinknode(dir, nodes)
1042 lookupfn = makelookupmflinknode(dir, nodes)
1042
1043
1043 if self._ellipses:
1044 if self._ellipses:
1044 revs = _sortnodesellipsis(store, prunednodes, cl,
1045 revs = _sortnodesellipsis(store, prunednodes, cl,
1045 lookupfn)
1046 lookupfn)
1046 else:
1047 else:
1047 revs = _sortnodesnormal(store, prunednodes,
1048 revs = _sortnodesnormal(store, prunednodes,
1048 self._reorder)
1049 self._reorder)
1049
1050
1050 if dir:
1051 if dir:
1051 assert self.version == b'03'
1052 assert self.version == b'03'
1052 chunk = _fileheader(dir)
1053 chunk = _fileheader(dir)
1053 size += len(chunk)
1054 size += len(chunk)
1054 yield chunk
1055 yield chunk
1055
1056
1056 it = self.group(
1057 it = deltagroup(
1057 self._repo, revs, store, False, lookupfn,
1058 self._repo, revs, store, False, lookupfn,
1058 self._deltaparentfn, self._builddeltaheader,
1059 self._deltaparentfn, self._builddeltaheader,
1059 ellipses=self._ellipses,
1060 ellipses=self._ellipses,
1060 units=_('manifests'),
1061 units=_('manifests'),
1061 clrevtolocalrev=clrevtolocalrev,
1062 clrevtolocalrev=clrevtolocalrev,
1062 fullclnodes=self._fullclnodes,
1063 fullclnodes=self._fullclnodes,
1063 precomputedellipsis=self._precomputedellipsis)
1064 precomputedellipsis=self._precomputedellipsis)
1064
1065
1065 for chunk in it:
1066 for chunk in it:
1066 size += len(chunk)
1067 size += len(chunk)
1067 yield chunk
1068 yield chunk
1068
1069
1069 self._verbosenote(_('%8.i (manifests)\n') % size)
1070 self._verbosenote(_('%8.i (manifests)\n') % size)
1070 yield self._manifestsend
1071 yield self._manifestsend
1071
1072
1072 # The 'source' parameter is useful for extensions
1073 # The 'source' parameter is useful for extensions
1073 def generatefiles(self, changedfiles, commonrevs, source,
1074 def generatefiles(self, changedfiles, commonrevs, source,
1074 mfdicts, fastpathlinkrev, fnodes, clrevs):
1075 mfdicts, fastpathlinkrev, fnodes, clrevs):
1075 changedfiles = list(filter(self._filematcher, changedfiles))
1076 changedfiles = list(filter(self._filematcher, changedfiles))
1076
1077
1077 if not fastpathlinkrev:
1078 if not fastpathlinkrev:
1078 def normallinknodes(unused, fname):
1079 def normallinknodes(unused, fname):
1079 return fnodes.get(fname, {})
1080 return fnodes.get(fname, {})
1080 else:
1081 else:
1081 cln = self._repo.changelog.node
1082 cln = self._repo.changelog.node
1082
1083
1083 def normallinknodes(store, fname):
1084 def normallinknodes(store, fname):
1084 flinkrev = store.linkrev
1085 flinkrev = store.linkrev
1085 fnode = store.node
1086 fnode = store.node
1086 revs = ((r, flinkrev(r)) for r in store)
1087 revs = ((r, flinkrev(r)) for r in store)
1087 return dict((fnode(r), cln(lr))
1088 return dict((fnode(r), cln(lr))
1088 for r, lr in revs if lr in clrevs)
1089 for r, lr in revs if lr in clrevs)
1089
1090
1090 clrevtolocalrev = {}
1091 clrevtolocalrev = {}
1091
1092
1092 if self._isshallow:
1093 if self._isshallow:
1093 # In a shallow clone, the linknodes callback needs to also include
1094 # In a shallow clone, the linknodes callback needs to also include
1094 # those file nodes that are in the manifests we sent but weren't
1095 # those file nodes that are in the manifests we sent but weren't
1095 # introduced by those manifests.
1096 # introduced by those manifests.
1096 commonctxs = [self._repo[c] for c in commonrevs]
1097 commonctxs = [self._repo[c] for c in commonrevs]
1097 clrev = self._repo.changelog.rev
1098 clrev = self._repo.changelog.rev
1098
1099
1099 # Defining this function has a side-effect of overriding the
1100 # Defining this function has a side-effect of overriding the
1100 # function of the same name that was passed in as an argument.
1101 # function of the same name that was passed in as an argument.
1101 # TODO have caller pass in appropriate function.
1102 # TODO have caller pass in appropriate function.
1102 def linknodes(flog, fname):
1103 def linknodes(flog, fname):
1103 for c in commonctxs:
1104 for c in commonctxs:
1104 try:
1105 try:
1105 fnode = c.filenode(fname)
1106 fnode = c.filenode(fname)
1106 clrevtolocalrev[c.rev()] = flog.rev(fnode)
1107 clrevtolocalrev[c.rev()] = flog.rev(fnode)
1107 except error.ManifestLookupError:
1108 except error.ManifestLookupError:
1108 pass
1109 pass
1109 links = normallinknodes(flog, fname)
1110 links = normallinknodes(flog, fname)
1110 if len(links) != len(mfdicts):
1111 if len(links) != len(mfdicts):
1111 for mf, lr in mfdicts:
1112 for mf, lr in mfdicts:
1112 fnode = mf.get(fname, None)
1113 fnode = mf.get(fname, None)
1113 if fnode in links:
1114 if fnode in links:
1114 links[fnode] = min(links[fnode], lr, key=clrev)
1115 links[fnode] = min(links[fnode], lr, key=clrev)
1115 elif fnode:
1116 elif fnode:
1116 links[fnode] = lr
1117 links[fnode] = lr
1117 return links
1118 return links
1118 else:
1119 else:
1119 linknodes = normallinknodes
1120 linknodes = normallinknodes
1120
1121
1121 repo = self._repo
1122 repo = self._repo
1122 cl = repo.changelog
1123 cl = repo.changelog
1123 progress = repo.ui.makeprogress(_('bundling'), unit=_('files'),
1124 progress = repo.ui.makeprogress(_('bundling'), unit=_('files'),
1124 total=len(changedfiles))
1125 total=len(changedfiles))
1125 for i, fname in enumerate(sorted(changedfiles)):
1126 for i, fname in enumerate(sorted(changedfiles)):
1126 filerevlog = repo.file(fname)
1127 filerevlog = repo.file(fname)
1127 if not filerevlog:
1128 if not filerevlog:
1128 raise error.Abort(_("empty or missing file data for %s") %
1129 raise error.Abort(_("empty or missing file data for %s") %
1129 fname)
1130 fname)
1130
1131
1131 clrevtolocalrev.clear()
1132 clrevtolocalrev.clear()
1132
1133
1133 linkrevnodes = linknodes(filerevlog, fname)
1134 linkrevnodes = linknodes(filerevlog, fname)
1134 # Lookup for filenodes, we collected the linkrev nodes above in the
1135 # Lookup for filenodes, we collected the linkrev nodes above in the
1135 # fastpath case and with lookupmf in the slowpath case.
1136 # fastpath case and with lookupmf in the slowpath case.
1136 def lookupfilelog(x):
1137 def lookupfilelog(x):
1137 return linkrevnodes[x]
1138 return linkrevnodes[x]
1138
1139
1139 frev, flr = filerevlog.rev, filerevlog.linkrev
1140 frev, flr = filerevlog.rev, filerevlog.linkrev
1140 filenodes = [n for n in linkrevnodes
1141 filenodes = [n for n in linkrevnodes
1141 if flr(frev(n)) not in commonrevs]
1142 if flr(frev(n)) not in commonrevs]
1142
1143
1143 if filenodes:
1144 if filenodes:
1144 if self._ellipses:
1145 if self._ellipses:
1145 revs = _sortnodesellipsis(filerevlog, filenodes,
1146 revs = _sortnodesellipsis(filerevlog, filenodes,
1146 cl, lookupfilelog)
1147 cl, lookupfilelog)
1147 else:
1148 else:
1148 revs = _sortnodesnormal(filerevlog, filenodes,
1149 revs = _sortnodesnormal(filerevlog, filenodes,
1149 self._reorder)
1150 self._reorder)
1150
1151
1151 progress.update(i + 1, item=fname)
1152 progress.update(i + 1, item=fname)
1152 h = _fileheader(fname)
1153 h = _fileheader(fname)
1153 size = len(h)
1154 size = len(h)
1154 yield h
1155 yield h
1155
1156
1156 it = self.group(
1157 it = deltagroup(
1157 self._repo, revs, filerevlog, False, lookupfilelog,
1158 self._repo, revs, filerevlog, False, lookupfilelog,
1158 self._deltaparentfn, self._builddeltaheader,
1159 self._deltaparentfn, self._builddeltaheader,
1159 ellipses=self._ellipses,
1160 ellipses=self._ellipses,
1160 clrevtolocalrev=clrevtolocalrev,
1161 clrevtolocalrev=clrevtolocalrev,
1161 fullclnodes=self._fullclnodes,
1162 fullclnodes=self._fullclnodes,
1162 precomputedellipsis=self._precomputedellipsis)
1163 precomputedellipsis=self._precomputedellipsis)
1163
1164
1164 for chunk in it:
1165 for chunk in it:
1165 size += len(chunk)
1166 size += len(chunk)
1166 yield chunk
1167 yield chunk
1167 self._verbosenote(_('%8.i %s\n') % (size, fname))
1168 self._verbosenote(_('%8.i %s\n') % (size, fname))
1168 progress.complete()
1169 progress.complete()
1169
1170
1170 def _deltaparentprev(store, rev, p1, p2, prev):
1171 def _deltaparentprev(store, rev, p1, p2, prev):
1171 """Resolve a delta parent to the previous revision.
1172 """Resolve a delta parent to the previous revision.
1172
1173
1173 Used for version 1 changegroups, which don't support generaldelta.
1174 Used for version 1 changegroups, which don't support generaldelta.
1174 """
1175 """
1175 return prev
1176 return prev
1176
1177
1177 def _deltaparentgeneraldelta(store, rev, p1, p2, prev):
1178 def _deltaparentgeneraldelta(store, rev, p1, p2, prev):
1178 """Resolve a delta parent when general deltas are supported."""
1179 """Resolve a delta parent when general deltas are supported."""
1179 dp = store.deltaparent(rev)
1180 dp = store.deltaparent(rev)
1180 if dp == nullrev and store.storedeltachains:
1181 if dp == nullrev and store.storedeltachains:
1181 # Avoid sending full revisions when delta parent is null. Pick prev
1182 # Avoid sending full revisions when delta parent is null. Pick prev
1182 # in that case. It's tempting to pick p1 in this case, as p1 will
1183 # in that case. It's tempting to pick p1 in this case, as p1 will
1183 # be smaller in the common case. However, computing a delta against
1184 # be smaller in the common case. However, computing a delta against
1184 # p1 may require resolving the raw text of p1, which could be
1185 # p1 may require resolving the raw text of p1, which could be
1185 # expensive. The revlog caches should have prev cached, meaning
1186 # expensive. The revlog caches should have prev cached, meaning
1186 # less CPU for changegroup generation. There is likely room to add
1187 # less CPU for changegroup generation. There is likely room to add
1187 # a flag and/or config option to control this behavior.
1188 # a flag and/or config option to control this behavior.
1188 base = prev
1189 base = prev
1189 elif dp == nullrev:
1190 elif dp == nullrev:
1190 # revlog is configured to use full snapshot for a reason,
1191 # revlog is configured to use full snapshot for a reason,
1191 # stick to full snapshot.
1192 # stick to full snapshot.
1192 base = nullrev
1193 base = nullrev
1193 elif dp not in (p1, p2, prev):
1194 elif dp not in (p1, p2, prev):
1194 # Pick prev when we can't be sure remote has the base revision.
1195 # Pick prev when we can't be sure remote has the base revision.
1195 return prev
1196 return prev
1196 else:
1197 else:
1197 base = dp
1198 base = dp
1198
1199
1199 if base != nullrev and not store.candelta(base, rev):
1200 if base != nullrev and not store.candelta(base, rev):
1200 base = nullrev
1201 base = nullrev
1201
1202
1202 return base
1203 return base
1203
1204
1204 def _deltaparentellipses(store, rev, p1, p2, prev):
1205 def _deltaparentellipses(store, rev, p1, p2, prev):
1205 """Resolve a delta parent when in ellipses mode."""
1206 """Resolve a delta parent when in ellipses mode."""
1206 # TODO: send better deltas when in narrow mode.
1207 # TODO: send better deltas when in narrow mode.
1207 #
1208 #
1208 # changegroup.group() loops over revisions to send,
1209 # changegroup.group() loops over revisions to send,
1209 # including revisions we'll skip. What this means is that
1210 # including revisions we'll skip. What this means is that
1210 # `prev` will be a potentially useless delta base for all
1211 # `prev` will be a potentially useless delta base for all
1211 # ellipsis nodes, as the client likely won't have it. In
1212 # ellipsis nodes, as the client likely won't have it. In
1212 # the future we should do bookkeeping about which nodes
1213 # the future we should do bookkeeping about which nodes
1213 # have been sent to the client, and try to be
1214 # have been sent to the client, and try to be
1214 # significantly smarter about delta bases. This is
1215 # significantly smarter about delta bases. This is
1215 # slightly tricky because this same code has to work for
1216 # slightly tricky because this same code has to work for
1216 # all revlogs, and we don't have the linkrev/linknode here.
1217 # all revlogs, and we don't have the linkrev/linknode here.
1217 return p1
1218 return p1
1218
1219
1219 def _makecg1packer(repo, filematcher, bundlecaps, ellipses=False,
1220 def _makecg1packer(repo, filematcher, bundlecaps, ellipses=False,
1220 shallow=False, ellipsisroots=None, fullnodes=None):
1221 shallow=False, ellipsisroots=None, fullnodes=None):
1221 builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack(
1222 builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack(
1222 d.node, d.p1node, d.p2node, d.linknode)
1223 d.node, d.p1node, d.p2node, d.linknode)
1223
1224
1224 return cgpacker(repo, filematcher, b'01',
1225 return cgpacker(repo, filematcher, b'01',
1225 deltaparentfn=_deltaparentprev,
1226 deltaparentfn=_deltaparentprev,
1226 allowreorder=None,
1227 allowreorder=None,
1227 builddeltaheader=builddeltaheader,
1228 builddeltaheader=builddeltaheader,
1228 manifestsend=b'',
1229 manifestsend=b'',
1229 bundlecaps=bundlecaps,
1230 bundlecaps=bundlecaps,
1230 ellipses=ellipses,
1231 ellipses=ellipses,
1231 shallow=shallow,
1232 shallow=shallow,
1232 ellipsisroots=ellipsisroots,
1233 ellipsisroots=ellipsisroots,
1233 fullnodes=fullnodes)
1234 fullnodes=fullnodes)
1234
1235
1235 def _makecg2packer(repo, filematcher, bundlecaps, ellipses=False,
1236 def _makecg2packer(repo, filematcher, bundlecaps, ellipses=False,
1236 shallow=False, ellipsisroots=None, fullnodes=None):
1237 shallow=False, ellipsisroots=None, fullnodes=None):
1237 builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack(
1238 builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack(
1238 d.node, d.p1node, d.p2node, d.basenode, d.linknode)
1239 d.node, d.p1node, d.p2node, d.basenode, d.linknode)
1239
1240
1240 # Since generaldelta is directly supported by cg2, reordering
1241 # Since generaldelta is directly supported by cg2, reordering
1241 # generally doesn't help, so we disable it by default (treating
1242 # generally doesn't help, so we disable it by default (treating
1242 # bundle.reorder=auto just like bundle.reorder=False).
1243 # bundle.reorder=auto just like bundle.reorder=False).
1243 return cgpacker(repo, filematcher, b'02',
1244 return cgpacker(repo, filematcher, b'02',
1244 deltaparentfn=_deltaparentgeneraldelta,
1245 deltaparentfn=_deltaparentgeneraldelta,
1245 allowreorder=False,
1246 allowreorder=False,
1246 builddeltaheader=builddeltaheader,
1247 builddeltaheader=builddeltaheader,
1247 manifestsend=b'',
1248 manifestsend=b'',
1248 bundlecaps=bundlecaps,
1249 bundlecaps=bundlecaps,
1249 ellipses=ellipses,
1250 ellipses=ellipses,
1250 shallow=shallow,
1251 shallow=shallow,
1251 ellipsisroots=ellipsisroots,
1252 ellipsisroots=ellipsisroots,
1252 fullnodes=fullnodes)
1253 fullnodes=fullnodes)
1253
1254
1254 def _makecg3packer(repo, filematcher, bundlecaps, ellipses=False,
1255 def _makecg3packer(repo, filematcher, bundlecaps, ellipses=False,
1255 shallow=False, ellipsisroots=None, fullnodes=None):
1256 shallow=False, ellipsisroots=None, fullnodes=None):
1256 builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack(
1257 builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack(
1257 d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags)
1258 d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags)
1258
1259
1259 deltaparentfn = (_deltaparentellipses if ellipses
1260 deltaparentfn = (_deltaparentellipses if ellipses
1260 else _deltaparentgeneraldelta)
1261 else _deltaparentgeneraldelta)
1261
1262
1262 return cgpacker(repo, filematcher, b'03',
1263 return cgpacker(repo, filematcher, b'03',
1263 deltaparentfn=deltaparentfn,
1264 deltaparentfn=deltaparentfn,
1264 allowreorder=False,
1265 allowreorder=False,
1265 builddeltaheader=builddeltaheader,
1266 builddeltaheader=builddeltaheader,
1266 manifestsend=closechunk(),
1267 manifestsend=closechunk(),
1267 bundlecaps=bundlecaps,
1268 bundlecaps=bundlecaps,
1268 ellipses=ellipses,
1269 ellipses=ellipses,
1269 shallow=shallow,
1270 shallow=shallow,
1270 ellipsisroots=ellipsisroots,
1271 ellipsisroots=ellipsisroots,
1271 fullnodes=fullnodes)
1272 fullnodes=fullnodes)
1272
1273
1273 _packermap = {'01': (_makecg1packer, cg1unpacker),
1274 _packermap = {'01': (_makecg1packer, cg1unpacker),
1274 # cg2 adds support for exchanging generaldelta
1275 # cg2 adds support for exchanging generaldelta
1275 '02': (_makecg2packer, cg2unpacker),
1276 '02': (_makecg2packer, cg2unpacker),
1276 # cg3 adds support for exchanging revlog flags and treemanifests
1277 # cg3 adds support for exchanging revlog flags and treemanifests
1277 '03': (_makecg3packer, cg3unpacker),
1278 '03': (_makecg3packer, cg3unpacker),
1278 }
1279 }
1279
1280
1280 def allsupportedversions(repo):
1281 def allsupportedversions(repo):
1281 versions = set(_packermap.keys())
1282 versions = set(_packermap.keys())
1282 if not (repo.ui.configbool('experimental', 'changegroup3') or
1283 if not (repo.ui.configbool('experimental', 'changegroup3') or
1283 repo.ui.configbool('experimental', 'treemanifest') or
1284 repo.ui.configbool('experimental', 'treemanifest') or
1284 'treemanifest' in repo.requirements):
1285 'treemanifest' in repo.requirements):
1285 versions.discard('03')
1286 versions.discard('03')
1286 return versions
1287 return versions
1287
1288
1288 # Changegroup versions that can be applied to the repo
1289 # Changegroup versions that can be applied to the repo
1289 def supportedincomingversions(repo):
1290 def supportedincomingversions(repo):
1290 return allsupportedversions(repo)
1291 return allsupportedversions(repo)
1291
1292
1292 # Changegroup versions that can be created from the repo
1293 # Changegroup versions that can be created from the repo
1293 def supportedoutgoingversions(repo):
1294 def supportedoutgoingversions(repo):
1294 versions = allsupportedversions(repo)
1295 versions = allsupportedversions(repo)
1295 if 'treemanifest' in repo.requirements:
1296 if 'treemanifest' in repo.requirements:
1296 # Versions 01 and 02 support only flat manifests and it's just too
1297 # Versions 01 and 02 support only flat manifests and it's just too
1297 # expensive to convert between the flat manifest and tree manifest on
1298 # expensive to convert between the flat manifest and tree manifest on
1298 # the fly. Since tree manifests are hashed differently, all of history
1299 # the fly. Since tree manifests are hashed differently, all of history
1299 # would have to be converted. Instead, we simply don't even pretend to
1300 # would have to be converted. Instead, we simply don't even pretend to
1300 # support versions 01 and 02.
1301 # support versions 01 and 02.
1301 versions.discard('01')
1302 versions.discard('01')
1302 versions.discard('02')
1303 versions.discard('02')
1303 if repository.NARROW_REQUIREMENT in repo.requirements:
1304 if repository.NARROW_REQUIREMENT in repo.requirements:
1304 # Versions 01 and 02 don't support revlog flags, and we need to
1305 # Versions 01 and 02 don't support revlog flags, and we need to
1305 # support that for stripping and unbundling to work.
1306 # support that for stripping and unbundling to work.
1306 versions.discard('01')
1307 versions.discard('01')
1307 versions.discard('02')
1308 versions.discard('02')
1308 if LFS_REQUIREMENT in repo.requirements:
1309 if LFS_REQUIREMENT in repo.requirements:
1309 # Versions 01 and 02 don't support revlog flags, and we need to
1310 # Versions 01 and 02 don't support revlog flags, and we need to
1310 # mark LFS entries with REVIDX_EXTSTORED.
1311 # mark LFS entries with REVIDX_EXTSTORED.
1311 versions.discard('01')
1312 versions.discard('01')
1312 versions.discard('02')
1313 versions.discard('02')
1313
1314
1314 return versions
1315 return versions
1315
1316
1316 def localversion(repo):
1317 def localversion(repo):
1317 # Finds the best version to use for bundles that are meant to be used
1318 # Finds the best version to use for bundles that are meant to be used
1318 # locally, such as those from strip and shelve, and temporary bundles.
1319 # locally, such as those from strip and shelve, and temporary bundles.
1319 return max(supportedoutgoingversions(repo))
1320 return max(supportedoutgoingversions(repo))
1320
1321
1321 def safeversion(repo):
1322 def safeversion(repo):
1322 # Finds the smallest version that it's safe to assume clients of the repo
1323 # Finds the smallest version that it's safe to assume clients of the repo
1323 # will support. For example, all hg versions that support generaldelta also
1324 # will support. For example, all hg versions that support generaldelta also
1324 # support changegroup 02.
1325 # support changegroup 02.
1325 versions = supportedoutgoingversions(repo)
1326 versions = supportedoutgoingversions(repo)
1326 if 'generaldelta' in repo.requirements:
1327 if 'generaldelta' in repo.requirements:
1327 versions.discard('01')
1328 versions.discard('01')
1328 assert versions
1329 assert versions
1329 return min(versions)
1330 return min(versions)
1330
1331
1331 def getbundler(version, repo, bundlecaps=None, filematcher=None,
1332 def getbundler(version, repo, bundlecaps=None, filematcher=None,
1332 ellipses=False, shallow=False, ellipsisroots=None,
1333 ellipses=False, shallow=False, ellipsisroots=None,
1333 fullnodes=None):
1334 fullnodes=None):
1334 assert version in supportedoutgoingversions(repo)
1335 assert version in supportedoutgoingversions(repo)
1335
1336
1336 if filematcher is None:
1337 if filematcher is None:
1337 filematcher = matchmod.alwaysmatcher(repo.root, '')
1338 filematcher = matchmod.alwaysmatcher(repo.root, '')
1338
1339
1339 if version == '01' and not filematcher.always():
1340 if version == '01' and not filematcher.always():
1340 raise error.ProgrammingError('version 01 changegroups do not support '
1341 raise error.ProgrammingError('version 01 changegroups do not support '
1341 'sparse file matchers')
1342 'sparse file matchers')
1342
1343
1343 if ellipses and version in (b'01', b'02'):
1344 if ellipses and version in (b'01', b'02'):
1344 raise error.Abort(
1345 raise error.Abort(
1345 _('ellipsis nodes require at least cg3 on client and server, '
1346 _('ellipsis nodes require at least cg3 on client and server, '
1346 'but negotiated version %s') % version)
1347 'but negotiated version %s') % version)
1347
1348
1348 # Requested files could include files not in the local store. So
1349 # Requested files could include files not in the local store. So
1349 # filter those out.
1350 # filter those out.
1350 filematcher = matchmod.intersectmatchers(repo.narrowmatch(),
1351 filematcher = matchmod.intersectmatchers(repo.narrowmatch(),
1351 filematcher)
1352 filematcher)
1352
1353
1353 fn = _packermap[version][0]
1354 fn = _packermap[version][0]
1354 return fn(repo, filematcher, bundlecaps, ellipses=ellipses,
1355 return fn(repo, filematcher, bundlecaps, ellipses=ellipses,
1355 shallow=shallow, ellipsisroots=ellipsisroots,
1356 shallow=shallow, ellipsisroots=ellipsisroots,
1356 fullnodes=fullnodes)
1357 fullnodes=fullnodes)
1357
1358
1358 def getunbundler(version, fh, alg, extras=None):
1359 def getunbundler(version, fh, alg, extras=None):
1359 return _packermap[version][1](fh, alg, extras=extras)
1360 return _packermap[version][1](fh, alg, extras=extras)
1360
1361
1361 def _changegroupinfo(repo, nodes, source):
1362 def _changegroupinfo(repo, nodes, source):
1362 if repo.ui.verbose or source == 'bundle':
1363 if repo.ui.verbose or source == 'bundle':
1363 repo.ui.status(_("%d changesets found\n") % len(nodes))
1364 repo.ui.status(_("%d changesets found\n") % len(nodes))
1364 if repo.ui.debugflag:
1365 if repo.ui.debugflag:
1365 repo.ui.debug("list of changesets:\n")
1366 repo.ui.debug("list of changesets:\n")
1366 for node in nodes:
1367 for node in nodes:
1367 repo.ui.debug("%s\n" % hex(node))
1368 repo.ui.debug("%s\n" % hex(node))
1368
1369
1369 def makechangegroup(repo, outgoing, version, source, fastpath=False,
1370 def makechangegroup(repo, outgoing, version, source, fastpath=False,
1370 bundlecaps=None):
1371 bundlecaps=None):
1371 cgstream = makestream(repo, outgoing, version, source,
1372 cgstream = makestream(repo, outgoing, version, source,
1372 fastpath=fastpath, bundlecaps=bundlecaps)
1373 fastpath=fastpath, bundlecaps=bundlecaps)
1373 return getunbundler(version, util.chunkbuffer(cgstream), None,
1374 return getunbundler(version, util.chunkbuffer(cgstream), None,
1374 {'clcount': len(outgoing.missing) })
1375 {'clcount': len(outgoing.missing) })
1375
1376
1376 def makestream(repo, outgoing, version, source, fastpath=False,
1377 def makestream(repo, outgoing, version, source, fastpath=False,
1377 bundlecaps=None, filematcher=None):
1378 bundlecaps=None, filematcher=None):
1378 bundler = getbundler(version, repo, bundlecaps=bundlecaps,
1379 bundler = getbundler(version, repo, bundlecaps=bundlecaps,
1379 filematcher=filematcher)
1380 filematcher=filematcher)
1380
1381
1381 repo = repo.unfiltered()
1382 repo = repo.unfiltered()
1382 commonrevs = outgoing.common
1383 commonrevs = outgoing.common
1383 csets = outgoing.missing
1384 csets = outgoing.missing
1384 heads = outgoing.missingheads
1385 heads = outgoing.missingheads
1385 # We go through the fast path if we get told to, or if all (unfiltered
1386 # We go through the fast path if we get told to, or if all (unfiltered
1386 # heads have been requested (since we then know there all linkrevs will
1387 # heads have been requested (since we then know there all linkrevs will
1387 # be pulled by the client).
1388 # be pulled by the client).
1388 heads.sort()
1389 heads.sort()
1389 fastpathlinkrev = fastpath or (
1390 fastpathlinkrev = fastpath or (
1390 repo.filtername is None and heads == sorted(repo.heads()))
1391 repo.filtername is None and heads == sorted(repo.heads()))
1391
1392
1392 repo.hook('preoutgoing', throw=True, source=source)
1393 repo.hook('preoutgoing', throw=True, source=source)
1393 _changegroupinfo(repo, csets, source)
1394 _changegroupinfo(repo, csets, source)
1394 return bundler.generate(commonrevs, csets, fastpathlinkrev, source)
1395 return bundler.generate(commonrevs, csets, fastpathlinkrev, source)
1395
1396
1396 def _addchangegroupfiles(repo, source, revmap, trp, expectedfiles, needfiles):
1397 def _addchangegroupfiles(repo, source, revmap, trp, expectedfiles, needfiles):
1397 revisions = 0
1398 revisions = 0
1398 files = 0
1399 files = 0
1399 progress = repo.ui.makeprogress(_('files'), unit=_('files'),
1400 progress = repo.ui.makeprogress(_('files'), unit=_('files'),
1400 total=expectedfiles)
1401 total=expectedfiles)
1401 for chunkdata in iter(source.filelogheader, {}):
1402 for chunkdata in iter(source.filelogheader, {}):
1402 files += 1
1403 files += 1
1403 f = chunkdata["filename"]
1404 f = chunkdata["filename"]
1404 repo.ui.debug("adding %s revisions\n" % f)
1405 repo.ui.debug("adding %s revisions\n" % f)
1405 progress.increment()
1406 progress.increment()
1406 fl = repo.file(f)
1407 fl = repo.file(f)
1407 o = len(fl)
1408 o = len(fl)
1408 try:
1409 try:
1409 deltas = source.deltaiter()
1410 deltas = source.deltaiter()
1410 if not fl.addgroup(deltas, revmap, trp):
1411 if not fl.addgroup(deltas, revmap, trp):
1411 raise error.Abort(_("received file revlog group is empty"))
1412 raise error.Abort(_("received file revlog group is empty"))
1412 except error.CensoredBaseError as e:
1413 except error.CensoredBaseError as e:
1413 raise error.Abort(_("received delta base is censored: %s") % e)
1414 raise error.Abort(_("received delta base is censored: %s") % e)
1414 revisions += len(fl) - o
1415 revisions += len(fl) - o
1415 if f in needfiles:
1416 if f in needfiles:
1416 needs = needfiles[f]
1417 needs = needfiles[f]
1417 for new in pycompat.xrange(o, len(fl)):
1418 for new in pycompat.xrange(o, len(fl)):
1418 n = fl.node(new)
1419 n = fl.node(new)
1419 if n in needs:
1420 if n in needs:
1420 needs.remove(n)
1421 needs.remove(n)
1421 else:
1422 else:
1422 raise error.Abort(
1423 raise error.Abort(
1423 _("received spurious file revlog entry"))
1424 _("received spurious file revlog entry"))
1424 if not needs:
1425 if not needs:
1425 del needfiles[f]
1426 del needfiles[f]
1426 progress.complete()
1427 progress.complete()
1427
1428
1428 for f, needs in needfiles.iteritems():
1429 for f, needs in needfiles.iteritems():
1429 fl = repo.file(f)
1430 fl = repo.file(f)
1430 for n in needs:
1431 for n in needs:
1431 try:
1432 try:
1432 fl.rev(n)
1433 fl.rev(n)
1433 except error.LookupError:
1434 except error.LookupError:
1434 raise error.Abort(
1435 raise error.Abort(
1435 _('missing file data for %s:%s - run hg verify') %
1436 _('missing file data for %s:%s - run hg verify') %
1436 (f, hex(n)))
1437 (f, hex(n)))
1437
1438
1438 return revisions, files
1439 return revisions, files
General Comments 0
You need to be logged in to leave comments. Login now