##// END OF EJS Templates
changegroup: inline _revchunk() into group()...
Gregory Szorc -
r39020:a3105433 default
parent child Browse files
Show More
@@ -1,1438 +1,1435
1 # changegroup.py - Mercurial changegroup manipulation functions
1 # changegroup.py - Mercurial changegroup manipulation functions
2 #
2 #
3 # Copyright 2006 Matt Mackall <mpm@selenic.com>
3 # Copyright 2006 Matt Mackall <mpm@selenic.com>
4 #
4 #
5 # This software may be used and distributed according to the terms of the
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
6 # GNU General Public License version 2 or any later version.
7
7
8 from __future__ import absolute_import
8 from __future__ import absolute_import
9
9
10 import os
10 import os
11 import struct
11 import struct
12 import weakref
12 import weakref
13
13
14 from .i18n import _
14 from .i18n import _
15 from .node import (
15 from .node import (
16 hex,
16 hex,
17 nullid,
17 nullid,
18 nullrev,
18 nullrev,
19 short,
19 short,
20 )
20 )
21
21
22 from .thirdparty import (
22 from .thirdparty import (
23 attr,
23 attr,
24 )
24 )
25
25
26 from . import (
26 from . import (
27 dagutil,
27 dagutil,
28 error,
28 error,
29 manifest,
29 manifest,
30 match as matchmod,
30 match as matchmod,
31 mdiff,
31 mdiff,
32 phases,
32 phases,
33 pycompat,
33 pycompat,
34 repository,
34 repository,
35 revlog,
35 revlog,
36 util,
36 util,
37 )
37 )
38
38
39 from .utils import (
39 from .utils import (
40 stringutil,
40 stringutil,
41 )
41 )
42
42
43 _CHANGEGROUPV1_DELTA_HEADER = struct.Struct("20s20s20s20s")
43 _CHANGEGROUPV1_DELTA_HEADER = struct.Struct("20s20s20s20s")
44 _CHANGEGROUPV2_DELTA_HEADER = struct.Struct("20s20s20s20s20s")
44 _CHANGEGROUPV2_DELTA_HEADER = struct.Struct("20s20s20s20s20s")
45 _CHANGEGROUPV3_DELTA_HEADER = struct.Struct(">20s20s20s20s20sH")
45 _CHANGEGROUPV3_DELTA_HEADER = struct.Struct(">20s20s20s20s20sH")
46
46
47 LFS_REQUIREMENT = 'lfs'
47 LFS_REQUIREMENT = 'lfs'
48
48
49 readexactly = util.readexactly
49 readexactly = util.readexactly
50
50
51 def getchunk(stream):
51 def getchunk(stream):
52 """return the next chunk from stream as a string"""
52 """return the next chunk from stream as a string"""
53 d = readexactly(stream, 4)
53 d = readexactly(stream, 4)
54 l = struct.unpack(">l", d)[0]
54 l = struct.unpack(">l", d)[0]
55 if l <= 4:
55 if l <= 4:
56 if l:
56 if l:
57 raise error.Abort(_("invalid chunk length %d") % l)
57 raise error.Abort(_("invalid chunk length %d") % l)
58 return ""
58 return ""
59 return readexactly(stream, l - 4)
59 return readexactly(stream, l - 4)
60
60
61 def chunkheader(length):
61 def chunkheader(length):
62 """return a changegroup chunk header (string)"""
62 """return a changegroup chunk header (string)"""
63 return struct.pack(">l", length + 4)
63 return struct.pack(">l", length + 4)
64
64
65 def closechunk():
65 def closechunk():
66 """return a changegroup chunk header (string) for a zero-length chunk"""
66 """return a changegroup chunk header (string) for a zero-length chunk"""
67 return struct.pack(">l", 0)
67 return struct.pack(">l", 0)
68
68
69 def _fileheader(path):
69 def _fileheader(path):
70 """Obtain a changegroup chunk header for a named path."""
70 """Obtain a changegroup chunk header for a named path."""
71 return chunkheader(len(path)) + path
71 return chunkheader(len(path)) + path
72
72
73 def writechunks(ui, chunks, filename, vfs=None):
73 def writechunks(ui, chunks, filename, vfs=None):
74 """Write chunks to a file and return its filename.
74 """Write chunks to a file and return its filename.
75
75
76 The stream is assumed to be a bundle file.
76 The stream is assumed to be a bundle file.
77 Existing files will not be overwritten.
77 Existing files will not be overwritten.
78 If no filename is specified, a temporary file is created.
78 If no filename is specified, a temporary file is created.
79 """
79 """
80 fh = None
80 fh = None
81 cleanup = None
81 cleanup = None
82 try:
82 try:
83 if filename:
83 if filename:
84 if vfs:
84 if vfs:
85 fh = vfs.open(filename, "wb")
85 fh = vfs.open(filename, "wb")
86 else:
86 else:
87 # Increase default buffer size because default is usually
87 # Increase default buffer size because default is usually
88 # small (4k is common on Linux).
88 # small (4k is common on Linux).
89 fh = open(filename, "wb", 131072)
89 fh = open(filename, "wb", 131072)
90 else:
90 else:
91 fd, filename = pycompat.mkstemp(prefix="hg-bundle-", suffix=".hg")
91 fd, filename = pycompat.mkstemp(prefix="hg-bundle-", suffix=".hg")
92 fh = os.fdopen(fd, r"wb")
92 fh = os.fdopen(fd, r"wb")
93 cleanup = filename
93 cleanup = filename
94 for c in chunks:
94 for c in chunks:
95 fh.write(c)
95 fh.write(c)
96 cleanup = None
96 cleanup = None
97 return filename
97 return filename
98 finally:
98 finally:
99 if fh is not None:
99 if fh is not None:
100 fh.close()
100 fh.close()
101 if cleanup is not None:
101 if cleanup is not None:
102 if filename and vfs:
102 if filename and vfs:
103 vfs.unlink(cleanup)
103 vfs.unlink(cleanup)
104 else:
104 else:
105 os.unlink(cleanup)
105 os.unlink(cleanup)
106
106
107 class cg1unpacker(object):
107 class cg1unpacker(object):
108 """Unpacker for cg1 changegroup streams.
108 """Unpacker for cg1 changegroup streams.
109
109
110 A changegroup unpacker handles the framing of the revision data in
110 A changegroup unpacker handles the framing of the revision data in
111 the wire format. Most consumers will want to use the apply()
111 the wire format. Most consumers will want to use the apply()
112 method to add the changes from the changegroup to a repository.
112 method to add the changes from the changegroup to a repository.
113
113
114 If you're forwarding a changegroup unmodified to another consumer,
114 If you're forwarding a changegroup unmodified to another consumer,
115 use getchunks(), which returns an iterator of changegroup
115 use getchunks(), which returns an iterator of changegroup
116 chunks. This is mostly useful for cases where you need to know the
116 chunks. This is mostly useful for cases where you need to know the
117 data stream has ended by observing the end of the changegroup.
117 data stream has ended by observing the end of the changegroup.
118
118
119 deltachunk() is useful only if you're applying delta data. Most
119 deltachunk() is useful only if you're applying delta data. Most
120 consumers should prefer apply() instead.
120 consumers should prefer apply() instead.
121
121
122 A few other public methods exist. Those are used only for
122 A few other public methods exist. Those are used only for
123 bundlerepo and some debug commands - their use is discouraged.
123 bundlerepo and some debug commands - their use is discouraged.
124 """
124 """
125 deltaheader = _CHANGEGROUPV1_DELTA_HEADER
125 deltaheader = _CHANGEGROUPV1_DELTA_HEADER
126 deltaheadersize = deltaheader.size
126 deltaheadersize = deltaheader.size
127 version = '01'
127 version = '01'
128 _grouplistcount = 1 # One list of files after the manifests
128 _grouplistcount = 1 # One list of files after the manifests
129
129
130 def __init__(self, fh, alg, extras=None):
130 def __init__(self, fh, alg, extras=None):
131 if alg is None:
131 if alg is None:
132 alg = 'UN'
132 alg = 'UN'
133 if alg not in util.compengines.supportedbundletypes:
133 if alg not in util.compengines.supportedbundletypes:
134 raise error.Abort(_('unknown stream compression type: %s')
134 raise error.Abort(_('unknown stream compression type: %s')
135 % alg)
135 % alg)
136 if alg == 'BZ':
136 if alg == 'BZ':
137 alg = '_truncatedBZ'
137 alg = '_truncatedBZ'
138
138
139 compengine = util.compengines.forbundletype(alg)
139 compengine = util.compengines.forbundletype(alg)
140 self._stream = compengine.decompressorreader(fh)
140 self._stream = compengine.decompressorreader(fh)
141 self._type = alg
141 self._type = alg
142 self.extras = extras or {}
142 self.extras = extras or {}
143 self.callback = None
143 self.callback = None
144
144
145 # These methods (compressed, read, seek, tell) all appear to only
145 # These methods (compressed, read, seek, tell) all appear to only
146 # be used by bundlerepo, but it's a little hard to tell.
146 # be used by bundlerepo, but it's a little hard to tell.
147 def compressed(self):
147 def compressed(self):
148 return self._type is not None and self._type != 'UN'
148 return self._type is not None and self._type != 'UN'
149 def read(self, l):
149 def read(self, l):
150 return self._stream.read(l)
150 return self._stream.read(l)
151 def seek(self, pos):
151 def seek(self, pos):
152 return self._stream.seek(pos)
152 return self._stream.seek(pos)
153 def tell(self):
153 def tell(self):
154 return self._stream.tell()
154 return self._stream.tell()
155 def close(self):
155 def close(self):
156 return self._stream.close()
156 return self._stream.close()
157
157
158 def _chunklength(self):
158 def _chunklength(self):
159 d = readexactly(self._stream, 4)
159 d = readexactly(self._stream, 4)
160 l = struct.unpack(">l", d)[0]
160 l = struct.unpack(">l", d)[0]
161 if l <= 4:
161 if l <= 4:
162 if l:
162 if l:
163 raise error.Abort(_("invalid chunk length %d") % l)
163 raise error.Abort(_("invalid chunk length %d") % l)
164 return 0
164 return 0
165 if self.callback:
165 if self.callback:
166 self.callback()
166 self.callback()
167 return l - 4
167 return l - 4
168
168
169 def changelogheader(self):
169 def changelogheader(self):
170 """v10 does not have a changelog header chunk"""
170 """v10 does not have a changelog header chunk"""
171 return {}
171 return {}
172
172
173 def manifestheader(self):
173 def manifestheader(self):
174 """v10 does not have a manifest header chunk"""
174 """v10 does not have a manifest header chunk"""
175 return {}
175 return {}
176
176
177 def filelogheader(self):
177 def filelogheader(self):
178 """return the header of the filelogs chunk, v10 only has the filename"""
178 """return the header of the filelogs chunk, v10 only has the filename"""
179 l = self._chunklength()
179 l = self._chunklength()
180 if not l:
180 if not l:
181 return {}
181 return {}
182 fname = readexactly(self._stream, l)
182 fname = readexactly(self._stream, l)
183 return {'filename': fname}
183 return {'filename': fname}
184
184
185 def _deltaheader(self, headertuple, prevnode):
185 def _deltaheader(self, headertuple, prevnode):
186 node, p1, p2, cs = headertuple
186 node, p1, p2, cs = headertuple
187 if prevnode is None:
187 if prevnode is None:
188 deltabase = p1
188 deltabase = p1
189 else:
189 else:
190 deltabase = prevnode
190 deltabase = prevnode
191 flags = 0
191 flags = 0
192 return node, p1, p2, deltabase, cs, flags
192 return node, p1, p2, deltabase, cs, flags
193
193
194 def deltachunk(self, prevnode):
194 def deltachunk(self, prevnode):
195 l = self._chunklength()
195 l = self._chunklength()
196 if not l:
196 if not l:
197 return {}
197 return {}
198 headerdata = readexactly(self._stream, self.deltaheadersize)
198 headerdata = readexactly(self._stream, self.deltaheadersize)
199 header = self.deltaheader.unpack(headerdata)
199 header = self.deltaheader.unpack(headerdata)
200 delta = readexactly(self._stream, l - self.deltaheadersize)
200 delta = readexactly(self._stream, l - self.deltaheadersize)
201 node, p1, p2, deltabase, cs, flags = self._deltaheader(header, prevnode)
201 node, p1, p2, deltabase, cs, flags = self._deltaheader(header, prevnode)
202 return (node, p1, p2, cs, deltabase, delta, flags)
202 return (node, p1, p2, cs, deltabase, delta, flags)
203
203
204 def getchunks(self):
204 def getchunks(self):
205 """returns all the chunks contains in the bundle
205 """returns all the chunks contains in the bundle
206
206
207 Used when you need to forward the binary stream to a file or another
207 Used when you need to forward the binary stream to a file or another
208 network API. To do so, it parse the changegroup data, otherwise it will
208 network API. To do so, it parse the changegroup data, otherwise it will
209 block in case of sshrepo because it don't know the end of the stream.
209 block in case of sshrepo because it don't know the end of the stream.
210 """
210 """
211 # For changegroup 1 and 2, we expect 3 parts: changelog, manifestlog,
211 # For changegroup 1 and 2, we expect 3 parts: changelog, manifestlog,
212 # and a list of filelogs. For changegroup 3, we expect 4 parts:
212 # and a list of filelogs. For changegroup 3, we expect 4 parts:
213 # changelog, manifestlog, a list of tree manifestlogs, and a list of
213 # changelog, manifestlog, a list of tree manifestlogs, and a list of
214 # filelogs.
214 # filelogs.
215 #
215 #
216 # Changelog and manifestlog parts are terminated with empty chunks. The
216 # Changelog and manifestlog parts are terminated with empty chunks. The
217 # tree and file parts are a list of entry sections. Each entry section
217 # tree and file parts are a list of entry sections. Each entry section
218 # is a series of chunks terminating in an empty chunk. The list of these
218 # is a series of chunks terminating in an empty chunk. The list of these
219 # entry sections is terminated in yet another empty chunk, so we know
219 # entry sections is terminated in yet another empty chunk, so we know
220 # we've reached the end of the tree/file list when we reach an empty
220 # we've reached the end of the tree/file list when we reach an empty
221 # chunk that was proceeded by no non-empty chunks.
221 # chunk that was proceeded by no non-empty chunks.
222
222
223 parts = 0
223 parts = 0
224 while parts < 2 + self._grouplistcount:
224 while parts < 2 + self._grouplistcount:
225 noentries = True
225 noentries = True
226 while True:
226 while True:
227 chunk = getchunk(self)
227 chunk = getchunk(self)
228 if not chunk:
228 if not chunk:
229 # The first two empty chunks represent the end of the
229 # The first two empty chunks represent the end of the
230 # changelog and the manifestlog portions. The remaining
230 # changelog and the manifestlog portions. The remaining
231 # empty chunks represent either A) the end of individual
231 # empty chunks represent either A) the end of individual
232 # tree or file entries in the file list, or B) the end of
232 # tree or file entries in the file list, or B) the end of
233 # the entire list. It's the end of the entire list if there
233 # the entire list. It's the end of the entire list if there
234 # were no entries (i.e. noentries is True).
234 # were no entries (i.e. noentries is True).
235 if parts < 2:
235 if parts < 2:
236 parts += 1
236 parts += 1
237 elif noentries:
237 elif noentries:
238 parts += 1
238 parts += 1
239 break
239 break
240 noentries = False
240 noentries = False
241 yield chunkheader(len(chunk))
241 yield chunkheader(len(chunk))
242 pos = 0
242 pos = 0
243 while pos < len(chunk):
243 while pos < len(chunk):
244 next = pos + 2**20
244 next = pos + 2**20
245 yield chunk[pos:next]
245 yield chunk[pos:next]
246 pos = next
246 pos = next
247 yield closechunk()
247 yield closechunk()
248
248
249 def _unpackmanifests(self, repo, revmap, trp, prog):
249 def _unpackmanifests(self, repo, revmap, trp, prog):
250 self.callback = prog.increment
250 self.callback = prog.increment
251 # no need to check for empty manifest group here:
251 # no need to check for empty manifest group here:
252 # if the result of the merge of 1 and 2 is the same in 3 and 4,
252 # if the result of the merge of 1 and 2 is the same in 3 and 4,
253 # no new manifest will be created and the manifest group will
253 # no new manifest will be created and the manifest group will
254 # be empty during the pull
254 # be empty during the pull
255 self.manifestheader()
255 self.manifestheader()
256 deltas = self.deltaiter()
256 deltas = self.deltaiter()
257 repo.manifestlog.addgroup(deltas, revmap, trp)
257 repo.manifestlog.addgroup(deltas, revmap, trp)
258 prog.complete()
258 prog.complete()
259 self.callback = None
259 self.callback = None
260
260
261 def apply(self, repo, tr, srctype, url, targetphase=phases.draft,
261 def apply(self, repo, tr, srctype, url, targetphase=phases.draft,
262 expectedtotal=None):
262 expectedtotal=None):
263 """Add the changegroup returned by source.read() to this repo.
263 """Add the changegroup returned by source.read() to this repo.
264 srctype is a string like 'push', 'pull', or 'unbundle'. url is
264 srctype is a string like 'push', 'pull', or 'unbundle'. url is
265 the URL of the repo where this changegroup is coming from.
265 the URL of the repo where this changegroup is coming from.
266
266
267 Return an integer summarizing the change to this repo:
267 Return an integer summarizing the change to this repo:
268 - nothing changed or no source: 0
268 - nothing changed or no source: 0
269 - more heads than before: 1+added heads (2..n)
269 - more heads than before: 1+added heads (2..n)
270 - fewer heads than before: -1-removed heads (-2..-n)
270 - fewer heads than before: -1-removed heads (-2..-n)
271 - number of heads stays the same: 1
271 - number of heads stays the same: 1
272 """
272 """
273 repo = repo.unfiltered()
273 repo = repo.unfiltered()
274 def csmap(x):
274 def csmap(x):
275 repo.ui.debug("add changeset %s\n" % short(x))
275 repo.ui.debug("add changeset %s\n" % short(x))
276 return len(cl)
276 return len(cl)
277
277
278 def revmap(x):
278 def revmap(x):
279 return cl.rev(x)
279 return cl.rev(x)
280
280
281 changesets = files = revisions = 0
281 changesets = files = revisions = 0
282
282
283 try:
283 try:
284 # The transaction may already carry source information. In this
284 # The transaction may already carry source information. In this
285 # case we use the top level data. We overwrite the argument
285 # case we use the top level data. We overwrite the argument
286 # because we need to use the top level value (if they exist)
286 # because we need to use the top level value (if they exist)
287 # in this function.
287 # in this function.
288 srctype = tr.hookargs.setdefault('source', srctype)
288 srctype = tr.hookargs.setdefault('source', srctype)
289 url = tr.hookargs.setdefault('url', url)
289 url = tr.hookargs.setdefault('url', url)
290 repo.hook('prechangegroup',
290 repo.hook('prechangegroup',
291 throw=True, **pycompat.strkwargs(tr.hookargs))
291 throw=True, **pycompat.strkwargs(tr.hookargs))
292
292
293 # write changelog data to temp files so concurrent readers
293 # write changelog data to temp files so concurrent readers
294 # will not see an inconsistent view
294 # will not see an inconsistent view
295 cl = repo.changelog
295 cl = repo.changelog
296 cl.delayupdate(tr)
296 cl.delayupdate(tr)
297 oldheads = set(cl.heads())
297 oldheads = set(cl.heads())
298
298
299 trp = weakref.proxy(tr)
299 trp = weakref.proxy(tr)
300 # pull off the changeset group
300 # pull off the changeset group
301 repo.ui.status(_("adding changesets\n"))
301 repo.ui.status(_("adding changesets\n"))
302 clstart = len(cl)
302 clstart = len(cl)
303 progress = repo.ui.makeprogress(_('changesets'), unit=_('chunks'),
303 progress = repo.ui.makeprogress(_('changesets'), unit=_('chunks'),
304 total=expectedtotal)
304 total=expectedtotal)
305 self.callback = progress.increment
305 self.callback = progress.increment
306
306
307 efiles = set()
307 efiles = set()
308 def onchangelog(cl, node):
308 def onchangelog(cl, node):
309 efiles.update(cl.readfiles(node))
309 efiles.update(cl.readfiles(node))
310
310
311 self.changelogheader()
311 self.changelogheader()
312 deltas = self.deltaiter()
312 deltas = self.deltaiter()
313 cgnodes = cl.addgroup(deltas, csmap, trp, addrevisioncb=onchangelog)
313 cgnodes = cl.addgroup(deltas, csmap, trp, addrevisioncb=onchangelog)
314 efiles = len(efiles)
314 efiles = len(efiles)
315
315
316 if not cgnodes:
316 if not cgnodes:
317 repo.ui.develwarn('applied empty changegroup',
317 repo.ui.develwarn('applied empty changegroup',
318 config='warn-empty-changegroup')
318 config='warn-empty-changegroup')
319 clend = len(cl)
319 clend = len(cl)
320 changesets = clend - clstart
320 changesets = clend - clstart
321 progress.complete()
321 progress.complete()
322 self.callback = None
322 self.callback = None
323
323
324 # pull off the manifest group
324 # pull off the manifest group
325 repo.ui.status(_("adding manifests\n"))
325 repo.ui.status(_("adding manifests\n"))
326 # We know that we'll never have more manifests than we had
326 # We know that we'll never have more manifests than we had
327 # changesets.
327 # changesets.
328 progress = repo.ui.makeprogress(_('manifests'), unit=_('chunks'),
328 progress = repo.ui.makeprogress(_('manifests'), unit=_('chunks'),
329 total=changesets)
329 total=changesets)
330 self._unpackmanifests(repo, revmap, trp, progress)
330 self._unpackmanifests(repo, revmap, trp, progress)
331
331
332 needfiles = {}
332 needfiles = {}
333 if repo.ui.configbool('server', 'validate'):
333 if repo.ui.configbool('server', 'validate'):
334 cl = repo.changelog
334 cl = repo.changelog
335 ml = repo.manifestlog
335 ml = repo.manifestlog
336 # validate incoming csets have their manifests
336 # validate incoming csets have their manifests
337 for cset in pycompat.xrange(clstart, clend):
337 for cset in pycompat.xrange(clstart, clend):
338 mfnode = cl.changelogrevision(cset).manifest
338 mfnode = cl.changelogrevision(cset).manifest
339 mfest = ml[mfnode].readdelta()
339 mfest = ml[mfnode].readdelta()
340 # store file cgnodes we must see
340 # store file cgnodes we must see
341 for f, n in mfest.iteritems():
341 for f, n in mfest.iteritems():
342 needfiles.setdefault(f, set()).add(n)
342 needfiles.setdefault(f, set()).add(n)
343
343
344 # process the files
344 # process the files
345 repo.ui.status(_("adding file changes\n"))
345 repo.ui.status(_("adding file changes\n"))
346 newrevs, newfiles = _addchangegroupfiles(
346 newrevs, newfiles = _addchangegroupfiles(
347 repo, self, revmap, trp, efiles, needfiles)
347 repo, self, revmap, trp, efiles, needfiles)
348 revisions += newrevs
348 revisions += newrevs
349 files += newfiles
349 files += newfiles
350
350
351 deltaheads = 0
351 deltaheads = 0
352 if oldheads:
352 if oldheads:
353 heads = cl.heads()
353 heads = cl.heads()
354 deltaheads = len(heads) - len(oldheads)
354 deltaheads = len(heads) - len(oldheads)
355 for h in heads:
355 for h in heads:
356 if h not in oldheads and repo[h].closesbranch():
356 if h not in oldheads and repo[h].closesbranch():
357 deltaheads -= 1
357 deltaheads -= 1
358 htext = ""
358 htext = ""
359 if deltaheads:
359 if deltaheads:
360 htext = _(" (%+d heads)") % deltaheads
360 htext = _(" (%+d heads)") % deltaheads
361
361
362 repo.ui.status(_("added %d changesets"
362 repo.ui.status(_("added %d changesets"
363 " with %d changes to %d files%s\n")
363 " with %d changes to %d files%s\n")
364 % (changesets, revisions, files, htext))
364 % (changesets, revisions, files, htext))
365 repo.invalidatevolatilesets()
365 repo.invalidatevolatilesets()
366
366
367 if changesets > 0:
367 if changesets > 0:
368 if 'node' not in tr.hookargs:
368 if 'node' not in tr.hookargs:
369 tr.hookargs['node'] = hex(cl.node(clstart))
369 tr.hookargs['node'] = hex(cl.node(clstart))
370 tr.hookargs['node_last'] = hex(cl.node(clend - 1))
370 tr.hookargs['node_last'] = hex(cl.node(clend - 1))
371 hookargs = dict(tr.hookargs)
371 hookargs = dict(tr.hookargs)
372 else:
372 else:
373 hookargs = dict(tr.hookargs)
373 hookargs = dict(tr.hookargs)
374 hookargs['node'] = hex(cl.node(clstart))
374 hookargs['node'] = hex(cl.node(clstart))
375 hookargs['node_last'] = hex(cl.node(clend - 1))
375 hookargs['node_last'] = hex(cl.node(clend - 1))
376 repo.hook('pretxnchangegroup',
376 repo.hook('pretxnchangegroup',
377 throw=True, **pycompat.strkwargs(hookargs))
377 throw=True, **pycompat.strkwargs(hookargs))
378
378
379 added = [cl.node(r) for r in pycompat.xrange(clstart, clend)]
379 added = [cl.node(r) for r in pycompat.xrange(clstart, clend)]
380 phaseall = None
380 phaseall = None
381 if srctype in ('push', 'serve'):
381 if srctype in ('push', 'serve'):
382 # Old servers can not push the boundary themselves.
382 # Old servers can not push the boundary themselves.
383 # New servers won't push the boundary if changeset already
383 # New servers won't push the boundary if changeset already
384 # exists locally as secret
384 # exists locally as secret
385 #
385 #
386 # We should not use added here but the list of all change in
386 # We should not use added here but the list of all change in
387 # the bundle
387 # the bundle
388 if repo.publishing():
388 if repo.publishing():
389 targetphase = phaseall = phases.public
389 targetphase = phaseall = phases.public
390 else:
390 else:
391 # closer target phase computation
391 # closer target phase computation
392
392
393 # Those changesets have been pushed from the
393 # Those changesets have been pushed from the
394 # outside, their phases are going to be pushed
394 # outside, their phases are going to be pushed
395 # alongside. Therefor `targetphase` is
395 # alongside. Therefor `targetphase` is
396 # ignored.
396 # ignored.
397 targetphase = phaseall = phases.draft
397 targetphase = phaseall = phases.draft
398 if added:
398 if added:
399 phases.registernew(repo, tr, targetphase, added)
399 phases.registernew(repo, tr, targetphase, added)
400 if phaseall is not None:
400 if phaseall is not None:
401 phases.advanceboundary(repo, tr, phaseall, cgnodes)
401 phases.advanceboundary(repo, tr, phaseall, cgnodes)
402
402
403 if changesets > 0:
403 if changesets > 0:
404
404
405 def runhooks():
405 def runhooks():
406 # These hooks run when the lock releases, not when the
406 # These hooks run when the lock releases, not when the
407 # transaction closes. So it's possible for the changelog
407 # transaction closes. So it's possible for the changelog
408 # to have changed since we last saw it.
408 # to have changed since we last saw it.
409 if clstart >= len(repo):
409 if clstart >= len(repo):
410 return
410 return
411
411
412 repo.hook("changegroup", **pycompat.strkwargs(hookargs))
412 repo.hook("changegroup", **pycompat.strkwargs(hookargs))
413
413
414 for n in added:
414 for n in added:
415 args = hookargs.copy()
415 args = hookargs.copy()
416 args['node'] = hex(n)
416 args['node'] = hex(n)
417 del args['node_last']
417 del args['node_last']
418 repo.hook("incoming", **pycompat.strkwargs(args))
418 repo.hook("incoming", **pycompat.strkwargs(args))
419
419
420 newheads = [h for h in repo.heads()
420 newheads = [h for h in repo.heads()
421 if h not in oldheads]
421 if h not in oldheads]
422 repo.ui.log("incoming",
422 repo.ui.log("incoming",
423 "%d incoming changes - new heads: %s\n",
423 "%d incoming changes - new heads: %s\n",
424 len(added),
424 len(added),
425 ', '.join([hex(c[:6]) for c in newheads]))
425 ', '.join([hex(c[:6]) for c in newheads]))
426
426
427 tr.addpostclose('changegroup-runhooks-%020i' % clstart,
427 tr.addpostclose('changegroup-runhooks-%020i' % clstart,
428 lambda tr: repo._afterlock(runhooks))
428 lambda tr: repo._afterlock(runhooks))
429 finally:
429 finally:
430 repo.ui.flush()
430 repo.ui.flush()
431 # never return 0 here:
431 # never return 0 here:
432 if deltaheads < 0:
432 if deltaheads < 0:
433 ret = deltaheads - 1
433 ret = deltaheads - 1
434 else:
434 else:
435 ret = deltaheads + 1
435 ret = deltaheads + 1
436 return ret
436 return ret
437
437
438 def deltaiter(self):
438 def deltaiter(self):
439 """
439 """
440 returns an iterator of the deltas in this changegroup
440 returns an iterator of the deltas in this changegroup
441
441
442 Useful for passing to the underlying storage system to be stored.
442 Useful for passing to the underlying storage system to be stored.
443 """
443 """
444 chain = None
444 chain = None
445 for chunkdata in iter(lambda: self.deltachunk(chain), {}):
445 for chunkdata in iter(lambda: self.deltachunk(chain), {}):
446 # Chunkdata: (node, p1, p2, cs, deltabase, delta, flags)
446 # Chunkdata: (node, p1, p2, cs, deltabase, delta, flags)
447 yield chunkdata
447 yield chunkdata
448 chain = chunkdata[0]
448 chain = chunkdata[0]
449
449
450 class cg2unpacker(cg1unpacker):
450 class cg2unpacker(cg1unpacker):
451 """Unpacker for cg2 streams.
451 """Unpacker for cg2 streams.
452
452
453 cg2 streams add support for generaldelta, so the delta header
453 cg2 streams add support for generaldelta, so the delta header
454 format is slightly different. All other features about the data
454 format is slightly different. All other features about the data
455 remain the same.
455 remain the same.
456 """
456 """
457 deltaheader = _CHANGEGROUPV2_DELTA_HEADER
457 deltaheader = _CHANGEGROUPV2_DELTA_HEADER
458 deltaheadersize = deltaheader.size
458 deltaheadersize = deltaheader.size
459 version = '02'
459 version = '02'
460
460
461 def _deltaheader(self, headertuple, prevnode):
461 def _deltaheader(self, headertuple, prevnode):
462 node, p1, p2, deltabase, cs = headertuple
462 node, p1, p2, deltabase, cs = headertuple
463 flags = 0
463 flags = 0
464 return node, p1, p2, deltabase, cs, flags
464 return node, p1, p2, deltabase, cs, flags
465
465
466 class cg3unpacker(cg2unpacker):
466 class cg3unpacker(cg2unpacker):
467 """Unpacker for cg3 streams.
467 """Unpacker for cg3 streams.
468
468
469 cg3 streams add support for exchanging treemanifests and revlog
469 cg3 streams add support for exchanging treemanifests and revlog
470 flags. It adds the revlog flags to the delta header and an empty chunk
470 flags. It adds the revlog flags to the delta header and an empty chunk
471 separating manifests and files.
471 separating manifests and files.
472 """
472 """
473 deltaheader = _CHANGEGROUPV3_DELTA_HEADER
473 deltaheader = _CHANGEGROUPV3_DELTA_HEADER
474 deltaheadersize = deltaheader.size
474 deltaheadersize = deltaheader.size
475 version = '03'
475 version = '03'
476 _grouplistcount = 2 # One list of manifests and one list of files
476 _grouplistcount = 2 # One list of manifests and one list of files
477
477
478 def _deltaheader(self, headertuple, prevnode):
478 def _deltaheader(self, headertuple, prevnode):
479 node, p1, p2, deltabase, cs, flags = headertuple
479 node, p1, p2, deltabase, cs, flags = headertuple
480 return node, p1, p2, deltabase, cs, flags
480 return node, p1, p2, deltabase, cs, flags
481
481
482 def _unpackmanifests(self, repo, revmap, trp, prog):
482 def _unpackmanifests(self, repo, revmap, trp, prog):
483 super(cg3unpacker, self)._unpackmanifests(repo, revmap, trp, prog)
483 super(cg3unpacker, self)._unpackmanifests(repo, revmap, trp, prog)
484 for chunkdata in iter(self.filelogheader, {}):
484 for chunkdata in iter(self.filelogheader, {}):
485 # If we get here, there are directory manifests in the changegroup
485 # If we get here, there are directory manifests in the changegroup
486 d = chunkdata["filename"]
486 d = chunkdata["filename"]
487 repo.ui.debug("adding %s revisions\n" % d)
487 repo.ui.debug("adding %s revisions\n" % d)
488 dirlog = repo.manifestlog._revlog.dirlog(d)
488 dirlog = repo.manifestlog._revlog.dirlog(d)
489 deltas = self.deltaiter()
489 deltas = self.deltaiter()
490 if not dirlog.addgroup(deltas, revmap, trp):
490 if not dirlog.addgroup(deltas, revmap, trp):
491 raise error.Abort(_("received dir revlog group is empty"))
491 raise error.Abort(_("received dir revlog group is empty"))
492
492
493 class headerlessfixup(object):
493 class headerlessfixup(object):
494 def __init__(self, fh, h):
494 def __init__(self, fh, h):
495 self._h = h
495 self._h = h
496 self._fh = fh
496 self._fh = fh
497 def read(self, n):
497 def read(self, n):
498 if self._h:
498 if self._h:
499 d, self._h = self._h[:n], self._h[n:]
499 d, self._h = self._h[:n], self._h[n:]
500 if len(d) < n:
500 if len(d) < n:
501 d += readexactly(self._fh, n - len(d))
501 d += readexactly(self._fh, n - len(d))
502 return d
502 return d
503 return readexactly(self._fh, n)
503 return readexactly(self._fh, n)
504
504
505 @attr.s(slots=True, frozen=True)
505 @attr.s(slots=True, frozen=True)
506 class revisiondelta(object):
506 class revisiondelta(object):
507 """Describes a delta entry in a changegroup.
507 """Describes a delta entry in a changegroup.
508
508
509 Captured data is sufficient to serialize the delta into multiple
509 Captured data is sufficient to serialize the delta into multiple
510 formats.
510 formats.
511 """
511 """
512 # 20 byte node of this revision.
512 # 20 byte node of this revision.
513 node = attr.ib()
513 node = attr.ib()
514 # 20 byte nodes of parent revisions.
514 # 20 byte nodes of parent revisions.
515 p1node = attr.ib()
515 p1node = attr.ib()
516 p2node = attr.ib()
516 p2node = attr.ib()
517 # 20 byte node of node this delta is against.
517 # 20 byte node of node this delta is against.
518 basenode = attr.ib()
518 basenode = attr.ib()
519 # 20 byte node of changeset revision this delta is associated with.
519 # 20 byte node of changeset revision this delta is associated with.
520 linknode = attr.ib()
520 linknode = attr.ib()
521 # 2 bytes of flags to apply to revision data.
521 # 2 bytes of flags to apply to revision data.
522 flags = attr.ib()
522 flags = attr.ib()
523 # Iterable of chunks holding raw delta data.
523 # Iterable of chunks holding raw delta data.
524 deltachunks = attr.ib()
524 deltachunks = attr.ib()
525
525
526 def _sortnodesnormal(store, nodes, reorder):
526 def _sortnodesnormal(store, nodes, reorder):
527 """Sort nodes for changegroup generation and turn into revnums."""
527 """Sort nodes for changegroup generation and turn into revnums."""
528 # for generaldelta revlogs, we linearize the revs; this will both be
528 # for generaldelta revlogs, we linearize the revs; this will both be
529 # much quicker and generate a much smaller bundle
529 # much quicker and generate a much smaller bundle
530 if (store._generaldelta and reorder is None) or reorder:
530 if (store._generaldelta and reorder is None) or reorder:
531 dag = dagutil.revlogdag(store)
531 dag = dagutil.revlogdag(store)
532 return dag.linearize(set(store.rev(n) for n in nodes))
532 return dag.linearize(set(store.rev(n) for n in nodes))
533 else:
533 else:
534 return sorted([store.rev(n) for n in nodes])
534 return sorted([store.rev(n) for n in nodes])
535
535
536 def _sortnodesellipsis(store, nodes, clnodetorev, lookup):
536 def _sortnodesellipsis(store, nodes, clnodetorev, lookup):
537 """Sort nodes for changegroup generation and turn into revnums."""
537 """Sort nodes for changegroup generation and turn into revnums."""
538 # Ellipses serving mode.
538 # Ellipses serving mode.
539 #
539 #
540 # In a perfect world, we'd generate better ellipsis-ified graphs
540 # In a perfect world, we'd generate better ellipsis-ified graphs
541 # for non-changelog revlogs. In practice, we haven't started doing
541 # for non-changelog revlogs. In practice, we haven't started doing
542 # that yet, so the resulting DAGs for the manifestlog and filelogs
542 # that yet, so the resulting DAGs for the manifestlog and filelogs
543 # are actually full of bogus parentage on all the ellipsis
543 # are actually full of bogus parentage on all the ellipsis
544 # nodes. This has the side effect that, while the contents are
544 # nodes. This has the side effect that, while the contents are
545 # correct, the individual DAGs might be completely out of whack in
545 # correct, the individual DAGs might be completely out of whack in
546 # a case like 882681bc3166 and its ancestors (back about 10
546 # a case like 882681bc3166 and its ancestors (back about 10
547 # revisions or so) in the main hg repo.
547 # revisions or so) in the main hg repo.
548 #
548 #
549 # The one invariant we *know* holds is that the new (potentially
549 # The one invariant we *know* holds is that the new (potentially
550 # bogus) DAG shape will be valid if we order the nodes in the
550 # bogus) DAG shape will be valid if we order the nodes in the
551 # order that they're introduced in dramatis personae by the
551 # order that they're introduced in dramatis personae by the
552 # changelog, so what we do is we sort the non-changelog histories
552 # changelog, so what we do is we sort the non-changelog histories
553 # by the order in which they are used by the changelog.
553 # by the order in which they are used by the changelog.
554 key = lambda n: clnodetorev[lookup(n)]
554 key = lambda n: clnodetorev[lookup(n)]
555 return [store.rev(n) for n in sorted(nodes, key=key)]
555 return [store.rev(n) for n in sorted(nodes, key=key)]
556
556
557 class cgpacker(object):
557 class cgpacker(object):
558 def __init__(self, repo, filematcher, version, allowreorder,
558 def __init__(self, repo, filematcher, version, allowreorder,
559 deltaparentfn, builddeltaheader, manifestsend,
559 deltaparentfn, builddeltaheader, manifestsend,
560 bundlecaps=None, ellipses=False,
560 bundlecaps=None, ellipses=False,
561 shallow=False, ellipsisroots=None, fullnodes=None):
561 shallow=False, ellipsisroots=None, fullnodes=None):
562 """Given a source repo, construct a bundler.
562 """Given a source repo, construct a bundler.
563
563
564 filematcher is a matcher that matches on files to include in the
564 filematcher is a matcher that matches on files to include in the
565 changegroup. Used to facilitate sparse changegroups.
565 changegroup. Used to facilitate sparse changegroups.
566
566
567 allowreorder controls whether reordering of revisions is allowed.
567 allowreorder controls whether reordering of revisions is allowed.
568 This value is used when ``bundle.reorder`` is ``auto`` or isn't
568 This value is used when ``bundle.reorder`` is ``auto`` or isn't
569 set.
569 set.
570
570
571 deltaparentfn is a callable that resolves the delta parent for
571 deltaparentfn is a callable that resolves the delta parent for
572 a specific revision.
572 a specific revision.
573
573
574 builddeltaheader is a callable that constructs the header for a group
574 builddeltaheader is a callable that constructs the header for a group
575 delta.
575 delta.
576
576
577 manifestsend is a chunk to send after manifests have been fully emitted.
577 manifestsend is a chunk to send after manifests have been fully emitted.
578
578
579 ellipses indicates whether ellipsis serving mode is enabled.
579 ellipses indicates whether ellipsis serving mode is enabled.
580
580
581 bundlecaps is optional and can be used to specify the set of
581 bundlecaps is optional and can be used to specify the set of
582 capabilities which can be used to build the bundle. While bundlecaps is
582 capabilities which can be used to build the bundle. While bundlecaps is
583 unused in core Mercurial, extensions rely on this feature to communicate
583 unused in core Mercurial, extensions rely on this feature to communicate
584 capabilities to customize the changegroup packer.
584 capabilities to customize the changegroup packer.
585
585
586 shallow indicates whether shallow data might be sent. The packer may
586 shallow indicates whether shallow data might be sent. The packer may
587 need to pack file contents not introduced by the changes being packed.
587 need to pack file contents not introduced by the changes being packed.
588
588
589 fullnodes is the list of nodes which should not be ellipsis nodes. We
589 fullnodes is the list of nodes which should not be ellipsis nodes. We
590 store this rather than the set of nodes that should be ellipsis because
590 store this rather than the set of nodes that should be ellipsis because
591 for very large histories we expect this to be significantly smaller.
591 for very large histories we expect this to be significantly smaller.
592 """
592 """
593 assert filematcher
593 assert filematcher
594 self._filematcher = filematcher
594 self._filematcher = filematcher
595
595
596 self.version = version
596 self.version = version
597 self._deltaparentfn = deltaparentfn
597 self._deltaparentfn = deltaparentfn
598 self._builddeltaheader = builddeltaheader
598 self._builddeltaheader = builddeltaheader
599 self._manifestsend = manifestsend
599 self._manifestsend = manifestsend
600 self._ellipses = ellipses
600 self._ellipses = ellipses
601
601
602 # Set of capabilities we can use to build the bundle.
602 # Set of capabilities we can use to build the bundle.
603 if bundlecaps is None:
603 if bundlecaps is None:
604 bundlecaps = set()
604 bundlecaps = set()
605 self._bundlecaps = bundlecaps
605 self._bundlecaps = bundlecaps
606 self._isshallow = shallow
606 self._isshallow = shallow
607 self._fullnodes = fullnodes
607 self._fullnodes = fullnodes
608
608
609 # Maps ellipsis revs to their roots at the changelog level.
609 # Maps ellipsis revs to their roots at the changelog level.
610 self._precomputedellipsis = ellipsisroots
610 self._precomputedellipsis = ellipsisroots
611
611
612 # experimental config: bundle.reorder
612 # experimental config: bundle.reorder
613 reorder = repo.ui.config('bundle', 'reorder')
613 reorder = repo.ui.config('bundle', 'reorder')
614 if reorder == 'auto':
614 if reorder == 'auto':
615 self._reorder = allowreorder
615 self._reorder = allowreorder
616 else:
616 else:
617 self._reorder = stringutil.parsebool(reorder)
617 self._reorder = stringutil.parsebool(reorder)
618
618
619 self._repo = repo
619 self._repo = repo
620
620
621 if self._repo.ui.verbose and not self._repo.ui.debugflag:
621 if self._repo.ui.verbose and not self._repo.ui.debugflag:
622 self._verbosenote = self._repo.ui.note
622 self._verbosenote = self._repo.ui.note
623 else:
623 else:
624 self._verbosenote = lambda s: None
624 self._verbosenote = lambda s: None
625
625
626 # Maps CL revs to per-revlog revisions. Cleared in close() at
626 # Maps CL revs to per-revlog revisions. Cleared in close() at
627 # the end of each group.
627 # the end of each group.
628 self._clrevtolocalrev = {}
628 self._clrevtolocalrev = {}
629 self._nextclrevtolocalrev = {}
629 self._nextclrevtolocalrev = {}
630
630
631 # Maps changelog nodes to changelog revs. Filled in once
631 # Maps changelog nodes to changelog revs. Filled in once
632 # during changelog stage and then left unmodified.
632 # during changelog stage and then left unmodified.
633 self._clnodetorev = {}
633 self._clnodetorev = {}
634
634
635 def _close(self):
635 def _close(self):
636 # Ellipses serving mode.
636 # Ellipses serving mode.
637 self._clrevtolocalrev.clear()
637 self._clrevtolocalrev.clear()
638 if self._nextclrevtolocalrev is not None:
638 if self._nextclrevtolocalrev is not None:
639 self._clrevtolocalrev = self._nextclrevtolocalrev
639 self._clrevtolocalrev = self._nextclrevtolocalrev
640 self._nextclrevtolocalrev = None
640 self._nextclrevtolocalrev = None
641
641
642 return closechunk()
642 return closechunk()
643
643
644 def group(self, revs, store, ischangelog, lookup, units=None):
644 def group(self, revs, store, ischangelog, lookup, units=None):
645 """Calculate a delta group, yielding a sequence of changegroup chunks
645 """Calculate a delta group, yielding a sequence of changegroup chunks
646 (strings).
646 (strings).
647
647
648 Given a list of changeset revs, return a set of deltas and
648 Given a list of changeset revs, return a set of deltas and
649 metadata corresponding to nodes. The first delta is
649 metadata corresponding to nodes. The first delta is
650 first parent(nodelist[0]) -> nodelist[0], the receiver is
650 first parent(nodelist[0]) -> nodelist[0], the receiver is
651 guaranteed to have this parent as it has all history before
651 guaranteed to have this parent as it has all history before
652 these changesets. In the case firstparent is nullrev the
652 these changesets. In the case firstparent is nullrev the
653 changegroup starts with a full revision.
653 changegroup starts with a full revision.
654
654
655 If units is not None, progress detail will be generated, units specifies
655 If units is not None, progress detail will be generated, units specifies
656 the type of revlog that is touched (changelog, manifest, etc.).
656 the type of revlog that is touched (changelog, manifest, etc.).
657 """
657 """
658 # if we don't have any revisions touched by these changesets, bail
658 # if we don't have any revisions touched by these changesets, bail
659 if len(revs) == 0:
659 if len(revs) == 0:
660 yield self._close()
660 yield self._close()
661 return
661 return
662
662
663 # add the parent of the first rev
663 # add the parent of the first rev
664 p = store.parentrevs(revs[0])[0]
664 p = store.parentrevs(revs[0])[0]
665 revs.insert(0, p)
665 revs.insert(0, p)
666
666
667 # build deltas
667 # build deltas
668 progress = None
668 progress = None
669 if units is not None:
669 if units is not None:
670 progress = self._repo.ui.makeprogress(_('bundling'), unit=units,
670 progress = self._repo.ui.makeprogress(_('bundling'), unit=units,
671 total=(len(revs) - 1))
671 total=(len(revs) - 1))
672 for r in pycompat.xrange(len(revs) - 1):
672 for r in pycompat.xrange(len(revs) - 1):
673 if progress:
673 if progress:
674 progress.update(r + 1)
674 progress.update(r + 1)
675 prev, curr = revs[r], revs[r + 1]
675 prev, curr = revs[r], revs[r + 1]
676 linknode = lookup(store.node(curr))
676 linknode = lookup(store.node(curr))
677 for c in self._revchunk(store, ischangelog, curr, prev, linknode):
677
678 yield c
678 if self._ellipses:
679 delta = self._revisiondeltanarrow(store, ischangelog,
680 curr, prev, linknode)
681 else:
682 delta = self._revisiondeltanormal(store, ischangelog,
683 curr, prev, linknode)
684
685 if not delta:
686 continue
687
688 meta = self._builddeltaheader(delta)
689 l = len(meta) + sum(len(x) for x in delta.deltachunks)
690 yield chunkheader(l)
691 yield meta
692 for x in delta.deltachunks:
693 yield x
679
694
680 if progress:
695 if progress:
681 progress.complete()
696 progress.complete()
682 yield self._close()
697 yield self._close()
683
698
684 # filter any nodes that claim to be part of the known set
699 # filter any nodes that claim to be part of the known set
685 def _prune(self, store, missing, commonrevs):
700 def _prune(self, store, missing, commonrevs):
686 # TODO this violates storage abstraction for manifests.
701 # TODO this violates storage abstraction for manifests.
687 if isinstance(store, manifest.manifestrevlog):
702 if isinstance(store, manifest.manifestrevlog):
688 if not self._filematcher.visitdir(store._dir[:-1] or '.'):
703 if not self._filematcher.visitdir(store._dir[:-1] or '.'):
689 return []
704 return []
690
705
691 rr, rl = store.rev, store.linkrev
706 rr, rl = store.rev, store.linkrev
692 return [n for n in missing if rl(rr(n)) not in commonrevs]
707 return [n for n in missing if rl(rr(n)) not in commonrevs]
693
708
694 def _packmanifests(self, dir, dirlog, revs, lookuplinknode):
709 def _packmanifests(self, dir, dirlog, revs, lookuplinknode):
695 """Pack manifests into a changegroup stream.
710 """Pack manifests into a changegroup stream.
696
711
697 Encodes the directory name in the output so multiple manifests
712 Encodes the directory name in the output so multiple manifests
698 can be sent. Multiple manifests is not supported by cg1 and cg2.
713 can be sent. Multiple manifests is not supported by cg1 and cg2.
699 """
714 """
700 if dir:
715 if dir:
701 assert self.version == b'03'
716 assert self.version == b'03'
702 yield _fileheader(dir)
717 yield _fileheader(dir)
703
718
704 for chunk in self.group(revs, dirlog, False, lookuplinknode,
719 for chunk in self.group(revs, dirlog, False, lookuplinknode,
705 units=_('manifests')):
720 units=_('manifests')):
706 yield chunk
721 yield chunk
707
722
708 def generate(self, commonrevs, clnodes, fastpathlinkrev, source):
723 def generate(self, commonrevs, clnodes, fastpathlinkrev, source):
709 """Yield a sequence of changegroup byte chunks."""
724 """Yield a sequence of changegroup byte chunks."""
710
725
711 repo = self._repo
726 repo = self._repo
712 cl = repo.changelog
727 cl = repo.changelog
713
728
714 self._verbosenote(_('uncompressed size of bundle content:\n'))
729 self._verbosenote(_('uncompressed size of bundle content:\n'))
715 size = 0
730 size = 0
716
731
717 clstate, chunks = self._generatechangelog(cl, clnodes)
732 clstate, chunks = self._generatechangelog(cl, clnodes)
718 for chunk in chunks:
733 for chunk in chunks:
719 size += len(chunk)
734 size += len(chunk)
720 yield chunk
735 yield chunk
721
736
722 self._verbosenote(_('%8.i (changelog)\n') % size)
737 self._verbosenote(_('%8.i (changelog)\n') % size)
723
738
724 clrevorder = clstate['clrevorder']
739 clrevorder = clstate['clrevorder']
725 mfs = clstate['mfs']
740 mfs = clstate['mfs']
726 changedfiles = clstate['changedfiles']
741 changedfiles = clstate['changedfiles']
727
742
728 # We need to make sure that the linkrev in the changegroup refers to
743 # We need to make sure that the linkrev in the changegroup refers to
729 # the first changeset that introduced the manifest or file revision.
744 # the first changeset that introduced the manifest or file revision.
730 # The fastpath is usually safer than the slowpath, because the filelogs
745 # The fastpath is usually safer than the slowpath, because the filelogs
731 # are walked in revlog order.
746 # are walked in revlog order.
732 #
747 #
733 # When taking the slowpath with reorder=None and the manifest revlog
748 # When taking the slowpath with reorder=None and the manifest revlog
734 # uses generaldelta, the manifest may be walked in the "wrong" order.
749 # uses generaldelta, the manifest may be walked in the "wrong" order.
735 # Without 'clrevorder', we would get an incorrect linkrev (see fix in
750 # Without 'clrevorder', we would get an incorrect linkrev (see fix in
736 # cc0ff93d0c0c).
751 # cc0ff93d0c0c).
737 #
752 #
738 # When taking the fastpath, we are only vulnerable to reordering
753 # When taking the fastpath, we are only vulnerable to reordering
739 # of the changelog itself. The changelog never uses generaldelta, so
754 # of the changelog itself. The changelog never uses generaldelta, so
740 # it is only reordered when reorder=True. To handle this case, we
755 # it is only reordered when reorder=True. To handle this case, we
741 # simply take the slowpath, which already has the 'clrevorder' logic.
756 # simply take the slowpath, which already has the 'clrevorder' logic.
742 # This was also fixed in cc0ff93d0c0c.
757 # This was also fixed in cc0ff93d0c0c.
743 fastpathlinkrev = fastpathlinkrev and not self._reorder
758 fastpathlinkrev = fastpathlinkrev and not self._reorder
744 # Treemanifests don't work correctly with fastpathlinkrev
759 # Treemanifests don't work correctly with fastpathlinkrev
745 # either, because we don't discover which directory nodes to
760 # either, because we don't discover which directory nodes to
746 # send along with files. This could probably be fixed.
761 # send along with files. This could probably be fixed.
747 fastpathlinkrev = fastpathlinkrev and (
762 fastpathlinkrev = fastpathlinkrev and (
748 'treemanifest' not in repo.requirements)
763 'treemanifest' not in repo.requirements)
749
764
750 fnodes = {} # needed file nodes
765 fnodes = {} # needed file nodes
751
766
752 for chunk in self.generatemanifests(commonrevs, clrevorder,
767 for chunk in self.generatemanifests(commonrevs, clrevorder,
753 fastpathlinkrev, mfs, fnodes, source):
768 fastpathlinkrev, mfs, fnodes, source):
754 yield chunk
769 yield chunk
755
770
756 mfdicts = None
771 mfdicts = None
757 if self._ellipses and self._isshallow:
772 if self._ellipses and self._isshallow:
758 mfdicts = [(self._repo.manifestlog[n].read(), lr)
773 mfdicts = [(self._repo.manifestlog[n].read(), lr)
759 for (n, lr) in mfs.iteritems()]
774 for (n, lr) in mfs.iteritems()]
760
775
761 mfs.clear()
776 mfs.clear()
762 clrevs = set(cl.rev(x) for x in clnodes)
777 clrevs = set(cl.rev(x) for x in clnodes)
763
778
764 if not fastpathlinkrev:
779 if not fastpathlinkrev:
765 def linknodes(unused, fname):
780 def linknodes(unused, fname):
766 return fnodes.get(fname, {})
781 return fnodes.get(fname, {})
767 else:
782 else:
768 cln = cl.node
783 cln = cl.node
769 def linknodes(filerevlog, fname):
784 def linknodes(filerevlog, fname):
770 llr = filerevlog.linkrev
785 llr = filerevlog.linkrev
771 fln = filerevlog.node
786 fln = filerevlog.node
772 revs = ((r, llr(r)) for r in filerevlog)
787 revs = ((r, llr(r)) for r in filerevlog)
773 return dict((fln(r), cln(lr)) for r, lr in revs if lr in clrevs)
788 return dict((fln(r), cln(lr)) for r, lr in revs if lr in clrevs)
774
789
775 for chunk in self.generatefiles(changedfiles, linknodes, commonrevs,
790 for chunk in self.generatefiles(changedfiles, linknodes, commonrevs,
776 source, mfdicts):
791 source, mfdicts):
777 yield chunk
792 yield chunk
778
793
779 yield self._close()
794 yield self._close()
780
795
781 if clnodes:
796 if clnodes:
782 repo.hook('outgoing', node=hex(clnodes[0]), source=source)
797 repo.hook('outgoing', node=hex(clnodes[0]), source=source)
783
798
784 def _generatechangelog(self, cl, nodes):
799 def _generatechangelog(self, cl, nodes):
785 """Generate data for changelog chunks.
800 """Generate data for changelog chunks.
786
801
787 Returns a 2-tuple of a dict containing state and an iterable of
802 Returns a 2-tuple of a dict containing state and an iterable of
788 byte chunks. The state will not be fully populated until the
803 byte chunks. The state will not be fully populated until the
789 chunk stream has been fully consumed.
804 chunk stream has been fully consumed.
790 """
805 """
791 clrevorder = {}
806 clrevorder = {}
792 mfs = {} # needed manifests
807 mfs = {} # needed manifests
793 mfl = self._repo.manifestlog
808 mfl = self._repo.manifestlog
794 # TODO violates storage abstraction.
809 # TODO violates storage abstraction.
795 mfrevlog = mfl._revlog
810 mfrevlog = mfl._revlog
796 changedfiles = set()
811 changedfiles = set()
797
812
798 # Callback for the changelog, used to collect changed files and
813 # Callback for the changelog, used to collect changed files and
799 # manifest nodes.
814 # manifest nodes.
800 # Returns the linkrev node (identity in the changelog case).
815 # Returns the linkrev node (identity in the changelog case).
801 def lookupcl(x):
816 def lookupcl(x):
802 c = cl.read(x)
817 c = cl.read(x)
803 clrevorder[x] = len(clrevorder)
818 clrevorder[x] = len(clrevorder)
804
819
805 if self._ellipses:
820 if self._ellipses:
806 # Only update mfs if x is going to be sent. Otherwise we
821 # Only update mfs if x is going to be sent. Otherwise we
807 # end up with bogus linkrevs specified for manifests and
822 # end up with bogus linkrevs specified for manifests and
808 # we skip some manifest nodes that we should otherwise
823 # we skip some manifest nodes that we should otherwise
809 # have sent.
824 # have sent.
810 if (x in self._fullnodes
825 if (x in self._fullnodes
811 or cl.rev(x) in self._precomputedellipsis):
826 or cl.rev(x) in self._precomputedellipsis):
812 n = c[0]
827 n = c[0]
813 # Record the first changeset introducing this manifest
828 # Record the first changeset introducing this manifest
814 # version.
829 # version.
815 mfs.setdefault(n, x)
830 mfs.setdefault(n, x)
816 # Set this narrow-specific dict so we have the lowest
831 # Set this narrow-specific dict so we have the lowest
817 # manifest revnum to look up for this cl revnum. (Part of
832 # manifest revnum to look up for this cl revnum. (Part of
818 # mapping changelog ellipsis parents to manifest ellipsis
833 # mapping changelog ellipsis parents to manifest ellipsis
819 # parents)
834 # parents)
820 self._nextclrevtolocalrev.setdefault(cl.rev(x),
835 self._nextclrevtolocalrev.setdefault(cl.rev(x),
821 mfrevlog.rev(n))
836 mfrevlog.rev(n))
822 # We can't trust the changed files list in the changeset if the
837 # We can't trust the changed files list in the changeset if the
823 # client requested a shallow clone.
838 # client requested a shallow clone.
824 if self._isshallow:
839 if self._isshallow:
825 changedfiles.update(mfl[c[0]].read().keys())
840 changedfiles.update(mfl[c[0]].read().keys())
826 else:
841 else:
827 changedfiles.update(c[3])
842 changedfiles.update(c[3])
828 else:
843 else:
829
844
830 n = c[0]
845 n = c[0]
831 # record the first changeset introducing this manifest version
846 # record the first changeset introducing this manifest version
832 mfs.setdefault(n, x)
847 mfs.setdefault(n, x)
833 # Record a complete list of potentially-changed files in
848 # Record a complete list of potentially-changed files in
834 # this manifest.
849 # this manifest.
835 changedfiles.update(c[3])
850 changedfiles.update(c[3])
836
851
837 return x
852 return x
838
853
839 # Changelog doesn't benefit from reordering revisions. So send out
854 # Changelog doesn't benefit from reordering revisions. So send out
840 # revisions in store order.
855 # revisions in store order.
841 revs = sorted(cl.rev(n) for n in nodes)
856 revs = sorted(cl.rev(n) for n in nodes)
842
857
843 state = {
858 state = {
844 'clrevorder': clrevorder,
859 'clrevorder': clrevorder,
845 'mfs': mfs,
860 'mfs': mfs,
846 'changedfiles': changedfiles,
861 'changedfiles': changedfiles,
847 }
862 }
848
863
849 gen = self.group(revs, cl, True, lookupcl, units=_('changesets'))
864 gen = self.group(revs, cl, True, lookupcl, units=_('changesets'))
850
865
851 return state, gen
866 return state, gen
852
867
853 def generatemanifests(self, commonrevs, clrevorder, fastpathlinkrev, mfs,
868 def generatemanifests(self, commonrevs, clrevorder, fastpathlinkrev, mfs,
854 fnodes, source):
869 fnodes, source):
855 """Returns an iterator of changegroup chunks containing manifests.
870 """Returns an iterator of changegroup chunks containing manifests.
856
871
857 `source` is unused here, but is used by extensions like remotefilelog to
872 `source` is unused here, but is used by extensions like remotefilelog to
858 change what is sent based in pulls vs pushes, etc.
873 change what is sent based in pulls vs pushes, etc.
859 """
874 """
860 repo = self._repo
875 repo = self._repo
861 mfl = repo.manifestlog
876 mfl = repo.manifestlog
862 dirlog = mfl._revlog.dirlog
877 dirlog = mfl._revlog.dirlog
863 tmfnodes = {'': mfs}
878 tmfnodes = {'': mfs}
864
879
865 # Callback for the manifest, used to collect linkrevs for filelog
880 # Callback for the manifest, used to collect linkrevs for filelog
866 # revisions.
881 # revisions.
867 # Returns the linkrev node (collected in lookupcl).
882 # Returns the linkrev node (collected in lookupcl).
868 def makelookupmflinknode(dir, nodes):
883 def makelookupmflinknode(dir, nodes):
869 if fastpathlinkrev:
884 if fastpathlinkrev:
870 assert not dir
885 assert not dir
871 return mfs.__getitem__
886 return mfs.__getitem__
872
887
873 def lookupmflinknode(x):
888 def lookupmflinknode(x):
874 """Callback for looking up the linknode for manifests.
889 """Callback for looking up the linknode for manifests.
875
890
876 Returns the linkrev node for the specified manifest.
891 Returns the linkrev node for the specified manifest.
877
892
878 SIDE EFFECT:
893 SIDE EFFECT:
879
894
880 1) fclnodes gets populated with the list of relevant
895 1) fclnodes gets populated with the list of relevant
881 file nodes if we're not using fastpathlinkrev
896 file nodes if we're not using fastpathlinkrev
882 2) When treemanifests are in use, collects treemanifest nodes
897 2) When treemanifests are in use, collects treemanifest nodes
883 to send
898 to send
884
899
885 Note that this means manifests must be completely sent to
900 Note that this means manifests must be completely sent to
886 the client before you can trust the list of files and
901 the client before you can trust the list of files and
887 treemanifests to send.
902 treemanifests to send.
888 """
903 """
889 clnode = nodes[x]
904 clnode = nodes[x]
890 mdata = mfl.get(dir, x).readfast(shallow=True)
905 mdata = mfl.get(dir, x).readfast(shallow=True)
891 for p, n, fl in mdata.iterentries():
906 for p, n, fl in mdata.iterentries():
892 if fl == 't': # subdirectory manifest
907 if fl == 't': # subdirectory manifest
893 subdir = dir + p + '/'
908 subdir = dir + p + '/'
894 tmfclnodes = tmfnodes.setdefault(subdir, {})
909 tmfclnodes = tmfnodes.setdefault(subdir, {})
895 tmfclnode = tmfclnodes.setdefault(n, clnode)
910 tmfclnode = tmfclnodes.setdefault(n, clnode)
896 if clrevorder[clnode] < clrevorder[tmfclnode]:
911 if clrevorder[clnode] < clrevorder[tmfclnode]:
897 tmfclnodes[n] = clnode
912 tmfclnodes[n] = clnode
898 else:
913 else:
899 f = dir + p
914 f = dir + p
900 fclnodes = fnodes.setdefault(f, {})
915 fclnodes = fnodes.setdefault(f, {})
901 fclnode = fclnodes.setdefault(n, clnode)
916 fclnode = fclnodes.setdefault(n, clnode)
902 if clrevorder[clnode] < clrevorder[fclnode]:
917 if clrevorder[clnode] < clrevorder[fclnode]:
903 fclnodes[n] = clnode
918 fclnodes[n] = clnode
904 return clnode
919 return clnode
905 return lookupmflinknode
920 return lookupmflinknode
906
921
907 size = 0
922 size = 0
908 while tmfnodes:
923 while tmfnodes:
909 dir, nodes = tmfnodes.popitem()
924 dir, nodes = tmfnodes.popitem()
910 store = dirlog(dir)
925 store = dirlog(dir)
911 prunednodes = self._prune(store, nodes, commonrevs)
926 prunednodes = self._prune(store, nodes, commonrevs)
912 if not dir or prunednodes:
927 if not dir or prunednodes:
913 lookupfn = makelookupmflinknode(dir, nodes)
928 lookupfn = makelookupmflinknode(dir, nodes)
914
929
915 if self._ellipses:
930 if self._ellipses:
916 revs = _sortnodesellipsis(store, prunednodes,
931 revs = _sortnodesellipsis(store, prunednodes,
917 self._clnodetorev, lookupfn)
932 self._clnodetorev, lookupfn)
918 else:
933 else:
919 revs = _sortnodesnormal(store, prunednodes,
934 revs = _sortnodesnormal(store, prunednodes,
920 self._reorder)
935 self._reorder)
921
936
922 for x in self._packmanifests(dir, store, revs, lookupfn):
937 for x in self._packmanifests(dir, store, revs, lookupfn):
923 size += len(x)
938 size += len(x)
924 yield x
939 yield x
925 self._verbosenote(_('%8.i (manifests)\n') % size)
940 self._verbosenote(_('%8.i (manifests)\n') % size)
926 yield self._manifestsend
941 yield self._manifestsend
927
942
928 # The 'source' parameter is useful for extensions
943 # The 'source' parameter is useful for extensions
929 def generatefiles(self, changedfiles, linknodes, commonrevs, source,
944 def generatefiles(self, changedfiles, linknodes, commonrevs, source,
930 mfdicts):
945 mfdicts):
931 changedfiles = list(filter(self._filematcher, changedfiles))
946 changedfiles = list(filter(self._filematcher, changedfiles))
932
947
933 if self._isshallow:
948 if self._isshallow:
934 # In a shallow clone, the linknodes callback needs to also include
949 # In a shallow clone, the linknodes callback needs to also include
935 # those file nodes that are in the manifests we sent but weren't
950 # those file nodes that are in the manifests we sent but weren't
936 # introduced by those manifests.
951 # introduced by those manifests.
937 commonctxs = [self._repo[c] for c in commonrevs]
952 commonctxs = [self._repo[c] for c in commonrevs]
938 oldlinknodes = linknodes
953 oldlinknodes = linknodes
939 clrev = self._repo.changelog.rev
954 clrev = self._repo.changelog.rev
940
955
941 # Defining this function has a side-effect of overriding the
956 # Defining this function has a side-effect of overriding the
942 # function of the same name that was passed in as an argument.
957 # function of the same name that was passed in as an argument.
943 # TODO have caller pass in appropriate function.
958 # TODO have caller pass in appropriate function.
944 def linknodes(flog, fname):
959 def linknodes(flog, fname):
945 for c in commonctxs:
960 for c in commonctxs:
946 try:
961 try:
947 fnode = c.filenode(fname)
962 fnode = c.filenode(fname)
948 self._clrevtolocalrev[c.rev()] = flog.rev(fnode)
963 self._clrevtolocalrev[c.rev()] = flog.rev(fnode)
949 except error.ManifestLookupError:
964 except error.ManifestLookupError:
950 pass
965 pass
951 links = oldlinknodes(flog, fname)
966 links = oldlinknodes(flog, fname)
952 if len(links) != len(mfdicts):
967 if len(links) != len(mfdicts):
953 for mf, lr in mfdicts:
968 for mf, lr in mfdicts:
954 fnode = mf.get(fname, None)
969 fnode = mf.get(fname, None)
955 if fnode in links:
970 if fnode in links:
956 links[fnode] = min(links[fnode], lr, key=clrev)
971 links[fnode] = min(links[fnode], lr, key=clrev)
957 elif fnode:
972 elif fnode:
958 links[fnode] = lr
973 links[fnode] = lr
959 return links
974 return links
960
975
961 return self._generatefiles(changedfiles, linknodes, commonrevs, source)
976 return self._generatefiles(changedfiles, linknodes, commonrevs, source)
962
977
963 def _generatefiles(self, changedfiles, linknodes, commonrevs, source):
978 def _generatefiles(self, changedfiles, linknodes, commonrevs, source):
964 repo = self._repo
979 repo = self._repo
965 progress = repo.ui.makeprogress(_('bundling'), unit=_('files'),
980 progress = repo.ui.makeprogress(_('bundling'), unit=_('files'),
966 total=len(changedfiles))
981 total=len(changedfiles))
967 for i, fname in enumerate(sorted(changedfiles)):
982 for i, fname in enumerate(sorted(changedfiles)):
968 filerevlog = repo.file(fname)
983 filerevlog = repo.file(fname)
969 if not filerevlog:
984 if not filerevlog:
970 raise error.Abort(_("empty or missing file data for %s") %
985 raise error.Abort(_("empty or missing file data for %s") %
971 fname)
986 fname)
972
987
973 linkrevnodes = linknodes(filerevlog, fname)
988 linkrevnodes = linknodes(filerevlog, fname)
974 # Lookup for filenodes, we collected the linkrev nodes above in the
989 # Lookup for filenodes, we collected the linkrev nodes above in the
975 # fastpath case and with lookupmf in the slowpath case.
990 # fastpath case and with lookupmf in the slowpath case.
976 def lookupfilelog(x):
991 def lookupfilelog(x):
977 return linkrevnodes[x]
992 return linkrevnodes[x]
978
993
979 filenodes = self._prune(filerevlog, linkrevnodes, commonrevs)
994 filenodes = self._prune(filerevlog, linkrevnodes, commonrevs)
980 if filenodes:
995 if filenodes:
981 if self._ellipses:
996 if self._ellipses:
982 revs = _sortnodesellipsis(filerevlog, filenodes,
997 revs = _sortnodesellipsis(filerevlog, filenodes,
983 self._clnodetorev, lookupfilelog)
998 self._clnodetorev, lookupfilelog)
984 else:
999 else:
985 revs = _sortnodesnormal(filerevlog, filenodes,
1000 revs = _sortnodesnormal(filerevlog, filenodes,
986 self._reorder)
1001 self._reorder)
987
1002
988 progress.update(i + 1, item=fname)
1003 progress.update(i + 1, item=fname)
989 h = _fileheader(fname)
1004 h = _fileheader(fname)
990 size = len(h)
1005 size = len(h)
991 yield h
1006 yield h
992 for chunk in self.group(revs, filerevlog, False, lookupfilelog):
1007 for chunk in self.group(revs, filerevlog, False, lookupfilelog):
993 size += len(chunk)
1008 size += len(chunk)
994 yield chunk
1009 yield chunk
995 self._verbosenote(_('%8.i %s\n') % (size, fname))
1010 self._verbosenote(_('%8.i %s\n') % (size, fname))
996 progress.complete()
1011 progress.complete()
997
1012
998 def _revchunk(self, store, ischangelog, rev, prev, linknode):
999 if self._ellipses:
1000 fn = self._revisiondeltanarrow
1001 else:
1002 fn = self._revisiondeltanormal
1003
1004 delta = fn(store, ischangelog, rev, prev, linknode)
1005 if not delta:
1006 return
1007
1008 meta = self._builddeltaheader(delta)
1009 l = len(meta) + sum(len(x) for x in delta.deltachunks)
1010
1011 yield chunkheader(l)
1012 yield meta
1013 for x in delta.deltachunks:
1014 yield x
1015
1016 def _revisiondeltanormal(self, store, ischangelog, rev, prev, linknode):
1013 def _revisiondeltanormal(self, store, ischangelog, rev, prev, linknode):
1017 node = store.node(rev)
1014 node = store.node(rev)
1018 p1, p2 = store.parentrevs(rev)
1015 p1, p2 = store.parentrevs(rev)
1019 base = self._deltaparentfn(store, rev, p1, p2, prev)
1016 base = self._deltaparentfn(store, rev, p1, p2, prev)
1020
1017
1021 prefix = ''
1018 prefix = ''
1022 if store.iscensored(base) or store.iscensored(rev):
1019 if store.iscensored(base) or store.iscensored(rev):
1023 try:
1020 try:
1024 delta = store.revision(node, raw=True)
1021 delta = store.revision(node, raw=True)
1025 except error.CensoredNodeError as e:
1022 except error.CensoredNodeError as e:
1026 delta = e.tombstone
1023 delta = e.tombstone
1027 if base == nullrev:
1024 if base == nullrev:
1028 prefix = mdiff.trivialdiffheader(len(delta))
1025 prefix = mdiff.trivialdiffheader(len(delta))
1029 else:
1026 else:
1030 baselen = store.rawsize(base)
1027 baselen = store.rawsize(base)
1031 prefix = mdiff.replacediffheader(baselen, len(delta))
1028 prefix = mdiff.replacediffheader(baselen, len(delta))
1032 elif base == nullrev:
1029 elif base == nullrev:
1033 delta = store.revision(node, raw=True)
1030 delta = store.revision(node, raw=True)
1034 prefix = mdiff.trivialdiffheader(len(delta))
1031 prefix = mdiff.trivialdiffheader(len(delta))
1035 else:
1032 else:
1036 delta = store.revdiff(base, rev)
1033 delta = store.revdiff(base, rev)
1037 p1n, p2n = store.parents(node)
1034 p1n, p2n = store.parents(node)
1038
1035
1039 return revisiondelta(
1036 return revisiondelta(
1040 node=node,
1037 node=node,
1041 p1node=p1n,
1038 p1node=p1n,
1042 p2node=p2n,
1039 p2node=p2n,
1043 basenode=store.node(base),
1040 basenode=store.node(base),
1044 linknode=linknode,
1041 linknode=linknode,
1045 flags=store.flags(rev),
1042 flags=store.flags(rev),
1046 deltachunks=(prefix, delta),
1043 deltachunks=(prefix, delta),
1047 )
1044 )
1048
1045
1049 def _revisiondeltanarrow(self, store, ischangelog, rev, prev, linknode):
1046 def _revisiondeltanarrow(self, store, ischangelog, rev, prev, linknode):
1050 # build up some mapping information that's useful later. See
1047 # build up some mapping information that's useful later. See
1051 # the local() nested function below.
1048 # the local() nested function below.
1052 if ischangelog:
1049 if ischangelog:
1053 self._clnodetorev[linknode] = rev
1050 self._clnodetorev[linknode] = rev
1054 linkrev = rev
1051 linkrev = rev
1055 self._clrevtolocalrev[linkrev] = rev
1052 self._clrevtolocalrev[linkrev] = rev
1056 else:
1053 else:
1057 linkrev = self._clnodetorev[linknode]
1054 linkrev = self._clnodetorev[linknode]
1058 self._clrevtolocalrev[linkrev] = rev
1055 self._clrevtolocalrev[linkrev] = rev
1059
1056
1060 # This is a node to send in full, because the changeset it
1057 # This is a node to send in full, because the changeset it
1061 # corresponds to was a full changeset.
1058 # corresponds to was a full changeset.
1062 if linknode in self._fullnodes:
1059 if linknode in self._fullnodes:
1063 return self._revisiondeltanormal(store, ischangelog, rev, prev,
1060 return self._revisiondeltanormal(store, ischangelog, rev, prev,
1064 linknode)
1061 linknode)
1065
1062
1066 # At this point, a node can either be one we should skip or an
1063 # At this point, a node can either be one we should skip or an
1067 # ellipsis. If it's not an ellipsis, bail immediately.
1064 # ellipsis. If it's not an ellipsis, bail immediately.
1068 if linkrev not in self._precomputedellipsis:
1065 if linkrev not in self._precomputedellipsis:
1069 return
1066 return
1070
1067
1071 linkparents = self._precomputedellipsis[linkrev]
1068 linkparents = self._precomputedellipsis[linkrev]
1072 def local(clrev):
1069 def local(clrev):
1073 """Turn a changelog revnum into a local revnum.
1070 """Turn a changelog revnum into a local revnum.
1074
1071
1075 The ellipsis dag is stored as revnums on the changelog,
1072 The ellipsis dag is stored as revnums on the changelog,
1076 but when we're producing ellipsis entries for
1073 but when we're producing ellipsis entries for
1077 non-changelog revlogs, we need to turn those numbers into
1074 non-changelog revlogs, we need to turn those numbers into
1078 something local. This does that for us, and during the
1075 something local. This does that for us, and during the
1079 changelog sending phase will also expand the stored
1076 changelog sending phase will also expand the stored
1080 mappings as needed.
1077 mappings as needed.
1081 """
1078 """
1082 if clrev == nullrev:
1079 if clrev == nullrev:
1083 return nullrev
1080 return nullrev
1084
1081
1085 if ischangelog:
1082 if ischangelog:
1086 # If we're doing the changelog, it's possible that we
1083 # If we're doing the changelog, it's possible that we
1087 # have a parent that is already on the client, and we
1084 # have a parent that is already on the client, and we
1088 # need to store some extra mapping information so that
1085 # need to store some extra mapping information so that
1089 # our contained ellipsis nodes will be able to resolve
1086 # our contained ellipsis nodes will be able to resolve
1090 # their parents.
1087 # their parents.
1091 if clrev not in self._clrevtolocalrev:
1088 if clrev not in self._clrevtolocalrev:
1092 clnode = store.node(clrev)
1089 clnode = store.node(clrev)
1093 self._clnodetorev[clnode] = clrev
1090 self._clnodetorev[clnode] = clrev
1094 return clrev
1091 return clrev
1095
1092
1096 # Walk the ellipsis-ized changelog breadth-first looking for a
1093 # Walk the ellipsis-ized changelog breadth-first looking for a
1097 # change that has been linked from the current revlog.
1094 # change that has been linked from the current revlog.
1098 #
1095 #
1099 # For a flat manifest revlog only a single step should be necessary
1096 # For a flat manifest revlog only a single step should be necessary
1100 # as all relevant changelog entries are relevant to the flat
1097 # as all relevant changelog entries are relevant to the flat
1101 # manifest.
1098 # manifest.
1102 #
1099 #
1103 # For a filelog or tree manifest dirlog however not every changelog
1100 # For a filelog or tree manifest dirlog however not every changelog
1104 # entry will have been relevant, so we need to skip some changelog
1101 # entry will have been relevant, so we need to skip some changelog
1105 # nodes even after ellipsis-izing.
1102 # nodes even after ellipsis-izing.
1106 walk = [clrev]
1103 walk = [clrev]
1107 while walk:
1104 while walk:
1108 p = walk[0]
1105 p = walk[0]
1109 walk = walk[1:]
1106 walk = walk[1:]
1110 if p in self._clrevtolocalrev:
1107 if p in self._clrevtolocalrev:
1111 return self._clrevtolocalrev[p]
1108 return self._clrevtolocalrev[p]
1112 elif p in self._fullnodes:
1109 elif p in self._fullnodes:
1113 walk.extend([pp for pp in self._repo.changelog.parentrevs(p)
1110 walk.extend([pp for pp in self._repo.changelog.parentrevs(p)
1114 if pp != nullrev])
1111 if pp != nullrev])
1115 elif p in self._precomputedellipsis:
1112 elif p in self._precomputedellipsis:
1116 walk.extend([pp for pp in self._precomputedellipsis[p]
1113 walk.extend([pp for pp in self._precomputedellipsis[p]
1117 if pp != nullrev])
1114 if pp != nullrev])
1118 else:
1115 else:
1119 # In this case, we've got an ellipsis with parents
1116 # In this case, we've got an ellipsis with parents
1120 # outside the current bundle (likely an
1117 # outside the current bundle (likely an
1121 # incremental pull). We "know" that we can use the
1118 # incremental pull). We "know" that we can use the
1122 # value of this same revlog at whatever revision
1119 # value of this same revlog at whatever revision
1123 # is pointed to by linknode. "Know" is in scare
1120 # is pointed to by linknode. "Know" is in scare
1124 # quotes because I haven't done enough examination
1121 # quotes because I haven't done enough examination
1125 # of edge cases to convince myself this is really
1122 # of edge cases to convince myself this is really
1126 # a fact - it works for all the (admittedly
1123 # a fact - it works for all the (admittedly
1127 # thorough) cases in our testsuite, but I would be
1124 # thorough) cases in our testsuite, but I would be
1128 # somewhat unsurprised to find a case in the wild
1125 # somewhat unsurprised to find a case in the wild
1129 # where this breaks down a bit. That said, I don't
1126 # where this breaks down a bit. That said, I don't
1130 # know if it would hurt anything.
1127 # know if it would hurt anything.
1131 for i in pycompat.xrange(rev, 0, -1):
1128 for i in pycompat.xrange(rev, 0, -1):
1132 if store.linkrev(i) == clrev:
1129 if store.linkrev(i) == clrev:
1133 return i
1130 return i
1134 # We failed to resolve a parent for this node, so
1131 # We failed to resolve a parent for this node, so
1135 # we crash the changegroup construction.
1132 # we crash the changegroup construction.
1136 raise error.Abort(
1133 raise error.Abort(
1137 'unable to resolve parent while packing %r %r'
1134 'unable to resolve parent while packing %r %r'
1138 ' for changeset %r' % (store.indexfile, rev, clrev))
1135 ' for changeset %r' % (store.indexfile, rev, clrev))
1139
1136
1140 return nullrev
1137 return nullrev
1141
1138
1142 if not linkparents or (
1139 if not linkparents or (
1143 store.parentrevs(rev) == (nullrev, nullrev)):
1140 store.parentrevs(rev) == (nullrev, nullrev)):
1144 p1, p2 = nullrev, nullrev
1141 p1, p2 = nullrev, nullrev
1145 elif len(linkparents) == 1:
1142 elif len(linkparents) == 1:
1146 p1, = sorted(local(p) for p in linkparents)
1143 p1, = sorted(local(p) for p in linkparents)
1147 p2 = nullrev
1144 p2 = nullrev
1148 else:
1145 else:
1149 p1, p2 = sorted(local(p) for p in linkparents)
1146 p1, p2 = sorted(local(p) for p in linkparents)
1150
1147
1151 n = store.node(rev)
1148 n = store.node(rev)
1152 p1n, p2n = store.node(p1), store.node(p2)
1149 p1n, p2n = store.node(p1), store.node(p2)
1153 flags = store.flags(rev)
1150 flags = store.flags(rev)
1154 flags |= revlog.REVIDX_ELLIPSIS
1151 flags |= revlog.REVIDX_ELLIPSIS
1155
1152
1156 # TODO: try and actually send deltas for ellipsis data blocks
1153 # TODO: try and actually send deltas for ellipsis data blocks
1157 data = store.revision(n)
1154 data = store.revision(n)
1158 diffheader = mdiff.trivialdiffheader(len(data))
1155 diffheader = mdiff.trivialdiffheader(len(data))
1159
1156
1160 return revisiondelta(
1157 return revisiondelta(
1161 node=n,
1158 node=n,
1162 p1node=p1n,
1159 p1node=p1n,
1163 p2node=p2n,
1160 p2node=p2n,
1164 basenode=nullid,
1161 basenode=nullid,
1165 linknode=linknode,
1162 linknode=linknode,
1166 flags=flags,
1163 flags=flags,
1167 deltachunks=(diffheader, data),
1164 deltachunks=(diffheader, data),
1168 )
1165 )
1169
1166
1170 def _deltaparentprev(store, rev, p1, p2, prev):
1167 def _deltaparentprev(store, rev, p1, p2, prev):
1171 """Resolve a delta parent to the previous revision.
1168 """Resolve a delta parent to the previous revision.
1172
1169
1173 Used for version 1 changegroups, which don't support generaldelta.
1170 Used for version 1 changegroups, which don't support generaldelta.
1174 """
1171 """
1175 return prev
1172 return prev
1176
1173
1177 def _deltaparentgeneraldelta(store, rev, p1, p2, prev):
1174 def _deltaparentgeneraldelta(store, rev, p1, p2, prev):
1178 """Resolve a delta parent when general deltas are supported."""
1175 """Resolve a delta parent when general deltas are supported."""
1179 dp = store.deltaparent(rev)
1176 dp = store.deltaparent(rev)
1180 if dp == nullrev and store.storedeltachains:
1177 if dp == nullrev and store.storedeltachains:
1181 # Avoid sending full revisions when delta parent is null. Pick prev
1178 # Avoid sending full revisions when delta parent is null. Pick prev
1182 # in that case. It's tempting to pick p1 in this case, as p1 will
1179 # in that case. It's tempting to pick p1 in this case, as p1 will
1183 # be smaller in the common case. However, computing a delta against
1180 # be smaller in the common case. However, computing a delta against
1184 # p1 may require resolving the raw text of p1, which could be
1181 # p1 may require resolving the raw text of p1, which could be
1185 # expensive. The revlog caches should have prev cached, meaning
1182 # expensive. The revlog caches should have prev cached, meaning
1186 # less CPU for changegroup generation. There is likely room to add
1183 # less CPU for changegroup generation. There is likely room to add
1187 # a flag and/or config option to control this behavior.
1184 # a flag and/or config option to control this behavior.
1188 base = prev
1185 base = prev
1189 elif dp == nullrev:
1186 elif dp == nullrev:
1190 # revlog is configured to use full snapshot for a reason,
1187 # revlog is configured to use full snapshot for a reason,
1191 # stick to full snapshot.
1188 # stick to full snapshot.
1192 base = nullrev
1189 base = nullrev
1193 elif dp not in (p1, p2, prev):
1190 elif dp not in (p1, p2, prev):
1194 # Pick prev when we can't be sure remote has the base revision.
1191 # Pick prev when we can't be sure remote has the base revision.
1195 return prev
1192 return prev
1196 else:
1193 else:
1197 base = dp
1194 base = dp
1198
1195
1199 if base != nullrev and not store.candelta(base, rev):
1196 if base != nullrev and not store.candelta(base, rev):
1200 base = nullrev
1197 base = nullrev
1201
1198
1202 return base
1199 return base
1203
1200
1204 def _deltaparentellipses(store, rev, p1, p2, prev):
1201 def _deltaparentellipses(store, rev, p1, p2, prev):
1205 """Resolve a delta parent when in ellipses mode."""
1202 """Resolve a delta parent when in ellipses mode."""
1206 # TODO: send better deltas when in narrow mode.
1203 # TODO: send better deltas when in narrow mode.
1207 #
1204 #
1208 # changegroup.group() loops over revisions to send,
1205 # changegroup.group() loops over revisions to send,
1209 # including revisions we'll skip. What this means is that
1206 # including revisions we'll skip. What this means is that
1210 # `prev` will be a potentially useless delta base for all
1207 # `prev` will be a potentially useless delta base for all
1211 # ellipsis nodes, as the client likely won't have it. In
1208 # ellipsis nodes, as the client likely won't have it. In
1212 # the future we should do bookkeeping about which nodes
1209 # the future we should do bookkeeping about which nodes
1213 # have been sent to the client, and try to be
1210 # have been sent to the client, and try to be
1214 # significantly smarter about delta bases. This is
1211 # significantly smarter about delta bases. This is
1215 # slightly tricky because this same code has to work for
1212 # slightly tricky because this same code has to work for
1216 # all revlogs, and we don't have the linkrev/linknode here.
1213 # all revlogs, and we don't have the linkrev/linknode here.
1217 return p1
1214 return p1
1218
1215
1219 def _makecg1packer(repo, filematcher, bundlecaps, ellipses=False,
1216 def _makecg1packer(repo, filematcher, bundlecaps, ellipses=False,
1220 shallow=False, ellipsisroots=None, fullnodes=None):
1217 shallow=False, ellipsisroots=None, fullnodes=None):
1221 builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack(
1218 builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack(
1222 d.node, d.p1node, d.p2node, d.linknode)
1219 d.node, d.p1node, d.p2node, d.linknode)
1223
1220
1224 return cgpacker(repo, filematcher, b'01',
1221 return cgpacker(repo, filematcher, b'01',
1225 deltaparentfn=_deltaparentprev,
1222 deltaparentfn=_deltaparentprev,
1226 allowreorder=None,
1223 allowreorder=None,
1227 builddeltaheader=builddeltaheader,
1224 builddeltaheader=builddeltaheader,
1228 manifestsend=b'',
1225 manifestsend=b'',
1229 bundlecaps=bundlecaps,
1226 bundlecaps=bundlecaps,
1230 ellipses=ellipses,
1227 ellipses=ellipses,
1231 shallow=shallow,
1228 shallow=shallow,
1232 ellipsisroots=ellipsisroots,
1229 ellipsisroots=ellipsisroots,
1233 fullnodes=fullnodes)
1230 fullnodes=fullnodes)
1234
1231
1235 def _makecg2packer(repo, filematcher, bundlecaps, ellipses=False,
1232 def _makecg2packer(repo, filematcher, bundlecaps, ellipses=False,
1236 shallow=False, ellipsisroots=None, fullnodes=None):
1233 shallow=False, ellipsisroots=None, fullnodes=None):
1237 builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack(
1234 builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack(
1238 d.node, d.p1node, d.p2node, d.basenode, d.linknode)
1235 d.node, d.p1node, d.p2node, d.basenode, d.linknode)
1239
1236
1240 # Since generaldelta is directly supported by cg2, reordering
1237 # Since generaldelta is directly supported by cg2, reordering
1241 # generally doesn't help, so we disable it by default (treating
1238 # generally doesn't help, so we disable it by default (treating
1242 # bundle.reorder=auto just like bundle.reorder=False).
1239 # bundle.reorder=auto just like bundle.reorder=False).
1243 return cgpacker(repo, filematcher, b'02',
1240 return cgpacker(repo, filematcher, b'02',
1244 deltaparentfn=_deltaparentgeneraldelta,
1241 deltaparentfn=_deltaparentgeneraldelta,
1245 allowreorder=False,
1242 allowreorder=False,
1246 builddeltaheader=builddeltaheader,
1243 builddeltaheader=builddeltaheader,
1247 manifestsend=b'',
1244 manifestsend=b'',
1248 bundlecaps=bundlecaps,
1245 bundlecaps=bundlecaps,
1249 ellipses=ellipses,
1246 ellipses=ellipses,
1250 shallow=shallow,
1247 shallow=shallow,
1251 ellipsisroots=ellipsisroots,
1248 ellipsisroots=ellipsisroots,
1252 fullnodes=fullnodes)
1249 fullnodes=fullnodes)
1253
1250
1254 def _makecg3packer(repo, filematcher, bundlecaps, ellipses=False,
1251 def _makecg3packer(repo, filematcher, bundlecaps, ellipses=False,
1255 shallow=False, ellipsisroots=None, fullnodes=None):
1252 shallow=False, ellipsisroots=None, fullnodes=None):
1256 builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack(
1253 builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack(
1257 d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags)
1254 d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags)
1258
1255
1259 deltaparentfn = (_deltaparentellipses if ellipses
1256 deltaparentfn = (_deltaparentellipses if ellipses
1260 else _deltaparentgeneraldelta)
1257 else _deltaparentgeneraldelta)
1261
1258
1262 return cgpacker(repo, filematcher, b'03',
1259 return cgpacker(repo, filematcher, b'03',
1263 deltaparentfn=deltaparentfn,
1260 deltaparentfn=deltaparentfn,
1264 allowreorder=False,
1261 allowreorder=False,
1265 builddeltaheader=builddeltaheader,
1262 builddeltaheader=builddeltaheader,
1266 manifestsend=closechunk(),
1263 manifestsend=closechunk(),
1267 bundlecaps=bundlecaps,
1264 bundlecaps=bundlecaps,
1268 ellipses=ellipses,
1265 ellipses=ellipses,
1269 shallow=shallow,
1266 shallow=shallow,
1270 ellipsisroots=ellipsisroots,
1267 ellipsisroots=ellipsisroots,
1271 fullnodes=fullnodes)
1268 fullnodes=fullnodes)
1272
1269
1273 _packermap = {'01': (_makecg1packer, cg1unpacker),
1270 _packermap = {'01': (_makecg1packer, cg1unpacker),
1274 # cg2 adds support for exchanging generaldelta
1271 # cg2 adds support for exchanging generaldelta
1275 '02': (_makecg2packer, cg2unpacker),
1272 '02': (_makecg2packer, cg2unpacker),
1276 # cg3 adds support for exchanging revlog flags and treemanifests
1273 # cg3 adds support for exchanging revlog flags and treemanifests
1277 '03': (_makecg3packer, cg3unpacker),
1274 '03': (_makecg3packer, cg3unpacker),
1278 }
1275 }
1279
1276
1280 def allsupportedversions(repo):
1277 def allsupportedversions(repo):
1281 versions = set(_packermap.keys())
1278 versions = set(_packermap.keys())
1282 if not (repo.ui.configbool('experimental', 'changegroup3') or
1279 if not (repo.ui.configbool('experimental', 'changegroup3') or
1283 repo.ui.configbool('experimental', 'treemanifest') or
1280 repo.ui.configbool('experimental', 'treemanifest') or
1284 'treemanifest' in repo.requirements):
1281 'treemanifest' in repo.requirements):
1285 versions.discard('03')
1282 versions.discard('03')
1286 return versions
1283 return versions
1287
1284
1288 # Changegroup versions that can be applied to the repo
1285 # Changegroup versions that can be applied to the repo
1289 def supportedincomingversions(repo):
1286 def supportedincomingversions(repo):
1290 return allsupportedversions(repo)
1287 return allsupportedversions(repo)
1291
1288
1292 # Changegroup versions that can be created from the repo
1289 # Changegroup versions that can be created from the repo
1293 def supportedoutgoingversions(repo):
1290 def supportedoutgoingversions(repo):
1294 versions = allsupportedversions(repo)
1291 versions = allsupportedversions(repo)
1295 if 'treemanifest' in repo.requirements:
1292 if 'treemanifest' in repo.requirements:
1296 # Versions 01 and 02 support only flat manifests and it's just too
1293 # Versions 01 and 02 support only flat manifests and it's just too
1297 # expensive to convert between the flat manifest and tree manifest on
1294 # expensive to convert between the flat manifest and tree manifest on
1298 # the fly. Since tree manifests are hashed differently, all of history
1295 # the fly. Since tree manifests are hashed differently, all of history
1299 # would have to be converted. Instead, we simply don't even pretend to
1296 # would have to be converted. Instead, we simply don't even pretend to
1300 # support versions 01 and 02.
1297 # support versions 01 and 02.
1301 versions.discard('01')
1298 versions.discard('01')
1302 versions.discard('02')
1299 versions.discard('02')
1303 if repository.NARROW_REQUIREMENT in repo.requirements:
1300 if repository.NARROW_REQUIREMENT in repo.requirements:
1304 # Versions 01 and 02 don't support revlog flags, and we need to
1301 # Versions 01 and 02 don't support revlog flags, and we need to
1305 # support that for stripping and unbundling to work.
1302 # support that for stripping and unbundling to work.
1306 versions.discard('01')
1303 versions.discard('01')
1307 versions.discard('02')
1304 versions.discard('02')
1308 if LFS_REQUIREMENT in repo.requirements:
1305 if LFS_REQUIREMENT in repo.requirements:
1309 # Versions 01 and 02 don't support revlog flags, and we need to
1306 # Versions 01 and 02 don't support revlog flags, and we need to
1310 # mark LFS entries with REVIDX_EXTSTORED.
1307 # mark LFS entries with REVIDX_EXTSTORED.
1311 versions.discard('01')
1308 versions.discard('01')
1312 versions.discard('02')
1309 versions.discard('02')
1313
1310
1314 return versions
1311 return versions
1315
1312
1316 def localversion(repo):
1313 def localversion(repo):
1317 # Finds the best version to use for bundles that are meant to be used
1314 # Finds the best version to use for bundles that are meant to be used
1318 # locally, such as those from strip and shelve, and temporary bundles.
1315 # locally, such as those from strip and shelve, and temporary bundles.
1319 return max(supportedoutgoingversions(repo))
1316 return max(supportedoutgoingversions(repo))
1320
1317
1321 def safeversion(repo):
1318 def safeversion(repo):
1322 # Finds the smallest version that it's safe to assume clients of the repo
1319 # Finds the smallest version that it's safe to assume clients of the repo
1323 # will support. For example, all hg versions that support generaldelta also
1320 # will support. For example, all hg versions that support generaldelta also
1324 # support changegroup 02.
1321 # support changegroup 02.
1325 versions = supportedoutgoingversions(repo)
1322 versions = supportedoutgoingversions(repo)
1326 if 'generaldelta' in repo.requirements:
1323 if 'generaldelta' in repo.requirements:
1327 versions.discard('01')
1324 versions.discard('01')
1328 assert versions
1325 assert versions
1329 return min(versions)
1326 return min(versions)
1330
1327
1331 def getbundler(version, repo, bundlecaps=None, filematcher=None,
1328 def getbundler(version, repo, bundlecaps=None, filematcher=None,
1332 ellipses=False, shallow=False, ellipsisroots=None,
1329 ellipses=False, shallow=False, ellipsisroots=None,
1333 fullnodes=None):
1330 fullnodes=None):
1334 assert version in supportedoutgoingversions(repo)
1331 assert version in supportedoutgoingversions(repo)
1335
1332
1336 if filematcher is None:
1333 if filematcher is None:
1337 filematcher = matchmod.alwaysmatcher(repo.root, '')
1334 filematcher = matchmod.alwaysmatcher(repo.root, '')
1338
1335
1339 if version == '01' and not filematcher.always():
1336 if version == '01' and not filematcher.always():
1340 raise error.ProgrammingError('version 01 changegroups do not support '
1337 raise error.ProgrammingError('version 01 changegroups do not support '
1341 'sparse file matchers')
1338 'sparse file matchers')
1342
1339
1343 if ellipses and version in (b'01', b'02'):
1340 if ellipses and version in (b'01', b'02'):
1344 raise error.Abort(
1341 raise error.Abort(
1345 _('ellipsis nodes require at least cg3 on client and server, '
1342 _('ellipsis nodes require at least cg3 on client and server, '
1346 'but negotiated version %s') % version)
1343 'but negotiated version %s') % version)
1347
1344
1348 # Requested files could include files not in the local store. So
1345 # Requested files could include files not in the local store. So
1349 # filter those out.
1346 # filter those out.
1350 filematcher = matchmod.intersectmatchers(repo.narrowmatch(),
1347 filematcher = matchmod.intersectmatchers(repo.narrowmatch(),
1351 filematcher)
1348 filematcher)
1352
1349
1353 fn = _packermap[version][0]
1350 fn = _packermap[version][0]
1354 return fn(repo, filematcher, bundlecaps, ellipses=ellipses,
1351 return fn(repo, filematcher, bundlecaps, ellipses=ellipses,
1355 shallow=shallow, ellipsisroots=ellipsisroots,
1352 shallow=shallow, ellipsisroots=ellipsisroots,
1356 fullnodes=fullnodes)
1353 fullnodes=fullnodes)
1357
1354
1358 def getunbundler(version, fh, alg, extras=None):
1355 def getunbundler(version, fh, alg, extras=None):
1359 return _packermap[version][1](fh, alg, extras=extras)
1356 return _packermap[version][1](fh, alg, extras=extras)
1360
1357
1361 def _changegroupinfo(repo, nodes, source):
1358 def _changegroupinfo(repo, nodes, source):
1362 if repo.ui.verbose or source == 'bundle':
1359 if repo.ui.verbose or source == 'bundle':
1363 repo.ui.status(_("%d changesets found\n") % len(nodes))
1360 repo.ui.status(_("%d changesets found\n") % len(nodes))
1364 if repo.ui.debugflag:
1361 if repo.ui.debugflag:
1365 repo.ui.debug("list of changesets:\n")
1362 repo.ui.debug("list of changesets:\n")
1366 for node in nodes:
1363 for node in nodes:
1367 repo.ui.debug("%s\n" % hex(node))
1364 repo.ui.debug("%s\n" % hex(node))
1368
1365
1369 def makechangegroup(repo, outgoing, version, source, fastpath=False,
1366 def makechangegroup(repo, outgoing, version, source, fastpath=False,
1370 bundlecaps=None):
1367 bundlecaps=None):
1371 cgstream = makestream(repo, outgoing, version, source,
1368 cgstream = makestream(repo, outgoing, version, source,
1372 fastpath=fastpath, bundlecaps=bundlecaps)
1369 fastpath=fastpath, bundlecaps=bundlecaps)
1373 return getunbundler(version, util.chunkbuffer(cgstream), None,
1370 return getunbundler(version, util.chunkbuffer(cgstream), None,
1374 {'clcount': len(outgoing.missing) })
1371 {'clcount': len(outgoing.missing) })
1375
1372
1376 def makestream(repo, outgoing, version, source, fastpath=False,
1373 def makestream(repo, outgoing, version, source, fastpath=False,
1377 bundlecaps=None, filematcher=None):
1374 bundlecaps=None, filematcher=None):
1378 bundler = getbundler(version, repo, bundlecaps=bundlecaps,
1375 bundler = getbundler(version, repo, bundlecaps=bundlecaps,
1379 filematcher=filematcher)
1376 filematcher=filematcher)
1380
1377
1381 repo = repo.unfiltered()
1378 repo = repo.unfiltered()
1382 commonrevs = outgoing.common
1379 commonrevs = outgoing.common
1383 csets = outgoing.missing
1380 csets = outgoing.missing
1384 heads = outgoing.missingheads
1381 heads = outgoing.missingheads
1385 # We go through the fast path if we get told to, or if all (unfiltered
1382 # We go through the fast path if we get told to, or if all (unfiltered
1386 # heads have been requested (since we then know there all linkrevs will
1383 # heads have been requested (since we then know there all linkrevs will
1387 # be pulled by the client).
1384 # be pulled by the client).
1388 heads.sort()
1385 heads.sort()
1389 fastpathlinkrev = fastpath or (
1386 fastpathlinkrev = fastpath or (
1390 repo.filtername is None and heads == sorted(repo.heads()))
1387 repo.filtername is None and heads == sorted(repo.heads()))
1391
1388
1392 repo.hook('preoutgoing', throw=True, source=source)
1389 repo.hook('preoutgoing', throw=True, source=source)
1393 _changegroupinfo(repo, csets, source)
1390 _changegroupinfo(repo, csets, source)
1394 return bundler.generate(commonrevs, csets, fastpathlinkrev, source)
1391 return bundler.generate(commonrevs, csets, fastpathlinkrev, source)
1395
1392
1396 def _addchangegroupfiles(repo, source, revmap, trp, expectedfiles, needfiles):
1393 def _addchangegroupfiles(repo, source, revmap, trp, expectedfiles, needfiles):
1397 revisions = 0
1394 revisions = 0
1398 files = 0
1395 files = 0
1399 progress = repo.ui.makeprogress(_('files'), unit=_('files'),
1396 progress = repo.ui.makeprogress(_('files'), unit=_('files'),
1400 total=expectedfiles)
1397 total=expectedfiles)
1401 for chunkdata in iter(source.filelogheader, {}):
1398 for chunkdata in iter(source.filelogheader, {}):
1402 files += 1
1399 files += 1
1403 f = chunkdata["filename"]
1400 f = chunkdata["filename"]
1404 repo.ui.debug("adding %s revisions\n" % f)
1401 repo.ui.debug("adding %s revisions\n" % f)
1405 progress.increment()
1402 progress.increment()
1406 fl = repo.file(f)
1403 fl = repo.file(f)
1407 o = len(fl)
1404 o = len(fl)
1408 try:
1405 try:
1409 deltas = source.deltaiter()
1406 deltas = source.deltaiter()
1410 if not fl.addgroup(deltas, revmap, trp):
1407 if not fl.addgroup(deltas, revmap, trp):
1411 raise error.Abort(_("received file revlog group is empty"))
1408 raise error.Abort(_("received file revlog group is empty"))
1412 except error.CensoredBaseError as e:
1409 except error.CensoredBaseError as e:
1413 raise error.Abort(_("received delta base is censored: %s") % e)
1410 raise error.Abort(_("received delta base is censored: %s") % e)
1414 revisions += len(fl) - o
1411 revisions += len(fl) - o
1415 if f in needfiles:
1412 if f in needfiles:
1416 needs = needfiles[f]
1413 needs = needfiles[f]
1417 for new in pycompat.xrange(o, len(fl)):
1414 for new in pycompat.xrange(o, len(fl)):
1418 n = fl.node(new)
1415 n = fl.node(new)
1419 if n in needs:
1416 if n in needs:
1420 needs.remove(n)
1417 needs.remove(n)
1421 else:
1418 else:
1422 raise error.Abort(
1419 raise error.Abort(
1423 _("received spurious file revlog entry"))
1420 _("received spurious file revlog entry"))
1424 if not needs:
1421 if not needs:
1425 del needfiles[f]
1422 del needfiles[f]
1426 progress.complete()
1423 progress.complete()
1427
1424
1428 for f, needs in needfiles.iteritems():
1425 for f, needs in needfiles.iteritems():
1429 fl = repo.file(f)
1426 fl = repo.file(f)
1430 for n in needs:
1427 for n in needs:
1431 try:
1428 try:
1432 fl.rev(n)
1429 fl.rev(n)
1433 except error.LookupError:
1430 except error.LookupError:
1434 raise error.Abort(
1431 raise error.Abort(
1435 _('missing file data for %s:%s - run hg verify') %
1432 _('missing file data for %s:%s - run hg verify') %
1436 (f, hex(n)))
1433 (f, hex(n)))
1437
1434
1438 return revisions, files
1435 return revisions, files
General Comments 0
You need to be logged in to leave comments. Login now