##// END OF EJS Templates
changegroup: specify ellipses mode explicitly...
Gregory Szorc -
r38944:1469584a default
parent child Browse files
Show More
@@ -1,1423 +1,1429 b''
1 # changegroup.py - Mercurial changegroup manipulation functions
1 # changegroup.py - Mercurial changegroup manipulation functions
2 #
2 #
3 # Copyright 2006 Matt Mackall <mpm@selenic.com>
3 # Copyright 2006 Matt Mackall <mpm@selenic.com>
4 #
4 #
5 # This software may be used and distributed according to the terms of the
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
6 # GNU General Public License version 2 or any later version.
7
7
8 from __future__ import absolute_import
8 from __future__ import absolute_import
9
9
10 import os
10 import os
11 import struct
11 import struct
12 import weakref
12 import weakref
13
13
14 from .i18n import _
14 from .i18n import _
15 from .node import (
15 from .node import (
16 hex,
16 hex,
17 nullid,
17 nullid,
18 nullrev,
18 nullrev,
19 short,
19 short,
20 )
20 )
21
21
22 from .thirdparty import (
22 from .thirdparty import (
23 attr,
23 attr,
24 )
24 )
25
25
26 from . import (
26 from . import (
27 dagutil,
27 dagutil,
28 error,
28 error,
29 manifest,
29 manifest,
30 match as matchmod,
30 match as matchmod,
31 mdiff,
31 mdiff,
32 phases,
32 phases,
33 pycompat,
33 pycompat,
34 repository,
34 repository,
35 revlog,
35 revlog,
36 util,
36 util,
37 )
37 )
38
38
39 from .utils import (
39 from .utils import (
40 stringutil,
40 stringutil,
41 )
41 )
42
42
43 _CHANGEGROUPV1_DELTA_HEADER = struct.Struct("20s20s20s20s")
43 _CHANGEGROUPV1_DELTA_HEADER = struct.Struct("20s20s20s20s")
44 _CHANGEGROUPV2_DELTA_HEADER = struct.Struct("20s20s20s20s20s")
44 _CHANGEGROUPV2_DELTA_HEADER = struct.Struct("20s20s20s20s20s")
45 _CHANGEGROUPV3_DELTA_HEADER = struct.Struct(">20s20s20s20s20sH")
45 _CHANGEGROUPV3_DELTA_HEADER = struct.Struct(">20s20s20s20s20sH")
46
46
47 LFS_REQUIREMENT = 'lfs'
47 LFS_REQUIREMENT = 'lfs'
48
48
49 readexactly = util.readexactly
49 readexactly = util.readexactly
50
50
51 def getchunk(stream):
51 def getchunk(stream):
52 """return the next chunk from stream as a string"""
52 """return the next chunk from stream as a string"""
53 d = readexactly(stream, 4)
53 d = readexactly(stream, 4)
54 l = struct.unpack(">l", d)[0]
54 l = struct.unpack(">l", d)[0]
55 if l <= 4:
55 if l <= 4:
56 if l:
56 if l:
57 raise error.Abort(_("invalid chunk length %d") % l)
57 raise error.Abort(_("invalid chunk length %d") % l)
58 return ""
58 return ""
59 return readexactly(stream, l - 4)
59 return readexactly(stream, l - 4)
60
60
61 def chunkheader(length):
61 def chunkheader(length):
62 """return a changegroup chunk header (string)"""
62 """return a changegroup chunk header (string)"""
63 return struct.pack(">l", length + 4)
63 return struct.pack(">l", length + 4)
64
64
65 def closechunk():
65 def closechunk():
66 """return a changegroup chunk header (string) for a zero-length chunk"""
66 """return a changegroup chunk header (string) for a zero-length chunk"""
67 return struct.pack(">l", 0)
67 return struct.pack(">l", 0)
68
68
69 def writechunks(ui, chunks, filename, vfs=None):
69 def writechunks(ui, chunks, filename, vfs=None):
70 """Write chunks to a file and return its filename.
70 """Write chunks to a file and return its filename.
71
71
72 The stream is assumed to be a bundle file.
72 The stream is assumed to be a bundle file.
73 Existing files will not be overwritten.
73 Existing files will not be overwritten.
74 If no filename is specified, a temporary file is created.
74 If no filename is specified, a temporary file is created.
75 """
75 """
76 fh = None
76 fh = None
77 cleanup = None
77 cleanup = None
78 try:
78 try:
79 if filename:
79 if filename:
80 if vfs:
80 if vfs:
81 fh = vfs.open(filename, "wb")
81 fh = vfs.open(filename, "wb")
82 else:
82 else:
83 # Increase default buffer size because default is usually
83 # Increase default buffer size because default is usually
84 # small (4k is common on Linux).
84 # small (4k is common on Linux).
85 fh = open(filename, "wb", 131072)
85 fh = open(filename, "wb", 131072)
86 else:
86 else:
87 fd, filename = pycompat.mkstemp(prefix="hg-bundle-", suffix=".hg")
87 fd, filename = pycompat.mkstemp(prefix="hg-bundle-", suffix=".hg")
88 fh = os.fdopen(fd, r"wb")
88 fh = os.fdopen(fd, r"wb")
89 cleanup = filename
89 cleanup = filename
90 for c in chunks:
90 for c in chunks:
91 fh.write(c)
91 fh.write(c)
92 cleanup = None
92 cleanup = None
93 return filename
93 return filename
94 finally:
94 finally:
95 if fh is not None:
95 if fh is not None:
96 fh.close()
96 fh.close()
97 if cleanup is not None:
97 if cleanup is not None:
98 if filename and vfs:
98 if filename and vfs:
99 vfs.unlink(cleanup)
99 vfs.unlink(cleanup)
100 else:
100 else:
101 os.unlink(cleanup)
101 os.unlink(cleanup)
102
102
103 class cg1unpacker(object):
103 class cg1unpacker(object):
104 """Unpacker for cg1 changegroup streams.
104 """Unpacker for cg1 changegroup streams.
105
105
106 A changegroup unpacker handles the framing of the revision data in
106 A changegroup unpacker handles the framing of the revision data in
107 the wire format. Most consumers will want to use the apply()
107 the wire format. Most consumers will want to use the apply()
108 method to add the changes from the changegroup to a repository.
108 method to add the changes from the changegroup to a repository.
109
109
110 If you're forwarding a changegroup unmodified to another consumer,
110 If you're forwarding a changegroup unmodified to another consumer,
111 use getchunks(), which returns an iterator of changegroup
111 use getchunks(), which returns an iterator of changegroup
112 chunks. This is mostly useful for cases where you need to know the
112 chunks. This is mostly useful for cases where you need to know the
113 data stream has ended by observing the end of the changegroup.
113 data stream has ended by observing the end of the changegroup.
114
114
115 deltachunk() is useful only if you're applying delta data. Most
115 deltachunk() is useful only if you're applying delta data. Most
116 consumers should prefer apply() instead.
116 consumers should prefer apply() instead.
117
117
118 A few other public methods exist. Those are used only for
118 A few other public methods exist. Those are used only for
119 bundlerepo and some debug commands - their use is discouraged.
119 bundlerepo and some debug commands - their use is discouraged.
120 """
120 """
121 deltaheader = _CHANGEGROUPV1_DELTA_HEADER
121 deltaheader = _CHANGEGROUPV1_DELTA_HEADER
122 deltaheadersize = deltaheader.size
122 deltaheadersize = deltaheader.size
123 version = '01'
123 version = '01'
124 _grouplistcount = 1 # One list of files after the manifests
124 _grouplistcount = 1 # One list of files after the manifests
125
125
126 def __init__(self, fh, alg, extras=None):
126 def __init__(self, fh, alg, extras=None):
127 if alg is None:
127 if alg is None:
128 alg = 'UN'
128 alg = 'UN'
129 if alg not in util.compengines.supportedbundletypes:
129 if alg not in util.compengines.supportedbundletypes:
130 raise error.Abort(_('unknown stream compression type: %s')
130 raise error.Abort(_('unknown stream compression type: %s')
131 % alg)
131 % alg)
132 if alg == 'BZ':
132 if alg == 'BZ':
133 alg = '_truncatedBZ'
133 alg = '_truncatedBZ'
134
134
135 compengine = util.compengines.forbundletype(alg)
135 compengine = util.compengines.forbundletype(alg)
136 self._stream = compengine.decompressorreader(fh)
136 self._stream = compengine.decompressorreader(fh)
137 self._type = alg
137 self._type = alg
138 self.extras = extras or {}
138 self.extras = extras or {}
139 self.callback = None
139 self.callback = None
140
140
141 # These methods (compressed, read, seek, tell) all appear to only
141 # These methods (compressed, read, seek, tell) all appear to only
142 # be used by bundlerepo, but it's a little hard to tell.
142 # be used by bundlerepo, but it's a little hard to tell.
143 def compressed(self):
143 def compressed(self):
144 return self._type is not None and self._type != 'UN'
144 return self._type is not None and self._type != 'UN'
145 def read(self, l):
145 def read(self, l):
146 return self._stream.read(l)
146 return self._stream.read(l)
147 def seek(self, pos):
147 def seek(self, pos):
148 return self._stream.seek(pos)
148 return self._stream.seek(pos)
149 def tell(self):
149 def tell(self):
150 return self._stream.tell()
150 return self._stream.tell()
151 def close(self):
151 def close(self):
152 return self._stream.close()
152 return self._stream.close()
153
153
154 def _chunklength(self):
154 def _chunklength(self):
155 d = readexactly(self._stream, 4)
155 d = readexactly(self._stream, 4)
156 l = struct.unpack(">l", d)[0]
156 l = struct.unpack(">l", d)[0]
157 if l <= 4:
157 if l <= 4:
158 if l:
158 if l:
159 raise error.Abort(_("invalid chunk length %d") % l)
159 raise error.Abort(_("invalid chunk length %d") % l)
160 return 0
160 return 0
161 if self.callback:
161 if self.callback:
162 self.callback()
162 self.callback()
163 return l - 4
163 return l - 4
164
164
165 def changelogheader(self):
165 def changelogheader(self):
166 """v10 does not have a changelog header chunk"""
166 """v10 does not have a changelog header chunk"""
167 return {}
167 return {}
168
168
169 def manifestheader(self):
169 def manifestheader(self):
170 """v10 does not have a manifest header chunk"""
170 """v10 does not have a manifest header chunk"""
171 return {}
171 return {}
172
172
173 def filelogheader(self):
173 def filelogheader(self):
174 """return the header of the filelogs chunk, v10 only has the filename"""
174 """return the header of the filelogs chunk, v10 only has the filename"""
175 l = self._chunklength()
175 l = self._chunklength()
176 if not l:
176 if not l:
177 return {}
177 return {}
178 fname = readexactly(self._stream, l)
178 fname = readexactly(self._stream, l)
179 return {'filename': fname}
179 return {'filename': fname}
180
180
181 def _deltaheader(self, headertuple, prevnode):
181 def _deltaheader(self, headertuple, prevnode):
182 node, p1, p2, cs = headertuple
182 node, p1, p2, cs = headertuple
183 if prevnode is None:
183 if prevnode is None:
184 deltabase = p1
184 deltabase = p1
185 else:
185 else:
186 deltabase = prevnode
186 deltabase = prevnode
187 flags = 0
187 flags = 0
188 return node, p1, p2, deltabase, cs, flags
188 return node, p1, p2, deltabase, cs, flags
189
189
190 def deltachunk(self, prevnode):
190 def deltachunk(self, prevnode):
191 l = self._chunklength()
191 l = self._chunklength()
192 if not l:
192 if not l:
193 return {}
193 return {}
194 headerdata = readexactly(self._stream, self.deltaheadersize)
194 headerdata = readexactly(self._stream, self.deltaheadersize)
195 header = self.deltaheader.unpack(headerdata)
195 header = self.deltaheader.unpack(headerdata)
196 delta = readexactly(self._stream, l - self.deltaheadersize)
196 delta = readexactly(self._stream, l - self.deltaheadersize)
197 node, p1, p2, deltabase, cs, flags = self._deltaheader(header, prevnode)
197 node, p1, p2, deltabase, cs, flags = self._deltaheader(header, prevnode)
198 return (node, p1, p2, cs, deltabase, delta, flags)
198 return (node, p1, p2, cs, deltabase, delta, flags)
199
199
200 def getchunks(self):
200 def getchunks(self):
201 """returns all the chunks contains in the bundle
201 """returns all the chunks contains in the bundle
202
202
203 Used when you need to forward the binary stream to a file or another
203 Used when you need to forward the binary stream to a file or another
204 network API. To do so, it parse the changegroup data, otherwise it will
204 network API. To do so, it parse the changegroup data, otherwise it will
205 block in case of sshrepo because it don't know the end of the stream.
205 block in case of sshrepo because it don't know the end of the stream.
206 """
206 """
207 # For changegroup 1 and 2, we expect 3 parts: changelog, manifestlog,
207 # For changegroup 1 and 2, we expect 3 parts: changelog, manifestlog,
208 # and a list of filelogs. For changegroup 3, we expect 4 parts:
208 # and a list of filelogs. For changegroup 3, we expect 4 parts:
209 # changelog, manifestlog, a list of tree manifestlogs, and a list of
209 # changelog, manifestlog, a list of tree manifestlogs, and a list of
210 # filelogs.
210 # filelogs.
211 #
211 #
212 # Changelog and manifestlog parts are terminated with empty chunks. The
212 # Changelog and manifestlog parts are terminated with empty chunks. The
213 # tree and file parts are a list of entry sections. Each entry section
213 # tree and file parts are a list of entry sections. Each entry section
214 # is a series of chunks terminating in an empty chunk. The list of these
214 # is a series of chunks terminating in an empty chunk. The list of these
215 # entry sections is terminated in yet another empty chunk, so we know
215 # entry sections is terminated in yet another empty chunk, so we know
216 # we've reached the end of the tree/file list when we reach an empty
216 # we've reached the end of the tree/file list when we reach an empty
217 # chunk that was proceeded by no non-empty chunks.
217 # chunk that was proceeded by no non-empty chunks.
218
218
219 parts = 0
219 parts = 0
220 while parts < 2 + self._grouplistcount:
220 while parts < 2 + self._grouplistcount:
221 noentries = True
221 noentries = True
222 while True:
222 while True:
223 chunk = getchunk(self)
223 chunk = getchunk(self)
224 if not chunk:
224 if not chunk:
225 # The first two empty chunks represent the end of the
225 # The first two empty chunks represent the end of the
226 # changelog and the manifestlog portions. The remaining
226 # changelog and the manifestlog portions. The remaining
227 # empty chunks represent either A) the end of individual
227 # empty chunks represent either A) the end of individual
228 # tree or file entries in the file list, or B) the end of
228 # tree or file entries in the file list, or B) the end of
229 # the entire list. It's the end of the entire list if there
229 # the entire list. It's the end of the entire list if there
230 # were no entries (i.e. noentries is True).
230 # were no entries (i.e. noentries is True).
231 if parts < 2:
231 if parts < 2:
232 parts += 1
232 parts += 1
233 elif noentries:
233 elif noentries:
234 parts += 1
234 parts += 1
235 break
235 break
236 noentries = False
236 noentries = False
237 yield chunkheader(len(chunk))
237 yield chunkheader(len(chunk))
238 pos = 0
238 pos = 0
239 while pos < len(chunk):
239 while pos < len(chunk):
240 next = pos + 2**20
240 next = pos + 2**20
241 yield chunk[pos:next]
241 yield chunk[pos:next]
242 pos = next
242 pos = next
243 yield closechunk()
243 yield closechunk()
244
244
245 def _unpackmanifests(self, repo, revmap, trp, prog):
245 def _unpackmanifests(self, repo, revmap, trp, prog):
246 self.callback = prog.increment
246 self.callback = prog.increment
247 # no need to check for empty manifest group here:
247 # no need to check for empty manifest group here:
248 # if the result of the merge of 1 and 2 is the same in 3 and 4,
248 # if the result of the merge of 1 and 2 is the same in 3 and 4,
249 # no new manifest will be created and the manifest group will
249 # no new manifest will be created and the manifest group will
250 # be empty during the pull
250 # be empty during the pull
251 self.manifestheader()
251 self.manifestheader()
252 deltas = self.deltaiter()
252 deltas = self.deltaiter()
253 repo.manifestlog.addgroup(deltas, revmap, trp)
253 repo.manifestlog.addgroup(deltas, revmap, trp)
254 prog.complete()
254 prog.complete()
255 self.callback = None
255 self.callback = None
256
256
257 def apply(self, repo, tr, srctype, url, targetphase=phases.draft,
257 def apply(self, repo, tr, srctype, url, targetphase=phases.draft,
258 expectedtotal=None):
258 expectedtotal=None):
259 """Add the changegroup returned by source.read() to this repo.
259 """Add the changegroup returned by source.read() to this repo.
260 srctype is a string like 'push', 'pull', or 'unbundle'. url is
260 srctype is a string like 'push', 'pull', or 'unbundle'. url is
261 the URL of the repo where this changegroup is coming from.
261 the URL of the repo where this changegroup is coming from.
262
262
263 Return an integer summarizing the change to this repo:
263 Return an integer summarizing the change to this repo:
264 - nothing changed or no source: 0
264 - nothing changed or no source: 0
265 - more heads than before: 1+added heads (2..n)
265 - more heads than before: 1+added heads (2..n)
266 - fewer heads than before: -1-removed heads (-2..-n)
266 - fewer heads than before: -1-removed heads (-2..-n)
267 - number of heads stays the same: 1
267 - number of heads stays the same: 1
268 """
268 """
269 repo = repo.unfiltered()
269 repo = repo.unfiltered()
270 def csmap(x):
270 def csmap(x):
271 repo.ui.debug("add changeset %s\n" % short(x))
271 repo.ui.debug("add changeset %s\n" % short(x))
272 return len(cl)
272 return len(cl)
273
273
274 def revmap(x):
274 def revmap(x):
275 return cl.rev(x)
275 return cl.rev(x)
276
276
277 changesets = files = revisions = 0
277 changesets = files = revisions = 0
278
278
279 try:
279 try:
280 # The transaction may already carry source information. In this
280 # The transaction may already carry source information. In this
281 # case we use the top level data. We overwrite the argument
281 # case we use the top level data. We overwrite the argument
282 # because we need to use the top level value (if they exist)
282 # because we need to use the top level value (if they exist)
283 # in this function.
283 # in this function.
284 srctype = tr.hookargs.setdefault('source', srctype)
284 srctype = tr.hookargs.setdefault('source', srctype)
285 url = tr.hookargs.setdefault('url', url)
285 url = tr.hookargs.setdefault('url', url)
286 repo.hook('prechangegroup',
286 repo.hook('prechangegroup',
287 throw=True, **pycompat.strkwargs(tr.hookargs))
287 throw=True, **pycompat.strkwargs(tr.hookargs))
288
288
289 # write changelog data to temp files so concurrent readers
289 # write changelog data to temp files so concurrent readers
290 # will not see an inconsistent view
290 # will not see an inconsistent view
291 cl = repo.changelog
291 cl = repo.changelog
292 cl.delayupdate(tr)
292 cl.delayupdate(tr)
293 oldheads = set(cl.heads())
293 oldheads = set(cl.heads())
294
294
295 trp = weakref.proxy(tr)
295 trp = weakref.proxy(tr)
296 # pull off the changeset group
296 # pull off the changeset group
297 repo.ui.status(_("adding changesets\n"))
297 repo.ui.status(_("adding changesets\n"))
298 clstart = len(cl)
298 clstart = len(cl)
299 progress = repo.ui.makeprogress(_('changesets'), unit=_('chunks'),
299 progress = repo.ui.makeprogress(_('changesets'), unit=_('chunks'),
300 total=expectedtotal)
300 total=expectedtotal)
301 self.callback = progress.increment
301 self.callback = progress.increment
302
302
303 efiles = set()
303 efiles = set()
304 def onchangelog(cl, node):
304 def onchangelog(cl, node):
305 efiles.update(cl.readfiles(node))
305 efiles.update(cl.readfiles(node))
306
306
307 self.changelogheader()
307 self.changelogheader()
308 deltas = self.deltaiter()
308 deltas = self.deltaiter()
309 cgnodes = cl.addgroup(deltas, csmap, trp, addrevisioncb=onchangelog)
309 cgnodes = cl.addgroup(deltas, csmap, trp, addrevisioncb=onchangelog)
310 efiles = len(efiles)
310 efiles = len(efiles)
311
311
312 if not cgnodes:
312 if not cgnodes:
313 repo.ui.develwarn('applied empty changegroup',
313 repo.ui.develwarn('applied empty changegroup',
314 config='warn-empty-changegroup')
314 config='warn-empty-changegroup')
315 clend = len(cl)
315 clend = len(cl)
316 changesets = clend - clstart
316 changesets = clend - clstart
317 progress.complete()
317 progress.complete()
318 self.callback = None
318 self.callback = None
319
319
320 # pull off the manifest group
320 # pull off the manifest group
321 repo.ui.status(_("adding manifests\n"))
321 repo.ui.status(_("adding manifests\n"))
322 # We know that we'll never have more manifests than we had
322 # We know that we'll never have more manifests than we had
323 # changesets.
323 # changesets.
324 progress = repo.ui.makeprogress(_('manifests'), unit=_('chunks'),
324 progress = repo.ui.makeprogress(_('manifests'), unit=_('chunks'),
325 total=changesets)
325 total=changesets)
326 self._unpackmanifests(repo, revmap, trp, progress)
326 self._unpackmanifests(repo, revmap, trp, progress)
327
327
328 needfiles = {}
328 needfiles = {}
329 if repo.ui.configbool('server', 'validate'):
329 if repo.ui.configbool('server', 'validate'):
330 cl = repo.changelog
330 cl = repo.changelog
331 ml = repo.manifestlog
331 ml = repo.manifestlog
332 # validate incoming csets have their manifests
332 # validate incoming csets have their manifests
333 for cset in pycompat.xrange(clstart, clend):
333 for cset in pycompat.xrange(clstart, clend):
334 mfnode = cl.changelogrevision(cset).manifest
334 mfnode = cl.changelogrevision(cset).manifest
335 mfest = ml[mfnode].readdelta()
335 mfest = ml[mfnode].readdelta()
336 # store file cgnodes we must see
336 # store file cgnodes we must see
337 for f, n in mfest.iteritems():
337 for f, n in mfest.iteritems():
338 needfiles.setdefault(f, set()).add(n)
338 needfiles.setdefault(f, set()).add(n)
339
339
340 # process the files
340 # process the files
341 repo.ui.status(_("adding file changes\n"))
341 repo.ui.status(_("adding file changes\n"))
342 newrevs, newfiles = _addchangegroupfiles(
342 newrevs, newfiles = _addchangegroupfiles(
343 repo, self, revmap, trp, efiles, needfiles)
343 repo, self, revmap, trp, efiles, needfiles)
344 revisions += newrevs
344 revisions += newrevs
345 files += newfiles
345 files += newfiles
346
346
347 deltaheads = 0
347 deltaheads = 0
348 if oldheads:
348 if oldheads:
349 heads = cl.heads()
349 heads = cl.heads()
350 deltaheads = len(heads) - len(oldheads)
350 deltaheads = len(heads) - len(oldheads)
351 for h in heads:
351 for h in heads:
352 if h not in oldheads and repo[h].closesbranch():
352 if h not in oldheads and repo[h].closesbranch():
353 deltaheads -= 1
353 deltaheads -= 1
354 htext = ""
354 htext = ""
355 if deltaheads:
355 if deltaheads:
356 htext = _(" (%+d heads)") % deltaheads
356 htext = _(" (%+d heads)") % deltaheads
357
357
358 repo.ui.status(_("added %d changesets"
358 repo.ui.status(_("added %d changesets"
359 " with %d changes to %d files%s\n")
359 " with %d changes to %d files%s\n")
360 % (changesets, revisions, files, htext))
360 % (changesets, revisions, files, htext))
361 repo.invalidatevolatilesets()
361 repo.invalidatevolatilesets()
362
362
363 if changesets > 0:
363 if changesets > 0:
364 if 'node' not in tr.hookargs:
364 if 'node' not in tr.hookargs:
365 tr.hookargs['node'] = hex(cl.node(clstart))
365 tr.hookargs['node'] = hex(cl.node(clstart))
366 tr.hookargs['node_last'] = hex(cl.node(clend - 1))
366 tr.hookargs['node_last'] = hex(cl.node(clend - 1))
367 hookargs = dict(tr.hookargs)
367 hookargs = dict(tr.hookargs)
368 else:
368 else:
369 hookargs = dict(tr.hookargs)
369 hookargs = dict(tr.hookargs)
370 hookargs['node'] = hex(cl.node(clstart))
370 hookargs['node'] = hex(cl.node(clstart))
371 hookargs['node_last'] = hex(cl.node(clend - 1))
371 hookargs['node_last'] = hex(cl.node(clend - 1))
372 repo.hook('pretxnchangegroup',
372 repo.hook('pretxnchangegroup',
373 throw=True, **pycompat.strkwargs(hookargs))
373 throw=True, **pycompat.strkwargs(hookargs))
374
374
375 added = [cl.node(r) for r in pycompat.xrange(clstart, clend)]
375 added = [cl.node(r) for r in pycompat.xrange(clstart, clend)]
376 phaseall = None
376 phaseall = None
377 if srctype in ('push', 'serve'):
377 if srctype in ('push', 'serve'):
378 # Old servers can not push the boundary themselves.
378 # Old servers can not push the boundary themselves.
379 # New servers won't push the boundary if changeset already
379 # New servers won't push the boundary if changeset already
380 # exists locally as secret
380 # exists locally as secret
381 #
381 #
382 # We should not use added here but the list of all change in
382 # We should not use added here but the list of all change in
383 # the bundle
383 # the bundle
384 if repo.publishing():
384 if repo.publishing():
385 targetphase = phaseall = phases.public
385 targetphase = phaseall = phases.public
386 else:
386 else:
387 # closer target phase computation
387 # closer target phase computation
388
388
389 # Those changesets have been pushed from the
389 # Those changesets have been pushed from the
390 # outside, their phases are going to be pushed
390 # outside, their phases are going to be pushed
391 # alongside. Therefor `targetphase` is
391 # alongside. Therefor `targetphase` is
392 # ignored.
392 # ignored.
393 targetphase = phaseall = phases.draft
393 targetphase = phaseall = phases.draft
394 if added:
394 if added:
395 phases.registernew(repo, tr, targetphase, added)
395 phases.registernew(repo, tr, targetphase, added)
396 if phaseall is not None:
396 if phaseall is not None:
397 phases.advanceboundary(repo, tr, phaseall, cgnodes)
397 phases.advanceboundary(repo, tr, phaseall, cgnodes)
398
398
399 if changesets > 0:
399 if changesets > 0:
400
400
401 def runhooks():
401 def runhooks():
402 # These hooks run when the lock releases, not when the
402 # These hooks run when the lock releases, not when the
403 # transaction closes. So it's possible for the changelog
403 # transaction closes. So it's possible for the changelog
404 # to have changed since we last saw it.
404 # to have changed since we last saw it.
405 if clstart >= len(repo):
405 if clstart >= len(repo):
406 return
406 return
407
407
408 repo.hook("changegroup", **pycompat.strkwargs(hookargs))
408 repo.hook("changegroup", **pycompat.strkwargs(hookargs))
409
409
410 for n in added:
410 for n in added:
411 args = hookargs.copy()
411 args = hookargs.copy()
412 args['node'] = hex(n)
412 args['node'] = hex(n)
413 del args['node_last']
413 del args['node_last']
414 repo.hook("incoming", **pycompat.strkwargs(args))
414 repo.hook("incoming", **pycompat.strkwargs(args))
415
415
416 newheads = [h for h in repo.heads()
416 newheads = [h for h in repo.heads()
417 if h not in oldheads]
417 if h not in oldheads]
418 repo.ui.log("incoming",
418 repo.ui.log("incoming",
419 "%d incoming changes - new heads: %s\n",
419 "%d incoming changes - new heads: %s\n",
420 len(added),
420 len(added),
421 ', '.join([hex(c[:6]) for c in newheads]))
421 ', '.join([hex(c[:6]) for c in newheads]))
422
422
423 tr.addpostclose('changegroup-runhooks-%020i' % clstart,
423 tr.addpostclose('changegroup-runhooks-%020i' % clstart,
424 lambda tr: repo._afterlock(runhooks))
424 lambda tr: repo._afterlock(runhooks))
425 finally:
425 finally:
426 repo.ui.flush()
426 repo.ui.flush()
427 # never return 0 here:
427 # never return 0 here:
428 if deltaheads < 0:
428 if deltaheads < 0:
429 ret = deltaheads - 1
429 ret = deltaheads - 1
430 else:
430 else:
431 ret = deltaheads + 1
431 ret = deltaheads + 1
432 return ret
432 return ret
433
433
434 def deltaiter(self):
434 def deltaiter(self):
435 """
435 """
436 returns an iterator of the deltas in this changegroup
436 returns an iterator of the deltas in this changegroup
437
437
438 Useful for passing to the underlying storage system to be stored.
438 Useful for passing to the underlying storage system to be stored.
439 """
439 """
440 chain = None
440 chain = None
441 for chunkdata in iter(lambda: self.deltachunk(chain), {}):
441 for chunkdata in iter(lambda: self.deltachunk(chain), {}):
442 # Chunkdata: (node, p1, p2, cs, deltabase, delta, flags)
442 # Chunkdata: (node, p1, p2, cs, deltabase, delta, flags)
443 yield chunkdata
443 yield chunkdata
444 chain = chunkdata[0]
444 chain = chunkdata[0]
445
445
446 class cg2unpacker(cg1unpacker):
446 class cg2unpacker(cg1unpacker):
447 """Unpacker for cg2 streams.
447 """Unpacker for cg2 streams.
448
448
449 cg2 streams add support for generaldelta, so the delta header
449 cg2 streams add support for generaldelta, so the delta header
450 format is slightly different. All other features about the data
450 format is slightly different. All other features about the data
451 remain the same.
451 remain the same.
452 """
452 """
453 deltaheader = _CHANGEGROUPV2_DELTA_HEADER
453 deltaheader = _CHANGEGROUPV2_DELTA_HEADER
454 deltaheadersize = deltaheader.size
454 deltaheadersize = deltaheader.size
455 version = '02'
455 version = '02'
456
456
457 def _deltaheader(self, headertuple, prevnode):
457 def _deltaheader(self, headertuple, prevnode):
458 node, p1, p2, deltabase, cs = headertuple
458 node, p1, p2, deltabase, cs = headertuple
459 flags = 0
459 flags = 0
460 return node, p1, p2, deltabase, cs, flags
460 return node, p1, p2, deltabase, cs, flags
461
461
462 class cg3unpacker(cg2unpacker):
462 class cg3unpacker(cg2unpacker):
463 """Unpacker for cg3 streams.
463 """Unpacker for cg3 streams.
464
464
465 cg3 streams add support for exchanging treemanifests and revlog
465 cg3 streams add support for exchanging treemanifests and revlog
466 flags. It adds the revlog flags to the delta header and an empty chunk
466 flags. It adds the revlog flags to the delta header and an empty chunk
467 separating manifests and files.
467 separating manifests and files.
468 """
468 """
469 deltaheader = _CHANGEGROUPV3_DELTA_HEADER
469 deltaheader = _CHANGEGROUPV3_DELTA_HEADER
470 deltaheadersize = deltaheader.size
470 deltaheadersize = deltaheader.size
471 version = '03'
471 version = '03'
472 _grouplistcount = 2 # One list of manifests and one list of files
472 _grouplistcount = 2 # One list of manifests and one list of files
473
473
474 def _deltaheader(self, headertuple, prevnode):
474 def _deltaheader(self, headertuple, prevnode):
475 node, p1, p2, deltabase, cs, flags = headertuple
475 node, p1, p2, deltabase, cs, flags = headertuple
476 return node, p1, p2, deltabase, cs, flags
476 return node, p1, p2, deltabase, cs, flags
477
477
478 def _unpackmanifests(self, repo, revmap, trp, prog):
478 def _unpackmanifests(self, repo, revmap, trp, prog):
479 super(cg3unpacker, self)._unpackmanifests(repo, revmap, trp, prog)
479 super(cg3unpacker, self)._unpackmanifests(repo, revmap, trp, prog)
480 for chunkdata in iter(self.filelogheader, {}):
480 for chunkdata in iter(self.filelogheader, {}):
481 # If we get here, there are directory manifests in the changegroup
481 # If we get here, there are directory manifests in the changegroup
482 d = chunkdata["filename"]
482 d = chunkdata["filename"]
483 repo.ui.debug("adding %s revisions\n" % d)
483 repo.ui.debug("adding %s revisions\n" % d)
484 dirlog = repo.manifestlog._revlog.dirlog(d)
484 dirlog = repo.manifestlog._revlog.dirlog(d)
485 deltas = self.deltaiter()
485 deltas = self.deltaiter()
486 if not dirlog.addgroup(deltas, revmap, trp):
486 if not dirlog.addgroup(deltas, revmap, trp):
487 raise error.Abort(_("received dir revlog group is empty"))
487 raise error.Abort(_("received dir revlog group is empty"))
488
488
489 class headerlessfixup(object):
489 class headerlessfixup(object):
490 def __init__(self, fh, h):
490 def __init__(self, fh, h):
491 self._h = h
491 self._h = h
492 self._fh = fh
492 self._fh = fh
493 def read(self, n):
493 def read(self, n):
494 if self._h:
494 if self._h:
495 d, self._h = self._h[:n], self._h[n:]
495 d, self._h = self._h[:n], self._h[n:]
496 if len(d) < n:
496 if len(d) < n:
497 d += readexactly(self._fh, n - len(d))
497 d += readexactly(self._fh, n - len(d))
498 return d
498 return d
499 return readexactly(self._fh, n)
499 return readexactly(self._fh, n)
500
500
501 @attr.s(slots=True, frozen=True)
501 @attr.s(slots=True, frozen=True)
502 class revisiondelta(object):
502 class revisiondelta(object):
503 """Describes a delta entry in a changegroup.
503 """Describes a delta entry in a changegroup.
504
504
505 Captured data is sufficient to serialize the delta into multiple
505 Captured data is sufficient to serialize the delta into multiple
506 formats.
506 formats.
507 """
507 """
508 # 20 byte node of this revision.
508 # 20 byte node of this revision.
509 node = attr.ib()
509 node = attr.ib()
510 # 20 byte nodes of parent revisions.
510 # 20 byte nodes of parent revisions.
511 p1node = attr.ib()
511 p1node = attr.ib()
512 p2node = attr.ib()
512 p2node = attr.ib()
513 # 20 byte node of node this delta is against.
513 # 20 byte node of node this delta is against.
514 basenode = attr.ib()
514 basenode = attr.ib()
515 # 20 byte node of changeset revision this delta is associated with.
515 # 20 byte node of changeset revision this delta is associated with.
516 linknode = attr.ib()
516 linknode = attr.ib()
517 # 2 bytes of flags to apply to revision data.
517 # 2 bytes of flags to apply to revision data.
518 flags = attr.ib()
518 flags = attr.ib()
519 # Iterable of chunks holding raw delta data.
519 # Iterable of chunks holding raw delta data.
520 deltachunks = attr.ib()
520 deltachunks = attr.ib()
521
521
522 class cgpacker(object):
522 class cgpacker(object):
523 def __init__(self, repo, filematcher, version, allowreorder,
523 def __init__(self, repo, filematcher, version, allowreorder,
524 useprevdelta, builddeltaheader, manifestsend,
524 useprevdelta, builddeltaheader, manifestsend,
525 sendtreemanifests, bundlecaps=None, shallow=False,
525 sendtreemanifests, bundlecaps=None, ellipses=False,
526 ellipsisroots=None):
526 shallow=False, ellipsisroots=None):
527 """Given a source repo, construct a bundler.
527 """Given a source repo, construct a bundler.
528
528
529 filematcher is a matcher that matches on files to include in the
529 filematcher is a matcher that matches on files to include in the
530 changegroup. Used to facilitate sparse changegroups.
530 changegroup. Used to facilitate sparse changegroups.
531
531
532 allowreorder controls whether reordering of revisions is allowed.
532 allowreorder controls whether reordering of revisions is allowed.
533 This value is used when ``bundle.reorder`` is ``auto`` or isn't
533 This value is used when ``bundle.reorder`` is ``auto`` or isn't
534 set.
534 set.
535
535
536 useprevdelta controls whether revisions should always delta against
536 useprevdelta controls whether revisions should always delta against
537 the previous revision in the changegroup.
537 the previous revision in the changegroup.
538
538
539 builddeltaheader is a callable that constructs the header for a group
539 builddeltaheader is a callable that constructs the header for a group
540 delta.
540 delta.
541
541
542 manifestsend is a chunk to send after manifests have been fully emitted.
542 manifestsend is a chunk to send after manifests have been fully emitted.
543
543
544 sendtreemanifests indicates whether tree manifests should be emitted.
544 sendtreemanifests indicates whether tree manifests should be emitted.
545
545
546 ellipses indicates whether ellipsis serving mode is enabled.
547
546 bundlecaps is optional and can be used to specify the set of
548 bundlecaps is optional and can be used to specify the set of
547 capabilities which can be used to build the bundle. While bundlecaps is
549 capabilities which can be used to build the bundle. While bundlecaps is
548 unused in core Mercurial, extensions rely on this feature to communicate
550 unused in core Mercurial, extensions rely on this feature to communicate
549 capabilities to customize the changegroup packer.
551 capabilities to customize the changegroup packer.
550
552
551 shallow indicates whether shallow data might be sent. The packer may
553 shallow indicates whether shallow data might be sent. The packer may
552 need to pack file contents not introduced by the changes being packed.
554 need to pack file contents not introduced by the changes being packed.
553 """
555 """
554 assert filematcher
556 assert filematcher
555 self._filematcher = filematcher
557 self._filematcher = filematcher
556
558
557 self.version = version
559 self.version = version
558 self._useprevdelta = useprevdelta
560 self._useprevdelta = useprevdelta
559 self._builddeltaheader = builddeltaheader
561 self._builddeltaheader = builddeltaheader
560 self._manifestsend = manifestsend
562 self._manifestsend = manifestsend
561 self._sendtreemanifests = sendtreemanifests
563 self._sendtreemanifests = sendtreemanifests
564 self._ellipses = ellipses
562
565
563 # Set of capabilities we can use to build the bundle.
566 # Set of capabilities we can use to build the bundle.
564 if bundlecaps is None:
567 if bundlecaps is None:
565 bundlecaps = set()
568 bundlecaps = set()
566 self._bundlecaps = bundlecaps
569 self._bundlecaps = bundlecaps
567 self._isshallow = shallow
570 self._isshallow = shallow
568
571
569 # Maps ellipsis revs to their roots at the changelog level.
572 # Maps ellipsis revs to their roots at the changelog level.
570 self._precomputedellipsis = ellipsisroots
573 self._precomputedellipsis = ellipsisroots
571
574
572 # experimental config: bundle.reorder
575 # experimental config: bundle.reorder
573 reorder = repo.ui.config('bundle', 'reorder')
576 reorder = repo.ui.config('bundle', 'reorder')
574 if reorder == 'auto':
577 if reorder == 'auto':
575 self._reorder = allowreorder
578 self._reorder = allowreorder
576 else:
579 else:
577 self._reorder = stringutil.parsebool(reorder)
580 self._reorder = stringutil.parsebool(reorder)
578
581
579 self._repo = repo
582 self._repo = repo
580
583
581 if self._repo.ui.verbose and not self._repo.ui.debugflag:
584 if self._repo.ui.verbose and not self._repo.ui.debugflag:
582 self._verbosenote = self._repo.ui.note
585 self._verbosenote = self._repo.ui.note
583 else:
586 else:
584 self._verbosenote = lambda s: None
587 self._verbosenote = lambda s: None
585
588
586 # TODO the functionality keyed off of this should probably be
589 # TODO the functionality keyed off of this should probably be
587 # controlled via arguments to group() that influence behavior.
590 # controlled via arguments to group() that influence behavior.
588 self._changelogdone = False
591 self._changelogdone = False
589
592
590 # Maps CL revs to per-revlog revisions. Cleared in close() at
593 # Maps CL revs to per-revlog revisions. Cleared in close() at
591 # the end of each group.
594 # the end of each group.
592 self._clrevtolocalrev = {}
595 self._clrevtolocalrev = {}
593 self._nextclrevtolocalrev = {}
596 self._nextclrevtolocalrev = {}
594
597
595 # Maps changelog nodes to changelog revs. Filled in once
598 # Maps changelog nodes to changelog revs. Filled in once
596 # during changelog stage and then left unmodified.
599 # during changelog stage and then left unmodified.
597 self._clnodetorev = {}
600 self._clnodetorev = {}
598
601
599 def _close(self):
602 def _close(self):
600 # Ellipses serving mode.
603 # Ellipses serving mode.
601 self._clrevtolocalrev.clear()
604 self._clrevtolocalrev.clear()
602 if self._nextclrevtolocalrev:
605 if self._nextclrevtolocalrev:
603 self.clrevtolocalrev = self._nextclrevtolocalrev
606 self.clrevtolocalrev = self._nextclrevtolocalrev
604 self._nextclrevtolocalrev.clear()
607 self._nextclrevtolocalrev.clear()
605 self._changelogdone = True
608 self._changelogdone = True
606
609
607 return closechunk()
610 return closechunk()
608
611
609 def _fileheader(self, fname):
612 def _fileheader(self, fname):
610 return chunkheader(len(fname)) + fname
613 return chunkheader(len(fname)) + fname
611
614
612 # Extracted both for clarity and for overriding in extensions.
615 # Extracted both for clarity and for overriding in extensions.
613 def _sortgroup(self, store, nodelist, lookup):
616 def _sortgroup(self, store, nodelist, lookup):
614 """Sort nodes for change group and turn them into revnums."""
617 """Sort nodes for change group and turn them into revnums."""
615 # Ellipses serving mode.
618 # Ellipses serving mode.
616 #
619 #
617 # In a perfect world, we'd generate better ellipsis-ified graphs
620 # In a perfect world, we'd generate better ellipsis-ified graphs
618 # for non-changelog revlogs. In practice, we haven't started doing
621 # for non-changelog revlogs. In practice, we haven't started doing
619 # that yet, so the resulting DAGs for the manifestlog and filelogs
622 # that yet, so the resulting DAGs for the manifestlog and filelogs
620 # are actually full of bogus parentage on all the ellipsis
623 # are actually full of bogus parentage on all the ellipsis
621 # nodes. This has the side effect that, while the contents are
624 # nodes. This has the side effect that, while the contents are
622 # correct, the individual DAGs might be completely out of whack in
625 # correct, the individual DAGs might be completely out of whack in
623 # a case like 882681bc3166 and its ancestors (back about 10
626 # a case like 882681bc3166 and its ancestors (back about 10
624 # revisions or so) in the main hg repo.
627 # revisions or so) in the main hg repo.
625 #
628 #
626 # The one invariant we *know* holds is that the new (potentially
629 # The one invariant we *know* holds is that the new (potentially
627 # bogus) DAG shape will be valid if we order the nodes in the
630 # bogus) DAG shape will be valid if we order the nodes in the
628 # order that they're introduced in dramatis personae by the
631 # order that they're introduced in dramatis personae by the
629 # changelog, so what we do is we sort the non-changelog histories
632 # changelog, so what we do is we sort the non-changelog histories
630 # by the order in which they are used by the changelog.
633 # by the order in which they are used by the changelog.
631 if util.safehasattr(self, '_full_nodes') and self._clnodetorev:
634 if self._ellipses and self._clnodetorev:
632 key = lambda n: self._clnodetorev[lookup(n)]
635 key = lambda n: self._clnodetorev[lookup(n)]
633 return [store.rev(n) for n in sorted(nodelist, key=key)]
636 return [store.rev(n) for n in sorted(nodelist, key=key)]
634
637
635 # for generaldelta revlogs, we linearize the revs; this will both be
638 # for generaldelta revlogs, we linearize the revs; this will both be
636 # much quicker and generate a much smaller bundle
639 # much quicker and generate a much smaller bundle
637 if (store._generaldelta and self._reorder is None) or self._reorder:
640 if (store._generaldelta and self._reorder is None) or self._reorder:
638 dag = dagutil.revlogdag(store)
641 dag = dagutil.revlogdag(store)
639 return dag.linearize(set(store.rev(n) for n in nodelist))
642 return dag.linearize(set(store.rev(n) for n in nodelist))
640 else:
643 else:
641 return sorted([store.rev(n) for n in nodelist])
644 return sorted([store.rev(n) for n in nodelist])
642
645
643 def group(self, nodelist, store, lookup, units=None):
646 def group(self, nodelist, store, lookup, units=None):
644 """Calculate a delta group, yielding a sequence of changegroup chunks
647 """Calculate a delta group, yielding a sequence of changegroup chunks
645 (strings).
648 (strings).
646
649
647 Given a list of changeset revs, return a set of deltas and
650 Given a list of changeset revs, return a set of deltas and
648 metadata corresponding to nodes. The first delta is
651 metadata corresponding to nodes. The first delta is
649 first parent(nodelist[0]) -> nodelist[0], the receiver is
652 first parent(nodelist[0]) -> nodelist[0], the receiver is
650 guaranteed to have this parent as it has all history before
653 guaranteed to have this parent as it has all history before
651 these changesets. In the case firstparent is nullrev the
654 these changesets. In the case firstparent is nullrev the
652 changegroup starts with a full revision.
655 changegroup starts with a full revision.
653
656
654 If units is not None, progress detail will be generated, units specifies
657 If units is not None, progress detail will be generated, units specifies
655 the type of revlog that is touched (changelog, manifest, etc.).
658 the type of revlog that is touched (changelog, manifest, etc.).
656 """
659 """
657 # if we don't have any revisions touched by these changesets, bail
660 # if we don't have any revisions touched by these changesets, bail
658 if len(nodelist) == 0:
661 if len(nodelist) == 0:
659 yield self._close()
662 yield self._close()
660 return
663 return
661
664
662 revs = self._sortgroup(store, nodelist, lookup)
665 revs = self._sortgroup(store, nodelist, lookup)
663
666
664 # add the parent of the first rev
667 # add the parent of the first rev
665 p = store.parentrevs(revs[0])[0]
668 p = store.parentrevs(revs[0])[0]
666 revs.insert(0, p)
669 revs.insert(0, p)
667
670
668 # build deltas
671 # build deltas
669 progress = None
672 progress = None
670 if units is not None:
673 if units is not None:
671 progress = self._repo.ui.makeprogress(_('bundling'), unit=units,
674 progress = self._repo.ui.makeprogress(_('bundling'), unit=units,
672 total=(len(revs) - 1))
675 total=(len(revs) - 1))
673 for r in pycompat.xrange(len(revs) - 1):
676 for r in pycompat.xrange(len(revs) - 1):
674 if progress:
677 if progress:
675 progress.update(r + 1)
678 progress.update(r + 1)
676 prev, curr = revs[r], revs[r + 1]
679 prev, curr = revs[r], revs[r + 1]
677 linknode = lookup(store.node(curr))
680 linknode = lookup(store.node(curr))
678 for c in self._revchunk(store, curr, prev, linknode):
681 for c in self._revchunk(store, curr, prev, linknode):
679 yield c
682 yield c
680
683
681 if progress:
684 if progress:
682 progress.complete()
685 progress.complete()
683 yield self._close()
686 yield self._close()
684
687
685 # filter any nodes that claim to be part of the known set
688 # filter any nodes that claim to be part of the known set
686 def _prune(self, store, missing, commonrevs):
689 def _prune(self, store, missing, commonrevs):
687 # TODO this violates storage abstraction for manifests.
690 # TODO this violates storage abstraction for manifests.
688 if isinstance(store, manifest.manifestrevlog):
691 if isinstance(store, manifest.manifestrevlog):
689 if not self._filematcher.visitdir(store._dir[:-1] or '.'):
692 if not self._filematcher.visitdir(store._dir[:-1] or '.'):
690 return []
693 return []
691
694
692 rr, rl = store.rev, store.linkrev
695 rr, rl = store.rev, store.linkrev
693 return [n for n in missing if rl(rr(n)) not in commonrevs]
696 return [n for n in missing if rl(rr(n)) not in commonrevs]
694
697
695 def _packmanifests(self, dir, mfnodes, lookuplinknode):
698 def _packmanifests(self, dir, mfnodes, lookuplinknode):
696 """Pack flat manifests into a changegroup stream."""
699 """Pack flat manifests into a changegroup stream."""
697 assert not dir
700 assert not dir
698 for chunk in self.group(mfnodes, self._repo.manifestlog._revlog,
701 for chunk in self.group(mfnodes, self._repo.manifestlog._revlog,
699 lookuplinknode, units=_('manifests')):
702 lookuplinknode, units=_('manifests')):
700 yield chunk
703 yield chunk
701
704
702 def _packtreemanifests(self, dir, mfnodes, lookuplinknode):
705 def _packtreemanifests(self, dir, mfnodes, lookuplinknode):
703 """Version of _packmanifests that operates on directory manifests.
706 """Version of _packmanifests that operates on directory manifests.
704
707
705 Encodes the directory name in the output so multiple manifests
708 Encodes the directory name in the output so multiple manifests
706 can be sent.
709 can be sent.
707 """
710 """
708 assert self.version == b'03'
711 assert self.version == b'03'
709
712
710 if dir:
713 if dir:
711 yield self._fileheader(dir)
714 yield self._fileheader(dir)
712
715
713 # TODO violates storage abstractions by assuming revlogs.
716 # TODO violates storage abstractions by assuming revlogs.
714 dirlog = self._repo.manifestlog._revlog.dirlog(dir)
717 dirlog = self._repo.manifestlog._revlog.dirlog(dir)
715 for chunk in self.group(mfnodes, dirlog, lookuplinknode,
718 for chunk in self.group(mfnodes, dirlog, lookuplinknode,
716 units=_('manifests')):
719 units=_('manifests')):
717 yield chunk
720 yield chunk
718
721
719 def generate(self, commonrevs, clnodes, fastpathlinkrev, source):
722 def generate(self, commonrevs, clnodes, fastpathlinkrev, source):
720 '''yield a sequence of changegroup chunks (strings)'''
723 '''yield a sequence of changegroup chunks (strings)'''
721 repo = self._repo
724 repo = self._repo
722 cl = repo.changelog
725 cl = repo.changelog
723
726
724 clrevorder = {}
727 clrevorder = {}
725 mfs = {} # needed manifests
728 mfs = {} # needed manifests
726 fnodes = {} # needed file nodes
729 fnodes = {} # needed file nodes
727 mfl = repo.manifestlog
730 mfl = repo.manifestlog
728 # TODO violates storage abstraction.
731 # TODO violates storage abstraction.
729 mfrevlog = mfl._revlog
732 mfrevlog = mfl._revlog
730 changedfiles = set()
733 changedfiles = set()
731
734
732 ellipsesmode = util.safehasattr(self, '_full_nodes')
733
734 # Callback for the changelog, used to collect changed files and
735 # Callback for the changelog, used to collect changed files and
735 # manifest nodes.
736 # manifest nodes.
736 # Returns the linkrev node (identity in the changelog case).
737 # Returns the linkrev node (identity in the changelog case).
737 def lookupcl(x):
738 def lookupcl(x):
738 c = cl.read(x)
739 c = cl.read(x)
739 clrevorder[x] = len(clrevorder)
740 clrevorder[x] = len(clrevorder)
740
741
741 if ellipsesmode:
742 if self._ellipses:
742 # Only update mfs if x is going to be sent. Otherwise we
743 # Only update mfs if x is going to be sent. Otherwise we
743 # end up with bogus linkrevs specified for manifests and
744 # end up with bogus linkrevs specified for manifests and
744 # we skip some manifest nodes that we should otherwise
745 # we skip some manifest nodes that we should otherwise
745 # have sent.
746 # have sent.
746 if (x in self._full_nodes
747 if (x in self._full_nodes
747 or cl.rev(x) in self._precomputedellipsis):
748 or cl.rev(x) in self._precomputedellipsis):
748 n = c[0]
749 n = c[0]
749 # Record the first changeset introducing this manifest
750 # Record the first changeset introducing this manifest
750 # version.
751 # version.
751 mfs.setdefault(n, x)
752 mfs.setdefault(n, x)
752 # Set this narrow-specific dict so we have the lowest
753 # Set this narrow-specific dict so we have the lowest
753 # manifest revnum to look up for this cl revnum. (Part of
754 # manifest revnum to look up for this cl revnum. (Part of
754 # mapping changelog ellipsis parents to manifest ellipsis
755 # mapping changelog ellipsis parents to manifest ellipsis
755 # parents)
756 # parents)
756 self._nextclrevtolocalrev.setdefault(cl.rev(x),
757 self._nextclrevtolocalrev.setdefault(cl.rev(x),
757 mfrevlog.rev(n))
758 mfrevlog.rev(n))
758 # We can't trust the changed files list in the changeset if the
759 # We can't trust the changed files list in the changeset if the
759 # client requested a shallow clone.
760 # client requested a shallow clone.
760 if self._isshallow:
761 if self._isshallow:
761 changedfiles.update(mfl[c[0]].read().keys())
762 changedfiles.update(mfl[c[0]].read().keys())
762 else:
763 else:
763 changedfiles.update(c[3])
764 changedfiles.update(c[3])
764 else:
765 else:
765
766
766 n = c[0]
767 n = c[0]
767 # record the first changeset introducing this manifest version
768 # record the first changeset introducing this manifest version
768 mfs.setdefault(n, x)
769 mfs.setdefault(n, x)
769 # Record a complete list of potentially-changed files in
770 # Record a complete list of potentially-changed files in
770 # this manifest.
771 # this manifest.
771 changedfiles.update(c[3])
772 changedfiles.update(c[3])
772
773
773 return x
774 return x
774
775
775 self._verbosenote(_('uncompressed size of bundle content:\n'))
776 self._verbosenote(_('uncompressed size of bundle content:\n'))
776 size = 0
777 size = 0
777 for chunk in self.group(clnodes, cl, lookupcl, units=_('changesets')):
778 for chunk in self.group(clnodes, cl, lookupcl, units=_('changesets')):
778 size += len(chunk)
779 size += len(chunk)
779 yield chunk
780 yield chunk
780 self._verbosenote(_('%8.i (changelog)\n') % size)
781 self._verbosenote(_('%8.i (changelog)\n') % size)
781
782
782 # We need to make sure that the linkrev in the changegroup refers to
783 # We need to make sure that the linkrev in the changegroup refers to
783 # the first changeset that introduced the manifest or file revision.
784 # the first changeset that introduced the manifest or file revision.
784 # The fastpath is usually safer than the slowpath, because the filelogs
785 # The fastpath is usually safer than the slowpath, because the filelogs
785 # are walked in revlog order.
786 # are walked in revlog order.
786 #
787 #
787 # When taking the slowpath with reorder=None and the manifest revlog
788 # When taking the slowpath with reorder=None and the manifest revlog
788 # uses generaldelta, the manifest may be walked in the "wrong" order.
789 # uses generaldelta, the manifest may be walked in the "wrong" order.
789 # Without 'clrevorder', we would get an incorrect linkrev (see fix in
790 # Without 'clrevorder', we would get an incorrect linkrev (see fix in
790 # cc0ff93d0c0c).
791 # cc0ff93d0c0c).
791 #
792 #
792 # When taking the fastpath, we are only vulnerable to reordering
793 # When taking the fastpath, we are only vulnerable to reordering
793 # of the changelog itself. The changelog never uses generaldelta, so
794 # of the changelog itself. The changelog never uses generaldelta, so
794 # it is only reordered when reorder=True. To handle this case, we
795 # it is only reordered when reorder=True. To handle this case, we
795 # simply take the slowpath, which already has the 'clrevorder' logic.
796 # simply take the slowpath, which already has the 'clrevorder' logic.
796 # This was also fixed in cc0ff93d0c0c.
797 # This was also fixed in cc0ff93d0c0c.
797 fastpathlinkrev = fastpathlinkrev and not self._reorder
798 fastpathlinkrev = fastpathlinkrev and not self._reorder
798 # Treemanifests don't work correctly with fastpathlinkrev
799 # Treemanifests don't work correctly with fastpathlinkrev
799 # either, because we don't discover which directory nodes to
800 # either, because we don't discover which directory nodes to
800 # send along with files. This could probably be fixed.
801 # send along with files. This could probably be fixed.
801 fastpathlinkrev = fastpathlinkrev and (
802 fastpathlinkrev = fastpathlinkrev and (
802 'treemanifest' not in repo.requirements)
803 'treemanifest' not in repo.requirements)
803
804
804 for chunk in self.generatemanifests(commonrevs, clrevorder,
805 for chunk in self.generatemanifests(commonrevs, clrevorder,
805 fastpathlinkrev, mfs, fnodes, source):
806 fastpathlinkrev, mfs, fnodes, source):
806 yield chunk
807 yield chunk
807
808
808 if ellipsesmode:
809 if self._ellipses:
809 mfdicts = None
810 mfdicts = None
810 if self._isshallow:
811 if self._isshallow:
811 mfdicts = [(self._repo.manifestlog[n].read(), lr)
812 mfdicts = [(self._repo.manifestlog[n].read(), lr)
812 for (n, lr) in mfs.iteritems()]
813 for (n, lr) in mfs.iteritems()]
813
814
814 mfs.clear()
815 mfs.clear()
815 clrevs = set(cl.rev(x) for x in clnodes)
816 clrevs = set(cl.rev(x) for x in clnodes)
816
817
817 if not fastpathlinkrev:
818 if not fastpathlinkrev:
818 def linknodes(unused, fname):
819 def linknodes(unused, fname):
819 return fnodes.get(fname, {})
820 return fnodes.get(fname, {})
820 else:
821 else:
821 cln = cl.node
822 cln = cl.node
822 def linknodes(filerevlog, fname):
823 def linknodes(filerevlog, fname):
823 llr = filerevlog.linkrev
824 llr = filerevlog.linkrev
824 fln = filerevlog.node
825 fln = filerevlog.node
825 revs = ((r, llr(r)) for r in filerevlog)
826 revs = ((r, llr(r)) for r in filerevlog)
826 return dict((fln(r), cln(lr)) for r, lr in revs if lr in clrevs)
827 return dict((fln(r), cln(lr)) for r, lr in revs if lr in clrevs)
827
828
828 if ellipsesmode:
829 if self._ellipses:
829 # We need to pass the mfdicts variable down into
830 # We need to pass the mfdicts variable down into
830 # generatefiles(), but more than one command might have
831 # generatefiles(), but more than one command might have
831 # wrapped generatefiles so we can't modify the function
832 # wrapped generatefiles so we can't modify the function
832 # signature. Instead, we pass the data to ourselves using an
833 # signature. Instead, we pass the data to ourselves using an
833 # instance attribute. I'm sorry.
834 # instance attribute. I'm sorry.
834 self._mfdicts = mfdicts
835 self._mfdicts = mfdicts
835
836
836 for chunk in self.generatefiles(changedfiles, linknodes, commonrevs,
837 for chunk in self.generatefiles(changedfiles, linknodes, commonrevs,
837 source):
838 source):
838 yield chunk
839 yield chunk
839
840
840 yield self._close()
841 yield self._close()
841
842
842 if clnodes:
843 if clnodes:
843 repo.hook('outgoing', node=hex(clnodes[0]), source=source)
844 repo.hook('outgoing', node=hex(clnodes[0]), source=source)
844
845
845 def generatemanifests(self, commonrevs, clrevorder, fastpathlinkrev, mfs,
846 def generatemanifests(self, commonrevs, clrevorder, fastpathlinkrev, mfs,
846 fnodes, source):
847 fnodes, source):
847 """Returns an iterator of changegroup chunks containing manifests.
848 """Returns an iterator of changegroup chunks containing manifests.
848
849
849 `source` is unused here, but is used by extensions like remotefilelog to
850 `source` is unused here, but is used by extensions like remotefilelog to
850 change what is sent based in pulls vs pushes, etc.
851 change what is sent based in pulls vs pushes, etc.
851 """
852 """
852 repo = self._repo
853 repo = self._repo
853 mfl = repo.manifestlog
854 mfl = repo.manifestlog
854 dirlog = mfl._revlog.dirlog
855 dirlog = mfl._revlog.dirlog
855 tmfnodes = {'': mfs}
856 tmfnodes = {'': mfs}
856
857
857 # Callback for the manifest, used to collect linkrevs for filelog
858 # Callback for the manifest, used to collect linkrevs for filelog
858 # revisions.
859 # revisions.
859 # Returns the linkrev node (collected in lookupcl).
860 # Returns the linkrev node (collected in lookupcl).
860 def makelookupmflinknode(dir, nodes):
861 def makelookupmflinknode(dir, nodes):
861 if fastpathlinkrev:
862 if fastpathlinkrev:
862 assert not dir
863 assert not dir
863 return mfs.__getitem__
864 return mfs.__getitem__
864
865
865 def lookupmflinknode(x):
866 def lookupmflinknode(x):
866 """Callback for looking up the linknode for manifests.
867 """Callback for looking up the linknode for manifests.
867
868
868 Returns the linkrev node for the specified manifest.
869 Returns the linkrev node for the specified manifest.
869
870
870 SIDE EFFECT:
871 SIDE EFFECT:
871
872
872 1) fclnodes gets populated with the list of relevant
873 1) fclnodes gets populated with the list of relevant
873 file nodes if we're not using fastpathlinkrev
874 file nodes if we're not using fastpathlinkrev
874 2) When treemanifests are in use, collects treemanifest nodes
875 2) When treemanifests are in use, collects treemanifest nodes
875 to send
876 to send
876
877
877 Note that this means manifests must be completely sent to
878 Note that this means manifests must be completely sent to
878 the client before you can trust the list of files and
879 the client before you can trust the list of files and
879 treemanifests to send.
880 treemanifests to send.
880 """
881 """
881 clnode = nodes[x]
882 clnode = nodes[x]
882 mdata = mfl.get(dir, x).readfast(shallow=True)
883 mdata = mfl.get(dir, x).readfast(shallow=True)
883 for p, n, fl in mdata.iterentries():
884 for p, n, fl in mdata.iterentries():
884 if fl == 't': # subdirectory manifest
885 if fl == 't': # subdirectory manifest
885 subdir = dir + p + '/'
886 subdir = dir + p + '/'
886 tmfclnodes = tmfnodes.setdefault(subdir, {})
887 tmfclnodes = tmfnodes.setdefault(subdir, {})
887 tmfclnode = tmfclnodes.setdefault(n, clnode)
888 tmfclnode = tmfclnodes.setdefault(n, clnode)
888 if clrevorder[clnode] < clrevorder[tmfclnode]:
889 if clrevorder[clnode] < clrevorder[tmfclnode]:
889 tmfclnodes[n] = clnode
890 tmfclnodes[n] = clnode
890 else:
891 else:
891 f = dir + p
892 f = dir + p
892 fclnodes = fnodes.setdefault(f, {})
893 fclnodes = fnodes.setdefault(f, {})
893 fclnode = fclnodes.setdefault(n, clnode)
894 fclnode = fclnodes.setdefault(n, clnode)
894 if clrevorder[clnode] < clrevorder[fclnode]:
895 if clrevorder[clnode] < clrevorder[fclnode]:
895 fclnodes[n] = clnode
896 fclnodes[n] = clnode
896 return clnode
897 return clnode
897 return lookupmflinknode
898 return lookupmflinknode
898
899
899 fn = (self._packtreemanifests if self._sendtreemanifests
900 fn = (self._packtreemanifests if self._sendtreemanifests
900 else self._packmanifests)
901 else self._packmanifests)
901 size = 0
902 size = 0
902 while tmfnodes:
903 while tmfnodes:
903 dir, nodes = tmfnodes.popitem()
904 dir, nodes = tmfnodes.popitem()
904 prunednodes = self._prune(dirlog(dir), nodes, commonrevs)
905 prunednodes = self._prune(dirlog(dir), nodes, commonrevs)
905 if not dir or prunednodes:
906 if not dir or prunednodes:
906 for x in fn(dir, prunednodes, makelookupmflinknode(dir, nodes)):
907 for x in fn(dir, prunednodes, makelookupmflinknode(dir, nodes)):
907 size += len(x)
908 size += len(x)
908 yield x
909 yield x
909 self._verbosenote(_('%8.i (manifests)\n') % size)
910 self._verbosenote(_('%8.i (manifests)\n') % size)
910 yield self._manifestsend
911 yield self._manifestsend
911
912
912 # The 'source' parameter is useful for extensions
913 # The 'source' parameter is useful for extensions
913 def generatefiles(self, changedfiles, linknodes, commonrevs, source):
914 def generatefiles(self, changedfiles, linknodes, commonrevs, source):
914 changedfiles = list(filter(self._filematcher, changedfiles))
915 changedfiles = list(filter(self._filematcher, changedfiles))
915
916
916 if self._isshallow:
917 if self._isshallow:
917 # See comment in generate() for why this sadness is a thing.
918 # See comment in generate() for why this sadness is a thing.
918 mfdicts = self._mfdicts
919 mfdicts = self._mfdicts
919 del self._mfdicts
920 del self._mfdicts
920 # In a shallow clone, the linknodes callback needs to also include
921 # In a shallow clone, the linknodes callback needs to also include
921 # those file nodes that are in the manifests we sent but weren't
922 # those file nodes that are in the manifests we sent but weren't
922 # introduced by those manifests.
923 # introduced by those manifests.
923 commonctxs = [self._repo[c] for c in commonrevs]
924 commonctxs = [self._repo[c] for c in commonrevs]
924 oldlinknodes = linknodes
925 oldlinknodes = linknodes
925 clrev = self._repo.changelog.rev
926 clrev = self._repo.changelog.rev
926
927
927 # Defining this function has a side-effect of overriding the
928 # Defining this function has a side-effect of overriding the
928 # function of the same name that was passed in as an argument.
929 # function of the same name that was passed in as an argument.
929 # TODO have caller pass in appropriate function.
930 # TODO have caller pass in appropriate function.
930 def linknodes(flog, fname):
931 def linknodes(flog, fname):
931 for c in commonctxs:
932 for c in commonctxs:
932 try:
933 try:
933 fnode = c.filenode(fname)
934 fnode = c.filenode(fname)
934 self._clrevtolocalrev[c.rev()] = flog.rev(fnode)
935 self._clrevtolocalrev[c.rev()] = flog.rev(fnode)
935 except error.ManifestLookupError:
936 except error.ManifestLookupError:
936 pass
937 pass
937 links = oldlinknodes(flog, fname)
938 links = oldlinknodes(flog, fname)
938 if len(links) != len(mfdicts):
939 if len(links) != len(mfdicts):
939 for mf, lr in mfdicts:
940 for mf, lr in mfdicts:
940 fnode = mf.get(fname, None)
941 fnode = mf.get(fname, None)
941 if fnode in links:
942 if fnode in links:
942 links[fnode] = min(links[fnode], lr, key=clrev)
943 links[fnode] = min(links[fnode], lr, key=clrev)
943 elif fnode:
944 elif fnode:
944 links[fnode] = lr
945 links[fnode] = lr
945 return links
946 return links
946
947
947 return self._generatefiles(changedfiles, linknodes, commonrevs, source)
948 return self._generatefiles(changedfiles, linknodes, commonrevs, source)
948
949
949 def _generatefiles(self, changedfiles, linknodes, commonrevs, source):
950 def _generatefiles(self, changedfiles, linknodes, commonrevs, source):
950 repo = self._repo
951 repo = self._repo
951 progress = repo.ui.makeprogress(_('bundling'), unit=_('files'),
952 progress = repo.ui.makeprogress(_('bundling'), unit=_('files'),
952 total=len(changedfiles))
953 total=len(changedfiles))
953 for i, fname in enumerate(sorted(changedfiles)):
954 for i, fname in enumerate(sorted(changedfiles)):
954 filerevlog = repo.file(fname)
955 filerevlog = repo.file(fname)
955 if not filerevlog:
956 if not filerevlog:
956 raise error.Abort(_("empty or missing file data for %s") %
957 raise error.Abort(_("empty or missing file data for %s") %
957 fname)
958 fname)
958
959
959 linkrevnodes = linknodes(filerevlog, fname)
960 linkrevnodes = linknodes(filerevlog, fname)
960 # Lookup for filenodes, we collected the linkrev nodes above in the
961 # Lookup for filenodes, we collected the linkrev nodes above in the
961 # fastpath case and with lookupmf in the slowpath case.
962 # fastpath case and with lookupmf in the slowpath case.
962 def lookupfilelog(x):
963 def lookupfilelog(x):
963 return linkrevnodes[x]
964 return linkrevnodes[x]
964
965
965 filenodes = self._prune(filerevlog, linkrevnodes, commonrevs)
966 filenodes = self._prune(filerevlog, linkrevnodes, commonrevs)
966 if filenodes:
967 if filenodes:
967 progress.update(i + 1, item=fname)
968 progress.update(i + 1, item=fname)
968 h = self._fileheader(fname)
969 h = self._fileheader(fname)
969 size = len(h)
970 size = len(h)
970 yield h
971 yield h
971 for chunk in self.group(filenodes, filerevlog, lookupfilelog):
972 for chunk in self.group(filenodes, filerevlog, lookupfilelog):
972 size += len(chunk)
973 size += len(chunk)
973 yield chunk
974 yield chunk
974 self._verbosenote(_('%8.i %s\n') % (size, fname))
975 self._verbosenote(_('%8.i %s\n') % (size, fname))
975 progress.complete()
976 progress.complete()
976
977
977 def _deltaparent(self, store, rev, p1, p2, prev):
978 def _deltaparent(self, store, rev, p1, p2, prev):
978 if self._useprevdelta:
979 if self._useprevdelta:
979 if not store.candelta(prev, rev):
980 if not store.candelta(prev, rev):
980 raise error.ProgrammingError(
981 raise error.ProgrammingError(
981 'cg1 should not be used in this case')
982 'cg1 should not be used in this case')
982 return prev
983 return prev
983
984
984 # Narrow ellipses mode.
985 # Narrow ellipses mode.
985 if util.safehasattr(self, '_full_nodes'):
986 if self._ellipses:
986 # TODO: send better deltas when in narrow mode.
987 # TODO: send better deltas when in narrow mode.
987 #
988 #
988 # changegroup.group() loops over revisions to send,
989 # changegroup.group() loops over revisions to send,
989 # including revisions we'll skip. What this means is that
990 # including revisions we'll skip. What this means is that
990 # `prev` will be a potentially useless delta base for all
991 # `prev` will be a potentially useless delta base for all
991 # ellipsis nodes, as the client likely won't have it. In
992 # ellipsis nodes, as the client likely won't have it. In
992 # the future we should do bookkeeping about which nodes
993 # the future we should do bookkeeping about which nodes
993 # have been sent to the client, and try to be
994 # have been sent to the client, and try to be
994 # significantly smarter about delta bases. This is
995 # significantly smarter about delta bases. This is
995 # slightly tricky because this same code has to work for
996 # slightly tricky because this same code has to work for
996 # all revlogs, and we don't have the linkrev/linknode here.
997 # all revlogs, and we don't have the linkrev/linknode here.
997 return p1
998 return p1
998
999
999 dp = store.deltaparent(rev)
1000 dp = store.deltaparent(rev)
1000 if dp == nullrev and store.storedeltachains:
1001 if dp == nullrev and store.storedeltachains:
1001 # Avoid sending full revisions when delta parent is null. Pick prev
1002 # Avoid sending full revisions when delta parent is null. Pick prev
1002 # in that case. It's tempting to pick p1 in this case, as p1 will
1003 # in that case. It's tempting to pick p1 in this case, as p1 will
1003 # be smaller in the common case. However, computing a delta against
1004 # be smaller in the common case. However, computing a delta against
1004 # p1 may require resolving the raw text of p1, which could be
1005 # p1 may require resolving the raw text of p1, which could be
1005 # expensive. The revlog caches should have prev cached, meaning
1006 # expensive. The revlog caches should have prev cached, meaning
1006 # less CPU for changegroup generation. There is likely room to add
1007 # less CPU for changegroup generation. There is likely room to add
1007 # a flag and/or config option to control this behavior.
1008 # a flag and/or config option to control this behavior.
1008 base = prev
1009 base = prev
1009 elif dp == nullrev:
1010 elif dp == nullrev:
1010 # revlog is configured to use full snapshot for a reason,
1011 # revlog is configured to use full snapshot for a reason,
1011 # stick to full snapshot.
1012 # stick to full snapshot.
1012 base = nullrev
1013 base = nullrev
1013 elif dp not in (p1, p2, prev):
1014 elif dp not in (p1, p2, prev):
1014 # Pick prev when we can't be sure remote has the base revision.
1015 # Pick prev when we can't be sure remote has the base revision.
1015 return prev
1016 return prev
1016 else:
1017 else:
1017 base = dp
1018 base = dp
1018
1019
1019 if base != nullrev and not store.candelta(base, rev):
1020 if base != nullrev and not store.candelta(base, rev):
1020 base = nullrev
1021 base = nullrev
1021
1022
1022 return base
1023 return base
1023
1024
1024 def _revchunk(self, store, rev, prev, linknode):
1025 def _revchunk(self, store, rev, prev, linknode):
1025 if util.safehasattr(self, '_full_nodes'):
1026 if self._ellipses:
1026 fn = self._revisiondeltanarrow
1027 fn = self._revisiondeltanarrow
1027 else:
1028 else:
1028 fn = self._revisiondeltanormal
1029 fn = self._revisiondeltanormal
1029
1030
1030 delta = fn(store, rev, prev, linknode)
1031 delta = fn(store, rev, prev, linknode)
1031 if not delta:
1032 if not delta:
1032 return
1033 return
1033
1034
1034 meta = self._builddeltaheader(delta)
1035 meta = self._builddeltaheader(delta)
1035 l = len(meta) + sum(len(x) for x in delta.deltachunks)
1036 l = len(meta) + sum(len(x) for x in delta.deltachunks)
1036
1037
1037 yield chunkheader(l)
1038 yield chunkheader(l)
1038 yield meta
1039 yield meta
1039 for x in delta.deltachunks:
1040 for x in delta.deltachunks:
1040 yield x
1041 yield x
1041
1042
1042 def _revisiondeltanormal(self, store, rev, prev, linknode):
1043 def _revisiondeltanormal(self, store, rev, prev, linknode):
1043 node = store.node(rev)
1044 node = store.node(rev)
1044 p1, p2 = store.parentrevs(rev)
1045 p1, p2 = store.parentrevs(rev)
1045 base = self._deltaparent(store, rev, p1, p2, prev)
1046 base = self._deltaparent(store, rev, p1, p2, prev)
1046
1047
1047 prefix = ''
1048 prefix = ''
1048 if store.iscensored(base) or store.iscensored(rev):
1049 if store.iscensored(base) or store.iscensored(rev):
1049 try:
1050 try:
1050 delta = store.revision(node, raw=True)
1051 delta = store.revision(node, raw=True)
1051 except error.CensoredNodeError as e:
1052 except error.CensoredNodeError as e:
1052 delta = e.tombstone
1053 delta = e.tombstone
1053 if base == nullrev:
1054 if base == nullrev:
1054 prefix = mdiff.trivialdiffheader(len(delta))
1055 prefix = mdiff.trivialdiffheader(len(delta))
1055 else:
1056 else:
1056 baselen = store.rawsize(base)
1057 baselen = store.rawsize(base)
1057 prefix = mdiff.replacediffheader(baselen, len(delta))
1058 prefix = mdiff.replacediffheader(baselen, len(delta))
1058 elif base == nullrev:
1059 elif base == nullrev:
1059 delta = store.revision(node, raw=True)
1060 delta = store.revision(node, raw=True)
1060 prefix = mdiff.trivialdiffheader(len(delta))
1061 prefix = mdiff.trivialdiffheader(len(delta))
1061 else:
1062 else:
1062 delta = store.revdiff(base, rev)
1063 delta = store.revdiff(base, rev)
1063 p1n, p2n = store.parents(node)
1064 p1n, p2n = store.parents(node)
1064
1065
1065 return revisiondelta(
1066 return revisiondelta(
1066 node=node,
1067 node=node,
1067 p1node=p1n,
1068 p1node=p1n,
1068 p2node=p2n,
1069 p2node=p2n,
1069 basenode=store.node(base),
1070 basenode=store.node(base),
1070 linknode=linknode,
1071 linknode=linknode,
1071 flags=store.flags(rev),
1072 flags=store.flags(rev),
1072 deltachunks=(prefix, delta),
1073 deltachunks=(prefix, delta),
1073 )
1074 )
1074
1075
1075 def _revisiondeltanarrow(self, store, rev, prev, linknode):
1076 def _revisiondeltanarrow(self, store, rev, prev, linknode):
1076 # build up some mapping information that's useful later. See
1077 # build up some mapping information that's useful later. See
1077 # the local() nested function below.
1078 # the local() nested function below.
1078 if not self._changelogdone:
1079 if not self._changelogdone:
1079 self._clnodetorev[linknode] = rev
1080 self._clnodetorev[linknode] = rev
1080 linkrev = rev
1081 linkrev = rev
1081 self._clrevtolocalrev[linkrev] = rev
1082 self._clrevtolocalrev[linkrev] = rev
1082 else:
1083 else:
1083 linkrev = self._clnodetorev[linknode]
1084 linkrev = self._clnodetorev[linknode]
1084 self._clrevtolocalrev[linkrev] = rev
1085 self._clrevtolocalrev[linkrev] = rev
1085
1086
1086 # This is a node to send in full, because the changeset it
1087 # This is a node to send in full, because the changeset it
1087 # corresponds to was a full changeset.
1088 # corresponds to was a full changeset.
1088 if linknode in self._full_nodes:
1089 if linknode in self._full_nodes:
1089 return self._revisiondeltanormal(store, rev, prev, linknode)
1090 return self._revisiondeltanormal(store, rev, prev, linknode)
1090
1091
1091 # At this point, a node can either be one we should skip or an
1092 # At this point, a node can either be one we should skip or an
1092 # ellipsis. If it's not an ellipsis, bail immediately.
1093 # ellipsis. If it's not an ellipsis, bail immediately.
1093 if linkrev not in self._precomputedellipsis:
1094 if linkrev not in self._precomputedellipsis:
1094 return
1095 return
1095
1096
1096 linkparents = self._precomputedellipsis[linkrev]
1097 linkparents = self._precomputedellipsis[linkrev]
1097 def local(clrev):
1098 def local(clrev):
1098 """Turn a changelog revnum into a local revnum.
1099 """Turn a changelog revnum into a local revnum.
1099
1100
1100 The ellipsis dag is stored as revnums on the changelog,
1101 The ellipsis dag is stored as revnums on the changelog,
1101 but when we're producing ellipsis entries for
1102 but when we're producing ellipsis entries for
1102 non-changelog revlogs, we need to turn those numbers into
1103 non-changelog revlogs, we need to turn those numbers into
1103 something local. This does that for us, and during the
1104 something local. This does that for us, and during the
1104 changelog sending phase will also expand the stored
1105 changelog sending phase will also expand the stored
1105 mappings as needed.
1106 mappings as needed.
1106 """
1107 """
1107 if clrev == nullrev:
1108 if clrev == nullrev:
1108 return nullrev
1109 return nullrev
1109
1110
1110 if not self._changelogdone:
1111 if not self._changelogdone:
1111 # If we're doing the changelog, it's possible that we
1112 # If we're doing the changelog, it's possible that we
1112 # have a parent that is already on the client, and we
1113 # have a parent that is already on the client, and we
1113 # need to store some extra mapping information so that
1114 # need to store some extra mapping information so that
1114 # our contained ellipsis nodes will be able to resolve
1115 # our contained ellipsis nodes will be able to resolve
1115 # their parents.
1116 # their parents.
1116 if clrev not in self._clrevtolocalrev:
1117 if clrev not in self._clrevtolocalrev:
1117 clnode = store.node(clrev)
1118 clnode = store.node(clrev)
1118 self._clnodetorev[clnode] = clrev
1119 self._clnodetorev[clnode] = clrev
1119 return clrev
1120 return clrev
1120
1121
1121 # Walk the ellipsis-ized changelog breadth-first looking for a
1122 # Walk the ellipsis-ized changelog breadth-first looking for a
1122 # change that has been linked from the current revlog.
1123 # change that has been linked from the current revlog.
1123 #
1124 #
1124 # For a flat manifest revlog only a single step should be necessary
1125 # For a flat manifest revlog only a single step should be necessary
1125 # as all relevant changelog entries are relevant to the flat
1126 # as all relevant changelog entries are relevant to the flat
1126 # manifest.
1127 # manifest.
1127 #
1128 #
1128 # For a filelog or tree manifest dirlog however not every changelog
1129 # For a filelog or tree manifest dirlog however not every changelog
1129 # entry will have been relevant, so we need to skip some changelog
1130 # entry will have been relevant, so we need to skip some changelog
1130 # nodes even after ellipsis-izing.
1131 # nodes even after ellipsis-izing.
1131 walk = [clrev]
1132 walk = [clrev]
1132 while walk:
1133 while walk:
1133 p = walk[0]
1134 p = walk[0]
1134 walk = walk[1:]
1135 walk = walk[1:]
1135 if p in self._clrevtolocalrev:
1136 if p in self._clrevtolocalrev:
1136 return self._clrevtolocalrev[p]
1137 return self._clrevtolocalrev[p]
1137 elif p in self._full_nodes:
1138 elif p in self._full_nodes:
1138 walk.extend([pp for pp in self._repo.changelog.parentrevs(p)
1139 walk.extend([pp for pp in self._repo.changelog.parentrevs(p)
1139 if pp != nullrev])
1140 if pp != nullrev])
1140 elif p in self._precomputedellipsis:
1141 elif p in self._precomputedellipsis:
1141 walk.extend([pp for pp in self._precomputedellipsis[p]
1142 walk.extend([pp for pp in self._precomputedellipsis[p]
1142 if pp != nullrev])
1143 if pp != nullrev])
1143 else:
1144 else:
1144 # In this case, we've got an ellipsis with parents
1145 # In this case, we've got an ellipsis with parents
1145 # outside the current bundle (likely an
1146 # outside the current bundle (likely an
1146 # incremental pull). We "know" that we can use the
1147 # incremental pull). We "know" that we can use the
1147 # value of this same revlog at whatever revision
1148 # value of this same revlog at whatever revision
1148 # is pointed to by linknode. "Know" is in scare
1149 # is pointed to by linknode. "Know" is in scare
1149 # quotes because I haven't done enough examination
1150 # quotes because I haven't done enough examination
1150 # of edge cases to convince myself this is really
1151 # of edge cases to convince myself this is really
1151 # a fact - it works for all the (admittedly
1152 # a fact - it works for all the (admittedly
1152 # thorough) cases in our testsuite, but I would be
1153 # thorough) cases in our testsuite, but I would be
1153 # somewhat unsurprised to find a case in the wild
1154 # somewhat unsurprised to find a case in the wild
1154 # where this breaks down a bit. That said, I don't
1155 # where this breaks down a bit. That said, I don't
1155 # know if it would hurt anything.
1156 # know if it would hurt anything.
1156 for i in pycompat.xrange(rev, 0, -1):
1157 for i in pycompat.xrange(rev, 0, -1):
1157 if store.linkrev(i) == clrev:
1158 if store.linkrev(i) == clrev:
1158 return i
1159 return i
1159 # We failed to resolve a parent for this node, so
1160 # We failed to resolve a parent for this node, so
1160 # we crash the changegroup construction.
1161 # we crash the changegroup construction.
1161 raise error.Abort(
1162 raise error.Abort(
1162 'unable to resolve parent while packing %r %r'
1163 'unable to resolve parent while packing %r %r'
1163 ' for changeset %r' % (store.indexfile, rev, clrev))
1164 ' for changeset %r' % (store.indexfile, rev, clrev))
1164
1165
1165 return nullrev
1166 return nullrev
1166
1167
1167 if not linkparents or (
1168 if not linkparents or (
1168 store.parentrevs(rev) == (nullrev, nullrev)):
1169 store.parentrevs(rev) == (nullrev, nullrev)):
1169 p1, p2 = nullrev, nullrev
1170 p1, p2 = nullrev, nullrev
1170 elif len(linkparents) == 1:
1171 elif len(linkparents) == 1:
1171 p1, = sorted(local(p) for p in linkparents)
1172 p1, = sorted(local(p) for p in linkparents)
1172 p2 = nullrev
1173 p2 = nullrev
1173 else:
1174 else:
1174 p1, p2 = sorted(local(p) for p in linkparents)
1175 p1, p2 = sorted(local(p) for p in linkparents)
1175
1176
1176 n = store.node(rev)
1177 n = store.node(rev)
1177 p1n, p2n = store.node(p1), store.node(p2)
1178 p1n, p2n = store.node(p1), store.node(p2)
1178 flags = store.flags(rev)
1179 flags = store.flags(rev)
1179 flags |= revlog.REVIDX_ELLIPSIS
1180 flags |= revlog.REVIDX_ELLIPSIS
1180
1181
1181 # TODO: try and actually send deltas for ellipsis data blocks
1182 # TODO: try and actually send deltas for ellipsis data blocks
1182 data = store.revision(n)
1183 data = store.revision(n)
1183 diffheader = mdiff.trivialdiffheader(len(data))
1184 diffheader = mdiff.trivialdiffheader(len(data))
1184
1185
1185 return revisiondelta(
1186 return revisiondelta(
1186 node=n,
1187 node=n,
1187 p1node=p1n,
1188 p1node=p1n,
1188 p2node=p2n,
1189 p2node=p2n,
1189 basenode=nullid,
1190 basenode=nullid,
1190 linknode=linknode,
1191 linknode=linknode,
1191 flags=flags,
1192 flags=flags,
1192 deltachunks=(diffheader, data),
1193 deltachunks=(diffheader, data),
1193 )
1194 )
1194
1195
1195 def _makecg1packer(repo, filematcher, bundlecaps, shallow=False,
1196 def _makecg1packer(repo, filematcher, bundlecaps, ellipses=False,
1196 ellipsisroots=None):
1197 shallow=False, ellipsisroots=None):
1197 builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack(
1198 builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack(
1198 d.node, d.p1node, d.p2node, d.linknode)
1199 d.node, d.p1node, d.p2node, d.linknode)
1199
1200
1200 return cgpacker(repo, filematcher, b'01',
1201 return cgpacker(repo, filematcher, b'01',
1201 useprevdelta=True,
1202 useprevdelta=True,
1202 allowreorder=None,
1203 allowreorder=None,
1203 builddeltaheader=builddeltaheader,
1204 builddeltaheader=builddeltaheader,
1204 manifestsend=b'',
1205 manifestsend=b'',
1205 sendtreemanifests=False,
1206 sendtreemanifests=False,
1206 bundlecaps=bundlecaps,
1207 bundlecaps=bundlecaps,
1208 ellipses=ellipses,
1207 shallow=shallow,
1209 shallow=shallow,
1208 ellipsisroots=ellipsisroots)
1210 ellipsisroots=ellipsisroots)
1209
1211
1210 def _makecg2packer(repo, filematcher, bundlecaps, shallow=False,
1212 def _makecg2packer(repo, filematcher, bundlecaps, ellipses=False,
1211 ellipsisroots=None):
1213 shallow=False, ellipsisroots=None):
1212 builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack(
1214 builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack(
1213 d.node, d.p1node, d.p2node, d.basenode, d.linknode)
1215 d.node, d.p1node, d.p2node, d.basenode, d.linknode)
1214
1216
1215 # Since generaldelta is directly supported by cg2, reordering
1217 # Since generaldelta is directly supported by cg2, reordering
1216 # generally doesn't help, so we disable it by default (treating
1218 # generally doesn't help, so we disable it by default (treating
1217 # bundle.reorder=auto just like bundle.reorder=False).
1219 # bundle.reorder=auto just like bundle.reorder=False).
1218 return cgpacker(repo, filematcher, b'02',
1220 return cgpacker(repo, filematcher, b'02',
1219 useprevdelta=False,
1221 useprevdelta=False,
1220 allowreorder=False,
1222 allowreorder=False,
1221 builddeltaheader=builddeltaheader,
1223 builddeltaheader=builddeltaheader,
1222 manifestsend=b'',
1224 manifestsend=b'',
1223 sendtreemanifests=False,
1225 sendtreemanifests=False,
1224 bundlecaps=bundlecaps,
1226 bundlecaps=bundlecaps,
1227 ellipses=ellipses,
1225 shallow=shallow,
1228 shallow=shallow,
1226 ellipsisroots=ellipsisroots)
1229 ellipsisroots=ellipsisroots)
1227
1230
1228 def _makecg3packer(repo, filematcher, bundlecaps, shallow=False,
1231 def _makecg3packer(repo, filematcher, bundlecaps, ellipses=False,
1229 ellipsisroots=None):
1232 shallow=False, ellipsisroots=None):
1230 builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack(
1233 builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack(
1231 d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags)
1234 d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags)
1232
1235
1233 return cgpacker(repo, filematcher, b'03',
1236 return cgpacker(repo, filematcher, b'03',
1234 useprevdelta=False,
1237 useprevdelta=False,
1235 allowreorder=False,
1238 allowreorder=False,
1236 builddeltaheader=builddeltaheader,
1239 builddeltaheader=builddeltaheader,
1237 manifestsend=closechunk(),
1240 manifestsend=closechunk(),
1238 sendtreemanifests=True,
1241 sendtreemanifests=True,
1239 bundlecaps=bundlecaps,
1242 bundlecaps=bundlecaps,
1243 ellipses=ellipses,
1240 shallow=shallow,
1244 shallow=shallow,
1241 ellipsisroots=ellipsisroots)
1245 ellipsisroots=ellipsisroots)
1242
1246
1243 _packermap = {'01': (_makecg1packer, cg1unpacker),
1247 _packermap = {'01': (_makecg1packer, cg1unpacker),
1244 # cg2 adds support for exchanging generaldelta
1248 # cg2 adds support for exchanging generaldelta
1245 '02': (_makecg2packer, cg2unpacker),
1249 '02': (_makecg2packer, cg2unpacker),
1246 # cg3 adds support for exchanging revlog flags and treemanifests
1250 # cg3 adds support for exchanging revlog flags and treemanifests
1247 '03': (_makecg3packer, cg3unpacker),
1251 '03': (_makecg3packer, cg3unpacker),
1248 }
1252 }
1249
1253
1250 def allsupportedversions(repo):
1254 def allsupportedversions(repo):
1251 versions = set(_packermap.keys())
1255 versions = set(_packermap.keys())
1252 if not (repo.ui.configbool('experimental', 'changegroup3') or
1256 if not (repo.ui.configbool('experimental', 'changegroup3') or
1253 repo.ui.configbool('experimental', 'treemanifest') or
1257 repo.ui.configbool('experimental', 'treemanifest') or
1254 'treemanifest' in repo.requirements):
1258 'treemanifest' in repo.requirements):
1255 versions.discard('03')
1259 versions.discard('03')
1256 return versions
1260 return versions
1257
1261
1258 # Changegroup versions that can be applied to the repo
1262 # Changegroup versions that can be applied to the repo
1259 def supportedincomingversions(repo):
1263 def supportedincomingversions(repo):
1260 return allsupportedversions(repo)
1264 return allsupportedversions(repo)
1261
1265
1262 # Changegroup versions that can be created from the repo
1266 # Changegroup versions that can be created from the repo
1263 def supportedoutgoingversions(repo):
1267 def supportedoutgoingversions(repo):
1264 versions = allsupportedversions(repo)
1268 versions = allsupportedversions(repo)
1265 if 'treemanifest' in repo.requirements:
1269 if 'treemanifest' in repo.requirements:
1266 # Versions 01 and 02 support only flat manifests and it's just too
1270 # Versions 01 and 02 support only flat manifests and it's just too
1267 # expensive to convert between the flat manifest and tree manifest on
1271 # expensive to convert between the flat manifest and tree manifest on
1268 # the fly. Since tree manifests are hashed differently, all of history
1272 # the fly. Since tree manifests are hashed differently, all of history
1269 # would have to be converted. Instead, we simply don't even pretend to
1273 # would have to be converted. Instead, we simply don't even pretend to
1270 # support versions 01 and 02.
1274 # support versions 01 and 02.
1271 versions.discard('01')
1275 versions.discard('01')
1272 versions.discard('02')
1276 versions.discard('02')
1273 if repository.NARROW_REQUIREMENT in repo.requirements:
1277 if repository.NARROW_REQUIREMENT in repo.requirements:
1274 # Versions 01 and 02 don't support revlog flags, and we need to
1278 # Versions 01 and 02 don't support revlog flags, and we need to
1275 # support that for stripping and unbundling to work.
1279 # support that for stripping and unbundling to work.
1276 versions.discard('01')
1280 versions.discard('01')
1277 versions.discard('02')
1281 versions.discard('02')
1278 if LFS_REQUIREMENT in repo.requirements:
1282 if LFS_REQUIREMENT in repo.requirements:
1279 # Versions 01 and 02 don't support revlog flags, and we need to
1283 # Versions 01 and 02 don't support revlog flags, and we need to
1280 # mark LFS entries with REVIDX_EXTSTORED.
1284 # mark LFS entries with REVIDX_EXTSTORED.
1281 versions.discard('01')
1285 versions.discard('01')
1282 versions.discard('02')
1286 versions.discard('02')
1283
1287
1284 return versions
1288 return versions
1285
1289
1286 def localversion(repo):
1290 def localversion(repo):
1287 # Finds the best version to use for bundles that are meant to be used
1291 # Finds the best version to use for bundles that are meant to be used
1288 # locally, such as those from strip and shelve, and temporary bundles.
1292 # locally, such as those from strip and shelve, and temporary bundles.
1289 return max(supportedoutgoingversions(repo))
1293 return max(supportedoutgoingversions(repo))
1290
1294
1291 def safeversion(repo):
1295 def safeversion(repo):
1292 # Finds the smallest version that it's safe to assume clients of the repo
1296 # Finds the smallest version that it's safe to assume clients of the repo
1293 # will support. For example, all hg versions that support generaldelta also
1297 # will support. For example, all hg versions that support generaldelta also
1294 # support changegroup 02.
1298 # support changegroup 02.
1295 versions = supportedoutgoingversions(repo)
1299 versions = supportedoutgoingversions(repo)
1296 if 'generaldelta' in repo.requirements:
1300 if 'generaldelta' in repo.requirements:
1297 versions.discard('01')
1301 versions.discard('01')
1298 assert versions
1302 assert versions
1299 return min(versions)
1303 return min(versions)
1300
1304
1301 def getbundler(version, repo, bundlecaps=None, filematcher=None,
1305 def getbundler(version, repo, bundlecaps=None, filematcher=None,
1302 shallow=False, ellipsisroots=None):
1306 ellipses=False, shallow=False, ellipsisroots=None):
1303 assert version in supportedoutgoingversions(repo)
1307 assert version in supportedoutgoingversions(repo)
1304
1308
1305 if filematcher is None:
1309 if filematcher is None:
1306 filematcher = matchmod.alwaysmatcher(repo.root, '')
1310 filematcher = matchmod.alwaysmatcher(repo.root, '')
1307
1311
1308 if version == '01' and not filematcher.always():
1312 if version == '01' and not filematcher.always():
1309 raise error.ProgrammingError('version 01 changegroups do not support '
1313 raise error.ProgrammingError('version 01 changegroups do not support '
1310 'sparse file matchers')
1314 'sparse file matchers')
1311
1315
1316 if ellipses and version in (b'01', b'02'):
1317 raise error.Abort(
1318 _('ellipsis nodes require at least cg3 on client and server, '
1319 'but negotiated version %s') % version)
1320
1312 # Requested files could include files not in the local store. So
1321 # Requested files could include files not in the local store. So
1313 # filter those out.
1322 # filter those out.
1314 filematcher = matchmod.intersectmatchers(repo.narrowmatch(),
1323 filematcher = matchmod.intersectmatchers(repo.narrowmatch(),
1315 filematcher)
1324 filematcher)
1316
1325
1317 fn = _packermap[version][0]
1326 fn = _packermap[version][0]
1318 return fn(repo, filematcher, bundlecaps, shallow=shallow,
1327 return fn(repo, filematcher, bundlecaps, ellipses=ellipses,
1319 ellipsisroots=ellipsisroots)
1328 shallow=shallow, ellipsisroots=ellipsisroots)
1320
1329
1321 def getunbundler(version, fh, alg, extras=None):
1330 def getunbundler(version, fh, alg, extras=None):
1322 return _packermap[version][1](fh, alg, extras=extras)
1331 return _packermap[version][1](fh, alg, extras=extras)
1323
1332
1324 def _changegroupinfo(repo, nodes, source):
1333 def _changegroupinfo(repo, nodes, source):
1325 if repo.ui.verbose or source == 'bundle':
1334 if repo.ui.verbose or source == 'bundle':
1326 repo.ui.status(_("%d changesets found\n") % len(nodes))
1335 repo.ui.status(_("%d changesets found\n") % len(nodes))
1327 if repo.ui.debugflag:
1336 if repo.ui.debugflag:
1328 repo.ui.debug("list of changesets:\n")
1337 repo.ui.debug("list of changesets:\n")
1329 for node in nodes:
1338 for node in nodes:
1330 repo.ui.debug("%s\n" % hex(node))
1339 repo.ui.debug("%s\n" % hex(node))
1331
1340
1332 def makechangegroup(repo, outgoing, version, source, fastpath=False,
1341 def makechangegroup(repo, outgoing, version, source, fastpath=False,
1333 bundlecaps=None):
1342 bundlecaps=None):
1334 cgstream = makestream(repo, outgoing, version, source,
1343 cgstream = makestream(repo, outgoing, version, source,
1335 fastpath=fastpath, bundlecaps=bundlecaps)
1344 fastpath=fastpath, bundlecaps=bundlecaps)
1336 return getunbundler(version, util.chunkbuffer(cgstream), None,
1345 return getunbundler(version, util.chunkbuffer(cgstream), None,
1337 {'clcount': len(outgoing.missing) })
1346 {'clcount': len(outgoing.missing) })
1338
1347
1339 def makestream(repo, outgoing, version, source, fastpath=False,
1348 def makestream(repo, outgoing, version, source, fastpath=False,
1340 bundlecaps=None, filematcher=None):
1349 bundlecaps=None, filematcher=None):
1341 bundler = getbundler(version, repo, bundlecaps=bundlecaps,
1350 bundler = getbundler(version, repo, bundlecaps=bundlecaps,
1342 filematcher=filematcher)
1351 filematcher=filematcher)
1343
1352
1344 repo = repo.unfiltered()
1353 repo = repo.unfiltered()
1345 commonrevs = outgoing.common
1354 commonrevs = outgoing.common
1346 csets = outgoing.missing
1355 csets = outgoing.missing
1347 heads = outgoing.missingheads
1356 heads = outgoing.missingheads
1348 # We go through the fast path if we get told to, or if all (unfiltered
1357 # We go through the fast path if we get told to, or if all (unfiltered
1349 # heads have been requested (since we then know there all linkrevs will
1358 # heads have been requested (since we then know there all linkrevs will
1350 # be pulled by the client).
1359 # be pulled by the client).
1351 heads.sort()
1360 heads.sort()
1352 fastpathlinkrev = fastpath or (
1361 fastpathlinkrev = fastpath or (
1353 repo.filtername is None and heads == sorted(repo.heads()))
1362 repo.filtername is None and heads == sorted(repo.heads()))
1354
1363
1355 repo.hook('preoutgoing', throw=True, source=source)
1364 repo.hook('preoutgoing', throw=True, source=source)
1356 _changegroupinfo(repo, csets, source)
1365 _changegroupinfo(repo, csets, source)
1357 return bundler.generate(commonrevs, csets, fastpathlinkrev, source)
1366 return bundler.generate(commonrevs, csets, fastpathlinkrev, source)
1358
1367
1359 def _addchangegroupfiles(repo, source, revmap, trp, expectedfiles, needfiles):
1368 def _addchangegroupfiles(repo, source, revmap, trp, expectedfiles, needfiles):
1360 revisions = 0
1369 revisions = 0
1361 files = 0
1370 files = 0
1362 progress = repo.ui.makeprogress(_('files'), unit=_('files'),
1371 progress = repo.ui.makeprogress(_('files'), unit=_('files'),
1363 total=expectedfiles)
1372 total=expectedfiles)
1364 for chunkdata in iter(source.filelogheader, {}):
1373 for chunkdata in iter(source.filelogheader, {}):
1365 files += 1
1374 files += 1
1366 f = chunkdata["filename"]
1375 f = chunkdata["filename"]
1367 repo.ui.debug("adding %s revisions\n" % f)
1376 repo.ui.debug("adding %s revisions\n" % f)
1368 progress.increment()
1377 progress.increment()
1369 fl = repo.file(f)
1378 fl = repo.file(f)
1370 o = len(fl)
1379 o = len(fl)
1371 try:
1380 try:
1372 deltas = source.deltaiter()
1381 deltas = source.deltaiter()
1373 if not fl.addgroup(deltas, revmap, trp):
1382 if not fl.addgroup(deltas, revmap, trp):
1374 raise error.Abort(_("received file revlog group is empty"))
1383 raise error.Abort(_("received file revlog group is empty"))
1375 except error.CensoredBaseError as e:
1384 except error.CensoredBaseError as e:
1376 raise error.Abort(_("received delta base is censored: %s") % e)
1385 raise error.Abort(_("received delta base is censored: %s") % e)
1377 revisions += len(fl) - o
1386 revisions += len(fl) - o
1378 if f in needfiles:
1387 if f in needfiles:
1379 needs = needfiles[f]
1388 needs = needfiles[f]
1380 for new in pycompat.xrange(o, len(fl)):
1389 for new in pycompat.xrange(o, len(fl)):
1381 n = fl.node(new)
1390 n = fl.node(new)
1382 if n in needs:
1391 if n in needs:
1383 needs.remove(n)
1392 needs.remove(n)
1384 else:
1393 else:
1385 raise error.Abort(
1394 raise error.Abort(
1386 _("received spurious file revlog entry"))
1395 _("received spurious file revlog entry"))
1387 if not needs:
1396 if not needs:
1388 del needfiles[f]
1397 del needfiles[f]
1389 progress.complete()
1398 progress.complete()
1390
1399
1391 for f, needs in needfiles.iteritems():
1400 for f, needs in needfiles.iteritems():
1392 fl = repo.file(f)
1401 fl = repo.file(f)
1393 for n in needs:
1402 for n in needs:
1394 try:
1403 try:
1395 fl.rev(n)
1404 fl.rev(n)
1396 except error.LookupError:
1405 except error.LookupError:
1397 raise error.Abort(
1406 raise error.Abort(
1398 _('missing file data for %s:%s - run hg verify') %
1407 _('missing file data for %s:%s - run hg verify') %
1399 (f, hex(n)))
1408 (f, hex(n)))
1400
1409
1401 return revisions, files
1410 return revisions, files
1402
1411
1403 def _packellipsischangegroup(repo, common, match, relevant_nodes,
1412 def _packellipsischangegroup(repo, common, match, relevant_nodes,
1404 ellipsisroots, visitnodes, depth, source, version):
1413 ellipsisroots, visitnodes, depth, source, version):
1405 if version in ('01', '02'):
1406 raise error.Abort(
1407 'ellipsis nodes require at least cg3 on client and server, '
1408 'but negotiated version %s' % version)
1409 # We wrap cg1packer.revchunk, using a side channel to pass
1414 # We wrap cg1packer.revchunk, using a side channel to pass
1410 # relevant_nodes into that area. Then if linknode isn't in the
1415 # relevant_nodes into that area. Then if linknode isn't in the
1411 # set, we know we have an ellipsis node and we should defer
1416 # set, we know we have an ellipsis node and we should defer
1412 # sending that node's data. We override close() to detect
1417 # sending that node's data. We override close() to detect
1413 # pending ellipsis nodes and flush them.
1418 # pending ellipsis nodes and flush them.
1414 packer = getbundler(version, repo, filematcher=match,
1419 packer = getbundler(version, repo, filematcher=match,
1420 ellipses=True,
1415 shallow=depth is not None,
1421 shallow=depth is not None,
1416 ellipsisroots=ellipsisroots)
1422 ellipsisroots=ellipsisroots)
1417 # Give the packer the list of nodes which should not be
1423 # Give the packer the list of nodes which should not be
1418 # ellipsis nodes. We store this rather than the set of nodes
1424 # ellipsis nodes. We store this rather than the set of nodes
1419 # that should be an ellipsis because for very large histories
1425 # that should be an ellipsis because for very large histories
1420 # we expect this to be significantly smaller.
1426 # we expect this to be significantly smaller.
1421 packer._full_nodes = relevant_nodes
1427 packer._full_nodes = relevant_nodes
1422
1428
1423 return packer.generate(common, visitnodes, False, source)
1429 return packer.generate(common, visitnodes, False, source)
General Comments 0
You need to be logged in to leave comments. Login now