##// END OF EJS Templates
changegroup: introduce makelookupmflinknode(dir)...
Martin von Zweigbergk -
r28231:3faba927 default
parent child Browse files
Show More
@@ -1,1157 +1,1164 b''
1 # changegroup.py - Mercurial changegroup manipulation functions
1 # changegroup.py - Mercurial changegroup manipulation functions
2 #
2 #
3 # Copyright 2006 Matt Mackall <mpm@selenic.com>
3 # Copyright 2006 Matt Mackall <mpm@selenic.com>
4 #
4 #
5 # This software may be used and distributed according to the terms of the
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
6 # GNU General Public License version 2 or any later version.
7
7
8 from __future__ import absolute_import
8 from __future__ import absolute_import
9
9
10 import os
10 import os
11 import struct
11 import struct
12 import tempfile
12 import tempfile
13 import weakref
13 import weakref
14
14
15 from .i18n import _
15 from .i18n import _
16 from .node import (
16 from .node import (
17 hex,
17 hex,
18 nullid,
18 nullid,
19 nullrev,
19 nullrev,
20 short,
20 short,
21 )
21 )
22
22
23 from . import (
23 from . import (
24 branchmap,
24 branchmap,
25 dagutil,
25 dagutil,
26 discovery,
26 discovery,
27 error,
27 error,
28 mdiff,
28 mdiff,
29 phases,
29 phases,
30 util,
30 util,
31 )
31 )
32
32
33 _CHANGEGROUPV1_DELTA_HEADER = "20s20s20s20s"
33 _CHANGEGROUPV1_DELTA_HEADER = "20s20s20s20s"
34 _CHANGEGROUPV2_DELTA_HEADER = "20s20s20s20s20s"
34 _CHANGEGROUPV2_DELTA_HEADER = "20s20s20s20s20s"
35 _CHANGEGROUPV3_DELTA_HEADER = ">20s20s20s20s20sH"
35 _CHANGEGROUPV3_DELTA_HEADER = ">20s20s20s20s20sH"
36
36
37 def readexactly(stream, n):
37 def readexactly(stream, n):
38 '''read n bytes from stream.read and abort if less was available'''
38 '''read n bytes from stream.read and abort if less was available'''
39 s = stream.read(n)
39 s = stream.read(n)
40 if len(s) < n:
40 if len(s) < n:
41 raise error.Abort(_("stream ended unexpectedly"
41 raise error.Abort(_("stream ended unexpectedly"
42 " (got %d bytes, expected %d)")
42 " (got %d bytes, expected %d)")
43 % (len(s), n))
43 % (len(s), n))
44 return s
44 return s
45
45
46 def getchunk(stream):
46 def getchunk(stream):
47 """return the next chunk from stream as a string"""
47 """return the next chunk from stream as a string"""
48 d = readexactly(stream, 4)
48 d = readexactly(stream, 4)
49 l = struct.unpack(">l", d)[0]
49 l = struct.unpack(">l", d)[0]
50 if l <= 4:
50 if l <= 4:
51 if l:
51 if l:
52 raise error.Abort(_("invalid chunk length %d") % l)
52 raise error.Abort(_("invalid chunk length %d") % l)
53 return ""
53 return ""
54 return readexactly(stream, l - 4)
54 return readexactly(stream, l - 4)
55
55
56 def chunkheader(length):
56 def chunkheader(length):
57 """return a changegroup chunk header (string)"""
57 """return a changegroup chunk header (string)"""
58 return struct.pack(">l", length + 4)
58 return struct.pack(">l", length + 4)
59
59
60 def closechunk():
60 def closechunk():
61 """return a changegroup chunk header (string) for a zero-length chunk"""
61 """return a changegroup chunk header (string) for a zero-length chunk"""
62 return struct.pack(">l", 0)
62 return struct.pack(">l", 0)
63
63
64 def combineresults(results):
64 def combineresults(results):
65 """logic to combine 0 or more addchangegroup results into one"""
65 """logic to combine 0 or more addchangegroup results into one"""
66 changedheads = 0
66 changedheads = 0
67 result = 1
67 result = 1
68 for ret in results:
68 for ret in results:
69 # If any changegroup result is 0, return 0
69 # If any changegroup result is 0, return 0
70 if ret == 0:
70 if ret == 0:
71 result = 0
71 result = 0
72 break
72 break
73 if ret < -1:
73 if ret < -1:
74 changedheads += ret + 1
74 changedheads += ret + 1
75 elif ret > 1:
75 elif ret > 1:
76 changedheads += ret - 1
76 changedheads += ret - 1
77 if changedheads > 0:
77 if changedheads > 0:
78 result = 1 + changedheads
78 result = 1 + changedheads
79 elif changedheads < 0:
79 elif changedheads < 0:
80 result = -1 + changedheads
80 result = -1 + changedheads
81 return result
81 return result
82
82
83 bundletypes = {
83 bundletypes = {
84 "": ("", None), # only when using unbundle on ssh and old http servers
84 "": ("", None), # only when using unbundle on ssh and old http servers
85 # since the unification ssh accepts a header but there
85 # since the unification ssh accepts a header but there
86 # is no capability signaling it.
86 # is no capability signaling it.
87 "HG20": (), # special-cased below
87 "HG20": (), # special-cased below
88 "HG10UN": ("HG10UN", None),
88 "HG10UN": ("HG10UN", None),
89 "HG10BZ": ("HG10", 'BZ'),
89 "HG10BZ": ("HG10", 'BZ'),
90 "HG10GZ": ("HG10GZ", 'GZ'),
90 "HG10GZ": ("HG10GZ", 'GZ'),
91 }
91 }
92
92
93 # hgweb uses this list to communicate its preferred type
93 # hgweb uses this list to communicate its preferred type
94 bundlepriority = ['HG10GZ', 'HG10BZ', 'HG10UN']
94 bundlepriority = ['HG10GZ', 'HG10BZ', 'HG10UN']
95
95
96 def writechunks(ui, chunks, filename, vfs=None):
96 def writechunks(ui, chunks, filename, vfs=None):
97 """Write chunks to a file and return its filename.
97 """Write chunks to a file and return its filename.
98
98
99 The stream is assumed to be a bundle file.
99 The stream is assumed to be a bundle file.
100 Existing files will not be overwritten.
100 Existing files will not be overwritten.
101 If no filename is specified, a temporary file is created.
101 If no filename is specified, a temporary file is created.
102 """
102 """
103 fh = None
103 fh = None
104 cleanup = None
104 cleanup = None
105 try:
105 try:
106 if filename:
106 if filename:
107 if vfs:
107 if vfs:
108 fh = vfs.open(filename, "wb")
108 fh = vfs.open(filename, "wb")
109 else:
109 else:
110 fh = open(filename, "wb")
110 fh = open(filename, "wb")
111 else:
111 else:
112 fd, filename = tempfile.mkstemp(prefix="hg-bundle-", suffix=".hg")
112 fd, filename = tempfile.mkstemp(prefix="hg-bundle-", suffix=".hg")
113 fh = os.fdopen(fd, "wb")
113 fh = os.fdopen(fd, "wb")
114 cleanup = filename
114 cleanup = filename
115 for c in chunks:
115 for c in chunks:
116 fh.write(c)
116 fh.write(c)
117 cleanup = None
117 cleanup = None
118 return filename
118 return filename
119 finally:
119 finally:
120 if fh is not None:
120 if fh is not None:
121 fh.close()
121 fh.close()
122 if cleanup is not None:
122 if cleanup is not None:
123 if filename and vfs:
123 if filename and vfs:
124 vfs.unlink(cleanup)
124 vfs.unlink(cleanup)
125 else:
125 else:
126 os.unlink(cleanup)
126 os.unlink(cleanup)
127
127
128 def writebundle(ui, cg, filename, bundletype, vfs=None, compression=None):
128 def writebundle(ui, cg, filename, bundletype, vfs=None, compression=None):
129 """Write a bundle file and return its filename.
129 """Write a bundle file and return its filename.
130
130
131 Existing files will not be overwritten.
131 Existing files will not be overwritten.
132 If no filename is specified, a temporary file is created.
132 If no filename is specified, a temporary file is created.
133 bz2 compression can be turned off.
133 bz2 compression can be turned off.
134 The bundle file will be deleted in case of errors.
134 The bundle file will be deleted in case of errors.
135 """
135 """
136
136
137 if bundletype == "HG20":
137 if bundletype == "HG20":
138 from . import bundle2
138 from . import bundle2
139 bundle = bundle2.bundle20(ui)
139 bundle = bundle2.bundle20(ui)
140 bundle.setcompression(compression)
140 bundle.setcompression(compression)
141 part = bundle.newpart('changegroup', data=cg.getchunks())
141 part = bundle.newpart('changegroup', data=cg.getchunks())
142 part.addparam('version', cg.version)
142 part.addparam('version', cg.version)
143 chunkiter = bundle.getchunks()
143 chunkiter = bundle.getchunks()
144 else:
144 else:
145 # compression argument is only for the bundle2 case
145 # compression argument is only for the bundle2 case
146 assert compression is None
146 assert compression is None
147 if cg.version != '01':
147 if cg.version != '01':
148 raise error.Abort(_('old bundle types only supports v1 '
148 raise error.Abort(_('old bundle types only supports v1 '
149 'changegroups'))
149 'changegroups'))
150 header, comp = bundletypes[bundletype]
150 header, comp = bundletypes[bundletype]
151 if comp not in util.compressors:
151 if comp not in util.compressors:
152 raise error.Abort(_('unknown stream compression type: %s')
152 raise error.Abort(_('unknown stream compression type: %s')
153 % comp)
153 % comp)
154 z = util.compressors[comp]()
154 z = util.compressors[comp]()
155 subchunkiter = cg.getchunks()
155 subchunkiter = cg.getchunks()
156 def chunkiter():
156 def chunkiter():
157 yield header
157 yield header
158 for chunk in subchunkiter:
158 for chunk in subchunkiter:
159 yield z.compress(chunk)
159 yield z.compress(chunk)
160 yield z.flush()
160 yield z.flush()
161 chunkiter = chunkiter()
161 chunkiter = chunkiter()
162
162
163 # parse the changegroup data, otherwise we will block
163 # parse the changegroup data, otherwise we will block
164 # in case of sshrepo because we don't know the end of the stream
164 # in case of sshrepo because we don't know the end of the stream
165
165
166 # an empty chunkgroup is the end of the changegroup
166 # an empty chunkgroup is the end of the changegroup
167 # a changegroup has at least 2 chunkgroups (changelog and manifest).
167 # a changegroup has at least 2 chunkgroups (changelog and manifest).
168 # after that, an empty chunkgroup is the end of the changegroup
168 # after that, an empty chunkgroup is the end of the changegroup
169 return writechunks(ui, chunkiter, filename, vfs=vfs)
169 return writechunks(ui, chunkiter, filename, vfs=vfs)
170
170
171 class cg1unpacker(object):
171 class cg1unpacker(object):
172 """Unpacker for cg1 changegroup streams.
172 """Unpacker for cg1 changegroup streams.
173
173
174 A changegroup unpacker handles the framing of the revision data in
174 A changegroup unpacker handles the framing of the revision data in
175 the wire format. Most consumers will want to use the apply()
175 the wire format. Most consumers will want to use the apply()
176 method to add the changes from the changegroup to a repository.
176 method to add the changes from the changegroup to a repository.
177
177
178 If you're forwarding a changegroup unmodified to another consumer,
178 If you're forwarding a changegroup unmodified to another consumer,
179 use getchunks(), which returns an iterator of changegroup
179 use getchunks(), which returns an iterator of changegroup
180 chunks. This is mostly useful for cases where you need to know the
180 chunks. This is mostly useful for cases where you need to know the
181 data stream has ended by observing the end of the changegroup.
181 data stream has ended by observing the end of the changegroup.
182
182
183 deltachunk() is useful only if you're applying delta data. Most
183 deltachunk() is useful only if you're applying delta data. Most
184 consumers should prefer apply() instead.
184 consumers should prefer apply() instead.
185
185
186 A few other public methods exist. Those are used only for
186 A few other public methods exist. Those are used only for
187 bundlerepo and some debug commands - their use is discouraged.
187 bundlerepo and some debug commands - their use is discouraged.
188 """
188 """
189 deltaheader = _CHANGEGROUPV1_DELTA_HEADER
189 deltaheader = _CHANGEGROUPV1_DELTA_HEADER
190 deltaheadersize = struct.calcsize(deltaheader)
190 deltaheadersize = struct.calcsize(deltaheader)
191 version = '01'
191 version = '01'
192 _grouplistcount = 1 # One list of files after the manifests
192 _grouplistcount = 1 # One list of files after the manifests
193
193
194 def __init__(self, fh, alg):
194 def __init__(self, fh, alg):
195 if alg == 'UN':
195 if alg == 'UN':
196 alg = None # get more modern without breaking too much
196 alg = None # get more modern without breaking too much
197 if not alg in util.decompressors:
197 if not alg in util.decompressors:
198 raise error.Abort(_('unknown stream compression type: %s')
198 raise error.Abort(_('unknown stream compression type: %s')
199 % alg)
199 % alg)
200 if alg == 'BZ':
200 if alg == 'BZ':
201 alg = '_truncatedBZ'
201 alg = '_truncatedBZ'
202 self._stream = util.decompressors[alg](fh)
202 self._stream = util.decompressors[alg](fh)
203 self._type = alg
203 self._type = alg
204 self.callback = None
204 self.callback = None
205
205
206 # These methods (compressed, read, seek, tell) all appear to only
206 # These methods (compressed, read, seek, tell) all appear to only
207 # be used by bundlerepo, but it's a little hard to tell.
207 # be used by bundlerepo, but it's a little hard to tell.
208 def compressed(self):
208 def compressed(self):
209 return self._type is not None
209 return self._type is not None
210 def read(self, l):
210 def read(self, l):
211 return self._stream.read(l)
211 return self._stream.read(l)
212 def seek(self, pos):
212 def seek(self, pos):
213 return self._stream.seek(pos)
213 return self._stream.seek(pos)
214 def tell(self):
214 def tell(self):
215 return self._stream.tell()
215 return self._stream.tell()
216 def close(self):
216 def close(self):
217 return self._stream.close()
217 return self._stream.close()
218
218
219 def _chunklength(self):
219 def _chunklength(self):
220 d = readexactly(self._stream, 4)
220 d = readexactly(self._stream, 4)
221 l = struct.unpack(">l", d)[0]
221 l = struct.unpack(">l", d)[0]
222 if l <= 4:
222 if l <= 4:
223 if l:
223 if l:
224 raise error.Abort(_("invalid chunk length %d") % l)
224 raise error.Abort(_("invalid chunk length %d") % l)
225 return 0
225 return 0
226 if self.callback:
226 if self.callback:
227 self.callback()
227 self.callback()
228 return l - 4
228 return l - 4
229
229
230 def changelogheader(self):
230 def changelogheader(self):
231 """v10 does not have a changelog header chunk"""
231 """v10 does not have a changelog header chunk"""
232 return {}
232 return {}
233
233
234 def manifestheader(self):
234 def manifestheader(self):
235 """v10 does not have a manifest header chunk"""
235 """v10 does not have a manifest header chunk"""
236 return {}
236 return {}
237
237
238 def filelogheader(self):
238 def filelogheader(self):
239 """return the header of the filelogs chunk, v10 only has the filename"""
239 """return the header of the filelogs chunk, v10 only has the filename"""
240 l = self._chunklength()
240 l = self._chunklength()
241 if not l:
241 if not l:
242 return {}
242 return {}
243 fname = readexactly(self._stream, l)
243 fname = readexactly(self._stream, l)
244 return {'filename': fname}
244 return {'filename': fname}
245
245
246 def _deltaheader(self, headertuple, prevnode):
246 def _deltaheader(self, headertuple, prevnode):
247 node, p1, p2, cs = headertuple
247 node, p1, p2, cs = headertuple
248 if prevnode is None:
248 if prevnode is None:
249 deltabase = p1
249 deltabase = p1
250 else:
250 else:
251 deltabase = prevnode
251 deltabase = prevnode
252 flags = 0
252 flags = 0
253 return node, p1, p2, deltabase, cs, flags
253 return node, p1, p2, deltabase, cs, flags
254
254
255 def deltachunk(self, prevnode):
255 def deltachunk(self, prevnode):
256 l = self._chunklength()
256 l = self._chunklength()
257 if not l:
257 if not l:
258 return {}
258 return {}
259 headerdata = readexactly(self._stream, self.deltaheadersize)
259 headerdata = readexactly(self._stream, self.deltaheadersize)
260 header = struct.unpack(self.deltaheader, headerdata)
260 header = struct.unpack(self.deltaheader, headerdata)
261 delta = readexactly(self._stream, l - self.deltaheadersize)
261 delta = readexactly(self._stream, l - self.deltaheadersize)
262 node, p1, p2, deltabase, cs, flags = self._deltaheader(header, prevnode)
262 node, p1, p2, deltabase, cs, flags = self._deltaheader(header, prevnode)
263 return {'node': node, 'p1': p1, 'p2': p2, 'cs': cs,
263 return {'node': node, 'p1': p1, 'p2': p2, 'cs': cs,
264 'deltabase': deltabase, 'delta': delta, 'flags': flags}
264 'deltabase': deltabase, 'delta': delta, 'flags': flags}
265
265
266 def getchunks(self):
266 def getchunks(self):
267 """returns all the chunks contains in the bundle
267 """returns all the chunks contains in the bundle
268
268
269 Used when you need to forward the binary stream to a file or another
269 Used when you need to forward the binary stream to a file or another
270 network API. To do so, it parse the changegroup data, otherwise it will
270 network API. To do so, it parse the changegroup data, otherwise it will
271 block in case of sshrepo because it don't know the end of the stream.
271 block in case of sshrepo because it don't know the end of the stream.
272 """
272 """
273 # an empty chunkgroup is the end of the changegroup
273 # an empty chunkgroup is the end of the changegroup
274 # a changegroup has at least 2 chunkgroups (changelog and manifest).
274 # a changegroup has at least 2 chunkgroups (changelog and manifest).
275 # after that, changegroup versions 1 and 2 have a series of groups
275 # after that, changegroup versions 1 and 2 have a series of groups
276 # with one group per file. changegroup 3 has a series of directory
276 # with one group per file. changegroup 3 has a series of directory
277 # manifests before the files.
277 # manifests before the files.
278 count = 0
278 count = 0
279 emptycount = 0
279 emptycount = 0
280 while emptycount < self._grouplistcount:
280 while emptycount < self._grouplistcount:
281 empty = True
281 empty = True
282 count += 1
282 count += 1
283 while True:
283 while True:
284 chunk = getchunk(self)
284 chunk = getchunk(self)
285 if not chunk:
285 if not chunk:
286 if empty and count > 2:
286 if empty and count > 2:
287 emptycount += 1
287 emptycount += 1
288 break
288 break
289 empty = False
289 empty = False
290 yield chunkheader(len(chunk))
290 yield chunkheader(len(chunk))
291 pos = 0
291 pos = 0
292 while pos < len(chunk):
292 while pos < len(chunk):
293 next = pos + 2**20
293 next = pos + 2**20
294 yield chunk[pos:next]
294 yield chunk[pos:next]
295 pos = next
295 pos = next
296 yield closechunk()
296 yield closechunk()
297
297
298 def _unpackmanifests(self, repo, revmap, trp, prog, numchanges):
298 def _unpackmanifests(self, repo, revmap, trp, prog, numchanges):
299 # We know that we'll never have more manifests than we had
299 # We know that we'll never have more manifests than we had
300 # changesets.
300 # changesets.
301 self.callback = prog(_('manifests'), numchanges)
301 self.callback = prog(_('manifests'), numchanges)
302 # no need to check for empty manifest group here:
302 # no need to check for empty manifest group here:
303 # if the result of the merge of 1 and 2 is the same in 3 and 4,
303 # if the result of the merge of 1 and 2 is the same in 3 and 4,
304 # no new manifest will be created and the manifest group will
304 # no new manifest will be created and the manifest group will
305 # be empty during the pull
305 # be empty during the pull
306 self.manifestheader()
306 self.manifestheader()
307 repo.manifest.addgroup(self, revmap, trp)
307 repo.manifest.addgroup(self, revmap, trp)
308 repo.ui.progress(_('manifests'), None)
308 repo.ui.progress(_('manifests'), None)
309
309
310 def apply(self, repo, srctype, url, emptyok=False,
310 def apply(self, repo, srctype, url, emptyok=False,
311 targetphase=phases.draft, expectedtotal=None):
311 targetphase=phases.draft, expectedtotal=None):
312 """Add the changegroup returned by source.read() to this repo.
312 """Add the changegroup returned by source.read() to this repo.
313 srctype is a string like 'push', 'pull', or 'unbundle'. url is
313 srctype is a string like 'push', 'pull', or 'unbundle'. url is
314 the URL of the repo where this changegroup is coming from.
314 the URL of the repo where this changegroup is coming from.
315
315
316 Return an integer summarizing the change to this repo:
316 Return an integer summarizing the change to this repo:
317 - nothing changed or no source: 0
317 - nothing changed or no source: 0
318 - more heads than before: 1+added heads (2..n)
318 - more heads than before: 1+added heads (2..n)
319 - fewer heads than before: -1-removed heads (-2..-n)
319 - fewer heads than before: -1-removed heads (-2..-n)
320 - number of heads stays the same: 1
320 - number of heads stays the same: 1
321 """
321 """
322 repo = repo.unfiltered()
322 repo = repo.unfiltered()
323 def csmap(x):
323 def csmap(x):
324 repo.ui.debug("add changeset %s\n" % short(x))
324 repo.ui.debug("add changeset %s\n" % short(x))
325 return len(cl)
325 return len(cl)
326
326
327 def revmap(x):
327 def revmap(x):
328 return cl.rev(x)
328 return cl.rev(x)
329
329
330 changesets = files = revisions = 0
330 changesets = files = revisions = 0
331
331
332 try:
332 try:
333 with repo.transaction("\n".join([srctype,
333 with repo.transaction("\n".join([srctype,
334 util.hidepassword(url)])) as tr:
334 util.hidepassword(url)])) as tr:
335 # The transaction could have been created before and already
335 # The transaction could have been created before and already
336 # carries source information. In this case we use the top
336 # carries source information. In this case we use the top
337 # level data. We overwrite the argument because we need to use
337 # level data. We overwrite the argument because we need to use
338 # the top level value (if they exist) in this function.
338 # the top level value (if they exist) in this function.
339 srctype = tr.hookargs.setdefault('source', srctype)
339 srctype = tr.hookargs.setdefault('source', srctype)
340 url = tr.hookargs.setdefault('url', url)
340 url = tr.hookargs.setdefault('url', url)
341 repo.hook('prechangegroup', throw=True, **tr.hookargs)
341 repo.hook('prechangegroup', throw=True, **tr.hookargs)
342
342
343 # write changelog data to temp files so concurrent readers
343 # write changelog data to temp files so concurrent readers
344 # will not see an inconsistent view
344 # will not see an inconsistent view
345 cl = repo.changelog
345 cl = repo.changelog
346 cl.delayupdate(tr)
346 cl.delayupdate(tr)
347 oldheads = cl.heads()
347 oldheads = cl.heads()
348
348
349 trp = weakref.proxy(tr)
349 trp = weakref.proxy(tr)
350 # pull off the changeset group
350 # pull off the changeset group
351 repo.ui.status(_("adding changesets\n"))
351 repo.ui.status(_("adding changesets\n"))
352 clstart = len(cl)
352 clstart = len(cl)
353 class prog(object):
353 class prog(object):
354 def __init__(self, step, total):
354 def __init__(self, step, total):
355 self._step = step
355 self._step = step
356 self._total = total
356 self._total = total
357 self._count = 1
357 self._count = 1
358 def __call__(self):
358 def __call__(self):
359 repo.ui.progress(self._step, self._count,
359 repo.ui.progress(self._step, self._count,
360 unit=_('chunks'), total=self._total)
360 unit=_('chunks'), total=self._total)
361 self._count += 1
361 self._count += 1
362 self.callback = prog(_('changesets'), expectedtotal)
362 self.callback = prog(_('changesets'), expectedtotal)
363
363
364 efiles = set()
364 efiles = set()
365 def onchangelog(cl, node):
365 def onchangelog(cl, node):
366 efiles.update(cl.read(node)[3])
366 efiles.update(cl.read(node)[3])
367
367
368 self.changelogheader()
368 self.changelogheader()
369 srccontent = cl.addgroup(self, csmap, trp,
369 srccontent = cl.addgroup(self, csmap, trp,
370 addrevisioncb=onchangelog)
370 addrevisioncb=onchangelog)
371 efiles = len(efiles)
371 efiles = len(efiles)
372
372
373 if not (srccontent or emptyok):
373 if not (srccontent or emptyok):
374 raise error.Abort(_("received changelog group is empty"))
374 raise error.Abort(_("received changelog group is empty"))
375 clend = len(cl)
375 clend = len(cl)
376 changesets = clend - clstart
376 changesets = clend - clstart
377 repo.ui.progress(_('changesets'), None)
377 repo.ui.progress(_('changesets'), None)
378
378
379 # pull off the manifest group
379 # pull off the manifest group
380 repo.ui.status(_("adding manifests\n"))
380 repo.ui.status(_("adding manifests\n"))
381 self._unpackmanifests(repo, revmap, trp, prog, changesets)
381 self._unpackmanifests(repo, revmap, trp, prog, changesets)
382
382
383 needfiles = {}
383 needfiles = {}
384 if repo.ui.configbool('server', 'validate', default=False):
384 if repo.ui.configbool('server', 'validate', default=False):
385 # validate incoming csets have their manifests
385 # validate incoming csets have their manifests
386 for cset in xrange(clstart, clend):
386 for cset in xrange(clstart, clend):
387 mfnode = repo.changelog.read(
387 mfnode = repo.changelog.read(
388 repo.changelog.node(cset))[0]
388 repo.changelog.node(cset))[0]
389 mfest = repo.manifest.readdelta(mfnode)
389 mfest = repo.manifest.readdelta(mfnode)
390 # store file nodes we must see
390 # store file nodes we must see
391 for f, n in mfest.iteritems():
391 for f, n in mfest.iteritems():
392 needfiles.setdefault(f, set()).add(n)
392 needfiles.setdefault(f, set()).add(n)
393
393
394 # process the files
394 # process the files
395 repo.ui.status(_("adding file changes\n"))
395 repo.ui.status(_("adding file changes\n"))
396 self.callback = None
396 self.callback = None
397 pr = prog(_('files'), efiles)
397 pr = prog(_('files'), efiles)
398 newrevs, newfiles = _addchangegroupfiles(
398 newrevs, newfiles = _addchangegroupfiles(
399 repo, self, revmap, trp, pr, needfiles)
399 repo, self, revmap, trp, pr, needfiles)
400 revisions += newrevs
400 revisions += newrevs
401 files += newfiles
401 files += newfiles
402
402
403 dh = 0
403 dh = 0
404 if oldheads:
404 if oldheads:
405 heads = cl.heads()
405 heads = cl.heads()
406 dh = len(heads) - len(oldheads)
406 dh = len(heads) - len(oldheads)
407 for h in heads:
407 for h in heads:
408 if h not in oldheads and repo[h].closesbranch():
408 if h not in oldheads and repo[h].closesbranch():
409 dh -= 1
409 dh -= 1
410 htext = ""
410 htext = ""
411 if dh:
411 if dh:
412 htext = _(" (%+d heads)") % dh
412 htext = _(" (%+d heads)") % dh
413
413
414 repo.ui.status(_("added %d changesets"
414 repo.ui.status(_("added %d changesets"
415 " with %d changes to %d files%s\n")
415 " with %d changes to %d files%s\n")
416 % (changesets, revisions, files, htext))
416 % (changesets, revisions, files, htext))
417 repo.invalidatevolatilesets()
417 repo.invalidatevolatilesets()
418
418
419 if changesets > 0:
419 if changesets > 0:
420 if 'node' not in tr.hookargs:
420 if 'node' not in tr.hookargs:
421 tr.hookargs['node'] = hex(cl.node(clstart))
421 tr.hookargs['node'] = hex(cl.node(clstart))
422 tr.hookargs['node_last'] = hex(cl.node(clend - 1))
422 tr.hookargs['node_last'] = hex(cl.node(clend - 1))
423 hookargs = dict(tr.hookargs)
423 hookargs = dict(tr.hookargs)
424 else:
424 else:
425 hookargs = dict(tr.hookargs)
425 hookargs = dict(tr.hookargs)
426 hookargs['node'] = hex(cl.node(clstart))
426 hookargs['node'] = hex(cl.node(clstart))
427 hookargs['node_last'] = hex(cl.node(clend - 1))
427 hookargs['node_last'] = hex(cl.node(clend - 1))
428 repo.hook('pretxnchangegroup', throw=True, **hookargs)
428 repo.hook('pretxnchangegroup', throw=True, **hookargs)
429
429
430 added = [cl.node(r) for r in xrange(clstart, clend)]
430 added = [cl.node(r) for r in xrange(clstart, clend)]
431 publishing = repo.publishing()
431 publishing = repo.publishing()
432 if srctype in ('push', 'serve'):
432 if srctype in ('push', 'serve'):
433 # Old servers can not push the boundary themselves.
433 # Old servers can not push the boundary themselves.
434 # New servers won't push the boundary if changeset already
434 # New servers won't push the boundary if changeset already
435 # exists locally as secret
435 # exists locally as secret
436 #
436 #
437 # We should not use added here but the list of all change in
437 # We should not use added here but the list of all change in
438 # the bundle
438 # the bundle
439 if publishing:
439 if publishing:
440 phases.advanceboundary(repo, tr, phases.public,
440 phases.advanceboundary(repo, tr, phases.public,
441 srccontent)
441 srccontent)
442 else:
442 else:
443 # Those changesets have been pushed from the
443 # Those changesets have been pushed from the
444 # outside, their phases are going to be pushed
444 # outside, their phases are going to be pushed
445 # alongside. Therefor `targetphase` is
445 # alongside. Therefor `targetphase` is
446 # ignored.
446 # ignored.
447 phases.advanceboundary(repo, tr, phases.draft,
447 phases.advanceboundary(repo, tr, phases.draft,
448 srccontent)
448 srccontent)
449 phases.retractboundary(repo, tr, phases.draft, added)
449 phases.retractboundary(repo, tr, phases.draft, added)
450 elif srctype != 'strip':
450 elif srctype != 'strip':
451 # publishing only alter behavior during push
451 # publishing only alter behavior during push
452 #
452 #
453 # strip should not touch boundary at all
453 # strip should not touch boundary at all
454 phases.retractboundary(repo, tr, targetphase, added)
454 phases.retractboundary(repo, tr, targetphase, added)
455
455
456 if changesets > 0:
456 if changesets > 0:
457 if srctype != 'strip':
457 if srctype != 'strip':
458 # During strip, branchcache is invalid but
458 # During strip, branchcache is invalid but
459 # coming call to `destroyed` will repair it.
459 # coming call to `destroyed` will repair it.
460 # In other case we can safely update cache on
460 # In other case we can safely update cache on
461 # disk.
461 # disk.
462 branchmap.updatecache(repo.filtered('served'))
462 branchmap.updatecache(repo.filtered('served'))
463
463
464 def runhooks():
464 def runhooks():
465 # These hooks run when the lock releases, not when the
465 # These hooks run when the lock releases, not when the
466 # transaction closes. So it's possible for the changelog
466 # transaction closes. So it's possible for the changelog
467 # to have changed since we last saw it.
467 # to have changed since we last saw it.
468 if clstart >= len(repo):
468 if clstart >= len(repo):
469 return
469 return
470
470
471 # forcefully update the on-disk branch cache
471 # forcefully update the on-disk branch cache
472 repo.ui.debug("updating the branch cache\n")
472 repo.ui.debug("updating the branch cache\n")
473 repo.hook("changegroup", **hookargs)
473 repo.hook("changegroup", **hookargs)
474
474
475 for n in added:
475 for n in added:
476 args = hookargs.copy()
476 args = hookargs.copy()
477 args['node'] = hex(n)
477 args['node'] = hex(n)
478 del args['node_last']
478 del args['node_last']
479 repo.hook("incoming", **args)
479 repo.hook("incoming", **args)
480
480
481 newheads = [h for h in repo.heads()
481 newheads = [h for h in repo.heads()
482 if h not in oldheads]
482 if h not in oldheads]
483 repo.ui.log("incoming",
483 repo.ui.log("incoming",
484 "%s incoming changes - new heads: %s\n",
484 "%s incoming changes - new heads: %s\n",
485 len(added),
485 len(added),
486 ', '.join([hex(c[:6]) for c in newheads]))
486 ', '.join([hex(c[:6]) for c in newheads]))
487
487
488 tr.addpostclose('changegroup-runhooks-%020i' % clstart,
488 tr.addpostclose('changegroup-runhooks-%020i' % clstart,
489 lambda tr: repo._afterlock(runhooks))
489 lambda tr: repo._afterlock(runhooks))
490 finally:
490 finally:
491 repo.ui.flush()
491 repo.ui.flush()
492 # never return 0 here:
492 # never return 0 here:
493 if dh < 0:
493 if dh < 0:
494 return dh - 1
494 return dh - 1
495 else:
495 else:
496 return dh + 1
496 return dh + 1
497
497
498 class cg2unpacker(cg1unpacker):
498 class cg2unpacker(cg1unpacker):
499 """Unpacker for cg2 streams.
499 """Unpacker for cg2 streams.
500
500
501 cg2 streams add support for generaldelta, so the delta header
501 cg2 streams add support for generaldelta, so the delta header
502 format is slightly different. All other features about the data
502 format is slightly different. All other features about the data
503 remain the same.
503 remain the same.
504 """
504 """
505 deltaheader = _CHANGEGROUPV2_DELTA_HEADER
505 deltaheader = _CHANGEGROUPV2_DELTA_HEADER
506 deltaheadersize = struct.calcsize(deltaheader)
506 deltaheadersize = struct.calcsize(deltaheader)
507 version = '02'
507 version = '02'
508
508
509 def _deltaheader(self, headertuple, prevnode):
509 def _deltaheader(self, headertuple, prevnode):
510 node, p1, p2, deltabase, cs = headertuple
510 node, p1, p2, deltabase, cs = headertuple
511 flags = 0
511 flags = 0
512 return node, p1, p2, deltabase, cs, flags
512 return node, p1, p2, deltabase, cs, flags
513
513
514 class cg3unpacker(cg2unpacker):
514 class cg3unpacker(cg2unpacker):
515 """Unpacker for cg3 streams.
515 """Unpacker for cg3 streams.
516
516
517 cg3 streams add support for exchanging treemanifests and revlog
517 cg3 streams add support for exchanging treemanifests and revlog
518 flags. It adds the revlog flags to the delta header and an empty chunk
518 flags. It adds the revlog flags to the delta header and an empty chunk
519 separating manifests and files.
519 separating manifests and files.
520 """
520 """
521 deltaheader = _CHANGEGROUPV3_DELTA_HEADER
521 deltaheader = _CHANGEGROUPV3_DELTA_HEADER
522 deltaheadersize = struct.calcsize(deltaheader)
522 deltaheadersize = struct.calcsize(deltaheader)
523 version = '03'
523 version = '03'
524 _grouplistcount = 2 # One list of manifests and one list of files
524 _grouplistcount = 2 # One list of manifests and one list of files
525
525
526 def _deltaheader(self, headertuple, prevnode):
526 def _deltaheader(self, headertuple, prevnode):
527 node, p1, p2, deltabase, cs, flags = headertuple
527 node, p1, p2, deltabase, cs, flags = headertuple
528 return node, p1, p2, deltabase, cs, flags
528 return node, p1, p2, deltabase, cs, flags
529
529
530 def _unpackmanifests(self, repo, revmap, trp, prog, numchanges):
530 def _unpackmanifests(self, repo, revmap, trp, prog, numchanges):
531 super(cg3unpacker, self)._unpackmanifests(repo, revmap, trp, prog,
531 super(cg3unpacker, self)._unpackmanifests(repo, revmap, trp, prog,
532 numchanges)
532 numchanges)
533 while True:
533 while True:
534 chunkdata = self.filelogheader()
534 chunkdata = self.filelogheader()
535 if not chunkdata:
535 if not chunkdata:
536 break
536 break
537 # If we get here, there are directory manifests in the changegroup
537 # If we get here, there are directory manifests in the changegroup
538 d = chunkdata["filename"]
538 d = chunkdata["filename"]
539 repo.ui.debug("adding %s revisions\n" % d)
539 repo.ui.debug("adding %s revisions\n" % d)
540 dirlog = repo.manifest.dirlog(d)
540 dirlog = repo.manifest.dirlog(d)
541 if not dirlog.addgroup(self, revmap, trp):
541 if not dirlog.addgroup(self, revmap, trp):
542 raise error.Abort(_("received dir revlog group is empty"))
542 raise error.Abort(_("received dir revlog group is empty"))
543
543
544 class headerlessfixup(object):
544 class headerlessfixup(object):
545 def __init__(self, fh, h):
545 def __init__(self, fh, h):
546 self._h = h
546 self._h = h
547 self._fh = fh
547 self._fh = fh
548 def read(self, n):
548 def read(self, n):
549 if self._h:
549 if self._h:
550 d, self._h = self._h[:n], self._h[n:]
550 d, self._h = self._h[:n], self._h[n:]
551 if len(d) < n:
551 if len(d) < n:
552 d += readexactly(self._fh, n - len(d))
552 d += readexactly(self._fh, n - len(d))
553 return d
553 return d
554 return readexactly(self._fh, n)
554 return readexactly(self._fh, n)
555
555
556 def _moddirs(files):
556 def _moddirs(files):
557 """Given a set of modified files, find the list of modified directories.
557 """Given a set of modified files, find the list of modified directories.
558
558
559 This returns a list of (path to changed dir, changed dir) tuples,
559 This returns a list of (path to changed dir, changed dir) tuples,
560 as that's what the one client needs anyway.
560 as that's what the one client needs anyway.
561
561
562 >>> _moddirs(['a/b/c.py', 'a/b/c.txt', 'a/d/e/f/g.txt', 'i.txt', ])
562 >>> _moddirs(['a/b/c.py', 'a/b/c.txt', 'a/d/e/f/g.txt', 'i.txt', ])
563 [('/', 'a/'), ('a/', 'b/'), ('a/', 'd/'), ('a/d/', 'e/'), ('a/d/e/', 'f/')]
563 [('/', 'a/'), ('a/', 'b/'), ('a/', 'd/'), ('a/d/', 'e/'), ('a/d/e/', 'f/')]
564
564
565 """
565 """
566 alldirs = set()
566 alldirs = set()
567 for f in files:
567 for f in files:
568 path = f.split('/')[:-1]
568 path = f.split('/')[:-1]
569 for i in xrange(len(path) - 1, -1, -1):
569 for i in xrange(len(path) - 1, -1, -1):
570 dn = '/'.join(path[:i])
570 dn = '/'.join(path[:i])
571 current = dn + '/', path[i] + '/'
571 current = dn + '/', path[i] + '/'
572 if current in alldirs:
572 if current in alldirs:
573 break
573 break
574 alldirs.add(current)
574 alldirs.add(current)
575 return sorted(alldirs)
575 return sorted(alldirs)
576
576
577 class cg1packer(object):
577 class cg1packer(object):
578 deltaheader = _CHANGEGROUPV1_DELTA_HEADER
578 deltaheader = _CHANGEGROUPV1_DELTA_HEADER
579 version = '01'
579 version = '01'
580 def __init__(self, repo, bundlecaps=None):
580 def __init__(self, repo, bundlecaps=None):
581 """Given a source repo, construct a bundler.
581 """Given a source repo, construct a bundler.
582
582
583 bundlecaps is optional and can be used to specify the set of
583 bundlecaps is optional and can be used to specify the set of
584 capabilities which can be used to build the bundle.
584 capabilities which can be used to build the bundle.
585 """
585 """
586 # Set of capabilities we can use to build the bundle.
586 # Set of capabilities we can use to build the bundle.
587 if bundlecaps is None:
587 if bundlecaps is None:
588 bundlecaps = set()
588 bundlecaps = set()
589 self._bundlecaps = bundlecaps
589 self._bundlecaps = bundlecaps
590 # experimental config: bundle.reorder
590 # experimental config: bundle.reorder
591 reorder = repo.ui.config('bundle', 'reorder', 'auto')
591 reorder = repo.ui.config('bundle', 'reorder', 'auto')
592 if reorder == 'auto':
592 if reorder == 'auto':
593 reorder = None
593 reorder = None
594 else:
594 else:
595 reorder = util.parsebool(reorder)
595 reorder = util.parsebool(reorder)
596 self._repo = repo
596 self._repo = repo
597 self._reorder = reorder
597 self._reorder = reorder
598 self._progress = repo.ui.progress
598 self._progress = repo.ui.progress
599 if self._repo.ui.verbose and not self._repo.ui.debugflag:
599 if self._repo.ui.verbose and not self._repo.ui.debugflag:
600 self._verbosenote = self._repo.ui.note
600 self._verbosenote = self._repo.ui.note
601 else:
601 else:
602 self._verbosenote = lambda s: None
602 self._verbosenote = lambda s: None
603
603
604 def close(self):
604 def close(self):
605 return closechunk()
605 return closechunk()
606
606
607 def fileheader(self, fname):
607 def fileheader(self, fname):
608 return chunkheader(len(fname)) + fname
608 return chunkheader(len(fname)) + fname
609
609
610 def group(self, nodelist, revlog, lookup, units=None):
610 def group(self, nodelist, revlog, lookup, units=None):
611 """Calculate a delta group, yielding a sequence of changegroup chunks
611 """Calculate a delta group, yielding a sequence of changegroup chunks
612 (strings).
612 (strings).
613
613
614 Given a list of changeset revs, return a set of deltas and
614 Given a list of changeset revs, return a set of deltas and
615 metadata corresponding to nodes. The first delta is
615 metadata corresponding to nodes. The first delta is
616 first parent(nodelist[0]) -> nodelist[0], the receiver is
616 first parent(nodelist[0]) -> nodelist[0], the receiver is
617 guaranteed to have this parent as it has all history before
617 guaranteed to have this parent as it has all history before
618 these changesets. In the case firstparent is nullrev the
618 these changesets. In the case firstparent is nullrev the
619 changegroup starts with a full revision.
619 changegroup starts with a full revision.
620
620
621 If units is not None, progress detail will be generated, units specifies
621 If units is not None, progress detail will be generated, units specifies
622 the type of revlog that is touched (changelog, manifest, etc.).
622 the type of revlog that is touched (changelog, manifest, etc.).
623 """
623 """
624 # if we don't have any revisions touched by these changesets, bail
624 # if we don't have any revisions touched by these changesets, bail
625 if len(nodelist) == 0:
625 if len(nodelist) == 0:
626 yield self.close()
626 yield self.close()
627 return
627 return
628
628
629 # for generaldelta revlogs, we linearize the revs; this will both be
629 # for generaldelta revlogs, we linearize the revs; this will both be
630 # much quicker and generate a much smaller bundle
630 # much quicker and generate a much smaller bundle
631 if (revlog._generaldelta and self._reorder is None) or self._reorder:
631 if (revlog._generaldelta and self._reorder is None) or self._reorder:
632 dag = dagutil.revlogdag(revlog)
632 dag = dagutil.revlogdag(revlog)
633 revs = set(revlog.rev(n) for n in nodelist)
633 revs = set(revlog.rev(n) for n in nodelist)
634 revs = dag.linearize(revs)
634 revs = dag.linearize(revs)
635 else:
635 else:
636 revs = sorted([revlog.rev(n) for n in nodelist])
636 revs = sorted([revlog.rev(n) for n in nodelist])
637
637
638 # add the parent of the first rev
638 # add the parent of the first rev
639 p = revlog.parentrevs(revs[0])[0]
639 p = revlog.parentrevs(revs[0])[0]
640 revs.insert(0, p)
640 revs.insert(0, p)
641
641
642 # build deltas
642 # build deltas
643 total = len(revs) - 1
643 total = len(revs) - 1
644 msgbundling = _('bundling')
644 msgbundling = _('bundling')
645 for r in xrange(len(revs) - 1):
645 for r in xrange(len(revs) - 1):
646 if units is not None:
646 if units is not None:
647 self._progress(msgbundling, r + 1, unit=units, total=total)
647 self._progress(msgbundling, r + 1, unit=units, total=total)
648 prev, curr = revs[r], revs[r + 1]
648 prev, curr = revs[r], revs[r + 1]
649 linknode = lookup(revlog.node(curr))
649 linknode = lookup(revlog.node(curr))
650 for c in self.revchunk(revlog, curr, prev, linknode):
650 for c in self.revchunk(revlog, curr, prev, linknode):
651 yield c
651 yield c
652
652
653 if units is not None:
653 if units is not None:
654 self._progress(msgbundling, None)
654 self._progress(msgbundling, None)
655 yield self.close()
655 yield self.close()
656
656
657 # filter any nodes that claim to be part of the known set
657 # filter any nodes that claim to be part of the known set
658 def prune(self, revlog, missing, commonrevs):
658 def prune(self, revlog, missing, commonrevs):
659 rr, rl = revlog.rev, revlog.linkrev
659 rr, rl = revlog.rev, revlog.linkrev
660 return [n for n in missing if rl(rr(n)) not in commonrevs]
660 return [n for n in missing if rl(rr(n)) not in commonrevs]
661
661
662 def _packmanifests(self, dir, mfnodes, lookuplinknode):
662 def _packmanifests(self, dir, mfnodes, lookuplinknode):
663 """Pack flat manifests into a changegroup stream."""
663 """Pack flat manifests into a changegroup stream."""
664 assert not dir
664 assert not dir
665 for chunk in self.group(mfnodes, self._repo.manifest,
665 for chunk in self.group(mfnodes, self._repo.manifest,
666 lookuplinknode, units=_('manifests')):
666 lookuplinknode, units=_('manifests')):
667 yield chunk
667 yield chunk
668
668
669 def _manifestsdone(self):
669 def _manifestsdone(self):
670 return ''
670 return ''
671
671
672 def generate(self, commonrevs, clnodes, fastpathlinkrev, source):
672 def generate(self, commonrevs, clnodes, fastpathlinkrev, source):
673 '''yield a sequence of changegroup chunks (strings)'''
673 '''yield a sequence of changegroup chunks (strings)'''
674 repo = self._repo
674 repo = self._repo
675 cl = repo.changelog
675 cl = repo.changelog
676
676
677 clrevorder = {}
677 clrevorder = {}
678 mfs = {} # needed manifests
678 mfs = {} # needed manifests
679 fnodes = {} # needed file nodes
679 fnodes = {} # needed file nodes
680 # maps manifest node id -> set(changed files)
680 # maps manifest node id -> set(changed files)
681 mfchangedfiles = {}
681 mfchangedfiles = {}
682
682
683 # Callback for the changelog, used to collect changed files and manifest
683 # Callback for the changelog, used to collect changed files and manifest
684 # nodes.
684 # nodes.
685 # Returns the linkrev node (identity in the changelog case).
685 # Returns the linkrev node (identity in the changelog case).
686 def lookupcl(x):
686 def lookupcl(x):
687 c = cl.read(x)
687 c = cl.read(x)
688 clrevorder[x] = len(clrevorder)
688 clrevorder[x] = len(clrevorder)
689 n = c[0]
689 n = c[0]
690 # record the first changeset introducing this manifest version
690 # record the first changeset introducing this manifest version
691 mfs.setdefault(n, x)
691 mfs.setdefault(n, x)
692 # Record a complete list of potentially-changed files in
692 # Record a complete list of potentially-changed files in
693 # this manifest.
693 # this manifest.
694 mfchangedfiles.setdefault(n, set()).update(c[3])
694 mfchangedfiles.setdefault(n, set()).update(c[3])
695 return x
695 return x
696
696
697 self._verbosenote(_('uncompressed size of bundle content:\n'))
697 self._verbosenote(_('uncompressed size of bundle content:\n'))
698 size = 0
698 size = 0
699 for chunk in self.group(clnodes, cl, lookupcl, units=_('changesets')):
699 for chunk in self.group(clnodes, cl, lookupcl, units=_('changesets')):
700 size += len(chunk)
700 size += len(chunk)
701 yield chunk
701 yield chunk
702 self._verbosenote(_('%8.i (changelog)\n') % size)
702 self._verbosenote(_('%8.i (changelog)\n') % size)
703
703
704 # We need to make sure that the linkrev in the changegroup refers to
704 # We need to make sure that the linkrev in the changegroup refers to
705 # the first changeset that introduced the manifest or file revision.
705 # the first changeset that introduced the manifest or file revision.
706 # The fastpath is usually safer than the slowpath, because the filelogs
706 # The fastpath is usually safer than the slowpath, because the filelogs
707 # are walked in revlog order.
707 # are walked in revlog order.
708 #
708 #
709 # When taking the slowpath with reorder=None and the manifest revlog
709 # When taking the slowpath with reorder=None and the manifest revlog
710 # uses generaldelta, the manifest may be walked in the "wrong" order.
710 # uses generaldelta, the manifest may be walked in the "wrong" order.
711 # Without 'clrevorder', we would get an incorrect linkrev (see fix in
711 # Without 'clrevorder', we would get an incorrect linkrev (see fix in
712 # cc0ff93d0c0c).
712 # cc0ff93d0c0c).
713 #
713 #
714 # When taking the fastpath, we are only vulnerable to reordering
714 # When taking the fastpath, we are only vulnerable to reordering
715 # of the changelog itself. The changelog never uses generaldelta, so
715 # of the changelog itself. The changelog never uses generaldelta, so
716 # it is only reordered when reorder=True. To handle this case, we
716 # it is only reordered when reorder=True. To handle this case, we
717 # simply take the slowpath, which already has the 'clrevorder' logic.
717 # simply take the slowpath, which already has the 'clrevorder' logic.
718 # This was also fixed in cc0ff93d0c0c.
718 # This was also fixed in cc0ff93d0c0c.
719 fastpathlinkrev = fastpathlinkrev and not self._reorder
719 fastpathlinkrev = fastpathlinkrev and not self._reorder
720 # Treemanifests don't work correctly with fastpathlinkrev
720 # Treemanifests don't work correctly with fastpathlinkrev
721 # either, because we don't discover which directory nodes to
721 # either, because we don't discover which directory nodes to
722 # send along with files. This could probably be fixed.
722 # send along with files. This could probably be fixed.
723 fastpathlinkrev = fastpathlinkrev and (
723 fastpathlinkrev = fastpathlinkrev and (
724 'treemanifest' not in repo.requirements)
724 'treemanifest' not in repo.requirements)
725
725
726 for chunk in self.generatemanifests(commonrevs, clrevorder,
726 for chunk in self.generatemanifests(commonrevs, clrevorder,
727 fastpathlinkrev, mfs, mfchangedfiles, fnodes):
727 fastpathlinkrev, mfs, mfchangedfiles, fnodes):
728 yield chunk
728 yield chunk
729 mfs.clear()
729 mfs.clear()
730 clrevs = set(cl.rev(x) for x in clnodes)
730 clrevs = set(cl.rev(x) for x in clnodes)
731
731
732 if not fastpathlinkrev:
732 if not fastpathlinkrev:
733 def linknodes(unused, fname):
733 def linknodes(unused, fname):
734 return fnodes.get(fname, {})
734 return fnodes.get(fname, {})
735 else:
735 else:
736 cln = cl.node
736 cln = cl.node
737 def linknodes(filerevlog, fname):
737 def linknodes(filerevlog, fname):
738 llr = filerevlog.linkrev
738 llr = filerevlog.linkrev
739 fln = filerevlog.node
739 fln = filerevlog.node
740 revs = ((r, llr(r)) for r in filerevlog)
740 revs = ((r, llr(r)) for r in filerevlog)
741 return dict((fln(r), cln(lr)) for r, lr in revs if lr in clrevs)
741 return dict((fln(r), cln(lr)) for r, lr in revs if lr in clrevs)
742
742
743 changedfiles = set()
743 changedfiles = set()
744 for x in mfchangedfiles.itervalues():
744 for x in mfchangedfiles.itervalues():
745 changedfiles.update(x)
745 changedfiles.update(x)
746 for chunk in self.generatefiles(changedfiles, linknodes, commonrevs,
746 for chunk in self.generatefiles(changedfiles, linknodes, commonrevs,
747 source):
747 source):
748 yield chunk
748 yield chunk
749
749
750 yield self.close()
750 yield self.close()
751
751
752 if clnodes:
752 if clnodes:
753 repo.hook('outgoing', node=hex(clnodes[0]), source=source)
753 repo.hook('outgoing', node=hex(clnodes[0]), source=source)
754
754
755 def generatemanifests(self, commonrevs, clrevorder, fastpathlinkrev, mfs,
755 def generatemanifests(self, commonrevs, clrevorder, fastpathlinkrev, mfs,
756 mfchangedfiles, fnodes):
756 mfchangedfiles, fnodes):
757 repo = self._repo
757 repo = self._repo
758 ml = repo.manifest
758 ml = repo.manifest
759 tmfnodes = {}
759 tmfnodes = {}
760
760
761 # Callback for the manifest, used to collect linkrevs for filelog
761 # Callback for the manifest, used to collect linkrevs for filelog
762 # revisions.
762 # revisions.
763 # Returns the linkrev node (collected in lookupcl).
763 # Returns the linkrev node (collected in lookupcl).
764 def makelookupmflinknode(dir):
764 if fastpathlinkrev:
765 if fastpathlinkrev:
765 lookupmflinknode = mfs.__getitem__
766 assert not dir
766 else:
767 return mfs.__getitem__
768
769 if dir:
770 return tmfnodes[dir].get
771
767 def lookupmflinknode(x):
772 def lookupmflinknode(x):
768 """Callback for looking up the linknode for manifests.
773 """Callback for looking up the linknode for manifests.
769
774
770 Returns the linkrev node for the specified manifest.
775 Returns the linkrev node for the specified manifest.
771
776
772 SIDE EFFECT:
777 SIDE EFFECT:
773
778
774 1) fclnodes gets populated with the list of relevant
779 1) fclnodes gets populated with the list of relevant
775 file nodes if we're not using fastpathlinkrev
780 file nodes if we're not using fastpathlinkrev
776 2) When treemanifests are in use, collects treemanifest nodes
781 2) When treemanifests are in use, collects treemanifest nodes
777 to send
782 to send
778
783
779 Note that this means manifests must be completely sent to
784 Note that this means manifests must be completely sent to
780 the client before you can trust the list of files and
785 the client before you can trust the list of files and
781 treemanifests to send.
786 treemanifests to send.
782 """
787 """
783 clnode = mfs[x]
788 clnode = mfs[x]
784 # We no longer actually care about reading deltas of
789 # We no longer actually care about reading deltas of
785 # the manifest here, because we already know the list
790 # the manifest here, because we already know the list
786 # of changed files, so for treemanifests (which
791 # of changed files, so for treemanifests (which
787 # lazily-load anyway to *generate* a readdelta) we can
792 # lazily-load anyway to *generate* a readdelta) we can
788 # just load them with read() and then we'll actually
793 # just load them with read() and then we'll actually
789 # be able to correctly load node IDs from the
794 # be able to correctly load node IDs from the
790 # submanifest entries.
795 # submanifest entries.
791 if 'treemanifest' in repo.requirements:
796 if 'treemanifest' in repo.requirements:
792 mdata = ml.read(x)
797 mdata = ml.read(x)
793 else:
798 else:
794 mdata = ml.readfast(x)
799 mdata = ml.readfast(x)
795 for f in mfchangedfiles[x]:
800 for f in mfchangedfiles[x]:
796 try:
801 try:
797 n = mdata[f]
802 n = mdata[f]
798 except KeyError:
803 except KeyError:
799 continue
804 continue
800 # record the first changeset introducing this filelog
805 # record the first changeset introducing this filelog
801 # version
806 # version
802 fclnodes = fnodes.setdefault(f, {})
807 fclnodes = fnodes.setdefault(f, {})
803 fclnode = fclnodes.setdefault(n, clnode)
808 fclnode = fclnodes.setdefault(n, clnode)
804 if clrevorder[clnode] < clrevorder[fclnode]:
809 if clrevorder[clnode] < clrevorder[fclnode]:
805 fclnodes[n] = clnode
810 fclnodes[n] = clnode
806 # gather list of changed treemanifest nodes
811 # gather list of changed treemanifest nodes
807 if 'treemanifest' in repo.requirements:
812 if 'treemanifest' in repo.requirements:
808 submfs = {'/': mdata}
813 submfs = {'/': mdata}
809 for dn, bn in _moddirs(mfchangedfiles[x]):
814 for dn, bn in _moddirs(mfchangedfiles[x]):
810 try:
815 try:
811 submf = submfs[dn]
816 submf = submfs[dn]
812 submf = submf._dirs[bn]
817 submf = submf._dirs[bn]
813 except KeyError:
818 except KeyError:
814 continue # deleted directory, so nothing to send
819 continue # deleted directory, so nothing to send
815 submfs[submf.dir()] = submf
820 submfs[submf.dir()] = submf
816 tmfclnodes = tmfnodes.setdefault(submf.dir(), {})
821 tmfclnodes = tmfnodes.setdefault(submf.dir(), {})
817 tmfclnode = tmfclnodes.setdefault(submf._node, clnode)
822 tmfclnode = tmfclnodes.setdefault(submf._node, clnode)
818 if clrevorder[clnode] < clrevorder[tmfclnode]:
823 if clrevorder[clnode] < clrevorder[tmfclnode]:
819 tmfclnodes[n] = clnode
824 tmfclnodes[n] = clnode
820 return clnode
825 return clnode
826 return lookupmflinknode
821
827
822 mfnodes = self.prune(ml, mfs, commonrevs)
828 mfnodes = self.prune(ml, mfs, commonrevs)
823 size = 0
829 size = 0
824 for x in self._packmanifests('', mfnodes, lookupmflinknode):
830 for x in self._packmanifests('', mfnodes, makelookupmflinknode('')):
825 size += len(x)
831 size += len(x)
826 yield x
832 yield x
827 for dir, nodes in tmfnodes.iteritems():
833 for dir, nodes in tmfnodes.iteritems():
828 prunednodes = self.prune(ml.dirlog(dir), nodes, commonrevs)
834 prunednodes = self.prune(ml.dirlog(dir), nodes, commonrevs)
829 for x in self._packmanifests(dir, prunednodes, nodes.get):
835 for x in self._packmanifests(dir, prunednodes,
836 makelookupmflinknode(dir)):
830 size += len(x)
837 size += len(x)
831 yield x
838 yield x
832 self._verbosenote(_('%8.i (manifests)\n') % size)
839 self._verbosenote(_('%8.i (manifests)\n') % size)
833 yield self._manifestsdone()
840 yield self._manifestsdone()
834
841
835 # The 'source' parameter is useful for extensions
842 # The 'source' parameter is useful for extensions
836 def generatefiles(self, changedfiles, linknodes, commonrevs, source):
843 def generatefiles(self, changedfiles, linknodes, commonrevs, source):
837 repo = self._repo
844 repo = self._repo
838 progress = self._progress
845 progress = self._progress
839 msgbundling = _('bundling')
846 msgbundling = _('bundling')
840
847
841 total = len(changedfiles)
848 total = len(changedfiles)
842 # for progress output
849 # for progress output
843 msgfiles = _('files')
850 msgfiles = _('files')
844 for i, fname in enumerate(sorted(changedfiles)):
851 for i, fname in enumerate(sorted(changedfiles)):
845 filerevlog = repo.file(fname)
852 filerevlog = repo.file(fname)
846 if not filerevlog:
853 if not filerevlog:
847 raise error.Abort(_("empty or missing revlog for %s") % fname)
854 raise error.Abort(_("empty or missing revlog for %s") % fname)
848
855
849 linkrevnodes = linknodes(filerevlog, fname)
856 linkrevnodes = linknodes(filerevlog, fname)
850 # Lookup for filenodes, we collected the linkrev nodes above in the
857 # Lookup for filenodes, we collected the linkrev nodes above in the
851 # fastpath case and with lookupmf in the slowpath case.
858 # fastpath case and with lookupmf in the slowpath case.
852 def lookupfilelog(x):
859 def lookupfilelog(x):
853 return linkrevnodes[x]
860 return linkrevnodes[x]
854
861
855 filenodes = self.prune(filerevlog, linkrevnodes, commonrevs)
862 filenodes = self.prune(filerevlog, linkrevnodes, commonrevs)
856 if filenodes:
863 if filenodes:
857 progress(msgbundling, i + 1, item=fname, unit=msgfiles,
864 progress(msgbundling, i + 1, item=fname, unit=msgfiles,
858 total=total)
865 total=total)
859 h = self.fileheader(fname)
866 h = self.fileheader(fname)
860 size = len(h)
867 size = len(h)
861 yield h
868 yield h
862 for chunk in self.group(filenodes, filerevlog, lookupfilelog):
869 for chunk in self.group(filenodes, filerevlog, lookupfilelog):
863 size += len(chunk)
870 size += len(chunk)
864 yield chunk
871 yield chunk
865 self._verbosenote(_('%8.i %s\n') % (size, fname))
872 self._verbosenote(_('%8.i %s\n') % (size, fname))
866 progress(msgbundling, None)
873 progress(msgbundling, None)
867
874
868 def deltaparent(self, revlog, rev, p1, p2, prev):
875 def deltaparent(self, revlog, rev, p1, p2, prev):
869 return prev
876 return prev
870
877
871 def revchunk(self, revlog, rev, prev, linknode):
878 def revchunk(self, revlog, rev, prev, linknode):
872 node = revlog.node(rev)
879 node = revlog.node(rev)
873 p1, p2 = revlog.parentrevs(rev)
880 p1, p2 = revlog.parentrevs(rev)
874 base = self.deltaparent(revlog, rev, p1, p2, prev)
881 base = self.deltaparent(revlog, rev, p1, p2, prev)
875
882
876 prefix = ''
883 prefix = ''
877 if revlog.iscensored(base) or revlog.iscensored(rev):
884 if revlog.iscensored(base) or revlog.iscensored(rev):
878 try:
885 try:
879 delta = revlog.revision(node)
886 delta = revlog.revision(node)
880 except error.CensoredNodeError as e:
887 except error.CensoredNodeError as e:
881 delta = e.tombstone
888 delta = e.tombstone
882 if base == nullrev:
889 if base == nullrev:
883 prefix = mdiff.trivialdiffheader(len(delta))
890 prefix = mdiff.trivialdiffheader(len(delta))
884 else:
891 else:
885 baselen = revlog.rawsize(base)
892 baselen = revlog.rawsize(base)
886 prefix = mdiff.replacediffheader(baselen, len(delta))
893 prefix = mdiff.replacediffheader(baselen, len(delta))
887 elif base == nullrev:
894 elif base == nullrev:
888 delta = revlog.revision(node)
895 delta = revlog.revision(node)
889 prefix = mdiff.trivialdiffheader(len(delta))
896 prefix = mdiff.trivialdiffheader(len(delta))
890 else:
897 else:
891 delta = revlog.revdiff(base, rev)
898 delta = revlog.revdiff(base, rev)
892 p1n, p2n = revlog.parents(node)
899 p1n, p2n = revlog.parents(node)
893 basenode = revlog.node(base)
900 basenode = revlog.node(base)
894 flags = revlog.flags(rev)
901 flags = revlog.flags(rev)
895 meta = self.builddeltaheader(node, p1n, p2n, basenode, linknode, flags)
902 meta = self.builddeltaheader(node, p1n, p2n, basenode, linknode, flags)
896 meta += prefix
903 meta += prefix
897 l = len(meta) + len(delta)
904 l = len(meta) + len(delta)
898 yield chunkheader(l)
905 yield chunkheader(l)
899 yield meta
906 yield meta
900 yield delta
907 yield delta
901 def builddeltaheader(self, node, p1n, p2n, basenode, linknode, flags):
908 def builddeltaheader(self, node, p1n, p2n, basenode, linknode, flags):
902 # do nothing with basenode, it is implicitly the previous one in HG10
909 # do nothing with basenode, it is implicitly the previous one in HG10
903 # do nothing with flags, it is implicitly 0 for cg1 and cg2
910 # do nothing with flags, it is implicitly 0 for cg1 and cg2
904 return struct.pack(self.deltaheader, node, p1n, p2n, linknode)
911 return struct.pack(self.deltaheader, node, p1n, p2n, linknode)
905
912
906 class cg2packer(cg1packer):
913 class cg2packer(cg1packer):
907 version = '02'
914 version = '02'
908 deltaheader = _CHANGEGROUPV2_DELTA_HEADER
915 deltaheader = _CHANGEGROUPV2_DELTA_HEADER
909
916
910 def __init__(self, repo, bundlecaps=None):
917 def __init__(self, repo, bundlecaps=None):
911 super(cg2packer, self).__init__(repo, bundlecaps)
918 super(cg2packer, self).__init__(repo, bundlecaps)
912 if self._reorder is None:
919 if self._reorder is None:
913 # Since generaldelta is directly supported by cg2, reordering
920 # Since generaldelta is directly supported by cg2, reordering
914 # generally doesn't help, so we disable it by default (treating
921 # generally doesn't help, so we disable it by default (treating
915 # bundle.reorder=auto just like bundle.reorder=False).
922 # bundle.reorder=auto just like bundle.reorder=False).
916 self._reorder = False
923 self._reorder = False
917
924
918 def deltaparent(self, revlog, rev, p1, p2, prev):
925 def deltaparent(self, revlog, rev, p1, p2, prev):
919 dp = revlog.deltaparent(rev)
926 dp = revlog.deltaparent(rev)
920 # avoid storing full revisions; pick prev in those cases
927 # avoid storing full revisions; pick prev in those cases
921 # also pick prev when we can't be sure remote has dp
928 # also pick prev when we can't be sure remote has dp
922 if dp == nullrev or (dp != p1 and dp != p2 and dp != prev):
929 if dp == nullrev or (dp != p1 and dp != p2 and dp != prev):
923 return prev
930 return prev
924 return dp
931 return dp
925
932
926 def builddeltaheader(self, node, p1n, p2n, basenode, linknode, flags):
933 def builddeltaheader(self, node, p1n, p2n, basenode, linknode, flags):
927 # Do nothing with flags, it is implicitly 0 in cg1 and cg2
934 # Do nothing with flags, it is implicitly 0 in cg1 and cg2
928 return struct.pack(self.deltaheader, node, p1n, p2n, basenode, linknode)
935 return struct.pack(self.deltaheader, node, p1n, p2n, basenode, linknode)
929
936
930 class cg3packer(cg2packer):
937 class cg3packer(cg2packer):
931 version = '03'
938 version = '03'
932 deltaheader = _CHANGEGROUPV3_DELTA_HEADER
939 deltaheader = _CHANGEGROUPV3_DELTA_HEADER
933
940
934 def _packmanifests(self, dir, mfnodes, lookuplinknode):
941 def _packmanifests(self, dir, mfnodes, lookuplinknode):
935 if dir:
942 if dir:
936 yield self.fileheader(dir)
943 yield self.fileheader(dir)
937 for chunk in self.group(mfnodes, self._repo.manifest.dirlog(dir),
944 for chunk in self.group(mfnodes, self._repo.manifest.dirlog(dir),
938 lookuplinknode, units=_('manifests')):
945 lookuplinknode, units=_('manifests')):
939 yield chunk
946 yield chunk
940
947
941 def _manifestsdone(self):
948 def _manifestsdone(self):
942 return self.close()
949 return self.close()
943
950
944 def builddeltaheader(self, node, p1n, p2n, basenode, linknode, flags):
951 def builddeltaheader(self, node, p1n, p2n, basenode, linknode, flags):
945 return struct.pack(
952 return struct.pack(
946 self.deltaheader, node, p1n, p2n, basenode, linknode, flags)
953 self.deltaheader, node, p1n, p2n, basenode, linknode, flags)
947
954
948 _packermap = {'01': (cg1packer, cg1unpacker),
955 _packermap = {'01': (cg1packer, cg1unpacker),
949 # cg2 adds support for exchanging generaldelta
956 # cg2 adds support for exchanging generaldelta
950 '02': (cg2packer, cg2unpacker),
957 '02': (cg2packer, cg2unpacker),
951 # cg3 adds support for exchanging revlog flags and treemanifests
958 # cg3 adds support for exchanging revlog flags and treemanifests
952 '03': (cg3packer, cg3unpacker),
959 '03': (cg3packer, cg3unpacker),
953 }
960 }
954
961
955 def allsupportedversions(ui):
962 def allsupportedversions(ui):
956 versions = set(_packermap.keys())
963 versions = set(_packermap.keys())
957 versions.discard('03')
964 versions.discard('03')
958 if (ui.configbool('experimental', 'changegroup3') or
965 if (ui.configbool('experimental', 'changegroup3') or
959 ui.configbool('experimental', 'treemanifest')):
966 ui.configbool('experimental', 'treemanifest')):
960 versions.add('03')
967 versions.add('03')
961 return versions
968 return versions
962
969
963 # Changegroup versions that can be applied to the repo
970 # Changegroup versions that can be applied to the repo
964 def supportedincomingversions(repo):
971 def supportedincomingversions(repo):
965 versions = allsupportedversions(repo.ui)
972 versions = allsupportedversions(repo.ui)
966 if 'treemanifest' in repo.requirements:
973 if 'treemanifest' in repo.requirements:
967 versions.add('03')
974 versions.add('03')
968 return versions
975 return versions
969
976
970 # Changegroup versions that can be created from the repo
977 # Changegroup versions that can be created from the repo
971 def supportedoutgoingversions(repo):
978 def supportedoutgoingversions(repo):
972 versions = allsupportedversions(repo.ui)
979 versions = allsupportedversions(repo.ui)
973 if 'treemanifest' in repo.requirements:
980 if 'treemanifest' in repo.requirements:
974 # Versions 01 and 02 support only flat manifests and it's just too
981 # Versions 01 and 02 support only flat manifests and it's just too
975 # expensive to convert between the flat manifest and tree manifest on
982 # expensive to convert between the flat manifest and tree manifest on
976 # the fly. Since tree manifests are hashed differently, all of history
983 # the fly. Since tree manifests are hashed differently, all of history
977 # would have to be converted. Instead, we simply don't even pretend to
984 # would have to be converted. Instead, we simply don't even pretend to
978 # support versions 01 and 02.
985 # support versions 01 and 02.
979 versions.discard('01')
986 versions.discard('01')
980 versions.discard('02')
987 versions.discard('02')
981 versions.add('03')
988 versions.add('03')
982 return versions
989 return versions
983
990
984 def safeversion(repo):
991 def safeversion(repo):
985 # Finds the smallest version that it's safe to assume clients of the repo
992 # Finds the smallest version that it's safe to assume clients of the repo
986 # will support. For example, all hg versions that support generaldelta also
993 # will support. For example, all hg versions that support generaldelta also
987 # support changegroup 02.
994 # support changegroup 02.
988 versions = supportedoutgoingversions(repo)
995 versions = supportedoutgoingversions(repo)
989 if 'generaldelta' in repo.requirements:
996 if 'generaldelta' in repo.requirements:
990 versions.discard('01')
997 versions.discard('01')
991 assert versions
998 assert versions
992 return min(versions)
999 return min(versions)
993
1000
994 def getbundler(version, repo, bundlecaps=None):
1001 def getbundler(version, repo, bundlecaps=None):
995 assert version in supportedoutgoingversions(repo)
1002 assert version in supportedoutgoingversions(repo)
996 return _packermap[version][0](repo, bundlecaps)
1003 return _packermap[version][0](repo, bundlecaps)
997
1004
998 def getunbundler(version, fh, alg):
1005 def getunbundler(version, fh, alg):
999 return _packermap[version][1](fh, alg)
1006 return _packermap[version][1](fh, alg)
1000
1007
1001 def _changegroupinfo(repo, nodes, source):
1008 def _changegroupinfo(repo, nodes, source):
1002 if repo.ui.verbose or source == 'bundle':
1009 if repo.ui.verbose or source == 'bundle':
1003 repo.ui.status(_("%d changesets found\n") % len(nodes))
1010 repo.ui.status(_("%d changesets found\n") % len(nodes))
1004 if repo.ui.debugflag:
1011 if repo.ui.debugflag:
1005 repo.ui.debug("list of changesets:\n")
1012 repo.ui.debug("list of changesets:\n")
1006 for node in nodes:
1013 for node in nodes:
1007 repo.ui.debug("%s\n" % hex(node))
1014 repo.ui.debug("%s\n" % hex(node))
1008
1015
1009 def getsubsetraw(repo, outgoing, bundler, source, fastpath=False):
1016 def getsubsetraw(repo, outgoing, bundler, source, fastpath=False):
1010 repo = repo.unfiltered()
1017 repo = repo.unfiltered()
1011 commonrevs = outgoing.common
1018 commonrevs = outgoing.common
1012 csets = outgoing.missing
1019 csets = outgoing.missing
1013 heads = outgoing.missingheads
1020 heads = outgoing.missingheads
1014 # We go through the fast path if we get told to, or if all (unfiltered
1021 # We go through the fast path if we get told to, or if all (unfiltered
1015 # heads have been requested (since we then know there all linkrevs will
1022 # heads have been requested (since we then know there all linkrevs will
1016 # be pulled by the client).
1023 # be pulled by the client).
1017 heads.sort()
1024 heads.sort()
1018 fastpathlinkrev = fastpath or (
1025 fastpathlinkrev = fastpath or (
1019 repo.filtername is None and heads == sorted(repo.heads()))
1026 repo.filtername is None and heads == sorted(repo.heads()))
1020
1027
1021 repo.hook('preoutgoing', throw=True, source=source)
1028 repo.hook('preoutgoing', throw=True, source=source)
1022 _changegroupinfo(repo, csets, source)
1029 _changegroupinfo(repo, csets, source)
1023 return bundler.generate(commonrevs, csets, fastpathlinkrev, source)
1030 return bundler.generate(commonrevs, csets, fastpathlinkrev, source)
1024
1031
1025 def getsubset(repo, outgoing, bundler, source, fastpath=False):
1032 def getsubset(repo, outgoing, bundler, source, fastpath=False):
1026 gengroup = getsubsetraw(repo, outgoing, bundler, source, fastpath)
1033 gengroup = getsubsetraw(repo, outgoing, bundler, source, fastpath)
1027 return getunbundler(bundler.version, util.chunkbuffer(gengroup), None)
1034 return getunbundler(bundler.version, util.chunkbuffer(gengroup), None)
1028
1035
1029 def changegroupsubset(repo, roots, heads, source, version='01'):
1036 def changegroupsubset(repo, roots, heads, source, version='01'):
1030 """Compute a changegroup consisting of all the nodes that are
1037 """Compute a changegroup consisting of all the nodes that are
1031 descendants of any of the roots and ancestors of any of the heads.
1038 descendants of any of the roots and ancestors of any of the heads.
1032 Return a chunkbuffer object whose read() method will return
1039 Return a chunkbuffer object whose read() method will return
1033 successive changegroup chunks.
1040 successive changegroup chunks.
1034
1041
1035 It is fairly complex as determining which filenodes and which
1042 It is fairly complex as determining which filenodes and which
1036 manifest nodes need to be included for the changeset to be complete
1043 manifest nodes need to be included for the changeset to be complete
1037 is non-trivial.
1044 is non-trivial.
1038
1045
1039 Another wrinkle is doing the reverse, figuring out which changeset in
1046 Another wrinkle is doing the reverse, figuring out which changeset in
1040 the changegroup a particular filenode or manifestnode belongs to.
1047 the changegroup a particular filenode or manifestnode belongs to.
1041 """
1048 """
1042 cl = repo.changelog
1049 cl = repo.changelog
1043 if not roots:
1050 if not roots:
1044 roots = [nullid]
1051 roots = [nullid]
1045 discbases = []
1052 discbases = []
1046 for n in roots:
1053 for n in roots:
1047 discbases.extend([p for p in cl.parents(n) if p != nullid])
1054 discbases.extend([p for p in cl.parents(n) if p != nullid])
1048 # TODO: remove call to nodesbetween.
1055 # TODO: remove call to nodesbetween.
1049 csets, roots, heads = cl.nodesbetween(roots, heads)
1056 csets, roots, heads = cl.nodesbetween(roots, heads)
1050 included = set(csets)
1057 included = set(csets)
1051 discbases = [n for n in discbases if n not in included]
1058 discbases = [n for n in discbases if n not in included]
1052 outgoing = discovery.outgoing(cl, discbases, heads)
1059 outgoing = discovery.outgoing(cl, discbases, heads)
1053 bundler = getbundler(version, repo)
1060 bundler = getbundler(version, repo)
1054 return getsubset(repo, outgoing, bundler, source)
1061 return getsubset(repo, outgoing, bundler, source)
1055
1062
1056 def getlocalchangegroupraw(repo, source, outgoing, bundlecaps=None,
1063 def getlocalchangegroupraw(repo, source, outgoing, bundlecaps=None,
1057 version='01'):
1064 version='01'):
1058 """Like getbundle, but taking a discovery.outgoing as an argument.
1065 """Like getbundle, but taking a discovery.outgoing as an argument.
1059
1066
1060 This is only implemented for local repos and reuses potentially
1067 This is only implemented for local repos and reuses potentially
1061 precomputed sets in outgoing. Returns a raw changegroup generator."""
1068 precomputed sets in outgoing. Returns a raw changegroup generator."""
1062 if not outgoing.missing:
1069 if not outgoing.missing:
1063 return None
1070 return None
1064 bundler = getbundler(version, repo, bundlecaps)
1071 bundler = getbundler(version, repo, bundlecaps)
1065 return getsubsetraw(repo, outgoing, bundler, source)
1072 return getsubsetraw(repo, outgoing, bundler, source)
1066
1073
1067 def getlocalchangegroup(repo, source, outgoing, bundlecaps=None,
1074 def getlocalchangegroup(repo, source, outgoing, bundlecaps=None,
1068 version='01'):
1075 version='01'):
1069 """Like getbundle, but taking a discovery.outgoing as an argument.
1076 """Like getbundle, but taking a discovery.outgoing as an argument.
1070
1077
1071 This is only implemented for local repos and reuses potentially
1078 This is only implemented for local repos and reuses potentially
1072 precomputed sets in outgoing."""
1079 precomputed sets in outgoing."""
1073 if not outgoing.missing:
1080 if not outgoing.missing:
1074 return None
1081 return None
1075 bundler = getbundler(version, repo, bundlecaps)
1082 bundler = getbundler(version, repo, bundlecaps)
1076 return getsubset(repo, outgoing, bundler, source)
1083 return getsubset(repo, outgoing, bundler, source)
1077
1084
1078 def computeoutgoing(repo, heads, common):
1085 def computeoutgoing(repo, heads, common):
1079 """Computes which revs are outgoing given a set of common
1086 """Computes which revs are outgoing given a set of common
1080 and a set of heads.
1087 and a set of heads.
1081
1088
1082 This is a separate function so extensions can have access to
1089 This is a separate function so extensions can have access to
1083 the logic.
1090 the logic.
1084
1091
1085 Returns a discovery.outgoing object.
1092 Returns a discovery.outgoing object.
1086 """
1093 """
1087 cl = repo.changelog
1094 cl = repo.changelog
1088 if common:
1095 if common:
1089 hasnode = cl.hasnode
1096 hasnode = cl.hasnode
1090 common = [n for n in common if hasnode(n)]
1097 common = [n for n in common if hasnode(n)]
1091 else:
1098 else:
1092 common = [nullid]
1099 common = [nullid]
1093 if not heads:
1100 if not heads:
1094 heads = cl.heads()
1101 heads = cl.heads()
1095 return discovery.outgoing(cl, common, heads)
1102 return discovery.outgoing(cl, common, heads)
1096
1103
1097 def getchangegroup(repo, source, heads=None, common=None, bundlecaps=None,
1104 def getchangegroup(repo, source, heads=None, common=None, bundlecaps=None,
1098 version='01'):
1105 version='01'):
1099 """Like changegroupsubset, but returns the set difference between the
1106 """Like changegroupsubset, but returns the set difference between the
1100 ancestors of heads and the ancestors common.
1107 ancestors of heads and the ancestors common.
1101
1108
1102 If heads is None, use the local heads. If common is None, use [nullid].
1109 If heads is None, use the local heads. If common is None, use [nullid].
1103
1110
1104 The nodes in common might not all be known locally due to the way the
1111 The nodes in common might not all be known locally due to the way the
1105 current discovery protocol works.
1112 current discovery protocol works.
1106 """
1113 """
1107 outgoing = computeoutgoing(repo, heads, common)
1114 outgoing = computeoutgoing(repo, heads, common)
1108 return getlocalchangegroup(repo, source, outgoing, bundlecaps=bundlecaps,
1115 return getlocalchangegroup(repo, source, outgoing, bundlecaps=bundlecaps,
1109 version=version)
1116 version=version)
1110
1117
1111 def changegroup(repo, basenodes, source):
1118 def changegroup(repo, basenodes, source):
1112 # to avoid a race we use changegroupsubset() (issue1320)
1119 # to avoid a race we use changegroupsubset() (issue1320)
1113 return changegroupsubset(repo, basenodes, repo.heads(), source)
1120 return changegroupsubset(repo, basenodes, repo.heads(), source)
1114
1121
1115 def _addchangegroupfiles(repo, source, revmap, trp, pr, needfiles):
1122 def _addchangegroupfiles(repo, source, revmap, trp, pr, needfiles):
1116 revisions = 0
1123 revisions = 0
1117 files = 0
1124 files = 0
1118 while True:
1125 while True:
1119 chunkdata = source.filelogheader()
1126 chunkdata = source.filelogheader()
1120 if not chunkdata:
1127 if not chunkdata:
1121 break
1128 break
1122 f = chunkdata["filename"]
1129 f = chunkdata["filename"]
1123 repo.ui.debug("adding %s revisions\n" % f)
1130 repo.ui.debug("adding %s revisions\n" % f)
1124 pr()
1131 pr()
1125 fl = repo.file(f)
1132 fl = repo.file(f)
1126 o = len(fl)
1133 o = len(fl)
1127 try:
1134 try:
1128 if not fl.addgroup(source, revmap, trp):
1135 if not fl.addgroup(source, revmap, trp):
1129 raise error.Abort(_("received file revlog group is empty"))
1136 raise error.Abort(_("received file revlog group is empty"))
1130 except error.CensoredBaseError as e:
1137 except error.CensoredBaseError as e:
1131 raise error.Abort(_("received delta base is censored: %s") % e)
1138 raise error.Abort(_("received delta base is censored: %s") % e)
1132 revisions += len(fl) - o
1139 revisions += len(fl) - o
1133 files += 1
1140 files += 1
1134 if f in needfiles:
1141 if f in needfiles:
1135 needs = needfiles[f]
1142 needs = needfiles[f]
1136 for new in xrange(o, len(fl)):
1143 for new in xrange(o, len(fl)):
1137 n = fl.node(new)
1144 n = fl.node(new)
1138 if n in needs:
1145 if n in needs:
1139 needs.remove(n)
1146 needs.remove(n)
1140 else:
1147 else:
1141 raise error.Abort(
1148 raise error.Abort(
1142 _("received spurious file revlog entry"))
1149 _("received spurious file revlog entry"))
1143 if not needs:
1150 if not needs:
1144 del needfiles[f]
1151 del needfiles[f]
1145 repo.ui.progress(_('files'), None)
1152 repo.ui.progress(_('files'), None)
1146
1153
1147 for f, needs in needfiles.iteritems():
1154 for f, needs in needfiles.iteritems():
1148 fl = repo.file(f)
1155 fl = repo.file(f)
1149 for n in needs:
1156 for n in needs:
1150 try:
1157 try:
1151 fl.rev(n)
1158 fl.rev(n)
1152 except error.LookupError:
1159 except error.LookupError:
1153 raise error.Abort(
1160 raise error.Abort(
1154 _('missing file data for %s:%s - run hg verify') %
1161 _('missing file data for %s:%s - run hg verify') %
1155 (f, hex(n)))
1162 (f, hex(n)))
1156
1163
1157 return revisions, files
1164 return revisions, files
General Comments 0
You need to be logged in to leave comments. Login now