##// END OF EJS Templates
changegroup: call 'prechangegroup' hook before setting up write delay...
Pierre-Yves David -
r26881:6b1ea696 stable
parent child Browse files
Show More
@@ -1,952 +1,952 b''
1 # changegroup.py - Mercurial changegroup manipulation functions
1 # changegroup.py - Mercurial changegroup manipulation functions
2 #
2 #
3 # Copyright 2006 Matt Mackall <mpm@selenic.com>
3 # Copyright 2006 Matt Mackall <mpm@selenic.com>
4 #
4 #
5 # This software may be used and distributed according to the terms of the
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
6 # GNU General Public License version 2 or any later version.
7
7
8 from __future__ import absolute_import
8 from __future__ import absolute_import
9
9
10 import os
10 import os
11 import struct
11 import struct
12 import tempfile
12 import tempfile
13 import weakref
13 import weakref
14
14
15 from .i18n import _
15 from .i18n import _
16 from .node import (
16 from .node import (
17 hex,
17 hex,
18 nullid,
18 nullid,
19 nullrev,
19 nullrev,
20 short,
20 short,
21 )
21 )
22
22
23 from . import (
23 from . import (
24 branchmap,
24 branchmap,
25 dagutil,
25 dagutil,
26 discovery,
26 discovery,
27 error,
27 error,
28 mdiff,
28 mdiff,
29 phases,
29 phases,
30 util,
30 util,
31 )
31 )
32
32
33 _CHANGEGROUPV1_DELTA_HEADER = "20s20s20s20s"
33 _CHANGEGROUPV1_DELTA_HEADER = "20s20s20s20s"
34 _CHANGEGROUPV2_DELTA_HEADER = "20s20s20s20s20s"
34 _CHANGEGROUPV2_DELTA_HEADER = "20s20s20s20s20s"
35
35
36 def readexactly(stream, n):
36 def readexactly(stream, n):
37 '''read n bytes from stream.read and abort if less was available'''
37 '''read n bytes from stream.read and abort if less was available'''
38 s = stream.read(n)
38 s = stream.read(n)
39 if len(s) < n:
39 if len(s) < n:
40 raise error.Abort(_("stream ended unexpectedly"
40 raise error.Abort(_("stream ended unexpectedly"
41 " (got %d bytes, expected %d)")
41 " (got %d bytes, expected %d)")
42 % (len(s), n))
42 % (len(s), n))
43 return s
43 return s
44
44
45 def getchunk(stream):
45 def getchunk(stream):
46 """return the next chunk from stream as a string"""
46 """return the next chunk from stream as a string"""
47 d = readexactly(stream, 4)
47 d = readexactly(stream, 4)
48 l = struct.unpack(">l", d)[0]
48 l = struct.unpack(">l", d)[0]
49 if l <= 4:
49 if l <= 4:
50 if l:
50 if l:
51 raise error.Abort(_("invalid chunk length %d") % l)
51 raise error.Abort(_("invalid chunk length %d") % l)
52 return ""
52 return ""
53 return readexactly(stream, l - 4)
53 return readexactly(stream, l - 4)
54
54
55 def chunkheader(length):
55 def chunkheader(length):
56 """return a changegroup chunk header (string)"""
56 """return a changegroup chunk header (string)"""
57 return struct.pack(">l", length + 4)
57 return struct.pack(">l", length + 4)
58
58
59 def closechunk():
59 def closechunk():
60 """return a changegroup chunk header (string) for a zero-length chunk"""
60 """return a changegroup chunk header (string) for a zero-length chunk"""
61 return struct.pack(">l", 0)
61 return struct.pack(">l", 0)
62
62
63 def combineresults(results):
63 def combineresults(results):
64 """logic to combine 0 or more addchangegroup results into one"""
64 """logic to combine 0 or more addchangegroup results into one"""
65 changedheads = 0
65 changedheads = 0
66 result = 1
66 result = 1
67 for ret in results:
67 for ret in results:
68 # If any changegroup result is 0, return 0
68 # If any changegroup result is 0, return 0
69 if ret == 0:
69 if ret == 0:
70 result = 0
70 result = 0
71 break
71 break
72 if ret < -1:
72 if ret < -1:
73 changedheads += ret + 1
73 changedheads += ret + 1
74 elif ret > 1:
74 elif ret > 1:
75 changedheads += ret - 1
75 changedheads += ret - 1
76 if changedheads > 0:
76 if changedheads > 0:
77 result = 1 + changedheads
77 result = 1 + changedheads
78 elif changedheads < 0:
78 elif changedheads < 0:
79 result = -1 + changedheads
79 result = -1 + changedheads
80 return result
80 return result
81
81
82 bundletypes = {
82 bundletypes = {
83 "": ("", None), # only when using unbundle on ssh and old http servers
83 "": ("", None), # only when using unbundle on ssh and old http servers
84 # since the unification ssh accepts a header but there
84 # since the unification ssh accepts a header but there
85 # is no capability signaling it.
85 # is no capability signaling it.
86 "HG20": (), # special-cased below
86 "HG20": (), # special-cased below
87 "HG10UN": ("HG10UN", None),
87 "HG10UN": ("HG10UN", None),
88 "HG10BZ": ("HG10", 'BZ'),
88 "HG10BZ": ("HG10", 'BZ'),
89 "HG10GZ": ("HG10GZ", 'GZ'),
89 "HG10GZ": ("HG10GZ", 'GZ'),
90 }
90 }
91
91
92 # hgweb uses this list to communicate its preferred type
92 # hgweb uses this list to communicate its preferred type
93 bundlepriority = ['HG10GZ', 'HG10BZ', 'HG10UN']
93 bundlepriority = ['HG10GZ', 'HG10BZ', 'HG10UN']
94
94
95 def writechunks(ui, chunks, filename, vfs=None):
95 def writechunks(ui, chunks, filename, vfs=None):
96 """Write chunks to a file and return its filename.
96 """Write chunks to a file and return its filename.
97
97
98 The stream is assumed to be a bundle file.
98 The stream is assumed to be a bundle file.
99 Existing files will not be overwritten.
99 Existing files will not be overwritten.
100 If no filename is specified, a temporary file is created.
100 If no filename is specified, a temporary file is created.
101 """
101 """
102 fh = None
102 fh = None
103 cleanup = None
103 cleanup = None
104 try:
104 try:
105 if filename:
105 if filename:
106 if vfs:
106 if vfs:
107 fh = vfs.open(filename, "wb")
107 fh = vfs.open(filename, "wb")
108 else:
108 else:
109 fh = open(filename, "wb")
109 fh = open(filename, "wb")
110 else:
110 else:
111 fd, filename = tempfile.mkstemp(prefix="hg-bundle-", suffix=".hg")
111 fd, filename = tempfile.mkstemp(prefix="hg-bundle-", suffix=".hg")
112 fh = os.fdopen(fd, "wb")
112 fh = os.fdopen(fd, "wb")
113 cleanup = filename
113 cleanup = filename
114 for c in chunks:
114 for c in chunks:
115 fh.write(c)
115 fh.write(c)
116 cleanup = None
116 cleanup = None
117 return filename
117 return filename
118 finally:
118 finally:
119 if fh is not None:
119 if fh is not None:
120 fh.close()
120 fh.close()
121 if cleanup is not None:
121 if cleanup is not None:
122 if filename and vfs:
122 if filename and vfs:
123 vfs.unlink(cleanup)
123 vfs.unlink(cleanup)
124 else:
124 else:
125 os.unlink(cleanup)
125 os.unlink(cleanup)
126
126
127 def writebundle(ui, cg, filename, bundletype, vfs=None, compression=None):
127 def writebundle(ui, cg, filename, bundletype, vfs=None, compression=None):
128 """Write a bundle file and return its filename.
128 """Write a bundle file and return its filename.
129
129
130 Existing files will not be overwritten.
130 Existing files will not be overwritten.
131 If no filename is specified, a temporary file is created.
131 If no filename is specified, a temporary file is created.
132 bz2 compression can be turned off.
132 bz2 compression can be turned off.
133 The bundle file will be deleted in case of errors.
133 The bundle file will be deleted in case of errors.
134 """
134 """
135
135
136 if bundletype == "HG20":
136 if bundletype == "HG20":
137 from . import bundle2
137 from . import bundle2
138 bundle = bundle2.bundle20(ui)
138 bundle = bundle2.bundle20(ui)
139 bundle.setcompression(compression)
139 bundle.setcompression(compression)
140 part = bundle.newpart('changegroup', data=cg.getchunks())
140 part = bundle.newpart('changegroup', data=cg.getchunks())
141 part.addparam('version', cg.version)
141 part.addparam('version', cg.version)
142 chunkiter = bundle.getchunks()
142 chunkiter = bundle.getchunks()
143 else:
143 else:
144 # compression argument is only for the bundle2 case
144 # compression argument is only for the bundle2 case
145 assert compression is None
145 assert compression is None
146 if cg.version != '01':
146 if cg.version != '01':
147 raise error.Abort(_('old bundle types only supports v1 '
147 raise error.Abort(_('old bundle types only supports v1 '
148 'changegroups'))
148 'changegroups'))
149 header, comp = bundletypes[bundletype]
149 header, comp = bundletypes[bundletype]
150 if comp not in util.compressors:
150 if comp not in util.compressors:
151 raise error.Abort(_('unknown stream compression type: %s')
151 raise error.Abort(_('unknown stream compression type: %s')
152 % comp)
152 % comp)
153 z = util.compressors[comp]()
153 z = util.compressors[comp]()
154 subchunkiter = cg.getchunks()
154 subchunkiter = cg.getchunks()
155 def chunkiter():
155 def chunkiter():
156 yield header
156 yield header
157 for chunk in subchunkiter:
157 for chunk in subchunkiter:
158 yield z.compress(chunk)
158 yield z.compress(chunk)
159 yield z.flush()
159 yield z.flush()
160 chunkiter = chunkiter()
160 chunkiter = chunkiter()
161
161
162 # parse the changegroup data, otherwise we will block
162 # parse the changegroup data, otherwise we will block
163 # in case of sshrepo because we don't know the end of the stream
163 # in case of sshrepo because we don't know the end of the stream
164
164
165 # an empty chunkgroup is the end of the changegroup
165 # an empty chunkgroup is the end of the changegroup
166 # a changegroup has at least 2 chunkgroups (changelog and manifest).
166 # a changegroup has at least 2 chunkgroups (changelog and manifest).
167 # after that, an empty chunkgroup is the end of the changegroup
167 # after that, an empty chunkgroup is the end of the changegroup
168 return writechunks(ui, chunkiter, filename, vfs=vfs)
168 return writechunks(ui, chunkiter, filename, vfs=vfs)
169
169
170 class cg1unpacker(object):
170 class cg1unpacker(object):
171 """Unpacker for cg1 changegroup streams.
171 """Unpacker for cg1 changegroup streams.
172
172
173 A changegroup unpacker handles the framing of the revision data in
173 A changegroup unpacker handles the framing of the revision data in
174 the wire format. Most consumers will want to use the apply()
174 the wire format. Most consumers will want to use the apply()
175 method to add the changes from the changegroup to a repository.
175 method to add the changes from the changegroup to a repository.
176
176
177 If you're forwarding a changegroup unmodified to another consumer,
177 If you're forwarding a changegroup unmodified to another consumer,
178 use getchunks(), which returns an iterator of changegroup
178 use getchunks(), which returns an iterator of changegroup
179 chunks. This is mostly useful for cases where you need to know the
179 chunks. This is mostly useful for cases where you need to know the
180 data stream has ended by observing the end of the changegroup.
180 data stream has ended by observing the end of the changegroup.
181
181
182 deltachunk() is useful only if you're applying delta data. Most
182 deltachunk() is useful only if you're applying delta data. Most
183 consumers should prefer apply() instead.
183 consumers should prefer apply() instead.
184
184
185 A few other public methods exist. Those are used only for
185 A few other public methods exist. Those are used only for
186 bundlerepo and some debug commands - their use is discouraged.
186 bundlerepo and some debug commands - their use is discouraged.
187 """
187 """
188 deltaheader = _CHANGEGROUPV1_DELTA_HEADER
188 deltaheader = _CHANGEGROUPV1_DELTA_HEADER
189 deltaheadersize = struct.calcsize(deltaheader)
189 deltaheadersize = struct.calcsize(deltaheader)
190 version = '01'
190 version = '01'
191 def __init__(self, fh, alg):
191 def __init__(self, fh, alg):
192 if alg == 'UN':
192 if alg == 'UN':
193 alg = None # get more modern without breaking too much
193 alg = None # get more modern without breaking too much
194 if not alg in util.decompressors:
194 if not alg in util.decompressors:
195 raise error.Abort(_('unknown stream compression type: %s')
195 raise error.Abort(_('unknown stream compression type: %s')
196 % alg)
196 % alg)
197 if alg == 'BZ':
197 if alg == 'BZ':
198 alg = '_truncatedBZ'
198 alg = '_truncatedBZ'
199 self._stream = util.decompressors[alg](fh)
199 self._stream = util.decompressors[alg](fh)
200 self._type = alg
200 self._type = alg
201 self.callback = None
201 self.callback = None
202
202
203 # These methods (compressed, read, seek, tell) all appear to only
203 # These methods (compressed, read, seek, tell) all appear to only
204 # be used by bundlerepo, but it's a little hard to tell.
204 # be used by bundlerepo, but it's a little hard to tell.
205 def compressed(self):
205 def compressed(self):
206 return self._type is not None
206 return self._type is not None
207 def read(self, l):
207 def read(self, l):
208 return self._stream.read(l)
208 return self._stream.read(l)
209 def seek(self, pos):
209 def seek(self, pos):
210 return self._stream.seek(pos)
210 return self._stream.seek(pos)
211 def tell(self):
211 def tell(self):
212 return self._stream.tell()
212 return self._stream.tell()
213 def close(self):
213 def close(self):
214 return self._stream.close()
214 return self._stream.close()
215
215
216 def _chunklength(self):
216 def _chunklength(self):
217 d = readexactly(self._stream, 4)
217 d = readexactly(self._stream, 4)
218 l = struct.unpack(">l", d)[0]
218 l = struct.unpack(">l", d)[0]
219 if l <= 4:
219 if l <= 4:
220 if l:
220 if l:
221 raise error.Abort(_("invalid chunk length %d") % l)
221 raise error.Abort(_("invalid chunk length %d") % l)
222 return 0
222 return 0
223 if self.callback:
223 if self.callback:
224 self.callback()
224 self.callback()
225 return l - 4
225 return l - 4
226
226
227 def changelogheader(self):
227 def changelogheader(self):
228 """v10 does not have a changelog header chunk"""
228 """v10 does not have a changelog header chunk"""
229 return {}
229 return {}
230
230
231 def manifestheader(self):
231 def manifestheader(self):
232 """v10 does not have a manifest header chunk"""
232 """v10 does not have a manifest header chunk"""
233 return {}
233 return {}
234
234
235 def filelogheader(self):
235 def filelogheader(self):
236 """return the header of the filelogs chunk, v10 only has the filename"""
236 """return the header of the filelogs chunk, v10 only has the filename"""
237 l = self._chunklength()
237 l = self._chunklength()
238 if not l:
238 if not l:
239 return {}
239 return {}
240 fname = readexactly(self._stream, l)
240 fname = readexactly(self._stream, l)
241 return {'filename': fname}
241 return {'filename': fname}
242
242
243 def _deltaheader(self, headertuple, prevnode):
243 def _deltaheader(self, headertuple, prevnode):
244 node, p1, p2, cs = headertuple
244 node, p1, p2, cs = headertuple
245 if prevnode is None:
245 if prevnode is None:
246 deltabase = p1
246 deltabase = p1
247 else:
247 else:
248 deltabase = prevnode
248 deltabase = prevnode
249 return node, p1, p2, deltabase, cs
249 return node, p1, p2, deltabase, cs
250
250
251 def deltachunk(self, prevnode):
251 def deltachunk(self, prevnode):
252 l = self._chunklength()
252 l = self._chunklength()
253 if not l:
253 if not l:
254 return {}
254 return {}
255 headerdata = readexactly(self._stream, self.deltaheadersize)
255 headerdata = readexactly(self._stream, self.deltaheadersize)
256 header = struct.unpack(self.deltaheader, headerdata)
256 header = struct.unpack(self.deltaheader, headerdata)
257 delta = readexactly(self._stream, l - self.deltaheadersize)
257 delta = readexactly(self._stream, l - self.deltaheadersize)
258 node, p1, p2, deltabase, cs = self._deltaheader(header, prevnode)
258 node, p1, p2, deltabase, cs = self._deltaheader(header, prevnode)
259 return {'node': node, 'p1': p1, 'p2': p2, 'cs': cs,
259 return {'node': node, 'p1': p1, 'p2': p2, 'cs': cs,
260 'deltabase': deltabase, 'delta': delta}
260 'deltabase': deltabase, 'delta': delta}
261
261
262 def getchunks(self):
262 def getchunks(self):
263 """returns all the chunks contains in the bundle
263 """returns all the chunks contains in the bundle
264
264
265 Used when you need to forward the binary stream to a file or another
265 Used when you need to forward the binary stream to a file or another
266 network API. To do so, it parse the changegroup data, otherwise it will
266 network API. To do so, it parse the changegroup data, otherwise it will
267 block in case of sshrepo because it don't know the end of the stream.
267 block in case of sshrepo because it don't know the end of the stream.
268 """
268 """
269 # an empty chunkgroup is the end of the changegroup
269 # an empty chunkgroup is the end of the changegroup
270 # a changegroup has at least 2 chunkgroups (changelog and manifest).
270 # a changegroup has at least 2 chunkgroups (changelog and manifest).
271 # after that, an empty chunkgroup is the end of the changegroup
271 # after that, an empty chunkgroup is the end of the changegroup
272 empty = False
272 empty = False
273 count = 0
273 count = 0
274 while not empty or count <= 2:
274 while not empty or count <= 2:
275 empty = True
275 empty = True
276 count += 1
276 count += 1
277 while True:
277 while True:
278 chunk = getchunk(self)
278 chunk = getchunk(self)
279 if not chunk:
279 if not chunk:
280 break
280 break
281 empty = False
281 empty = False
282 yield chunkheader(len(chunk))
282 yield chunkheader(len(chunk))
283 pos = 0
283 pos = 0
284 while pos < len(chunk):
284 while pos < len(chunk):
285 next = pos + 2**20
285 next = pos + 2**20
286 yield chunk[pos:next]
286 yield chunk[pos:next]
287 pos = next
287 pos = next
288 yield closechunk()
288 yield closechunk()
289
289
290 def _unpackmanifests(self, repo, revmap, trp, prog, numchanges):
290 def _unpackmanifests(self, repo, revmap, trp, prog, numchanges):
291 # We know that we'll never have more manifests than we had
291 # We know that we'll never have more manifests than we had
292 # changesets.
292 # changesets.
293 self.callback = prog(_('manifests'), numchanges)
293 self.callback = prog(_('manifests'), numchanges)
294 # no need to check for empty manifest group here:
294 # no need to check for empty manifest group here:
295 # if the result of the merge of 1 and 2 is the same in 3 and 4,
295 # if the result of the merge of 1 and 2 is the same in 3 and 4,
296 # no new manifest will be created and the manifest group will
296 # no new manifest will be created and the manifest group will
297 # be empty during the pull
297 # be empty during the pull
298 self.manifestheader()
298 self.manifestheader()
299 repo.manifest.addgroup(self, revmap, trp)
299 repo.manifest.addgroup(self, revmap, trp)
300 repo.ui.progress(_('manifests'), None)
300 repo.ui.progress(_('manifests'), None)
301
301
302 def apply(self, repo, srctype, url, emptyok=False,
302 def apply(self, repo, srctype, url, emptyok=False,
303 targetphase=phases.draft, expectedtotal=None):
303 targetphase=phases.draft, expectedtotal=None):
304 """Add the changegroup returned by source.read() to this repo.
304 """Add the changegroup returned by source.read() to this repo.
305 srctype is a string like 'push', 'pull', or 'unbundle'. url is
305 srctype is a string like 'push', 'pull', or 'unbundle'. url is
306 the URL of the repo where this changegroup is coming from.
306 the URL of the repo where this changegroup is coming from.
307
307
308 Return an integer summarizing the change to this repo:
308 Return an integer summarizing the change to this repo:
309 - nothing changed or no source: 0
309 - nothing changed or no source: 0
310 - more heads than before: 1+added heads (2..n)
310 - more heads than before: 1+added heads (2..n)
311 - fewer heads than before: -1-removed heads (-2..-n)
311 - fewer heads than before: -1-removed heads (-2..-n)
312 - number of heads stays the same: 1
312 - number of heads stays the same: 1
313 """
313 """
314 repo = repo.unfiltered()
314 repo = repo.unfiltered()
315 def csmap(x):
315 def csmap(x):
316 repo.ui.debug("add changeset %s\n" % short(x))
316 repo.ui.debug("add changeset %s\n" % short(x))
317 return len(cl)
317 return len(cl)
318
318
319 def revmap(x):
319 def revmap(x):
320 return cl.rev(x)
320 return cl.rev(x)
321
321
322 changesets = files = revisions = 0
322 changesets = files = revisions = 0
323
323
324 tr = repo.transaction("\n".join([srctype, util.hidepassword(url)]))
324 tr = repo.transaction("\n".join([srctype, util.hidepassword(url)]))
325 try:
325 try:
326 # The transaction could have been created before and already
326 # The transaction could have been created before and already
327 # carries source information. In this case we use the top
327 # carries source information. In this case we use the top
328 # level data. We overwrite the argument because we need to use
328 # level data. We overwrite the argument because we need to use
329 # the top level value (if they exist) in this function.
329 # the top level value (if they exist) in this function.
330 srctype = tr.hookargs.setdefault('source', srctype)
330 srctype = tr.hookargs.setdefault('source', srctype)
331 url = tr.hookargs.setdefault('url', url)
331 url = tr.hookargs.setdefault('url', url)
332 repo.hook('prechangegroup', throw=True, **tr.hookargs)
332
333
333 # write changelog data to temp files so concurrent readers
334 # write changelog data to temp files so concurrent readers
334 # will not see an inconsistent view
335 # will not see an inconsistent view
335 cl = repo.changelog
336 cl = repo.changelog
336 cl.delayupdate(tr)
337 cl.delayupdate(tr)
337 oldheads = cl.heads()
338 oldheads = cl.heads()
338 repo.hook('prechangegroup', throw=True, **tr.hookargs)
339
339
340 trp = weakref.proxy(tr)
340 trp = weakref.proxy(tr)
341 # pull off the changeset group
341 # pull off the changeset group
342 repo.ui.status(_("adding changesets\n"))
342 repo.ui.status(_("adding changesets\n"))
343 clstart = len(cl)
343 clstart = len(cl)
344 class prog(object):
344 class prog(object):
345 def __init__(self, step, total):
345 def __init__(self, step, total):
346 self._step = step
346 self._step = step
347 self._total = total
347 self._total = total
348 self._count = 1
348 self._count = 1
349 def __call__(self):
349 def __call__(self):
350 repo.ui.progress(self._step, self._count, unit=_('chunks'),
350 repo.ui.progress(self._step, self._count, unit=_('chunks'),
351 total=self._total)
351 total=self._total)
352 self._count += 1
352 self._count += 1
353 self.callback = prog(_('changesets'), expectedtotal)
353 self.callback = prog(_('changesets'), expectedtotal)
354
354
355 efiles = set()
355 efiles = set()
356 def onchangelog(cl, node):
356 def onchangelog(cl, node):
357 efiles.update(cl.read(node)[3])
357 efiles.update(cl.read(node)[3])
358
358
359 self.changelogheader()
359 self.changelogheader()
360 srccontent = cl.addgroup(self, csmap, trp,
360 srccontent = cl.addgroup(self, csmap, trp,
361 addrevisioncb=onchangelog)
361 addrevisioncb=onchangelog)
362 efiles = len(efiles)
362 efiles = len(efiles)
363
363
364 if not (srccontent or emptyok):
364 if not (srccontent or emptyok):
365 raise error.Abort(_("received changelog group is empty"))
365 raise error.Abort(_("received changelog group is empty"))
366 clend = len(cl)
366 clend = len(cl)
367 changesets = clend - clstart
367 changesets = clend - clstart
368 repo.ui.progress(_('changesets'), None)
368 repo.ui.progress(_('changesets'), None)
369
369
370 # pull off the manifest group
370 # pull off the manifest group
371 repo.ui.status(_("adding manifests\n"))
371 repo.ui.status(_("adding manifests\n"))
372 self._unpackmanifests(repo, revmap, trp, prog, changesets)
372 self._unpackmanifests(repo, revmap, trp, prog, changesets)
373
373
374 needfiles = {}
374 needfiles = {}
375 if repo.ui.configbool('server', 'validate', default=False):
375 if repo.ui.configbool('server', 'validate', default=False):
376 # validate incoming csets have their manifests
376 # validate incoming csets have their manifests
377 for cset in xrange(clstart, clend):
377 for cset in xrange(clstart, clend):
378 mfnode = repo.changelog.read(repo.changelog.node(cset))[0]
378 mfnode = repo.changelog.read(repo.changelog.node(cset))[0]
379 mfest = repo.manifest.readdelta(mfnode)
379 mfest = repo.manifest.readdelta(mfnode)
380 # store file nodes we must see
380 # store file nodes we must see
381 for f, n in mfest.iteritems():
381 for f, n in mfest.iteritems():
382 needfiles.setdefault(f, set()).add(n)
382 needfiles.setdefault(f, set()).add(n)
383
383
384 # process the files
384 # process the files
385 repo.ui.status(_("adding file changes\n"))
385 repo.ui.status(_("adding file changes\n"))
386 self.callback = None
386 self.callback = None
387 pr = prog(_('files'), efiles)
387 pr = prog(_('files'), efiles)
388 newrevs, newfiles = _addchangegroupfiles(
388 newrevs, newfiles = _addchangegroupfiles(
389 repo, self, revmap, trp, pr, needfiles)
389 repo, self, revmap, trp, pr, needfiles)
390 revisions += newrevs
390 revisions += newrevs
391 files += newfiles
391 files += newfiles
392
392
393 dh = 0
393 dh = 0
394 if oldheads:
394 if oldheads:
395 heads = cl.heads()
395 heads = cl.heads()
396 dh = len(heads) - len(oldheads)
396 dh = len(heads) - len(oldheads)
397 for h in heads:
397 for h in heads:
398 if h not in oldheads and repo[h].closesbranch():
398 if h not in oldheads and repo[h].closesbranch():
399 dh -= 1
399 dh -= 1
400 htext = ""
400 htext = ""
401 if dh:
401 if dh:
402 htext = _(" (%+d heads)") % dh
402 htext = _(" (%+d heads)") % dh
403
403
404 repo.ui.status(_("added %d changesets"
404 repo.ui.status(_("added %d changesets"
405 " with %d changes to %d files%s\n")
405 " with %d changes to %d files%s\n")
406 % (changesets, revisions, files, htext))
406 % (changesets, revisions, files, htext))
407 repo.invalidatevolatilesets()
407 repo.invalidatevolatilesets()
408
408
409 # Call delayupdate again to ensure the transaction writepending
409 # Call delayupdate again to ensure the transaction writepending
410 # subscriptions are still in place.
410 # subscriptions are still in place.
411 cl.delayupdate(tr)
411 cl.delayupdate(tr)
412
412
413 if changesets > 0:
413 if changesets > 0:
414 if 'node' not in tr.hookargs:
414 if 'node' not in tr.hookargs:
415 tr.hookargs['node'] = hex(cl.node(clstart))
415 tr.hookargs['node'] = hex(cl.node(clstart))
416 hookargs = dict(tr.hookargs)
416 hookargs = dict(tr.hookargs)
417 else:
417 else:
418 hookargs = dict(tr.hookargs)
418 hookargs = dict(tr.hookargs)
419 hookargs['node'] = hex(cl.node(clstart))
419 hookargs['node'] = hex(cl.node(clstart))
420 repo.hook('pretxnchangegroup', throw=True, **hookargs)
420 repo.hook('pretxnchangegroup', throw=True, **hookargs)
421
421
422 added = [cl.node(r) for r in xrange(clstart, clend)]
422 added = [cl.node(r) for r in xrange(clstart, clend)]
423 publishing = repo.publishing()
423 publishing = repo.publishing()
424 if srctype in ('push', 'serve'):
424 if srctype in ('push', 'serve'):
425 # Old servers can not push the boundary themselves.
425 # Old servers can not push the boundary themselves.
426 # New servers won't push the boundary if changeset already
426 # New servers won't push the boundary if changeset already
427 # exists locally as secret
427 # exists locally as secret
428 #
428 #
429 # We should not use added here but the list of all change in
429 # We should not use added here but the list of all change in
430 # the bundle
430 # the bundle
431 if publishing:
431 if publishing:
432 phases.advanceboundary(repo, tr, phases.public, srccontent)
432 phases.advanceboundary(repo, tr, phases.public, srccontent)
433 else:
433 else:
434 # Those changesets have been pushed from the outside, their
434 # Those changesets have been pushed from the outside, their
435 # phases are going to be pushed alongside. Therefor
435 # phases are going to be pushed alongside. Therefor
436 # `targetphase` is ignored.
436 # `targetphase` is ignored.
437 phases.advanceboundary(repo, tr, phases.draft, srccontent)
437 phases.advanceboundary(repo, tr, phases.draft, srccontent)
438 phases.retractboundary(repo, tr, phases.draft, added)
438 phases.retractboundary(repo, tr, phases.draft, added)
439 elif srctype != 'strip':
439 elif srctype != 'strip':
440 # publishing only alter behavior during push
440 # publishing only alter behavior during push
441 #
441 #
442 # strip should not touch boundary at all
442 # strip should not touch boundary at all
443 phases.retractboundary(repo, tr, targetphase, added)
443 phases.retractboundary(repo, tr, targetphase, added)
444
444
445 if changesets > 0:
445 if changesets > 0:
446 if srctype != 'strip':
446 if srctype != 'strip':
447 # During strip, branchcache is invalid but coming call to
447 # During strip, branchcache is invalid but coming call to
448 # `destroyed` will repair it.
448 # `destroyed` will repair it.
449 # In other case we can safely update cache on disk.
449 # In other case we can safely update cache on disk.
450 branchmap.updatecache(repo.filtered('served'))
450 branchmap.updatecache(repo.filtered('served'))
451
451
452 def runhooks():
452 def runhooks():
453 # These hooks run when the lock releases, not when the
453 # These hooks run when the lock releases, not when the
454 # transaction closes. So it's possible for the changelog
454 # transaction closes. So it's possible for the changelog
455 # to have changed since we last saw it.
455 # to have changed since we last saw it.
456 if clstart >= len(repo):
456 if clstart >= len(repo):
457 return
457 return
458
458
459 # forcefully update the on-disk branch cache
459 # forcefully update the on-disk branch cache
460 repo.ui.debug("updating the branch cache\n")
460 repo.ui.debug("updating the branch cache\n")
461 repo.hook("changegroup", **hookargs)
461 repo.hook("changegroup", **hookargs)
462
462
463 for n in added:
463 for n in added:
464 args = hookargs.copy()
464 args = hookargs.copy()
465 args['node'] = hex(n)
465 args['node'] = hex(n)
466 repo.hook("incoming", **args)
466 repo.hook("incoming", **args)
467
467
468 newheads = [h for h in repo.heads() if h not in oldheads]
468 newheads = [h for h in repo.heads() if h not in oldheads]
469 repo.ui.log("incoming",
469 repo.ui.log("incoming",
470 "%s incoming changes - new heads: %s\n",
470 "%s incoming changes - new heads: %s\n",
471 len(added),
471 len(added),
472 ', '.join([hex(c[:6]) for c in newheads]))
472 ', '.join([hex(c[:6]) for c in newheads]))
473
473
474 tr.addpostclose('changegroup-runhooks-%020i' % clstart,
474 tr.addpostclose('changegroup-runhooks-%020i' % clstart,
475 lambda tr: repo._afterlock(runhooks))
475 lambda tr: repo._afterlock(runhooks))
476
476
477 tr.close()
477 tr.close()
478
478
479 finally:
479 finally:
480 tr.release()
480 tr.release()
481 repo.ui.flush()
481 repo.ui.flush()
482 # never return 0 here:
482 # never return 0 here:
483 if dh < 0:
483 if dh < 0:
484 return dh - 1
484 return dh - 1
485 else:
485 else:
486 return dh + 1
486 return dh + 1
487
487
488 class cg2unpacker(cg1unpacker):
488 class cg2unpacker(cg1unpacker):
489 """Unpacker for cg2 streams.
489 """Unpacker for cg2 streams.
490
490
491 cg2 streams add support for generaldelta, so the delta header
491 cg2 streams add support for generaldelta, so the delta header
492 format is slightly different. All other features about the data
492 format is slightly different. All other features about the data
493 remain the same.
493 remain the same.
494 """
494 """
495 deltaheader = _CHANGEGROUPV2_DELTA_HEADER
495 deltaheader = _CHANGEGROUPV2_DELTA_HEADER
496 deltaheadersize = struct.calcsize(deltaheader)
496 deltaheadersize = struct.calcsize(deltaheader)
497 version = '02'
497 version = '02'
498
498
499 def _deltaheader(self, headertuple, prevnode):
499 def _deltaheader(self, headertuple, prevnode):
500 node, p1, p2, deltabase, cs = headertuple
500 node, p1, p2, deltabase, cs = headertuple
501 return node, p1, p2, deltabase, cs
501 return node, p1, p2, deltabase, cs
502
502
503 class headerlessfixup(object):
503 class headerlessfixup(object):
504 def __init__(self, fh, h):
504 def __init__(self, fh, h):
505 self._h = h
505 self._h = h
506 self._fh = fh
506 self._fh = fh
507 def read(self, n):
507 def read(self, n):
508 if self._h:
508 if self._h:
509 d, self._h = self._h[:n], self._h[n:]
509 d, self._h = self._h[:n], self._h[n:]
510 if len(d) < n:
510 if len(d) < n:
511 d += readexactly(self._fh, n - len(d))
511 d += readexactly(self._fh, n - len(d))
512 return d
512 return d
513 return readexactly(self._fh, n)
513 return readexactly(self._fh, n)
514
514
515 class cg1packer(object):
515 class cg1packer(object):
516 deltaheader = _CHANGEGROUPV1_DELTA_HEADER
516 deltaheader = _CHANGEGROUPV1_DELTA_HEADER
517 version = '01'
517 version = '01'
518 def __init__(self, repo, bundlecaps=None):
518 def __init__(self, repo, bundlecaps=None):
519 """Given a source repo, construct a bundler.
519 """Given a source repo, construct a bundler.
520
520
521 bundlecaps is optional and can be used to specify the set of
521 bundlecaps is optional and can be used to specify the set of
522 capabilities which can be used to build the bundle.
522 capabilities which can be used to build the bundle.
523 """
523 """
524 # Set of capabilities we can use to build the bundle.
524 # Set of capabilities we can use to build the bundle.
525 if bundlecaps is None:
525 if bundlecaps is None:
526 bundlecaps = set()
526 bundlecaps = set()
527 self._bundlecaps = bundlecaps
527 self._bundlecaps = bundlecaps
528 # experimental config: bundle.reorder
528 # experimental config: bundle.reorder
529 reorder = repo.ui.config('bundle', 'reorder', 'auto')
529 reorder = repo.ui.config('bundle', 'reorder', 'auto')
530 if reorder == 'auto':
530 if reorder == 'auto':
531 reorder = None
531 reorder = None
532 else:
532 else:
533 reorder = util.parsebool(reorder)
533 reorder = util.parsebool(reorder)
534 self._repo = repo
534 self._repo = repo
535 self._reorder = reorder
535 self._reorder = reorder
536 self._progress = repo.ui.progress
536 self._progress = repo.ui.progress
537 if self._repo.ui.verbose and not self._repo.ui.debugflag:
537 if self._repo.ui.verbose and not self._repo.ui.debugflag:
538 self._verbosenote = self._repo.ui.note
538 self._verbosenote = self._repo.ui.note
539 else:
539 else:
540 self._verbosenote = lambda s: None
540 self._verbosenote = lambda s: None
541
541
542 def close(self):
542 def close(self):
543 return closechunk()
543 return closechunk()
544
544
545 def fileheader(self, fname):
545 def fileheader(self, fname):
546 return chunkheader(len(fname)) + fname
546 return chunkheader(len(fname)) + fname
547
547
548 def group(self, nodelist, revlog, lookup, units=None):
548 def group(self, nodelist, revlog, lookup, units=None):
549 """Calculate a delta group, yielding a sequence of changegroup chunks
549 """Calculate a delta group, yielding a sequence of changegroup chunks
550 (strings).
550 (strings).
551
551
552 Given a list of changeset revs, return a set of deltas and
552 Given a list of changeset revs, return a set of deltas and
553 metadata corresponding to nodes. The first delta is
553 metadata corresponding to nodes. The first delta is
554 first parent(nodelist[0]) -> nodelist[0], the receiver is
554 first parent(nodelist[0]) -> nodelist[0], the receiver is
555 guaranteed to have this parent as it has all history before
555 guaranteed to have this parent as it has all history before
556 these changesets. In the case firstparent is nullrev the
556 these changesets. In the case firstparent is nullrev the
557 changegroup starts with a full revision.
557 changegroup starts with a full revision.
558
558
559 If units is not None, progress detail will be generated, units specifies
559 If units is not None, progress detail will be generated, units specifies
560 the type of revlog that is touched (changelog, manifest, etc.).
560 the type of revlog that is touched (changelog, manifest, etc.).
561 """
561 """
562 # if we don't have any revisions touched by these changesets, bail
562 # if we don't have any revisions touched by these changesets, bail
563 if len(nodelist) == 0:
563 if len(nodelist) == 0:
564 yield self.close()
564 yield self.close()
565 return
565 return
566
566
567 # for generaldelta revlogs, we linearize the revs; this will both be
567 # for generaldelta revlogs, we linearize the revs; this will both be
568 # much quicker and generate a much smaller bundle
568 # much quicker and generate a much smaller bundle
569 if (revlog._generaldelta and self._reorder is None) or self._reorder:
569 if (revlog._generaldelta and self._reorder is None) or self._reorder:
570 dag = dagutil.revlogdag(revlog)
570 dag = dagutil.revlogdag(revlog)
571 revs = set(revlog.rev(n) for n in nodelist)
571 revs = set(revlog.rev(n) for n in nodelist)
572 revs = dag.linearize(revs)
572 revs = dag.linearize(revs)
573 else:
573 else:
574 revs = sorted([revlog.rev(n) for n in nodelist])
574 revs = sorted([revlog.rev(n) for n in nodelist])
575
575
576 # add the parent of the first rev
576 # add the parent of the first rev
577 p = revlog.parentrevs(revs[0])[0]
577 p = revlog.parentrevs(revs[0])[0]
578 revs.insert(0, p)
578 revs.insert(0, p)
579
579
580 # build deltas
580 # build deltas
581 total = len(revs) - 1
581 total = len(revs) - 1
582 msgbundling = _('bundling')
582 msgbundling = _('bundling')
583 for r in xrange(len(revs) - 1):
583 for r in xrange(len(revs) - 1):
584 if units is not None:
584 if units is not None:
585 self._progress(msgbundling, r + 1, unit=units, total=total)
585 self._progress(msgbundling, r + 1, unit=units, total=total)
586 prev, curr = revs[r], revs[r + 1]
586 prev, curr = revs[r], revs[r + 1]
587 linknode = lookup(revlog.node(curr))
587 linknode = lookup(revlog.node(curr))
588 for c in self.revchunk(revlog, curr, prev, linknode):
588 for c in self.revchunk(revlog, curr, prev, linknode):
589 yield c
589 yield c
590
590
591 if units is not None:
591 if units is not None:
592 self._progress(msgbundling, None)
592 self._progress(msgbundling, None)
593 yield self.close()
593 yield self.close()
594
594
595 # filter any nodes that claim to be part of the known set
595 # filter any nodes that claim to be part of the known set
596 def prune(self, revlog, missing, commonrevs):
596 def prune(self, revlog, missing, commonrevs):
597 rr, rl = revlog.rev, revlog.linkrev
597 rr, rl = revlog.rev, revlog.linkrev
598 return [n for n in missing if rl(rr(n)) not in commonrevs]
598 return [n for n in missing if rl(rr(n)) not in commonrevs]
599
599
600 def _packmanifests(self, mfnodes, lookuplinknode):
600 def _packmanifests(self, mfnodes, lookuplinknode):
601 """Pack flat manifests into a changegroup stream."""
601 """Pack flat manifests into a changegroup stream."""
602 ml = self._repo.manifest
602 ml = self._repo.manifest
603 size = 0
603 size = 0
604 for chunk in self.group(
604 for chunk in self.group(
605 mfnodes, ml, lookuplinknode, units=_('manifests')):
605 mfnodes, ml, lookuplinknode, units=_('manifests')):
606 size += len(chunk)
606 size += len(chunk)
607 yield chunk
607 yield chunk
608 self._verbosenote(_('%8.i (manifests)\n') % size)
608 self._verbosenote(_('%8.i (manifests)\n') % size)
609
609
610 def generate(self, commonrevs, clnodes, fastpathlinkrev, source):
610 def generate(self, commonrevs, clnodes, fastpathlinkrev, source):
611 '''yield a sequence of changegroup chunks (strings)'''
611 '''yield a sequence of changegroup chunks (strings)'''
612 repo = self._repo
612 repo = self._repo
613 cl = repo.changelog
613 cl = repo.changelog
614 ml = repo.manifest
614 ml = repo.manifest
615
615
616 clrevorder = {}
616 clrevorder = {}
617 mfs = {} # needed manifests
617 mfs = {} # needed manifests
618 fnodes = {} # needed file nodes
618 fnodes = {} # needed file nodes
619 changedfiles = set()
619 changedfiles = set()
620
620
621 # Callback for the changelog, used to collect changed files and manifest
621 # Callback for the changelog, used to collect changed files and manifest
622 # nodes.
622 # nodes.
623 # Returns the linkrev node (identity in the changelog case).
623 # Returns the linkrev node (identity in the changelog case).
624 def lookupcl(x):
624 def lookupcl(x):
625 c = cl.read(x)
625 c = cl.read(x)
626 clrevorder[x] = len(clrevorder)
626 clrevorder[x] = len(clrevorder)
627 changedfiles.update(c[3])
627 changedfiles.update(c[3])
628 # record the first changeset introducing this manifest version
628 # record the first changeset introducing this manifest version
629 mfs.setdefault(c[0], x)
629 mfs.setdefault(c[0], x)
630 return x
630 return x
631
631
632 self._verbosenote(_('uncompressed size of bundle content:\n'))
632 self._verbosenote(_('uncompressed size of bundle content:\n'))
633 size = 0
633 size = 0
634 for chunk in self.group(clnodes, cl, lookupcl, units=_('changesets')):
634 for chunk in self.group(clnodes, cl, lookupcl, units=_('changesets')):
635 size += len(chunk)
635 size += len(chunk)
636 yield chunk
636 yield chunk
637 self._verbosenote(_('%8.i (changelog)\n') % size)
637 self._verbosenote(_('%8.i (changelog)\n') % size)
638
638
639 # We need to make sure that the linkrev in the changegroup refers to
639 # We need to make sure that the linkrev in the changegroup refers to
640 # the first changeset that introduced the manifest or file revision.
640 # the first changeset that introduced the manifest or file revision.
641 # The fastpath is usually safer than the slowpath, because the filelogs
641 # The fastpath is usually safer than the slowpath, because the filelogs
642 # are walked in revlog order.
642 # are walked in revlog order.
643 #
643 #
644 # When taking the slowpath with reorder=None and the manifest revlog
644 # When taking the slowpath with reorder=None and the manifest revlog
645 # uses generaldelta, the manifest may be walked in the "wrong" order.
645 # uses generaldelta, the manifest may be walked in the "wrong" order.
646 # Without 'clrevorder', we would get an incorrect linkrev (see fix in
646 # Without 'clrevorder', we would get an incorrect linkrev (see fix in
647 # cc0ff93d0c0c).
647 # cc0ff93d0c0c).
648 #
648 #
649 # When taking the fastpath, we are only vulnerable to reordering
649 # When taking the fastpath, we are only vulnerable to reordering
650 # of the changelog itself. The changelog never uses generaldelta, so
650 # of the changelog itself. The changelog never uses generaldelta, so
651 # it is only reordered when reorder=True. To handle this case, we
651 # it is only reordered when reorder=True. To handle this case, we
652 # simply take the slowpath, which already has the 'clrevorder' logic.
652 # simply take the slowpath, which already has the 'clrevorder' logic.
653 # This was also fixed in cc0ff93d0c0c.
653 # This was also fixed in cc0ff93d0c0c.
654 fastpathlinkrev = fastpathlinkrev and not self._reorder
654 fastpathlinkrev = fastpathlinkrev and not self._reorder
655 # Callback for the manifest, used to collect linkrevs for filelog
655 # Callback for the manifest, used to collect linkrevs for filelog
656 # revisions.
656 # revisions.
657 # Returns the linkrev node (collected in lookupcl).
657 # Returns the linkrev node (collected in lookupcl).
658 def lookupmflinknode(x):
658 def lookupmflinknode(x):
659 clnode = mfs[x]
659 clnode = mfs[x]
660 if not fastpathlinkrev:
660 if not fastpathlinkrev:
661 mdata = ml.readfast(x)
661 mdata = ml.readfast(x)
662 for f, n in mdata.iteritems():
662 for f, n in mdata.iteritems():
663 if f in changedfiles:
663 if f in changedfiles:
664 # record the first changeset introducing this filelog
664 # record the first changeset introducing this filelog
665 # version
665 # version
666 fclnodes = fnodes.setdefault(f, {})
666 fclnodes = fnodes.setdefault(f, {})
667 fclnode = fclnodes.setdefault(n, clnode)
667 fclnode = fclnodes.setdefault(n, clnode)
668 if clrevorder[clnode] < clrevorder[fclnode]:
668 if clrevorder[clnode] < clrevorder[fclnode]:
669 fclnodes[n] = clnode
669 fclnodes[n] = clnode
670 return clnode
670 return clnode
671
671
672 mfnodes = self.prune(ml, mfs, commonrevs)
672 mfnodes = self.prune(ml, mfs, commonrevs)
673 for x in self._packmanifests(mfnodes, lookupmflinknode):
673 for x in self._packmanifests(mfnodes, lookupmflinknode):
674 yield x
674 yield x
675
675
676 mfs.clear()
676 mfs.clear()
677 clrevs = set(cl.rev(x) for x in clnodes)
677 clrevs = set(cl.rev(x) for x in clnodes)
678
678
679 def linknodes(filerevlog, fname):
679 def linknodes(filerevlog, fname):
680 if fastpathlinkrev:
680 if fastpathlinkrev:
681 llr = filerevlog.linkrev
681 llr = filerevlog.linkrev
682 def genfilenodes():
682 def genfilenodes():
683 for r in filerevlog:
683 for r in filerevlog:
684 linkrev = llr(r)
684 linkrev = llr(r)
685 if linkrev in clrevs:
685 if linkrev in clrevs:
686 yield filerevlog.node(r), cl.node(linkrev)
686 yield filerevlog.node(r), cl.node(linkrev)
687 return dict(genfilenodes())
687 return dict(genfilenodes())
688 return fnodes.get(fname, {})
688 return fnodes.get(fname, {})
689
689
690 for chunk in self.generatefiles(changedfiles, linknodes, commonrevs,
690 for chunk in self.generatefiles(changedfiles, linknodes, commonrevs,
691 source):
691 source):
692 yield chunk
692 yield chunk
693
693
694 yield self.close()
694 yield self.close()
695
695
696 if clnodes:
696 if clnodes:
697 repo.hook('outgoing', node=hex(clnodes[0]), source=source)
697 repo.hook('outgoing', node=hex(clnodes[0]), source=source)
698
698
699 # The 'source' parameter is useful for extensions
699 # The 'source' parameter is useful for extensions
700 def generatefiles(self, changedfiles, linknodes, commonrevs, source):
700 def generatefiles(self, changedfiles, linknodes, commonrevs, source):
701 repo = self._repo
701 repo = self._repo
702 progress = self._progress
702 progress = self._progress
703 msgbundling = _('bundling')
703 msgbundling = _('bundling')
704
704
705 total = len(changedfiles)
705 total = len(changedfiles)
706 # for progress output
706 # for progress output
707 msgfiles = _('files')
707 msgfiles = _('files')
708 for i, fname in enumerate(sorted(changedfiles)):
708 for i, fname in enumerate(sorted(changedfiles)):
709 filerevlog = repo.file(fname)
709 filerevlog = repo.file(fname)
710 if not filerevlog:
710 if not filerevlog:
711 raise error.Abort(_("empty or missing revlog for %s") % fname)
711 raise error.Abort(_("empty or missing revlog for %s") % fname)
712
712
713 linkrevnodes = linknodes(filerevlog, fname)
713 linkrevnodes = linknodes(filerevlog, fname)
714 # Lookup for filenodes, we collected the linkrev nodes above in the
714 # Lookup for filenodes, we collected the linkrev nodes above in the
715 # fastpath case and with lookupmf in the slowpath case.
715 # fastpath case and with lookupmf in the slowpath case.
716 def lookupfilelog(x):
716 def lookupfilelog(x):
717 return linkrevnodes[x]
717 return linkrevnodes[x]
718
718
719 filenodes = self.prune(filerevlog, linkrevnodes, commonrevs)
719 filenodes = self.prune(filerevlog, linkrevnodes, commonrevs)
720 if filenodes:
720 if filenodes:
721 progress(msgbundling, i + 1, item=fname, unit=msgfiles,
721 progress(msgbundling, i + 1, item=fname, unit=msgfiles,
722 total=total)
722 total=total)
723 h = self.fileheader(fname)
723 h = self.fileheader(fname)
724 size = len(h)
724 size = len(h)
725 yield h
725 yield h
726 for chunk in self.group(filenodes, filerevlog, lookupfilelog):
726 for chunk in self.group(filenodes, filerevlog, lookupfilelog):
727 size += len(chunk)
727 size += len(chunk)
728 yield chunk
728 yield chunk
729 self._verbosenote(_('%8.i %s\n') % (size, fname))
729 self._verbosenote(_('%8.i %s\n') % (size, fname))
730 progress(msgbundling, None)
730 progress(msgbundling, None)
731
731
732 def deltaparent(self, revlog, rev, p1, p2, prev):
732 def deltaparent(self, revlog, rev, p1, p2, prev):
733 return prev
733 return prev
734
734
735 def revchunk(self, revlog, rev, prev, linknode):
735 def revchunk(self, revlog, rev, prev, linknode):
736 node = revlog.node(rev)
736 node = revlog.node(rev)
737 p1, p2 = revlog.parentrevs(rev)
737 p1, p2 = revlog.parentrevs(rev)
738 base = self.deltaparent(revlog, rev, p1, p2, prev)
738 base = self.deltaparent(revlog, rev, p1, p2, prev)
739
739
740 prefix = ''
740 prefix = ''
741 if revlog.iscensored(base) or revlog.iscensored(rev):
741 if revlog.iscensored(base) or revlog.iscensored(rev):
742 try:
742 try:
743 delta = revlog.revision(node)
743 delta = revlog.revision(node)
744 except error.CensoredNodeError as e:
744 except error.CensoredNodeError as e:
745 delta = e.tombstone
745 delta = e.tombstone
746 if base == nullrev:
746 if base == nullrev:
747 prefix = mdiff.trivialdiffheader(len(delta))
747 prefix = mdiff.trivialdiffheader(len(delta))
748 else:
748 else:
749 baselen = revlog.rawsize(base)
749 baselen = revlog.rawsize(base)
750 prefix = mdiff.replacediffheader(baselen, len(delta))
750 prefix = mdiff.replacediffheader(baselen, len(delta))
751 elif base == nullrev:
751 elif base == nullrev:
752 delta = revlog.revision(node)
752 delta = revlog.revision(node)
753 prefix = mdiff.trivialdiffheader(len(delta))
753 prefix = mdiff.trivialdiffheader(len(delta))
754 else:
754 else:
755 delta = revlog.revdiff(base, rev)
755 delta = revlog.revdiff(base, rev)
756 p1n, p2n = revlog.parents(node)
756 p1n, p2n = revlog.parents(node)
757 basenode = revlog.node(base)
757 basenode = revlog.node(base)
758 meta = self.builddeltaheader(node, p1n, p2n, basenode, linknode)
758 meta = self.builddeltaheader(node, p1n, p2n, basenode, linknode)
759 meta += prefix
759 meta += prefix
760 l = len(meta) + len(delta)
760 l = len(meta) + len(delta)
761 yield chunkheader(l)
761 yield chunkheader(l)
762 yield meta
762 yield meta
763 yield delta
763 yield delta
764 def builddeltaheader(self, node, p1n, p2n, basenode, linknode):
764 def builddeltaheader(self, node, p1n, p2n, basenode, linknode):
765 # do nothing with basenode, it is implicitly the previous one in HG10
765 # do nothing with basenode, it is implicitly the previous one in HG10
766 return struct.pack(self.deltaheader, node, p1n, p2n, linknode)
766 return struct.pack(self.deltaheader, node, p1n, p2n, linknode)
767
767
768 class cg2packer(cg1packer):
768 class cg2packer(cg1packer):
769 version = '02'
769 version = '02'
770 deltaheader = _CHANGEGROUPV2_DELTA_HEADER
770 deltaheader = _CHANGEGROUPV2_DELTA_HEADER
771
771
772 def __init__(self, repo, bundlecaps=None):
772 def __init__(self, repo, bundlecaps=None):
773 super(cg2packer, self).__init__(repo, bundlecaps)
773 super(cg2packer, self).__init__(repo, bundlecaps)
774 if self._reorder is None:
774 if self._reorder is None:
775 # Since generaldelta is directly supported by cg2, reordering
775 # Since generaldelta is directly supported by cg2, reordering
776 # generally doesn't help, so we disable it by default (treating
776 # generally doesn't help, so we disable it by default (treating
777 # bundle.reorder=auto just like bundle.reorder=False).
777 # bundle.reorder=auto just like bundle.reorder=False).
778 self._reorder = False
778 self._reorder = False
779
779
780 def deltaparent(self, revlog, rev, p1, p2, prev):
780 def deltaparent(self, revlog, rev, p1, p2, prev):
781 dp = revlog.deltaparent(rev)
781 dp = revlog.deltaparent(rev)
782 # avoid storing full revisions; pick prev in those cases
782 # avoid storing full revisions; pick prev in those cases
783 # also pick prev when we can't be sure remote has dp
783 # also pick prev when we can't be sure remote has dp
784 if dp == nullrev or (dp != p1 and dp != p2 and dp != prev):
784 if dp == nullrev or (dp != p1 and dp != p2 and dp != prev):
785 return prev
785 return prev
786 return dp
786 return dp
787
787
788 def builddeltaheader(self, node, p1n, p2n, basenode, linknode):
788 def builddeltaheader(self, node, p1n, p2n, basenode, linknode):
789 return struct.pack(self.deltaheader, node, p1n, p2n, basenode, linknode)
789 return struct.pack(self.deltaheader, node, p1n, p2n, basenode, linknode)
790
790
791 packermap = {'01': (cg1packer, cg1unpacker),
791 packermap = {'01': (cg1packer, cg1unpacker),
792 # cg2 adds support for exchanging generaldelta
792 # cg2 adds support for exchanging generaldelta
793 '02': (cg2packer, cg2unpacker),
793 '02': (cg2packer, cg2unpacker),
794 }
794 }
795
795
796 def _changegroupinfo(repo, nodes, source):
796 def _changegroupinfo(repo, nodes, source):
797 if repo.ui.verbose or source == 'bundle':
797 if repo.ui.verbose or source == 'bundle':
798 repo.ui.status(_("%d changesets found\n") % len(nodes))
798 repo.ui.status(_("%d changesets found\n") % len(nodes))
799 if repo.ui.debugflag:
799 if repo.ui.debugflag:
800 repo.ui.debug("list of changesets:\n")
800 repo.ui.debug("list of changesets:\n")
801 for node in nodes:
801 for node in nodes:
802 repo.ui.debug("%s\n" % hex(node))
802 repo.ui.debug("%s\n" % hex(node))
803
803
804 def getsubsetraw(repo, outgoing, bundler, source, fastpath=False):
804 def getsubsetraw(repo, outgoing, bundler, source, fastpath=False):
805 repo = repo.unfiltered()
805 repo = repo.unfiltered()
806 commonrevs = outgoing.common
806 commonrevs = outgoing.common
807 csets = outgoing.missing
807 csets = outgoing.missing
808 heads = outgoing.missingheads
808 heads = outgoing.missingheads
809 # We go through the fast path if we get told to, or if all (unfiltered
809 # We go through the fast path if we get told to, or if all (unfiltered
810 # heads have been requested (since we then know there all linkrevs will
810 # heads have been requested (since we then know there all linkrevs will
811 # be pulled by the client).
811 # be pulled by the client).
812 heads.sort()
812 heads.sort()
813 fastpathlinkrev = fastpath or (
813 fastpathlinkrev = fastpath or (
814 repo.filtername is None and heads == sorted(repo.heads()))
814 repo.filtername is None and heads == sorted(repo.heads()))
815
815
816 repo.hook('preoutgoing', throw=True, source=source)
816 repo.hook('preoutgoing', throw=True, source=source)
817 _changegroupinfo(repo, csets, source)
817 _changegroupinfo(repo, csets, source)
818 return bundler.generate(commonrevs, csets, fastpathlinkrev, source)
818 return bundler.generate(commonrevs, csets, fastpathlinkrev, source)
819
819
820 def getsubset(repo, outgoing, bundler, source, fastpath=False):
820 def getsubset(repo, outgoing, bundler, source, fastpath=False):
821 gengroup = getsubsetraw(repo, outgoing, bundler, source, fastpath)
821 gengroup = getsubsetraw(repo, outgoing, bundler, source, fastpath)
822 return packermap[bundler.version][1](util.chunkbuffer(gengroup), None)
822 return packermap[bundler.version][1](util.chunkbuffer(gengroup), None)
823
823
824 def changegroupsubset(repo, roots, heads, source, version='01'):
824 def changegroupsubset(repo, roots, heads, source, version='01'):
825 """Compute a changegroup consisting of all the nodes that are
825 """Compute a changegroup consisting of all the nodes that are
826 descendants of any of the roots and ancestors of any of the heads.
826 descendants of any of the roots and ancestors of any of the heads.
827 Return a chunkbuffer object whose read() method will return
827 Return a chunkbuffer object whose read() method will return
828 successive changegroup chunks.
828 successive changegroup chunks.
829
829
830 It is fairly complex as determining which filenodes and which
830 It is fairly complex as determining which filenodes and which
831 manifest nodes need to be included for the changeset to be complete
831 manifest nodes need to be included for the changeset to be complete
832 is non-trivial.
832 is non-trivial.
833
833
834 Another wrinkle is doing the reverse, figuring out which changeset in
834 Another wrinkle is doing the reverse, figuring out which changeset in
835 the changegroup a particular filenode or manifestnode belongs to.
835 the changegroup a particular filenode or manifestnode belongs to.
836 """
836 """
837 cl = repo.changelog
837 cl = repo.changelog
838 if not roots:
838 if not roots:
839 roots = [nullid]
839 roots = [nullid]
840 discbases = []
840 discbases = []
841 for n in roots:
841 for n in roots:
842 discbases.extend([p for p in cl.parents(n) if p != nullid])
842 discbases.extend([p for p in cl.parents(n) if p != nullid])
843 # TODO: remove call to nodesbetween.
843 # TODO: remove call to nodesbetween.
844 csets, roots, heads = cl.nodesbetween(roots, heads)
844 csets, roots, heads = cl.nodesbetween(roots, heads)
845 included = set(csets)
845 included = set(csets)
846 discbases = [n for n in discbases if n not in included]
846 discbases = [n for n in discbases if n not in included]
847 outgoing = discovery.outgoing(cl, discbases, heads)
847 outgoing = discovery.outgoing(cl, discbases, heads)
848 bundler = packermap[version][0](repo)
848 bundler = packermap[version][0](repo)
849 return getsubset(repo, outgoing, bundler, source)
849 return getsubset(repo, outgoing, bundler, source)
850
850
851 def getlocalchangegroupraw(repo, source, outgoing, bundlecaps=None,
851 def getlocalchangegroupraw(repo, source, outgoing, bundlecaps=None,
852 version='01'):
852 version='01'):
853 """Like getbundle, but taking a discovery.outgoing as an argument.
853 """Like getbundle, but taking a discovery.outgoing as an argument.
854
854
855 This is only implemented for local repos and reuses potentially
855 This is only implemented for local repos and reuses potentially
856 precomputed sets in outgoing. Returns a raw changegroup generator."""
856 precomputed sets in outgoing. Returns a raw changegroup generator."""
857 if not outgoing.missing:
857 if not outgoing.missing:
858 return None
858 return None
859 bundler = packermap[version][0](repo, bundlecaps)
859 bundler = packermap[version][0](repo, bundlecaps)
860 return getsubsetraw(repo, outgoing, bundler, source)
860 return getsubsetraw(repo, outgoing, bundler, source)
861
861
862 def getlocalchangegroup(repo, source, outgoing, bundlecaps=None,
862 def getlocalchangegroup(repo, source, outgoing, bundlecaps=None,
863 version='01'):
863 version='01'):
864 """Like getbundle, but taking a discovery.outgoing as an argument.
864 """Like getbundle, but taking a discovery.outgoing as an argument.
865
865
866 This is only implemented for local repos and reuses potentially
866 This is only implemented for local repos and reuses potentially
867 precomputed sets in outgoing."""
867 precomputed sets in outgoing."""
868 if not outgoing.missing:
868 if not outgoing.missing:
869 return None
869 return None
870 bundler = packermap[version][0](repo, bundlecaps)
870 bundler = packermap[version][0](repo, bundlecaps)
871 return getsubset(repo, outgoing, bundler, source)
871 return getsubset(repo, outgoing, bundler, source)
872
872
873 def computeoutgoing(repo, heads, common):
873 def computeoutgoing(repo, heads, common):
874 """Computes which revs are outgoing given a set of common
874 """Computes which revs are outgoing given a set of common
875 and a set of heads.
875 and a set of heads.
876
876
877 This is a separate function so extensions can have access to
877 This is a separate function so extensions can have access to
878 the logic.
878 the logic.
879
879
880 Returns a discovery.outgoing object.
880 Returns a discovery.outgoing object.
881 """
881 """
882 cl = repo.changelog
882 cl = repo.changelog
883 if common:
883 if common:
884 hasnode = cl.hasnode
884 hasnode = cl.hasnode
885 common = [n for n in common if hasnode(n)]
885 common = [n for n in common if hasnode(n)]
886 else:
886 else:
887 common = [nullid]
887 common = [nullid]
888 if not heads:
888 if not heads:
889 heads = cl.heads()
889 heads = cl.heads()
890 return discovery.outgoing(cl, common, heads)
890 return discovery.outgoing(cl, common, heads)
891
891
892 def getchangegroup(repo, source, heads=None, common=None, bundlecaps=None,
892 def getchangegroup(repo, source, heads=None, common=None, bundlecaps=None,
893 version='01'):
893 version='01'):
894 """Like changegroupsubset, but returns the set difference between the
894 """Like changegroupsubset, but returns the set difference between the
895 ancestors of heads and the ancestors common.
895 ancestors of heads and the ancestors common.
896
896
897 If heads is None, use the local heads. If common is None, use [nullid].
897 If heads is None, use the local heads. If common is None, use [nullid].
898
898
899 The nodes in common might not all be known locally due to the way the
899 The nodes in common might not all be known locally due to the way the
900 current discovery protocol works.
900 current discovery protocol works.
901 """
901 """
902 outgoing = computeoutgoing(repo, heads, common)
902 outgoing = computeoutgoing(repo, heads, common)
903 return getlocalchangegroup(repo, source, outgoing, bundlecaps=bundlecaps,
903 return getlocalchangegroup(repo, source, outgoing, bundlecaps=bundlecaps,
904 version=version)
904 version=version)
905
905
906 def changegroup(repo, basenodes, source):
906 def changegroup(repo, basenodes, source):
907 # to avoid a race we use changegroupsubset() (issue1320)
907 # to avoid a race we use changegroupsubset() (issue1320)
908 return changegroupsubset(repo, basenodes, repo.heads(), source)
908 return changegroupsubset(repo, basenodes, repo.heads(), source)
909
909
910 def _addchangegroupfiles(repo, source, revmap, trp, pr, needfiles):
910 def _addchangegroupfiles(repo, source, revmap, trp, pr, needfiles):
911 revisions = 0
911 revisions = 0
912 files = 0
912 files = 0
913 while True:
913 while True:
914 chunkdata = source.filelogheader()
914 chunkdata = source.filelogheader()
915 if not chunkdata:
915 if not chunkdata:
916 break
916 break
917 f = chunkdata["filename"]
917 f = chunkdata["filename"]
918 repo.ui.debug("adding %s revisions\n" % f)
918 repo.ui.debug("adding %s revisions\n" % f)
919 pr()
919 pr()
920 fl = repo.file(f)
920 fl = repo.file(f)
921 o = len(fl)
921 o = len(fl)
922 try:
922 try:
923 if not fl.addgroup(source, revmap, trp):
923 if not fl.addgroup(source, revmap, trp):
924 raise error.Abort(_("received file revlog group is empty"))
924 raise error.Abort(_("received file revlog group is empty"))
925 except error.CensoredBaseError as e:
925 except error.CensoredBaseError as e:
926 raise error.Abort(_("received delta base is censored: %s") % e)
926 raise error.Abort(_("received delta base is censored: %s") % e)
927 revisions += len(fl) - o
927 revisions += len(fl) - o
928 files += 1
928 files += 1
929 if f in needfiles:
929 if f in needfiles:
930 needs = needfiles[f]
930 needs = needfiles[f]
931 for new in xrange(o, len(fl)):
931 for new in xrange(o, len(fl)):
932 n = fl.node(new)
932 n = fl.node(new)
933 if n in needs:
933 if n in needs:
934 needs.remove(n)
934 needs.remove(n)
935 else:
935 else:
936 raise error.Abort(
936 raise error.Abort(
937 _("received spurious file revlog entry"))
937 _("received spurious file revlog entry"))
938 if not needs:
938 if not needs:
939 del needfiles[f]
939 del needfiles[f]
940 repo.ui.progress(_('files'), None)
940 repo.ui.progress(_('files'), None)
941
941
942 for f, needs in needfiles.iteritems():
942 for f, needs in needfiles.iteritems():
943 fl = repo.file(f)
943 fl = repo.file(f)
944 for n in needs:
944 for n in needs:
945 try:
945 try:
946 fl.rev(n)
946 fl.rev(n)
947 except error.LookupError:
947 except error.LookupError:
948 raise error.Abort(
948 raise error.Abort(
949 _('missing file data for %s:%s - run hg verify') %
949 _('missing file data for %s:%s - run hg verify') %
950 (f, hex(n)))
950 (f, hex(n)))
951
951
952 return revisions, files
952 return revisions, files
General Comments 0
You need to be logged in to leave comments. Login now