Show More
@@ -0,0 +1,422 b'' | |||||
|
1 | # bundlecaches.py - utility to deal with pre-computed bundle for servers | |||
|
2 | # | |||
|
3 | # This software may be used and distributed according to the terms of the | |||
|
4 | # GNU General Public License version 2 or any later version. | |||
|
5 | ||||
|
6 | from .i18n import _ | |||
|
7 | ||||
|
8 | from .thirdparty import attr | |||
|
9 | ||||
|
10 | from . import ( | |||
|
11 | error, | |||
|
12 | sslutil, | |||
|
13 | util, | |||
|
14 | ) | |||
|
15 | from .utils import stringutil | |||
|
16 | ||||
|
17 | urlreq = util.urlreq | |||
|
18 | ||||
|
19 | ||||
|
20 | @attr.s | |||
|
21 | class bundlespec(object): | |||
|
22 | compression = attr.ib() | |||
|
23 | wirecompression = attr.ib() | |||
|
24 | version = attr.ib() | |||
|
25 | wireversion = attr.ib() | |||
|
26 | params = attr.ib() | |||
|
27 | contentopts = attr.ib() | |||
|
28 | ||||
|
29 | ||||
|
30 | # Maps bundle version human names to changegroup versions. | |||
|
31 | _bundlespeccgversions = { | |||
|
32 | b'v1': b'01', | |||
|
33 | b'v2': b'02', | |||
|
34 | b'packed1': b's1', | |||
|
35 | b'bundle2': b'02', # legacy | |||
|
36 | } | |||
|
37 | ||||
|
38 | # Maps bundle version with content opts to choose which part to bundle | |||
|
39 | _bundlespeccontentopts = { | |||
|
40 | b'v1': { | |||
|
41 | b'changegroup': True, | |||
|
42 | b'cg.version': b'01', | |||
|
43 | b'obsolescence': False, | |||
|
44 | b'phases': False, | |||
|
45 | b'tagsfnodescache': False, | |||
|
46 | b'revbranchcache': False, | |||
|
47 | }, | |||
|
48 | b'v2': { | |||
|
49 | b'changegroup': True, | |||
|
50 | b'cg.version': b'02', | |||
|
51 | b'obsolescence': False, | |||
|
52 | b'phases': False, | |||
|
53 | b'tagsfnodescache': True, | |||
|
54 | b'revbranchcache': True, | |||
|
55 | }, | |||
|
56 | b'packed1': {b'cg.version': b's1'}, | |||
|
57 | } | |||
|
58 | _bundlespeccontentopts[b'bundle2'] = _bundlespeccontentopts[b'v2'] | |||
|
59 | ||||
|
60 | _bundlespecvariants = { | |||
|
61 | b"streamv2": { | |||
|
62 | b"changegroup": False, | |||
|
63 | b"streamv2": True, | |||
|
64 | b"tagsfnodescache": False, | |||
|
65 | b"revbranchcache": False, | |||
|
66 | } | |||
|
67 | } | |||
|
68 | ||||
|
69 | # Compression engines allowed in version 1. THIS SHOULD NEVER CHANGE. | |||
|
70 | _bundlespecv1compengines = {b'gzip', b'bzip2', b'none'} | |||
|
71 | ||||
|
72 | ||||
|
73 | def parsebundlespec(repo, spec, strict=True): | |||
|
74 | """Parse a bundle string specification into parts. | |||
|
75 | ||||
|
76 | Bundle specifications denote a well-defined bundle/exchange format. | |||
|
77 | The content of a given specification should not change over time in | |||
|
78 | order to ensure that bundles produced by a newer version of Mercurial are | |||
|
79 | readable from an older version. | |||
|
80 | ||||
|
81 | The string currently has the form: | |||
|
82 | ||||
|
83 | <compression>-<type>[;<parameter0>[;<parameter1>]] | |||
|
84 | ||||
|
85 | Where <compression> is one of the supported compression formats | |||
|
86 | and <type> is (currently) a version string. A ";" can follow the type and | |||
|
87 | all text afterwards is interpreted as URI encoded, ";" delimited key=value | |||
|
88 | pairs. | |||
|
89 | ||||
|
90 | If ``strict`` is True (the default) <compression> is required. Otherwise, | |||
|
91 | it is optional. | |||
|
92 | ||||
|
93 | Returns a bundlespec object of (compression, version, parameters). | |||
|
94 | Compression will be ``None`` if not in strict mode and a compression isn't | |||
|
95 | defined. | |||
|
96 | ||||
|
97 | An ``InvalidBundleSpecification`` is raised when the specification is | |||
|
98 | not syntactically well formed. | |||
|
99 | ||||
|
100 | An ``UnsupportedBundleSpecification`` is raised when the compression or | |||
|
101 | bundle type/version is not recognized. | |||
|
102 | ||||
|
103 | Note: this function will likely eventually return a more complex data | |||
|
104 | structure, including bundle2 part information. | |||
|
105 | """ | |||
|
106 | ||||
|
107 | def parseparams(s): | |||
|
108 | if b';' not in s: | |||
|
109 | return s, {} | |||
|
110 | ||||
|
111 | params = {} | |||
|
112 | version, paramstr = s.split(b';', 1) | |||
|
113 | ||||
|
114 | for p in paramstr.split(b';'): | |||
|
115 | if b'=' not in p: | |||
|
116 | raise error.InvalidBundleSpecification( | |||
|
117 | _( | |||
|
118 | b'invalid bundle specification: ' | |||
|
119 | b'missing "=" in parameter: %s' | |||
|
120 | ) | |||
|
121 | % p | |||
|
122 | ) | |||
|
123 | ||||
|
124 | key, value = p.split(b'=', 1) | |||
|
125 | key = urlreq.unquote(key) | |||
|
126 | value = urlreq.unquote(value) | |||
|
127 | params[key] = value | |||
|
128 | ||||
|
129 | return version, params | |||
|
130 | ||||
|
131 | if strict and b'-' not in spec: | |||
|
132 | raise error.InvalidBundleSpecification( | |||
|
133 | _( | |||
|
134 | b'invalid bundle specification; ' | |||
|
135 | b'must be prefixed with compression: %s' | |||
|
136 | ) | |||
|
137 | % spec | |||
|
138 | ) | |||
|
139 | ||||
|
140 | if b'-' in spec: | |||
|
141 | compression, version = spec.split(b'-', 1) | |||
|
142 | ||||
|
143 | if compression not in util.compengines.supportedbundlenames: | |||
|
144 | raise error.UnsupportedBundleSpecification( | |||
|
145 | _(b'%s compression is not supported') % compression | |||
|
146 | ) | |||
|
147 | ||||
|
148 | version, params = parseparams(version) | |||
|
149 | ||||
|
150 | if version not in _bundlespeccgversions: | |||
|
151 | raise error.UnsupportedBundleSpecification( | |||
|
152 | _(b'%s is not a recognized bundle version') % version | |||
|
153 | ) | |||
|
154 | else: | |||
|
155 | # Value could be just the compression or just the version, in which | |||
|
156 | # case some defaults are assumed (but only when not in strict mode). | |||
|
157 | assert not strict | |||
|
158 | ||||
|
159 | spec, params = parseparams(spec) | |||
|
160 | ||||
|
161 | if spec in util.compengines.supportedbundlenames: | |||
|
162 | compression = spec | |||
|
163 | version = b'v1' | |||
|
164 | # Generaldelta repos require v2. | |||
|
165 | if b'generaldelta' in repo.requirements: | |||
|
166 | version = b'v2' | |||
|
167 | # Modern compression engines require v2. | |||
|
168 | if compression not in _bundlespecv1compengines: | |||
|
169 | version = b'v2' | |||
|
170 | elif spec in _bundlespeccgversions: | |||
|
171 | if spec == b'packed1': | |||
|
172 | compression = b'none' | |||
|
173 | else: | |||
|
174 | compression = b'bzip2' | |||
|
175 | version = spec | |||
|
176 | else: | |||
|
177 | raise error.UnsupportedBundleSpecification( | |||
|
178 | _(b'%s is not a recognized bundle specification') % spec | |||
|
179 | ) | |||
|
180 | ||||
|
181 | # Bundle version 1 only supports a known set of compression engines. | |||
|
182 | if version == b'v1' and compression not in _bundlespecv1compengines: | |||
|
183 | raise error.UnsupportedBundleSpecification( | |||
|
184 | _(b'compression engine %s is not supported on v1 bundles') | |||
|
185 | % compression | |||
|
186 | ) | |||
|
187 | ||||
|
188 | # The specification for packed1 can optionally declare the data formats | |||
|
189 | # required to apply it. If we see this metadata, compare against what the | |||
|
190 | # repo supports and error if the bundle isn't compatible. | |||
|
191 | if version == b'packed1' and b'requirements' in params: | |||
|
192 | requirements = set(params[b'requirements'].split(b',')) | |||
|
193 | missingreqs = requirements - repo.supportedformats | |||
|
194 | if missingreqs: | |||
|
195 | raise error.UnsupportedBundleSpecification( | |||
|
196 | _(b'missing support for repository features: %s') | |||
|
197 | % b', '.join(sorted(missingreqs)) | |||
|
198 | ) | |||
|
199 | ||||
|
200 | # Compute contentopts based on the version | |||
|
201 | contentopts = _bundlespeccontentopts.get(version, {}).copy() | |||
|
202 | ||||
|
203 | # Process the variants | |||
|
204 | if b"stream" in params and params[b"stream"] == b"v2": | |||
|
205 | variant = _bundlespecvariants[b"streamv2"] | |||
|
206 | contentopts.update(variant) | |||
|
207 | ||||
|
208 | engine = util.compengines.forbundlename(compression) | |||
|
209 | compression, wirecompression = engine.bundletype() | |||
|
210 | wireversion = _bundlespeccgversions[version] | |||
|
211 | ||||
|
212 | return bundlespec( | |||
|
213 | compression, wirecompression, version, wireversion, params, contentopts | |||
|
214 | ) | |||
|
215 | ||||
|
216 | ||||
|
217 | def parseclonebundlesmanifest(repo, s): | |||
|
218 | """Parses the raw text of a clone bundles manifest. | |||
|
219 | ||||
|
220 | Returns a list of dicts. The dicts have a ``URL`` key corresponding | |||
|
221 | to the URL and other keys are the attributes for the entry. | |||
|
222 | """ | |||
|
223 | m = [] | |||
|
224 | for line in s.splitlines(): | |||
|
225 | fields = line.split() | |||
|
226 | if not fields: | |||
|
227 | continue | |||
|
228 | attrs = {b'URL': fields[0]} | |||
|
229 | for rawattr in fields[1:]: | |||
|
230 | key, value = rawattr.split(b'=', 1) | |||
|
231 | key = util.urlreq.unquote(key) | |||
|
232 | value = util.urlreq.unquote(value) | |||
|
233 | attrs[key] = value | |||
|
234 | ||||
|
235 | # Parse BUNDLESPEC into components. This makes client-side | |||
|
236 | # preferences easier to specify since you can prefer a single | |||
|
237 | # component of the BUNDLESPEC. | |||
|
238 | if key == b'BUNDLESPEC': | |||
|
239 | try: | |||
|
240 | bundlespec = parsebundlespec(repo, value) | |||
|
241 | attrs[b'COMPRESSION'] = bundlespec.compression | |||
|
242 | attrs[b'VERSION'] = bundlespec.version | |||
|
243 | except error.InvalidBundleSpecification: | |||
|
244 | pass | |||
|
245 | except error.UnsupportedBundleSpecification: | |||
|
246 | pass | |||
|
247 | ||||
|
248 | m.append(attrs) | |||
|
249 | ||||
|
250 | return m | |||
|
251 | ||||
|
252 | ||||
|
253 | def isstreamclonespec(bundlespec): | |||
|
254 | # Stream clone v1 | |||
|
255 | if bundlespec.wirecompression == b'UN' and bundlespec.wireversion == b's1': | |||
|
256 | return True | |||
|
257 | ||||
|
258 | # Stream clone v2 | |||
|
259 | if ( | |||
|
260 | bundlespec.wirecompression == b'UN' | |||
|
261 | and bundlespec.wireversion == b'02' | |||
|
262 | and bundlespec.contentopts.get(b'streamv2') | |||
|
263 | ): | |||
|
264 | return True | |||
|
265 | ||||
|
266 | return False | |||
|
267 | ||||
|
268 | ||||
|
269 | def filterclonebundleentries(repo, entries, streamclonerequested=False): | |||
|
270 | """Remove incompatible clone bundle manifest entries. | |||
|
271 | ||||
|
272 | Accepts a list of entries parsed with ``parseclonebundlesmanifest`` | |||
|
273 | and returns a new list consisting of only the entries that this client | |||
|
274 | should be able to apply. | |||
|
275 | ||||
|
276 | There is no guarantee we'll be able to apply all returned entries because | |||
|
277 | the metadata we use to filter on may be missing or wrong. | |||
|
278 | """ | |||
|
279 | newentries = [] | |||
|
280 | for entry in entries: | |||
|
281 | spec = entry.get(b'BUNDLESPEC') | |||
|
282 | if spec: | |||
|
283 | try: | |||
|
284 | bundlespec = parsebundlespec(repo, spec, strict=True) | |||
|
285 | ||||
|
286 | # If a stream clone was requested, filter out non-streamclone | |||
|
287 | # entries. | |||
|
288 | if streamclonerequested and not isstreamclonespec(bundlespec): | |||
|
289 | repo.ui.debug( | |||
|
290 | b'filtering %s because not a stream clone\n' | |||
|
291 | % entry[b'URL'] | |||
|
292 | ) | |||
|
293 | continue | |||
|
294 | ||||
|
295 | except error.InvalidBundleSpecification as e: | |||
|
296 | repo.ui.debug(stringutil.forcebytestr(e) + b'\n') | |||
|
297 | continue | |||
|
298 | except error.UnsupportedBundleSpecification as e: | |||
|
299 | repo.ui.debug( | |||
|
300 | b'filtering %s because unsupported bundle ' | |||
|
301 | b'spec: %s\n' % (entry[b'URL'], stringutil.forcebytestr(e)) | |||
|
302 | ) | |||
|
303 | continue | |||
|
304 | # If we don't have a spec and requested a stream clone, we don't know | |||
|
305 | # what the entry is so don't attempt to apply it. | |||
|
306 | elif streamclonerequested: | |||
|
307 | repo.ui.debug( | |||
|
308 | b'filtering %s because cannot determine if a stream ' | |||
|
309 | b'clone bundle\n' % entry[b'URL'] | |||
|
310 | ) | |||
|
311 | continue | |||
|
312 | ||||
|
313 | if b'REQUIRESNI' in entry and not sslutil.hassni: | |||
|
314 | repo.ui.debug( | |||
|
315 | b'filtering %s because SNI not supported\n' % entry[b'URL'] | |||
|
316 | ) | |||
|
317 | continue | |||
|
318 | ||||
|
319 | if b'REQUIREDRAM' in entry: | |||
|
320 | try: | |||
|
321 | requiredram = util.sizetoint(entry[b'REQUIREDRAM']) | |||
|
322 | except error.ParseError: | |||
|
323 | repo.ui.debug( | |||
|
324 | b'filtering %s due to a bad REQUIREDRAM attribute\n' | |||
|
325 | % entry[b'URL'] | |||
|
326 | ) | |||
|
327 | continue | |||
|
328 | actualram = repo.ui.estimatememory() | |||
|
329 | if actualram is not None and actualram * 0.66 < requiredram: | |||
|
330 | repo.ui.debug( | |||
|
331 | b'filtering %s as it needs more than 2/3 of system memory\n' | |||
|
332 | % entry[b'URL'] | |||
|
333 | ) | |||
|
334 | continue | |||
|
335 | ||||
|
336 | newentries.append(entry) | |||
|
337 | ||||
|
338 | return newentries | |||
|
339 | ||||
|
340 | ||||
|
341 | class clonebundleentry(object): | |||
|
342 | """Represents an item in a clone bundles manifest. | |||
|
343 | ||||
|
344 | This rich class is needed to support sorting since sorted() in Python 3 | |||
|
345 | doesn't support ``cmp`` and our comparison is complex enough that ``key=`` | |||
|
346 | won't work. | |||
|
347 | """ | |||
|
348 | ||||
|
349 | def __init__(self, value, prefers): | |||
|
350 | self.value = value | |||
|
351 | self.prefers = prefers | |||
|
352 | ||||
|
353 | def _cmp(self, other): | |||
|
354 | for prefkey, prefvalue in self.prefers: | |||
|
355 | avalue = self.value.get(prefkey) | |||
|
356 | bvalue = other.value.get(prefkey) | |||
|
357 | ||||
|
358 | # Special case for b missing attribute and a matches exactly. | |||
|
359 | if avalue is not None and bvalue is None and avalue == prefvalue: | |||
|
360 | return -1 | |||
|
361 | ||||
|
362 | # Special case for a missing attribute and b matches exactly. | |||
|
363 | if bvalue is not None and avalue is None and bvalue == prefvalue: | |||
|
364 | return 1 | |||
|
365 | ||||
|
366 | # We can't compare unless attribute present on both. | |||
|
367 | if avalue is None or bvalue is None: | |||
|
368 | continue | |||
|
369 | ||||
|
370 | # Same values should fall back to next attribute. | |||
|
371 | if avalue == bvalue: | |||
|
372 | continue | |||
|
373 | ||||
|
374 | # Exact matches come first. | |||
|
375 | if avalue == prefvalue: | |||
|
376 | return -1 | |||
|
377 | if bvalue == prefvalue: | |||
|
378 | return 1 | |||
|
379 | ||||
|
380 | # Fall back to next attribute. | |||
|
381 | continue | |||
|
382 | ||||
|
383 | # If we got here we couldn't sort by attributes and prefers. Fall | |||
|
384 | # back to index order. | |||
|
385 | return 0 | |||
|
386 | ||||
|
387 | def __lt__(self, other): | |||
|
388 | return self._cmp(other) < 0 | |||
|
389 | ||||
|
390 | def __gt__(self, other): | |||
|
391 | return self._cmp(other) > 0 | |||
|
392 | ||||
|
393 | def __eq__(self, other): | |||
|
394 | return self._cmp(other) == 0 | |||
|
395 | ||||
|
396 | def __le__(self, other): | |||
|
397 | return self._cmp(other) <= 0 | |||
|
398 | ||||
|
399 | def __ge__(self, other): | |||
|
400 | return self._cmp(other) >= 0 | |||
|
401 | ||||
|
402 | def __ne__(self, other): | |||
|
403 | return self._cmp(other) != 0 | |||
|
404 | ||||
|
405 | ||||
|
406 | def sortclonebundleentries(ui, entries): | |||
|
407 | prefers = ui.configlist(b'ui', b'clonebundleprefers') | |||
|
408 | if not prefers: | |||
|
409 | return list(entries) | |||
|
410 | ||||
|
411 | def _split(p): | |||
|
412 | if b'=' not in p: | |||
|
413 | hint = _(b"each comma separated item should be key=value pairs") | |||
|
414 | raise error.Abort( | |||
|
415 | _(b"invalid ui.clonebundleprefers item: %s") % p, hint=hint | |||
|
416 | ) | |||
|
417 | return p.split(b'=', 1) | |||
|
418 | ||||
|
419 | prefers = [_split(p) for p in prefers] | |||
|
420 | ||||
|
421 | items = sorted(clonebundleentry(v, prefers) for v in entries) | |||
|
422 | return [i.value for i in items] |
@@ -127,10 +127,10 b' import sys' | |||||
127 | from mercurial.i18n import _ |
|
127 | from mercurial.i18n import _ | |
128 |
|
128 | |||
129 | from mercurial import ( |
|
129 | from mercurial import ( | |
|
130 | bundlecaches, | |||
130 | config, |
|
131 | config, | |
131 | context, |
|
132 | context, | |
132 | error, |
|
133 | error, | |
133 | exchange, |
|
|||
134 | extensions, |
|
134 | extensions, | |
135 | exthelper, |
|
135 | exthelper, | |
136 | filelog, |
|
136 | filelog, | |
@@ -351,7 +351,7 b' def _extsetup(ui):' | |||||
351 | # Make bundle choose changegroup3 instead of changegroup2. This affects |
|
351 | # Make bundle choose changegroup3 instead of changegroup2. This affects | |
352 | # "hg bundle" command. Note: it does not cover all bundle formats like |
|
352 | # "hg bundle" command. Note: it does not cover all bundle formats like | |
353 | # "packed1". Using "packed1" with lfs will likely cause trouble. |
|
353 | # "packed1". Using "packed1" with lfs will likely cause trouble. | |
354 |
|
|
354 | bundlecaches._bundlespeccontentopts[b"v2"][b"cg.version"] = b"03" | |
355 |
|
355 | |||
356 |
|
356 | |||
357 | @eh.filesetpredicate(b'lfs()') |
|
357 | @eh.filesetpredicate(b'lfs()') |
@@ -26,6 +26,7 b' from . import (' | |||||
26 | archival, |
|
26 | archival, | |
27 | bookmarks, |
|
27 | bookmarks, | |
28 | bundle2, |
|
28 | bundle2, | |
|
29 | bundlecaches, | |||
29 | changegroup, |
|
30 | changegroup, | |
30 | cmdutil, |
|
31 | cmdutil, | |
31 | copies, |
|
32 | copies, | |
@@ -1544,7 +1545,9 b' def bundle(ui, repo, fname, dest=None, *' | |||||
1544 |
|
1545 | |||
1545 | bundletype = opts.get(b'type', b'bzip2').lower() |
|
1546 | bundletype = opts.get(b'type', b'bzip2').lower() | |
1546 | try: |
|
1547 | try: | |
1547 |
bundlespec = |
|
1548 | bundlespec = bundlecaches.parsebundlespec( | |
|
1549 | repo, bundletype, strict=False | |||
|
1550 | ) | |||
1548 | except error.UnsupportedBundleSpecification as e: |
|
1551 | except error.UnsupportedBundleSpecification as e: | |
1549 | raise error.Abort( |
|
1552 | raise error.Abort( | |
1550 | pycompat.bytestr(e), |
|
1553 | pycompat.bytestr(e), |
@@ -16,10 +16,10 b' from .node import (' | |||||
16 | nullid, |
|
16 | nullid, | |
17 | nullrev, |
|
17 | nullrev, | |
18 | ) |
|
18 | ) | |
19 | from .thirdparty import attr |
|
|||
20 | from . import ( |
|
19 | from . import ( | |
21 | bookmarks as bookmod, |
|
20 | bookmarks as bookmod, | |
22 | bundle2, |
|
21 | bundle2, | |
|
22 | bundlecaches, | |||
23 | changegroup, |
|
23 | changegroup, | |
24 | discovery, |
|
24 | discovery, | |
25 | error, |
|
25 | error, | |
@@ -34,7 +34,6 b' from . import (' | |||||
34 | pycompat, |
|
34 | pycompat, | |
35 | requirements, |
|
35 | requirements, | |
36 | scmutil, |
|
36 | scmutil, | |
37 | sslutil, |
|
|||
38 | streamclone, |
|
37 | streamclone, | |
39 | url as urlmod, |
|
38 | url as urlmod, | |
40 | util, |
|
39 | util, | |
@@ -50,202 +49,6 b' urlreq = util.urlreq' | |||||
50 |
|
49 | |||
51 | _NARROWACL_SECTION = b'narrowacl' |
|
50 | _NARROWACL_SECTION = b'narrowacl' | |
52 |
|
51 | |||
53 | # Maps bundle version human names to changegroup versions. |
|
|||
54 | _bundlespeccgversions = { |
|
|||
55 | b'v1': b'01', |
|
|||
56 | b'v2': b'02', |
|
|||
57 | b'packed1': b's1', |
|
|||
58 | b'bundle2': b'02', # legacy |
|
|||
59 | } |
|
|||
60 |
|
||||
61 | # Maps bundle version with content opts to choose which part to bundle |
|
|||
62 | _bundlespeccontentopts = { |
|
|||
63 | b'v1': { |
|
|||
64 | b'changegroup': True, |
|
|||
65 | b'cg.version': b'01', |
|
|||
66 | b'obsolescence': False, |
|
|||
67 | b'phases': False, |
|
|||
68 | b'tagsfnodescache': False, |
|
|||
69 | b'revbranchcache': False, |
|
|||
70 | }, |
|
|||
71 | b'v2': { |
|
|||
72 | b'changegroup': True, |
|
|||
73 | b'cg.version': b'02', |
|
|||
74 | b'obsolescence': False, |
|
|||
75 | b'phases': False, |
|
|||
76 | b'tagsfnodescache': True, |
|
|||
77 | b'revbranchcache': True, |
|
|||
78 | }, |
|
|||
79 | b'packed1': {b'cg.version': b's1'}, |
|
|||
80 | } |
|
|||
81 | _bundlespeccontentopts[b'bundle2'] = _bundlespeccontentopts[b'v2'] |
|
|||
82 |
|
||||
83 | _bundlespecvariants = { |
|
|||
84 | b"streamv2": { |
|
|||
85 | b"changegroup": False, |
|
|||
86 | b"streamv2": True, |
|
|||
87 | b"tagsfnodescache": False, |
|
|||
88 | b"revbranchcache": False, |
|
|||
89 | } |
|
|||
90 | } |
|
|||
91 |
|
||||
92 | # Compression engines allowed in version 1. THIS SHOULD NEVER CHANGE. |
|
|||
93 | _bundlespecv1compengines = {b'gzip', b'bzip2', b'none'} |
|
|||
94 |
|
||||
95 |
|
||||
96 | @attr.s |
|
|||
97 | class bundlespec(object): |
|
|||
98 | compression = attr.ib() |
|
|||
99 | wirecompression = attr.ib() |
|
|||
100 | version = attr.ib() |
|
|||
101 | wireversion = attr.ib() |
|
|||
102 | params = attr.ib() |
|
|||
103 | contentopts = attr.ib() |
|
|||
104 |
|
||||
105 |
|
||||
106 | def parsebundlespec(repo, spec, strict=True): |
|
|||
107 | """Parse a bundle string specification into parts. |
|
|||
108 |
|
||||
109 | Bundle specifications denote a well-defined bundle/exchange format. |
|
|||
110 | The content of a given specification should not change over time in |
|
|||
111 | order to ensure that bundles produced by a newer version of Mercurial are |
|
|||
112 | readable from an older version. |
|
|||
113 |
|
||||
114 | The string currently has the form: |
|
|||
115 |
|
||||
116 | <compression>-<type>[;<parameter0>[;<parameter1>]] |
|
|||
117 |
|
||||
118 | Where <compression> is one of the supported compression formats |
|
|||
119 | and <type> is (currently) a version string. A ";" can follow the type and |
|
|||
120 | all text afterwards is interpreted as URI encoded, ";" delimited key=value |
|
|||
121 | pairs. |
|
|||
122 |
|
||||
123 | If ``strict`` is True (the default) <compression> is required. Otherwise, |
|
|||
124 | it is optional. |
|
|||
125 |
|
||||
126 | Returns a bundlespec object of (compression, version, parameters). |
|
|||
127 | Compression will be ``None`` if not in strict mode and a compression isn't |
|
|||
128 | defined. |
|
|||
129 |
|
||||
130 | An ``InvalidBundleSpecification`` is raised when the specification is |
|
|||
131 | not syntactically well formed. |
|
|||
132 |
|
||||
133 | An ``UnsupportedBundleSpecification`` is raised when the compression or |
|
|||
134 | bundle type/version is not recognized. |
|
|||
135 |
|
||||
136 | Note: this function will likely eventually return a more complex data |
|
|||
137 | structure, including bundle2 part information. |
|
|||
138 | """ |
|
|||
139 |
|
||||
140 | def parseparams(s): |
|
|||
141 | if b';' not in s: |
|
|||
142 | return s, {} |
|
|||
143 |
|
||||
144 | params = {} |
|
|||
145 | version, paramstr = s.split(b';', 1) |
|
|||
146 |
|
||||
147 | for p in paramstr.split(b';'): |
|
|||
148 | if b'=' not in p: |
|
|||
149 | raise error.InvalidBundleSpecification( |
|
|||
150 | _( |
|
|||
151 | b'invalid bundle specification: ' |
|
|||
152 | b'missing "=" in parameter: %s' |
|
|||
153 | ) |
|
|||
154 | % p |
|
|||
155 | ) |
|
|||
156 |
|
||||
157 | key, value = p.split(b'=', 1) |
|
|||
158 | key = urlreq.unquote(key) |
|
|||
159 | value = urlreq.unquote(value) |
|
|||
160 | params[key] = value |
|
|||
161 |
|
||||
162 | return version, params |
|
|||
163 |
|
||||
164 | if strict and b'-' not in spec: |
|
|||
165 | raise error.InvalidBundleSpecification( |
|
|||
166 | _( |
|
|||
167 | b'invalid bundle specification; ' |
|
|||
168 | b'must be prefixed with compression: %s' |
|
|||
169 | ) |
|
|||
170 | % spec |
|
|||
171 | ) |
|
|||
172 |
|
||||
173 | if b'-' in spec: |
|
|||
174 | compression, version = spec.split(b'-', 1) |
|
|||
175 |
|
||||
176 | if compression not in util.compengines.supportedbundlenames: |
|
|||
177 | raise error.UnsupportedBundleSpecification( |
|
|||
178 | _(b'%s compression is not supported') % compression |
|
|||
179 | ) |
|
|||
180 |
|
||||
181 | version, params = parseparams(version) |
|
|||
182 |
|
||||
183 | if version not in _bundlespeccgversions: |
|
|||
184 | raise error.UnsupportedBundleSpecification( |
|
|||
185 | _(b'%s is not a recognized bundle version') % version |
|
|||
186 | ) |
|
|||
187 | else: |
|
|||
188 | # Value could be just the compression or just the version, in which |
|
|||
189 | # case some defaults are assumed (but only when not in strict mode). |
|
|||
190 | assert not strict |
|
|||
191 |
|
||||
192 | spec, params = parseparams(spec) |
|
|||
193 |
|
||||
194 | if spec in util.compengines.supportedbundlenames: |
|
|||
195 | compression = spec |
|
|||
196 | version = b'v1' |
|
|||
197 | # Generaldelta repos require v2. |
|
|||
198 | if b'generaldelta' in repo.requirements: |
|
|||
199 | version = b'v2' |
|
|||
200 | # Modern compression engines require v2. |
|
|||
201 | if compression not in _bundlespecv1compengines: |
|
|||
202 | version = b'v2' |
|
|||
203 | elif spec in _bundlespeccgversions: |
|
|||
204 | if spec == b'packed1': |
|
|||
205 | compression = b'none' |
|
|||
206 | else: |
|
|||
207 | compression = b'bzip2' |
|
|||
208 | version = spec |
|
|||
209 | else: |
|
|||
210 | raise error.UnsupportedBundleSpecification( |
|
|||
211 | _(b'%s is not a recognized bundle specification') % spec |
|
|||
212 | ) |
|
|||
213 |
|
||||
214 | # Bundle version 1 only supports a known set of compression engines. |
|
|||
215 | if version == b'v1' and compression not in _bundlespecv1compengines: |
|
|||
216 | raise error.UnsupportedBundleSpecification( |
|
|||
217 | _(b'compression engine %s is not supported on v1 bundles') |
|
|||
218 | % compression |
|
|||
219 | ) |
|
|||
220 |
|
||||
221 | # The specification for packed1 can optionally declare the data formats |
|
|||
222 | # required to apply it. If we see this metadata, compare against what the |
|
|||
223 | # repo supports and error if the bundle isn't compatible. |
|
|||
224 | if version == b'packed1' and b'requirements' in params: |
|
|||
225 | requirements = set(params[b'requirements'].split(b',')) |
|
|||
226 | missingreqs = requirements - repo.supportedformats |
|
|||
227 | if missingreqs: |
|
|||
228 | raise error.UnsupportedBundleSpecification( |
|
|||
229 | _(b'missing support for repository features: %s') |
|
|||
230 | % b', '.join(sorted(missingreqs)) |
|
|||
231 | ) |
|
|||
232 |
|
||||
233 | # Compute contentopts based on the version |
|
|||
234 | contentopts = _bundlespeccontentopts.get(version, {}).copy() |
|
|||
235 |
|
||||
236 | # Process the variants |
|
|||
237 | if b"stream" in params and params[b"stream"] == b"v2": |
|
|||
238 | variant = _bundlespecvariants[b"streamv2"] |
|
|||
239 | contentopts.update(variant) |
|
|||
240 |
|
||||
241 | engine = util.compengines.forbundlename(compression) |
|
|||
242 | compression, wirecompression = engine.bundletype() |
|
|||
243 | wireversion = _bundlespeccgversions[version] |
|
|||
244 |
|
||||
245 | return bundlespec( |
|
|||
246 | compression, wirecompression, version, wireversion, params, contentopts |
|
|||
247 | ) |
|
|||
248 |
|
||||
249 |
|
52 | |||
250 | def readbundle(ui, fh, fname, vfs=None): |
|
53 | def readbundle(ui, fh, fname, vfs=None): | |
251 | header = changegroup.readexactly(fh, 4) |
|
54 | header = changegroup.readexactly(fh, 4) | |
@@ -2867,7 +2670,7 b' def _maybeapplyclonebundle(pullop):' | |||||
2867 | # attempt. |
|
2670 | # attempt. | |
2868 | pullop.clonebundleattempted = True |
|
2671 | pullop.clonebundleattempted = True | |
2869 |
|
2672 | |||
2870 | entries = parseclonebundlesmanifest(repo, res) |
|
2673 | entries = bundlecaches.parseclonebundlesmanifest(repo, res) | |
2871 | if not entries: |
|
2674 | if not entries: | |
2872 | repo.ui.note( |
|
2675 | repo.ui.note( | |
2873 | _( |
|
2676 | _( | |
@@ -2877,7 +2680,7 b' def _maybeapplyclonebundle(pullop):' | |||||
2877 | ) |
|
2680 | ) | |
2878 | return |
|
2681 | return | |
2879 |
|
2682 | |||
2880 | entries = filterclonebundleentries( |
|
2683 | entries = bundlecaches.filterclonebundleentries( | |
2881 | repo, entries, streamclonerequested=pullop.streamclonerequested |
|
2684 | repo, entries, streamclonerequested=pullop.streamclonerequested | |
2882 | ) |
|
2685 | ) | |
2883 |
|
2686 | |||
@@ -2898,7 +2701,7 b' def _maybeapplyclonebundle(pullop):' | |||||
2898 | ) |
|
2701 | ) | |
2899 | return |
|
2702 | return | |
2900 |
|
2703 | |||
2901 | entries = sortclonebundleentries(repo.ui, entries) |
|
2704 | entries = bundlecaches.sortclonebundleentries(repo.ui, entries) | |
2902 |
|
2705 | |||
2903 | url = entries[0][b'URL'] |
|
2706 | url = entries[0][b'URL'] | |
2904 | repo.ui.status(_(b'applying clone bundle from %s\n') % url) |
|
2707 | repo.ui.status(_(b'applying clone bundle from %s\n') % url) | |
@@ -2923,214 +2726,6 b' def _maybeapplyclonebundle(pullop):' | |||||
2923 | ) |
|
2726 | ) | |
2924 |
|
2727 | |||
2925 |
|
2728 | |||
2926 | def parseclonebundlesmanifest(repo, s): |
|
|||
2927 | """Parses the raw text of a clone bundles manifest. |
|
|||
2928 |
|
||||
2929 | Returns a list of dicts. The dicts have a ``URL`` key corresponding |
|
|||
2930 | to the URL and other keys are the attributes for the entry. |
|
|||
2931 | """ |
|
|||
2932 | m = [] |
|
|||
2933 | for line in s.splitlines(): |
|
|||
2934 | fields = line.split() |
|
|||
2935 | if not fields: |
|
|||
2936 | continue |
|
|||
2937 | attrs = {b'URL': fields[0]} |
|
|||
2938 | for rawattr in fields[1:]: |
|
|||
2939 | key, value = rawattr.split(b'=', 1) |
|
|||
2940 | key = urlreq.unquote(key) |
|
|||
2941 | value = urlreq.unquote(value) |
|
|||
2942 | attrs[key] = value |
|
|||
2943 |
|
||||
2944 | # Parse BUNDLESPEC into components. This makes client-side |
|
|||
2945 | # preferences easier to specify since you can prefer a single |
|
|||
2946 | # component of the BUNDLESPEC. |
|
|||
2947 | if key == b'BUNDLESPEC': |
|
|||
2948 | try: |
|
|||
2949 | bundlespec = parsebundlespec(repo, value) |
|
|||
2950 | attrs[b'COMPRESSION'] = bundlespec.compression |
|
|||
2951 | attrs[b'VERSION'] = bundlespec.version |
|
|||
2952 | except error.InvalidBundleSpecification: |
|
|||
2953 | pass |
|
|||
2954 | except error.UnsupportedBundleSpecification: |
|
|||
2955 | pass |
|
|||
2956 |
|
||||
2957 | m.append(attrs) |
|
|||
2958 |
|
||||
2959 | return m |
|
|||
2960 |
|
||||
2961 |
|
||||
2962 | def isstreamclonespec(bundlespec): |
|
|||
2963 | # Stream clone v1 |
|
|||
2964 | if bundlespec.wirecompression == b'UN' and bundlespec.wireversion == b's1': |
|
|||
2965 | return True |
|
|||
2966 |
|
||||
2967 | # Stream clone v2 |
|
|||
2968 | if ( |
|
|||
2969 | bundlespec.wirecompression == b'UN' |
|
|||
2970 | and bundlespec.wireversion == b'02' |
|
|||
2971 | and bundlespec.contentopts.get(b'streamv2') |
|
|||
2972 | ): |
|
|||
2973 | return True |
|
|||
2974 |
|
||||
2975 | return False |
|
|||
2976 |
|
||||
2977 |
|
||||
2978 | def filterclonebundleentries(repo, entries, streamclonerequested=False): |
|
|||
2979 | """Remove incompatible clone bundle manifest entries. |
|
|||
2980 |
|
||||
2981 | Accepts a list of entries parsed with ``parseclonebundlesmanifest`` |
|
|||
2982 | and returns a new list consisting of only the entries that this client |
|
|||
2983 | should be able to apply. |
|
|||
2984 |
|
||||
2985 | There is no guarantee we'll be able to apply all returned entries because |
|
|||
2986 | the metadata we use to filter on may be missing or wrong. |
|
|||
2987 | """ |
|
|||
2988 | newentries = [] |
|
|||
2989 | for entry in entries: |
|
|||
2990 | spec = entry.get(b'BUNDLESPEC') |
|
|||
2991 | if spec: |
|
|||
2992 | try: |
|
|||
2993 | bundlespec = parsebundlespec(repo, spec, strict=True) |
|
|||
2994 |
|
||||
2995 | # If a stream clone was requested, filter out non-streamclone |
|
|||
2996 | # entries. |
|
|||
2997 | if streamclonerequested and not isstreamclonespec(bundlespec): |
|
|||
2998 | repo.ui.debug( |
|
|||
2999 | b'filtering %s because not a stream clone\n' |
|
|||
3000 | % entry[b'URL'] |
|
|||
3001 | ) |
|
|||
3002 | continue |
|
|||
3003 |
|
||||
3004 | except error.InvalidBundleSpecification as e: |
|
|||
3005 | repo.ui.debug(stringutil.forcebytestr(e) + b'\n') |
|
|||
3006 | continue |
|
|||
3007 | except error.UnsupportedBundleSpecification as e: |
|
|||
3008 | repo.ui.debug( |
|
|||
3009 | b'filtering %s because unsupported bundle ' |
|
|||
3010 | b'spec: %s\n' % (entry[b'URL'], stringutil.forcebytestr(e)) |
|
|||
3011 | ) |
|
|||
3012 | continue |
|
|||
3013 | # If we don't have a spec and requested a stream clone, we don't know |
|
|||
3014 | # what the entry is so don't attempt to apply it. |
|
|||
3015 | elif streamclonerequested: |
|
|||
3016 | repo.ui.debug( |
|
|||
3017 | b'filtering %s because cannot determine if a stream ' |
|
|||
3018 | b'clone bundle\n' % entry[b'URL'] |
|
|||
3019 | ) |
|
|||
3020 | continue |
|
|||
3021 |
|
||||
3022 | if b'REQUIRESNI' in entry and not sslutil.hassni: |
|
|||
3023 | repo.ui.debug( |
|
|||
3024 | b'filtering %s because SNI not supported\n' % entry[b'URL'] |
|
|||
3025 | ) |
|
|||
3026 | continue |
|
|||
3027 |
|
||||
3028 | if b'REQUIREDRAM' in entry: |
|
|||
3029 | try: |
|
|||
3030 | requiredram = util.sizetoint(entry[b'REQUIREDRAM']) |
|
|||
3031 | except error.ParseError: |
|
|||
3032 | repo.ui.debug( |
|
|||
3033 | b'filtering %s due to a bad REQUIREDRAM attribute\n' |
|
|||
3034 | % entry[b'URL'] |
|
|||
3035 | ) |
|
|||
3036 | continue |
|
|||
3037 | actualram = repo.ui.estimatememory() |
|
|||
3038 | if actualram is not None and actualram * 0.66 < requiredram: |
|
|||
3039 | repo.ui.debug( |
|
|||
3040 | b'filtering %s as it needs more than 2/3 of system memory\n' |
|
|||
3041 | % entry[b'URL'] |
|
|||
3042 | ) |
|
|||
3043 | continue |
|
|||
3044 |
|
||||
3045 | newentries.append(entry) |
|
|||
3046 |
|
||||
3047 | return newentries |
|
|||
3048 |
|
||||
3049 |
|
||||
3050 | class clonebundleentry(object): |
|
|||
3051 | """Represents an item in a clone bundles manifest. |
|
|||
3052 |
|
||||
3053 | This rich class is needed to support sorting since sorted() in Python 3 |
|
|||
3054 | doesn't support ``cmp`` and our comparison is complex enough that ``key=`` |
|
|||
3055 | won't work. |
|
|||
3056 | """ |
|
|||
3057 |
|
||||
3058 | def __init__(self, value, prefers): |
|
|||
3059 | self.value = value |
|
|||
3060 | self.prefers = prefers |
|
|||
3061 |
|
||||
3062 | def _cmp(self, other): |
|
|||
3063 | for prefkey, prefvalue in self.prefers: |
|
|||
3064 | avalue = self.value.get(prefkey) |
|
|||
3065 | bvalue = other.value.get(prefkey) |
|
|||
3066 |
|
||||
3067 | # Special case for b missing attribute and a matches exactly. |
|
|||
3068 | if avalue is not None and bvalue is None and avalue == prefvalue: |
|
|||
3069 | return -1 |
|
|||
3070 |
|
||||
3071 | # Special case for a missing attribute and b matches exactly. |
|
|||
3072 | if bvalue is not None and avalue is None and bvalue == prefvalue: |
|
|||
3073 | return 1 |
|
|||
3074 |
|
||||
3075 | # We can't compare unless attribute present on both. |
|
|||
3076 | if avalue is None or bvalue is None: |
|
|||
3077 | continue |
|
|||
3078 |
|
||||
3079 | # Same values should fall back to next attribute. |
|
|||
3080 | if avalue == bvalue: |
|
|||
3081 | continue |
|
|||
3082 |
|
||||
3083 | # Exact matches come first. |
|
|||
3084 | if avalue == prefvalue: |
|
|||
3085 | return -1 |
|
|||
3086 | if bvalue == prefvalue: |
|
|||
3087 | return 1 |
|
|||
3088 |
|
||||
3089 | # Fall back to next attribute. |
|
|||
3090 | continue |
|
|||
3091 |
|
||||
3092 | # If we got here we couldn't sort by attributes and prefers. Fall |
|
|||
3093 | # back to index order. |
|
|||
3094 | return 0 |
|
|||
3095 |
|
||||
3096 | def __lt__(self, other): |
|
|||
3097 | return self._cmp(other) < 0 |
|
|||
3098 |
|
||||
3099 | def __gt__(self, other): |
|
|||
3100 | return self._cmp(other) > 0 |
|
|||
3101 |
|
||||
3102 | def __eq__(self, other): |
|
|||
3103 | return self._cmp(other) == 0 |
|
|||
3104 |
|
||||
3105 | def __le__(self, other): |
|
|||
3106 | return self._cmp(other) <= 0 |
|
|||
3107 |
|
||||
3108 | def __ge__(self, other): |
|
|||
3109 | return self._cmp(other) >= 0 |
|
|||
3110 |
|
||||
3111 | def __ne__(self, other): |
|
|||
3112 | return self._cmp(other) != 0 |
|
|||
3113 |
|
||||
3114 |
|
||||
3115 | def sortclonebundleentries(ui, entries): |
|
|||
3116 | prefers = ui.configlist(b'ui', b'clonebundleprefers') |
|
|||
3117 | if not prefers: |
|
|||
3118 | return list(entries) |
|
|||
3119 |
|
||||
3120 | def _split(p): |
|
|||
3121 | if b'=' not in p: |
|
|||
3122 | hint = _(b"each comma separated item should be key=value pairs") |
|
|||
3123 | raise error.Abort( |
|
|||
3124 | _(b"invalid ui.clonebundleprefers item: %s") % p, hint=hint |
|
|||
3125 | ) |
|
|||
3126 | return p.split(b'=', 1) |
|
|||
3127 |
|
||||
3128 | prefers = [_split(p) for p in prefers] |
|
|||
3129 |
|
||||
3130 | items = sorted(clonebundleentry(v, prefers) for v in entries) |
|
|||
3131 | return [i.value for i in items] |
|
|||
3132 |
|
||||
3133 |
|
||||
3134 | def trypullbundlefromurl(ui, repo, url): |
|
2729 | def trypullbundlefromurl(ui, repo, url): | |
3135 | """Attempt to apply a bundle from a URL.""" |
|
2730 | """Attempt to apply a bundle from a URL.""" | |
3136 | with repo.lock(), repo.transaction(b'bundleurl') as tr: |
|
2731 | with repo.lock(), repo.transaction(b'bundleurl') as tr: |
@@ -19,6 +19,7 b' from .pycompat import getattr' | |||||
19 |
|
19 | |||
20 | from . import ( |
|
20 | from . import ( | |
21 | bundle2, |
|
21 | bundle2, | |
|
22 | bundlecaches, | |||
22 | changegroup as changegroupmod, |
|
23 | changegroup as changegroupmod, | |
23 | discovery, |
|
24 | discovery, | |
24 | encoding, |
|
25 | encoding, | |
@@ -387,8 +388,8 b' def find_pullbundle(repo, proto, opts, c' | |||||
387 | manifest = repo.vfs.tryread(b'pullbundles.manifest') |
|
388 | manifest = repo.vfs.tryread(b'pullbundles.manifest') | |
388 | if not manifest: |
|
389 | if not manifest: | |
389 | return None |
|
390 | return None | |
390 |
res = |
|
391 | res = bundlecaches.parseclonebundlesmanifest(repo, manifest) | |
391 |
res = |
|
392 | res = bundlecaches.filterclonebundleentries(repo, res) | |
392 | if not res: |
|
393 | if not res: | |
393 | return None |
|
394 | return None | |
394 | cl = repo.unfiltered().changelog |
|
395 | cl = repo.unfiltered().changelog |
@@ -6,8 +6,8 b' import base64' | |||||
6 | import zlib |
|
6 | import zlib | |
7 |
|
7 | |||
8 | from mercurial import ( |
|
8 | from mercurial import ( | |
|
9 | bundlecaches, | |||
9 | changegroup, |
|
10 | changegroup, | |
10 | exchange, |
|
|||
11 | extensions, |
|
11 | extensions, | |
12 | revlog, |
|
12 | revlog, | |
13 | util, |
|
13 | util, | |
@@ -134,8 +134,8 b' def extsetup(ui):' | |||||
134 | revlog.REVIDX_FLAGS_ORDER.extend(flags) |
|
134 | revlog.REVIDX_FLAGS_ORDER.extend(flags) | |
135 |
|
135 | |||
136 | # Teach exchange to use changegroup 3 |
|
136 | # Teach exchange to use changegroup 3 | |
137 |
for k in |
|
137 | for k in bundlecaches._bundlespeccontentopts.keys(): | |
138 |
|
|
138 | bundlecaches._bundlespeccontentopts[k][b"cg.version"] = b"03" | |
139 |
|
139 | |||
140 | # Register flag processors for each extension |
|
140 | # Register flag processors for each extension | |
141 | flagutil.addflagprocessor( |
|
141 | flagutil.addflagprocessor( |
General Comments 0
You need to be logged in to leave comments.
Login now