Show More
@@ -1,465 +1,467 b'' | |||
|
1 | 1 | # bundlecaches.py - utility to deal with pre-computed bundle for servers |
|
2 | 2 | # |
|
3 | 3 | # This software may be used and distributed according to the terms of the |
|
4 | 4 | # GNU General Public License version 2 or any later version. |
|
5 | 5 | |
|
6 | 6 | import collections |
|
7 | 7 | |
|
8 | 8 | from .i18n import _ |
|
9 | 9 | |
|
10 | 10 | from .thirdparty import attr |
|
11 | 11 | |
|
12 | 12 | from . import ( |
|
13 | 13 | error, |
|
14 | 14 | requirements as requirementsmod, |
|
15 | 15 | sslutil, |
|
16 | 16 | util, |
|
17 | 17 | ) |
|
18 | 18 | from .utils import stringutil |
|
19 | 19 | |
|
20 | 20 | urlreq = util.urlreq |
|
21 | 21 | |
|
22 | 22 | CB_MANIFEST_FILE = b'clonebundles.manifest' |
|
23 | 23 | |
|
24 | 24 | |
|
25 | 25 | @attr.s |
|
26 | 26 | class bundlespec: |
|
27 | 27 | compression = attr.ib() |
|
28 | 28 | wirecompression = attr.ib() |
|
29 | 29 | version = attr.ib() |
|
30 | 30 | wireversion = attr.ib() |
|
31 | 31 | # parameters explicitly overwritten by the config or the specification |
|
32 | 32 | _explicit_params = attr.ib() |
|
33 | 33 | # default parameter for the version |
|
34 | 34 | # |
|
35 | 35 | # Keeping it separated is useful to check what was actually overwritten. |
|
36 | 36 | _default_opts = attr.ib() |
|
37 | 37 | |
|
38 | 38 | @property |
|
39 | 39 | def params(self): |
|
40 | 40 | return collections.ChainMap(self._explicit_params, self._default_opts) |
|
41 | 41 | |
|
42 | 42 | @property |
|
43 | 43 | def contentopts(self): |
|
44 | 44 | # kept for Backward Compatibility concerns. |
|
45 | 45 | return self.params |
|
46 | 46 | |
|
47 | 47 | def set_param(self, key, value, overwrite=True): |
|
48 | 48 | """Set a bundle parameter value. |
|
49 | 49 | |
|
50 | 50 | Will only overwrite if overwrite is true""" |
|
51 | 51 | if overwrite or key not in self._explicit_params: |
|
52 | 52 | self._explicit_params[key] = value |
|
53 | 53 | |
|
54 | 54 | |
|
55 | 55 | # Maps bundle version human names to changegroup versions. |
|
56 | 56 | _bundlespeccgversions = { |
|
57 | 57 | b'v1': b'01', |
|
58 | 58 | b'v2': b'02', |
|
59 | 59 | b'packed1': b's1', |
|
60 | 60 | b'bundle2': b'02', # legacy |
|
61 | 61 | } |
|
62 | 62 | |
|
63 | 63 | # Maps bundle version with content opts to choose which part to bundle |
|
64 | 64 | _bundlespeccontentopts = { |
|
65 | 65 | b'v1': { |
|
66 | 66 | b'changegroup': True, |
|
67 | 67 | b'cg.version': b'01', |
|
68 | 68 | b'obsolescence': False, |
|
69 | 69 | b'phases': False, |
|
70 | 70 | b'tagsfnodescache': False, |
|
71 | 71 | b'revbranchcache': False, |
|
72 | 72 | }, |
|
73 | 73 | b'v2': { |
|
74 | 74 | b'changegroup': True, |
|
75 | 75 | b'cg.version': b'02', |
|
76 | 76 | b'obsolescence': False, |
|
77 | 77 | b'phases': False, |
|
78 | 78 | b'tagsfnodescache': True, |
|
79 | 79 | b'revbranchcache': True, |
|
80 | 80 | }, |
|
81 | 81 | b'streamv2': { |
|
82 | 82 | b'changegroup': False, |
|
83 | 83 | b'cg.version': b'02', |
|
84 | 84 | b'obsolescence': False, |
|
85 | 85 | b'phases': False, |
|
86 | 86 | b"streamv2": True, |
|
87 | 87 | b'tagsfnodescache': False, |
|
88 | 88 | b'revbranchcache': False, |
|
89 | 89 | }, |
|
90 | 90 | b'packed1': { |
|
91 | 91 | b'cg.version': b's1', |
|
92 | 92 | }, |
|
93 | 93 | b'bundle2': { # legacy |
|
94 | 94 | b'cg.version': b'02', |
|
95 | 95 | }, |
|
96 | 96 | } |
|
97 | 97 | _bundlespeccontentopts[b'bundle2'] = _bundlespeccontentopts[b'v2'] |
|
98 | 98 | |
|
99 | 99 | _bundlespecvariants = {b"streamv2": {}} |
|
100 | 100 | |
|
101 | 101 | # Compression engines allowed in version 1. THIS SHOULD NEVER CHANGE. |
|
102 | 102 | _bundlespecv1compengines = {b'gzip', b'bzip2', b'none'} |
|
103 | 103 | |
|
104 | 104 | |
|
105 | def _parseparams(s): | |
|
106 | """parse bundlespec parameter section | |
|
107 | ||
|
108 | input: "comp-version;params" string | |
|
109 | ||
|
110 | return: (spec; {param_key: param_value}) | |
|
111 | """ | |
|
112 | if b';' not in s: | |
|
113 | return s, {} | |
|
114 | ||
|
115 | params = {} | |
|
116 | version, paramstr = s.split(b';', 1) | |
|
117 | ||
|
118 | err = _(b'invalid bundle specification: missing "=" in parameter: %s') | |
|
119 | for p in paramstr.split(b';'): | |
|
120 | if b'=' not in p: | |
|
121 | msg = err % p | |
|
122 | raise error.InvalidBundleSpecification(msg) | |
|
123 | ||
|
124 | key, value = p.split(b'=', 1) | |
|
125 | key = urlreq.unquote(key) | |
|
126 | value = urlreq.unquote(value) | |
|
127 | params[key] = value | |
|
128 | ||
|
129 | return version, params | |
|
130 | ||
|
131 | ||
|
105 | 132 | def parsebundlespec(repo, spec, strict=True): |
|
106 | 133 | """Parse a bundle string specification into parts. |
|
107 | 134 | |
|
108 | 135 | Bundle specifications denote a well-defined bundle/exchange format. |
|
109 | 136 | The content of a given specification should not change over time in |
|
110 | 137 | order to ensure that bundles produced by a newer version of Mercurial are |
|
111 | 138 | readable from an older version. |
|
112 | 139 | |
|
113 | 140 | The string currently has the form: |
|
114 | 141 | |
|
115 | 142 | <compression>-<type>[;<parameter0>[;<parameter1>]] |
|
116 | 143 | |
|
117 | 144 | Where <compression> is one of the supported compression formats |
|
118 | 145 | and <type> is (currently) a version string. A ";" can follow the type and |
|
119 | 146 | all text afterwards is interpreted as URI encoded, ";" delimited key=value |
|
120 | 147 | pairs. |
|
121 | 148 | |
|
122 | 149 | If ``strict`` is True (the default) <compression> is required. Otherwise, |
|
123 | 150 | it is optional. |
|
124 | 151 | |
|
125 | 152 | Returns a bundlespec object of (compression, version, parameters). |
|
126 | 153 | Compression will be ``None`` if not in strict mode and a compression isn't |
|
127 | 154 | defined. |
|
128 | 155 | |
|
129 | 156 | An ``InvalidBundleSpecification`` is raised when the specification is |
|
130 | 157 | not syntactically well formed. |
|
131 | 158 | |
|
132 | 159 | An ``UnsupportedBundleSpecification`` is raised when the compression or |
|
133 | 160 | bundle type/version is not recognized. |
|
134 | 161 | |
|
135 | 162 | Note: this function will likely eventually return a more complex data |
|
136 | 163 | structure, including bundle2 part information. |
|
137 | 164 | """ |
|
138 | ||
|
139 | def parseparams(s): | |
|
140 | if b';' not in s: | |
|
141 | return s, {} | |
|
142 | ||
|
143 | params = {} | |
|
144 | version, paramstr = s.split(b';', 1) | |
|
145 | ||
|
146 | for p in paramstr.split(b';'): | |
|
147 | if b'=' not in p: | |
|
148 | raise error.InvalidBundleSpecification( | |
|
149 | _( | |
|
150 | b'invalid bundle specification: ' | |
|
151 | b'missing "=" in parameter: %s' | |
|
152 | ) | |
|
153 | % p | |
|
154 | ) | |
|
155 | ||
|
156 | key, value = p.split(b'=', 1) | |
|
157 | key = urlreq.unquote(key) | |
|
158 | value = urlreq.unquote(value) | |
|
159 | params[key] = value | |
|
160 | ||
|
161 | return version, params | |
|
162 | ||
|
163 | 165 | if strict and b'-' not in spec: |
|
164 | 166 | raise error.InvalidBundleSpecification( |
|
165 | 167 | _( |
|
166 | 168 | b'invalid bundle specification; ' |
|
167 | 169 | b'must be prefixed with compression: %s' |
|
168 | 170 | ) |
|
169 | 171 | % spec |
|
170 | 172 | ) |
|
171 | 173 | |
|
172 | 174 | if b'-' in spec: |
|
173 | 175 | compression, version = spec.split(b'-', 1) |
|
174 | 176 | |
|
175 | 177 | if compression not in util.compengines.supportedbundlenames: |
|
176 | 178 | raise error.UnsupportedBundleSpecification( |
|
177 | 179 | _(b'%s compression is not supported') % compression |
|
178 | 180 | ) |
|
179 | 181 | |
|
180 | version, params = parseparams(version) | |
|
182 | version, params = _parseparams(version) | |
|
181 | 183 | |
|
182 | 184 | if version not in _bundlespeccontentopts: |
|
183 | 185 | raise error.UnsupportedBundleSpecification( |
|
184 | 186 | _(b'%s is not a recognized bundle version') % version |
|
185 | 187 | ) |
|
186 | 188 | else: |
|
187 | 189 | # Value could be just the compression or just the version, in which |
|
188 | 190 | # case some defaults are assumed (but only when not in strict mode). |
|
189 | 191 | assert not strict |
|
190 | 192 | |
|
191 | spec, params = parseparams(spec) | |
|
193 | spec, params = _parseparams(spec) | |
|
192 | 194 | |
|
193 | 195 | if spec in util.compengines.supportedbundlenames: |
|
194 | 196 | compression = spec |
|
195 | 197 | version = b'v1' |
|
196 | 198 | # Generaldelta repos require v2. |
|
197 | 199 | if requirementsmod.GENERALDELTA_REQUIREMENT in repo.requirements: |
|
198 | 200 | version = b'v2' |
|
199 | 201 | elif requirementsmod.REVLOGV2_REQUIREMENT in repo.requirements: |
|
200 | 202 | version = b'v2' |
|
201 | 203 | # Modern compression engines require v2. |
|
202 | 204 | if compression not in _bundlespecv1compengines: |
|
203 | 205 | version = b'v2' |
|
204 | 206 | elif spec in _bundlespeccontentopts: |
|
205 | 207 | if spec == b'packed1': |
|
206 | 208 | compression = b'none' |
|
207 | 209 | else: |
|
208 | 210 | compression = b'bzip2' |
|
209 | 211 | version = spec |
|
210 | 212 | else: |
|
211 | 213 | raise error.UnsupportedBundleSpecification( |
|
212 | 214 | _(b'%s is not a recognized bundle specification') % spec |
|
213 | 215 | ) |
|
214 | 216 | |
|
215 | 217 | # Bundle version 1 only supports a known set of compression engines. |
|
216 | 218 | if version == b'v1' and compression not in _bundlespecv1compengines: |
|
217 | 219 | raise error.UnsupportedBundleSpecification( |
|
218 | 220 | _(b'compression engine %s is not supported on v1 bundles') |
|
219 | 221 | % compression |
|
220 | 222 | ) |
|
221 | 223 | |
|
222 | 224 | # The specification for packed1 can optionally declare the data formats |
|
223 | 225 | # required to apply it. If we see this metadata, compare against what the |
|
224 | 226 | # repo supports and error if the bundle isn't compatible. |
|
225 | 227 | if version == b'packed1' and b'requirements' in params: |
|
226 | 228 | requirements = set(params[b'requirements'].split(b',')) |
|
227 | 229 | missingreqs = requirements - requirementsmod.STREAM_FIXED_REQUIREMENTS |
|
228 | 230 | if missingreqs: |
|
229 | 231 | raise error.UnsupportedBundleSpecification( |
|
230 | 232 | _(b'missing support for repository features: %s') |
|
231 | 233 | % b', '.join(sorted(missingreqs)) |
|
232 | 234 | ) |
|
233 | 235 | |
|
234 | 236 | # Compute contentopts based on the version |
|
235 | 237 | if b"stream" in params and params[b"stream"] == b"v2": |
|
236 | 238 | # That case is fishy as this mostly derails the version selection |
|
237 | 239 | # mechanism. `stream` bundles are quite specific and used differently |
|
238 | 240 | # as "normal" bundles. |
|
239 | 241 | # |
|
240 | 242 | # So we are pinning this to "v2", as this will likely be |
|
241 | 243 | # compatible forever. (see the next conditional). |
|
242 | 244 | # |
|
243 | 245 | # (we should probably define a cleaner way to do this and raise a |
|
244 | 246 | # warning when the old way is encounter) |
|
245 | 247 | version = b"streamv2" |
|
246 | 248 | contentopts = _bundlespeccontentopts.get(version, {}).copy() |
|
247 | 249 | if version == b"streamv2": |
|
248 | 250 | # streamv2 have been reported as "v2" for a while. |
|
249 | 251 | version = b"v2" |
|
250 | 252 | |
|
251 | 253 | engine = util.compengines.forbundlename(compression) |
|
252 | 254 | compression, wirecompression = engine.bundletype() |
|
253 | 255 | wireversion = _bundlespeccontentopts[version][b'cg.version'] |
|
254 | 256 | |
|
255 | 257 | return bundlespec( |
|
256 | 258 | compression, wirecompression, version, wireversion, params, contentopts |
|
257 | 259 | ) |
|
258 | 260 | |
|
259 | 261 | |
|
260 | 262 | def parseclonebundlesmanifest(repo, s): |
|
261 | 263 | """Parses the raw text of a clone bundles manifest. |
|
262 | 264 | |
|
263 | 265 | Returns a list of dicts. The dicts have a ``URL`` key corresponding |
|
264 | 266 | to the URL and other keys are the attributes for the entry. |
|
265 | 267 | """ |
|
266 | 268 | m = [] |
|
267 | 269 | for line in s.splitlines(): |
|
268 | 270 | fields = line.split() |
|
269 | 271 | if not fields: |
|
270 | 272 | continue |
|
271 | 273 | attrs = {b'URL': fields[0]} |
|
272 | 274 | for rawattr in fields[1:]: |
|
273 | 275 | key, value = rawattr.split(b'=', 1) |
|
274 | 276 | key = util.urlreq.unquote(key) |
|
275 | 277 | value = util.urlreq.unquote(value) |
|
276 | 278 | attrs[key] = value |
|
277 | 279 | |
|
278 | 280 | # Parse BUNDLESPEC into components. This makes client-side |
|
279 | 281 | # preferences easier to specify since you can prefer a single |
|
280 | 282 | # component of the BUNDLESPEC. |
|
281 | 283 | if key == b'BUNDLESPEC': |
|
282 | 284 | try: |
|
283 | 285 | bundlespec = parsebundlespec(repo, value) |
|
284 | 286 | attrs[b'COMPRESSION'] = bundlespec.compression |
|
285 | 287 | attrs[b'VERSION'] = bundlespec.version |
|
286 | 288 | except error.InvalidBundleSpecification: |
|
287 | 289 | pass |
|
288 | 290 | except error.UnsupportedBundleSpecification: |
|
289 | 291 | pass |
|
290 | 292 | |
|
291 | 293 | m.append(attrs) |
|
292 | 294 | |
|
293 | 295 | return m |
|
294 | 296 | |
|
295 | 297 | |
|
296 | 298 | def isstreamclonespec(bundlespec): |
|
297 | 299 | # Stream clone v1 |
|
298 | 300 | if bundlespec.wirecompression == b'UN' and bundlespec.wireversion == b's1': |
|
299 | 301 | return True |
|
300 | 302 | |
|
301 | 303 | # Stream clone v2 |
|
302 | 304 | if ( |
|
303 | 305 | bundlespec.wirecompression == b'UN' |
|
304 | 306 | and bundlespec.wireversion == b'02' |
|
305 | 307 | and bundlespec.contentopts.get(b'streamv2') |
|
306 | 308 | ): |
|
307 | 309 | return True |
|
308 | 310 | |
|
309 | 311 | return False |
|
310 | 312 | |
|
311 | 313 | |
|
312 | 314 | def filterclonebundleentries(repo, entries, streamclonerequested=False): |
|
313 | 315 | """Remove incompatible clone bundle manifest entries. |
|
314 | 316 | |
|
315 | 317 | Accepts a list of entries parsed with ``parseclonebundlesmanifest`` |
|
316 | 318 | and returns a new list consisting of only the entries that this client |
|
317 | 319 | should be able to apply. |
|
318 | 320 | |
|
319 | 321 | There is no guarantee we'll be able to apply all returned entries because |
|
320 | 322 | the metadata we use to filter on may be missing or wrong. |
|
321 | 323 | """ |
|
322 | 324 | newentries = [] |
|
323 | 325 | for entry in entries: |
|
324 | 326 | spec = entry.get(b'BUNDLESPEC') |
|
325 | 327 | if spec: |
|
326 | 328 | try: |
|
327 | 329 | bundlespec = parsebundlespec(repo, spec, strict=True) |
|
328 | 330 | |
|
329 | 331 | # If a stream clone was requested, filter out non-streamclone |
|
330 | 332 | # entries. |
|
331 | 333 | if streamclonerequested and not isstreamclonespec(bundlespec): |
|
332 | 334 | repo.ui.debug( |
|
333 | 335 | b'filtering %s because not a stream clone\n' |
|
334 | 336 | % entry[b'URL'] |
|
335 | 337 | ) |
|
336 | 338 | continue |
|
337 | 339 | |
|
338 | 340 | except error.InvalidBundleSpecification as e: |
|
339 | 341 | repo.ui.debug(stringutil.forcebytestr(e) + b'\n') |
|
340 | 342 | continue |
|
341 | 343 | except error.UnsupportedBundleSpecification as e: |
|
342 | 344 | repo.ui.debug( |
|
343 | 345 | b'filtering %s because unsupported bundle ' |
|
344 | 346 | b'spec: %s\n' % (entry[b'URL'], stringutil.forcebytestr(e)) |
|
345 | 347 | ) |
|
346 | 348 | continue |
|
347 | 349 | # If we don't have a spec and requested a stream clone, we don't know |
|
348 | 350 | # what the entry is so don't attempt to apply it. |
|
349 | 351 | elif streamclonerequested: |
|
350 | 352 | repo.ui.debug( |
|
351 | 353 | b'filtering %s because cannot determine if a stream ' |
|
352 | 354 | b'clone bundle\n' % entry[b'URL'] |
|
353 | 355 | ) |
|
354 | 356 | continue |
|
355 | 357 | |
|
356 | 358 | if b'REQUIRESNI' in entry and not sslutil.hassni: |
|
357 | 359 | repo.ui.debug( |
|
358 | 360 | b'filtering %s because SNI not supported\n' % entry[b'URL'] |
|
359 | 361 | ) |
|
360 | 362 | continue |
|
361 | 363 | |
|
362 | 364 | if b'REQUIREDRAM' in entry: |
|
363 | 365 | try: |
|
364 | 366 | requiredram = util.sizetoint(entry[b'REQUIREDRAM']) |
|
365 | 367 | except error.ParseError: |
|
366 | 368 | repo.ui.debug( |
|
367 | 369 | b'filtering %s due to a bad REQUIREDRAM attribute\n' |
|
368 | 370 | % entry[b'URL'] |
|
369 | 371 | ) |
|
370 | 372 | continue |
|
371 | 373 | actualram = repo.ui.estimatememory() |
|
372 | 374 | if actualram is not None and actualram * 0.66 < requiredram: |
|
373 | 375 | repo.ui.debug( |
|
374 | 376 | b'filtering %s as it needs more than 2/3 of system memory\n' |
|
375 | 377 | % entry[b'URL'] |
|
376 | 378 | ) |
|
377 | 379 | continue |
|
378 | 380 | |
|
379 | 381 | newentries.append(entry) |
|
380 | 382 | |
|
381 | 383 | return newentries |
|
382 | 384 | |
|
383 | 385 | |
|
384 | 386 | class clonebundleentry: |
|
385 | 387 | """Represents an item in a clone bundles manifest. |
|
386 | 388 | |
|
387 | 389 | This rich class is needed to support sorting since sorted() in Python 3 |
|
388 | 390 | doesn't support ``cmp`` and our comparison is complex enough that ``key=`` |
|
389 | 391 | won't work. |
|
390 | 392 | """ |
|
391 | 393 | |
|
392 | 394 | def __init__(self, value, prefers): |
|
393 | 395 | self.value = value |
|
394 | 396 | self.prefers = prefers |
|
395 | 397 | |
|
396 | 398 | def _cmp(self, other): |
|
397 | 399 | for prefkey, prefvalue in self.prefers: |
|
398 | 400 | avalue = self.value.get(prefkey) |
|
399 | 401 | bvalue = other.value.get(prefkey) |
|
400 | 402 | |
|
401 | 403 | # Special case for b missing attribute and a matches exactly. |
|
402 | 404 | if avalue is not None and bvalue is None and avalue == prefvalue: |
|
403 | 405 | return -1 |
|
404 | 406 | |
|
405 | 407 | # Special case for a missing attribute and b matches exactly. |
|
406 | 408 | if bvalue is not None and avalue is None and bvalue == prefvalue: |
|
407 | 409 | return 1 |
|
408 | 410 | |
|
409 | 411 | # We can't compare unless attribute present on both. |
|
410 | 412 | if avalue is None or bvalue is None: |
|
411 | 413 | continue |
|
412 | 414 | |
|
413 | 415 | # Same values should fall back to next attribute. |
|
414 | 416 | if avalue == bvalue: |
|
415 | 417 | continue |
|
416 | 418 | |
|
417 | 419 | # Exact matches come first. |
|
418 | 420 | if avalue == prefvalue: |
|
419 | 421 | return -1 |
|
420 | 422 | if bvalue == prefvalue: |
|
421 | 423 | return 1 |
|
422 | 424 | |
|
423 | 425 | # Fall back to next attribute. |
|
424 | 426 | continue |
|
425 | 427 | |
|
426 | 428 | # If we got here we couldn't sort by attributes and prefers. Fall |
|
427 | 429 | # back to index order. |
|
428 | 430 | return 0 |
|
429 | 431 | |
|
430 | 432 | def __lt__(self, other): |
|
431 | 433 | return self._cmp(other) < 0 |
|
432 | 434 | |
|
433 | 435 | def __gt__(self, other): |
|
434 | 436 | return self._cmp(other) > 0 |
|
435 | 437 | |
|
436 | 438 | def __eq__(self, other): |
|
437 | 439 | return self._cmp(other) == 0 |
|
438 | 440 | |
|
439 | 441 | def __le__(self, other): |
|
440 | 442 | return self._cmp(other) <= 0 |
|
441 | 443 | |
|
442 | 444 | def __ge__(self, other): |
|
443 | 445 | return self._cmp(other) >= 0 |
|
444 | 446 | |
|
445 | 447 | def __ne__(self, other): |
|
446 | 448 | return self._cmp(other) != 0 |
|
447 | 449 | |
|
448 | 450 | |
|
449 | 451 | def sortclonebundleentries(ui, entries): |
|
450 | 452 | prefers = ui.configlist(b'ui', b'clonebundleprefers') |
|
451 | 453 | if not prefers: |
|
452 | 454 | return list(entries) |
|
453 | 455 | |
|
454 | 456 | def _split(p): |
|
455 | 457 | if b'=' not in p: |
|
456 | 458 | hint = _(b"each comma separated item should be key=value pairs") |
|
457 | 459 | raise error.Abort( |
|
458 | 460 | _(b"invalid ui.clonebundleprefers item: %s") % p, hint=hint |
|
459 | 461 | ) |
|
460 | 462 | return p.split(b'=', 1) |
|
461 | 463 | |
|
462 | 464 | prefers = [_split(p) for p in prefers] |
|
463 | 465 | |
|
464 | 466 | items = sorted(clonebundleentry(v, prefers) for v in entries) |
|
465 | 467 | return [i.value for i in items] |
General Comments 0
You need to be logged in to leave comments.
Login now