Show More
@@ -1,427 +1,432 b'' | |||
|
1 | 1 | # bundlecaches.py - utility to deal with pre-computed bundle for servers |
|
2 | 2 | # |
|
3 | 3 | # This software may be used and distributed according to the terms of the |
|
4 | 4 | # GNU General Public License version 2 or any later version. |
|
5 | 5 | |
|
6 | 6 | from .i18n import _ |
|
7 | 7 | |
|
8 | 8 | from .thirdparty import attr |
|
9 | 9 | |
|
10 | 10 | from . import ( |
|
11 | 11 | error, |
|
12 | 12 | requirements as requirementsmod, |
|
13 | 13 | sslutil, |
|
14 | 14 | util, |
|
15 | 15 | ) |
|
16 | 16 | from .utils import stringutil |
|
17 | 17 | |
|
18 | 18 | urlreq = util.urlreq |
|
19 | 19 | |
|
20 | 20 | CB_MANIFEST_FILE = b'clonebundles.manifest' |
|
21 | 21 | |
|
22 | 22 | |
|
23 | 23 | @attr.s |
|
24 | 24 | class bundlespec: |
|
25 | 25 | compression = attr.ib() |
|
26 | 26 | wirecompression = attr.ib() |
|
27 | 27 | version = attr.ib() |
|
28 | 28 | wireversion = attr.ib() |
|
29 | 29 | params = attr.ib() |
|
30 | 30 | contentopts = attr.ib() |
|
31 | 31 | |
|
32 | 32 | |
|
33 | 33 | # Maps bundle version human names to changegroup versions. |
|
34 | 34 | _bundlespeccgversions = { |
|
35 | 35 | b'v1': b'01', |
|
36 | 36 | b'v2': b'02', |
|
37 | 37 | b'packed1': b's1', |
|
38 | 38 | b'bundle2': b'02', # legacy |
|
39 | 39 | } |
|
40 | 40 | |
|
41 | 41 | # Maps bundle version with content opts to choose which part to bundle |
|
42 | 42 | _bundlespeccontentopts = { |
|
43 | 43 | b'v1': { |
|
44 | 44 | b'changegroup': True, |
|
45 | 45 | b'cg.version': b'01', |
|
46 | 46 | b'obsolescence': False, |
|
47 | 47 | b'phases': False, |
|
48 | 48 | b'tagsfnodescache': False, |
|
49 | 49 | b'revbranchcache': False, |
|
50 | 50 | }, |
|
51 | 51 | b'v2': { |
|
52 | 52 | b'changegroup': True, |
|
53 | 53 | b'cg.version': b'02', |
|
54 | 54 | b'obsolescence': False, |
|
55 | 55 | b'phases': False, |
|
56 | 56 | b'tagsfnodescache': True, |
|
57 | 57 | b'revbranchcache': True, |
|
58 | 58 | }, |
|
59 |
b'packed1': { |
|
|
59 | b'packed1': { | |
|
60 | b'cg.version': b's1', | |
|
61 | }, | |
|
62 | b'bundle2': { # legacy | |
|
63 | b'cg.version': b'02', | |
|
64 | }, | |
|
60 | 65 | } |
|
61 | 66 | _bundlespeccontentopts[b'bundle2'] = _bundlespeccontentopts[b'v2'] |
|
62 | 67 | |
|
63 | 68 | _bundlespecvariants = { |
|
64 | 69 | b"streamv2": { |
|
65 | 70 | b"changegroup": False, |
|
66 | 71 | b"streamv2": True, |
|
67 | 72 | b"tagsfnodescache": False, |
|
68 | 73 | b"revbranchcache": False, |
|
69 | 74 | } |
|
70 | 75 | } |
|
71 | 76 | |
|
72 | 77 | # Compression engines allowed in version 1. THIS SHOULD NEVER CHANGE. |
|
73 | 78 | _bundlespecv1compengines = {b'gzip', b'bzip2', b'none'} |
|
74 | 79 | |
|
75 | 80 | |
|
76 | 81 | def parsebundlespec(repo, spec, strict=True): |
|
77 | 82 | """Parse a bundle string specification into parts. |
|
78 | 83 | |
|
79 | 84 | Bundle specifications denote a well-defined bundle/exchange format. |
|
80 | 85 | The content of a given specification should not change over time in |
|
81 | 86 | order to ensure that bundles produced by a newer version of Mercurial are |
|
82 | 87 | readable from an older version. |
|
83 | 88 | |
|
84 | 89 | The string currently has the form: |
|
85 | 90 | |
|
86 | 91 | <compression>-<type>[;<parameter0>[;<parameter1>]] |
|
87 | 92 | |
|
88 | 93 | Where <compression> is one of the supported compression formats |
|
89 | 94 | and <type> is (currently) a version string. A ";" can follow the type and |
|
90 | 95 | all text afterwards is interpreted as URI encoded, ";" delimited key=value |
|
91 | 96 | pairs. |
|
92 | 97 | |
|
93 | 98 | If ``strict`` is True (the default) <compression> is required. Otherwise, |
|
94 | 99 | it is optional. |
|
95 | 100 | |
|
96 | 101 | Returns a bundlespec object of (compression, version, parameters). |
|
97 | 102 | Compression will be ``None`` if not in strict mode and a compression isn't |
|
98 | 103 | defined. |
|
99 | 104 | |
|
100 | 105 | An ``InvalidBundleSpecification`` is raised when the specification is |
|
101 | 106 | not syntactically well formed. |
|
102 | 107 | |
|
103 | 108 | An ``UnsupportedBundleSpecification`` is raised when the compression or |
|
104 | 109 | bundle type/version is not recognized. |
|
105 | 110 | |
|
106 | 111 | Note: this function will likely eventually return a more complex data |
|
107 | 112 | structure, including bundle2 part information. |
|
108 | 113 | """ |
|
109 | 114 | |
|
110 | 115 | def parseparams(s): |
|
111 | 116 | if b';' not in s: |
|
112 | 117 | return s, {} |
|
113 | 118 | |
|
114 | 119 | params = {} |
|
115 | 120 | version, paramstr = s.split(b';', 1) |
|
116 | 121 | |
|
117 | 122 | for p in paramstr.split(b';'): |
|
118 | 123 | if b'=' not in p: |
|
119 | 124 | raise error.InvalidBundleSpecification( |
|
120 | 125 | _( |
|
121 | 126 | b'invalid bundle specification: ' |
|
122 | 127 | b'missing "=" in parameter: %s' |
|
123 | 128 | ) |
|
124 | 129 | % p |
|
125 | 130 | ) |
|
126 | 131 | |
|
127 | 132 | key, value = p.split(b'=', 1) |
|
128 | 133 | key = urlreq.unquote(key) |
|
129 | 134 | value = urlreq.unquote(value) |
|
130 | 135 | params[key] = value |
|
131 | 136 | |
|
132 | 137 | return version, params |
|
133 | 138 | |
|
134 | 139 | if strict and b'-' not in spec: |
|
135 | 140 | raise error.InvalidBundleSpecification( |
|
136 | 141 | _( |
|
137 | 142 | b'invalid bundle specification; ' |
|
138 | 143 | b'must be prefixed with compression: %s' |
|
139 | 144 | ) |
|
140 | 145 | % spec |
|
141 | 146 | ) |
|
142 | 147 | |
|
143 | 148 | if b'-' in spec: |
|
144 | 149 | compression, version = spec.split(b'-', 1) |
|
145 | 150 | |
|
146 | 151 | if compression not in util.compengines.supportedbundlenames: |
|
147 | 152 | raise error.UnsupportedBundleSpecification( |
|
148 | 153 | _(b'%s compression is not supported') % compression |
|
149 | 154 | ) |
|
150 | 155 | |
|
151 | 156 | version, params = parseparams(version) |
|
152 | 157 | |
|
153 |
if version not in _bundlespecc |
|
|
158 | if version not in _bundlespeccontentopts: | |
|
154 | 159 | raise error.UnsupportedBundleSpecification( |
|
155 | 160 | _(b'%s is not a recognized bundle version') % version |
|
156 | 161 | ) |
|
157 | 162 | else: |
|
158 | 163 | # Value could be just the compression or just the version, in which |
|
159 | 164 | # case some defaults are assumed (but only when not in strict mode). |
|
160 | 165 | assert not strict |
|
161 | 166 | |
|
162 | 167 | spec, params = parseparams(spec) |
|
163 | 168 | |
|
164 | 169 | if spec in util.compengines.supportedbundlenames: |
|
165 | 170 | compression = spec |
|
166 | 171 | version = b'v1' |
|
167 | 172 | # Generaldelta repos require v2. |
|
168 | 173 | if requirementsmod.GENERALDELTA_REQUIREMENT in repo.requirements: |
|
169 | 174 | version = b'v2' |
|
170 | 175 | elif requirementsmod.REVLOGV2_REQUIREMENT in repo.requirements: |
|
171 | 176 | version = b'v2' |
|
172 | 177 | # Modern compression engines require v2. |
|
173 | 178 | if compression not in _bundlespecv1compengines: |
|
174 | 179 | version = b'v2' |
|
175 |
elif spec in _bundlespecc |
|
|
180 | elif spec in _bundlespeccontentopts: | |
|
176 | 181 | if spec == b'packed1': |
|
177 | 182 | compression = b'none' |
|
178 | 183 | else: |
|
179 | 184 | compression = b'bzip2' |
|
180 | 185 | version = spec |
|
181 | 186 | else: |
|
182 | 187 | raise error.UnsupportedBundleSpecification( |
|
183 | 188 | _(b'%s is not a recognized bundle specification') % spec |
|
184 | 189 | ) |
|
185 | 190 | |
|
186 | 191 | # Bundle version 1 only supports a known set of compression engines. |
|
187 | 192 | if version == b'v1' and compression not in _bundlespecv1compengines: |
|
188 | 193 | raise error.UnsupportedBundleSpecification( |
|
189 | 194 | _(b'compression engine %s is not supported on v1 bundles') |
|
190 | 195 | % compression |
|
191 | 196 | ) |
|
192 | 197 | |
|
193 | 198 | # The specification for packed1 can optionally declare the data formats |
|
194 | 199 | # required to apply it. If we see this metadata, compare against what the |
|
195 | 200 | # repo supports and error if the bundle isn't compatible. |
|
196 | 201 | if version == b'packed1' and b'requirements' in params: |
|
197 | 202 | requirements = set(params[b'requirements'].split(b',')) |
|
198 | 203 | missingreqs = requirements - requirementsmod.STREAM_FIXED_REQUIREMENTS |
|
199 | 204 | if missingreqs: |
|
200 | 205 | raise error.UnsupportedBundleSpecification( |
|
201 | 206 | _(b'missing support for repository features: %s') |
|
202 | 207 | % b', '.join(sorted(missingreqs)) |
|
203 | 208 | ) |
|
204 | 209 | |
|
205 | 210 | # Compute contentopts based on the version |
|
206 | 211 | contentopts = _bundlespeccontentopts.get(version, {}).copy() |
|
207 | 212 | |
|
208 | 213 | # Process the variants |
|
209 | 214 | if b"stream" in params and params[b"stream"] == b"v2": |
|
210 | 215 | variant = _bundlespecvariants[b"streamv2"] |
|
211 | 216 | contentopts.update(variant) |
|
212 | 217 | |
|
213 | 218 | engine = util.compengines.forbundlename(compression) |
|
214 | 219 | compression, wirecompression = engine.bundletype() |
|
215 |
wireversion = _bundlespecc |
|
|
220 | wireversion = _bundlespeccontentopts[version][b'cg.version'] | |
|
216 | 221 | |
|
217 | 222 | return bundlespec( |
|
218 | 223 | compression, wirecompression, version, wireversion, params, contentopts |
|
219 | 224 | ) |
|
220 | 225 | |
|
221 | 226 | |
|
222 | 227 | def parseclonebundlesmanifest(repo, s): |
|
223 | 228 | """Parses the raw text of a clone bundles manifest. |
|
224 | 229 | |
|
225 | 230 | Returns a list of dicts. The dicts have a ``URL`` key corresponding |
|
226 | 231 | to the URL and other keys are the attributes for the entry. |
|
227 | 232 | """ |
|
228 | 233 | m = [] |
|
229 | 234 | for line in s.splitlines(): |
|
230 | 235 | fields = line.split() |
|
231 | 236 | if not fields: |
|
232 | 237 | continue |
|
233 | 238 | attrs = {b'URL': fields[0]} |
|
234 | 239 | for rawattr in fields[1:]: |
|
235 | 240 | key, value = rawattr.split(b'=', 1) |
|
236 | 241 | key = util.urlreq.unquote(key) |
|
237 | 242 | value = util.urlreq.unquote(value) |
|
238 | 243 | attrs[key] = value |
|
239 | 244 | |
|
240 | 245 | # Parse BUNDLESPEC into components. This makes client-side |
|
241 | 246 | # preferences easier to specify since you can prefer a single |
|
242 | 247 | # component of the BUNDLESPEC. |
|
243 | 248 | if key == b'BUNDLESPEC': |
|
244 | 249 | try: |
|
245 | 250 | bundlespec = parsebundlespec(repo, value) |
|
246 | 251 | attrs[b'COMPRESSION'] = bundlespec.compression |
|
247 | 252 | attrs[b'VERSION'] = bundlespec.version |
|
248 | 253 | except error.InvalidBundleSpecification: |
|
249 | 254 | pass |
|
250 | 255 | except error.UnsupportedBundleSpecification: |
|
251 | 256 | pass |
|
252 | 257 | |
|
253 | 258 | m.append(attrs) |
|
254 | 259 | |
|
255 | 260 | return m |
|
256 | 261 | |
|
257 | 262 | |
|
258 | 263 | def isstreamclonespec(bundlespec): |
|
259 | 264 | # Stream clone v1 |
|
260 | 265 | if bundlespec.wirecompression == b'UN' and bundlespec.wireversion == b's1': |
|
261 | 266 | return True |
|
262 | 267 | |
|
263 | 268 | # Stream clone v2 |
|
264 | 269 | if ( |
|
265 | 270 | bundlespec.wirecompression == b'UN' |
|
266 | 271 | and bundlespec.wireversion == b'02' |
|
267 | 272 | and bundlespec.contentopts.get(b'streamv2') |
|
268 | 273 | ): |
|
269 | 274 | return True |
|
270 | 275 | |
|
271 | 276 | return False |
|
272 | 277 | |
|
273 | 278 | |
|
274 | 279 | def filterclonebundleentries(repo, entries, streamclonerequested=False): |
|
275 | 280 | """Remove incompatible clone bundle manifest entries. |
|
276 | 281 | |
|
277 | 282 | Accepts a list of entries parsed with ``parseclonebundlesmanifest`` |
|
278 | 283 | and returns a new list consisting of only the entries that this client |
|
279 | 284 | should be able to apply. |
|
280 | 285 | |
|
281 | 286 | There is no guarantee we'll be able to apply all returned entries because |
|
282 | 287 | the metadata we use to filter on may be missing or wrong. |
|
283 | 288 | """ |
|
284 | 289 | newentries = [] |
|
285 | 290 | for entry in entries: |
|
286 | 291 | spec = entry.get(b'BUNDLESPEC') |
|
287 | 292 | if spec: |
|
288 | 293 | try: |
|
289 | 294 | bundlespec = parsebundlespec(repo, spec, strict=True) |
|
290 | 295 | |
|
291 | 296 | # If a stream clone was requested, filter out non-streamclone |
|
292 | 297 | # entries. |
|
293 | 298 | if streamclonerequested and not isstreamclonespec(bundlespec): |
|
294 | 299 | repo.ui.debug( |
|
295 | 300 | b'filtering %s because not a stream clone\n' |
|
296 | 301 | % entry[b'URL'] |
|
297 | 302 | ) |
|
298 | 303 | continue |
|
299 | 304 | |
|
300 | 305 | except error.InvalidBundleSpecification as e: |
|
301 | 306 | repo.ui.debug(stringutil.forcebytestr(e) + b'\n') |
|
302 | 307 | continue |
|
303 | 308 | except error.UnsupportedBundleSpecification as e: |
|
304 | 309 | repo.ui.debug( |
|
305 | 310 | b'filtering %s because unsupported bundle ' |
|
306 | 311 | b'spec: %s\n' % (entry[b'URL'], stringutil.forcebytestr(e)) |
|
307 | 312 | ) |
|
308 | 313 | continue |
|
309 | 314 | # If we don't have a spec and requested a stream clone, we don't know |
|
310 | 315 | # what the entry is so don't attempt to apply it. |
|
311 | 316 | elif streamclonerequested: |
|
312 | 317 | repo.ui.debug( |
|
313 | 318 | b'filtering %s because cannot determine if a stream ' |
|
314 | 319 | b'clone bundle\n' % entry[b'URL'] |
|
315 | 320 | ) |
|
316 | 321 | continue |
|
317 | 322 | |
|
318 | 323 | if b'REQUIRESNI' in entry and not sslutil.hassni: |
|
319 | 324 | repo.ui.debug( |
|
320 | 325 | b'filtering %s because SNI not supported\n' % entry[b'URL'] |
|
321 | 326 | ) |
|
322 | 327 | continue |
|
323 | 328 | |
|
324 | 329 | if b'REQUIREDRAM' in entry: |
|
325 | 330 | try: |
|
326 | 331 | requiredram = util.sizetoint(entry[b'REQUIREDRAM']) |
|
327 | 332 | except error.ParseError: |
|
328 | 333 | repo.ui.debug( |
|
329 | 334 | b'filtering %s due to a bad REQUIREDRAM attribute\n' |
|
330 | 335 | % entry[b'URL'] |
|
331 | 336 | ) |
|
332 | 337 | continue |
|
333 | 338 | actualram = repo.ui.estimatememory() |
|
334 | 339 | if actualram is not None and actualram * 0.66 < requiredram: |
|
335 | 340 | repo.ui.debug( |
|
336 | 341 | b'filtering %s as it needs more than 2/3 of system memory\n' |
|
337 | 342 | % entry[b'URL'] |
|
338 | 343 | ) |
|
339 | 344 | continue |
|
340 | 345 | |
|
341 | 346 | newentries.append(entry) |
|
342 | 347 | |
|
343 | 348 | return newentries |
|
344 | 349 | |
|
345 | 350 | |
|
346 | 351 | class clonebundleentry: |
|
347 | 352 | """Represents an item in a clone bundles manifest. |
|
348 | 353 | |
|
349 | 354 | This rich class is needed to support sorting since sorted() in Python 3 |
|
350 | 355 | doesn't support ``cmp`` and our comparison is complex enough that ``key=`` |
|
351 | 356 | won't work. |
|
352 | 357 | """ |
|
353 | 358 | |
|
354 | 359 | def __init__(self, value, prefers): |
|
355 | 360 | self.value = value |
|
356 | 361 | self.prefers = prefers |
|
357 | 362 | |
|
358 | 363 | def _cmp(self, other): |
|
359 | 364 | for prefkey, prefvalue in self.prefers: |
|
360 | 365 | avalue = self.value.get(prefkey) |
|
361 | 366 | bvalue = other.value.get(prefkey) |
|
362 | 367 | |
|
363 | 368 | # Special case for b missing attribute and a matches exactly. |
|
364 | 369 | if avalue is not None and bvalue is None and avalue == prefvalue: |
|
365 | 370 | return -1 |
|
366 | 371 | |
|
367 | 372 | # Special case for a missing attribute and b matches exactly. |
|
368 | 373 | if bvalue is not None and avalue is None and bvalue == prefvalue: |
|
369 | 374 | return 1 |
|
370 | 375 | |
|
371 | 376 | # We can't compare unless attribute present on both. |
|
372 | 377 | if avalue is None or bvalue is None: |
|
373 | 378 | continue |
|
374 | 379 | |
|
375 | 380 | # Same values should fall back to next attribute. |
|
376 | 381 | if avalue == bvalue: |
|
377 | 382 | continue |
|
378 | 383 | |
|
379 | 384 | # Exact matches come first. |
|
380 | 385 | if avalue == prefvalue: |
|
381 | 386 | return -1 |
|
382 | 387 | if bvalue == prefvalue: |
|
383 | 388 | return 1 |
|
384 | 389 | |
|
385 | 390 | # Fall back to next attribute. |
|
386 | 391 | continue |
|
387 | 392 | |
|
388 | 393 | # If we got here we couldn't sort by attributes and prefers. Fall |
|
389 | 394 | # back to index order. |
|
390 | 395 | return 0 |
|
391 | 396 | |
|
392 | 397 | def __lt__(self, other): |
|
393 | 398 | return self._cmp(other) < 0 |
|
394 | 399 | |
|
395 | 400 | def __gt__(self, other): |
|
396 | 401 | return self._cmp(other) > 0 |
|
397 | 402 | |
|
398 | 403 | def __eq__(self, other): |
|
399 | 404 | return self._cmp(other) == 0 |
|
400 | 405 | |
|
401 | 406 | def __le__(self, other): |
|
402 | 407 | return self._cmp(other) <= 0 |
|
403 | 408 | |
|
404 | 409 | def __ge__(self, other): |
|
405 | 410 | return self._cmp(other) >= 0 |
|
406 | 411 | |
|
407 | 412 | def __ne__(self, other): |
|
408 | 413 | return self._cmp(other) != 0 |
|
409 | 414 | |
|
410 | 415 | |
|
411 | 416 | def sortclonebundleentries(ui, entries): |
|
412 | 417 | prefers = ui.configlist(b'ui', b'clonebundleprefers') |
|
413 | 418 | if not prefers: |
|
414 | 419 | return list(entries) |
|
415 | 420 | |
|
416 | 421 | def _split(p): |
|
417 | 422 | if b'=' not in p: |
|
418 | 423 | hint = _(b"each comma separated item should be key=value pairs") |
|
419 | 424 | raise error.Abort( |
|
420 | 425 | _(b"invalid ui.clonebundleprefers item: %s") % p, hint=hint |
|
421 | 426 | ) |
|
422 | 427 | return p.split(b'=', 1) |
|
423 | 428 | |
|
424 | 429 | prefers = [_split(p) for p in prefers] |
|
425 | 430 | |
|
426 | 431 | items = sorted(clonebundleentry(v, prefers) for v in entries) |
|
427 | 432 | return [i.value for i in items] |
General Comments 0
You need to be logged in to leave comments.
Login now