##// END OF EJS Templates
engine: 'if not, else' -> 'if, else'...
Pulkit Goyal -
r47194:45c3a263 default
parent child Browse files
Show More
@@ -1,539 +1,539 b''
1 1 # upgrade.py - functions for in place upgrade of Mercurial repository
2 2 #
3 3 # Copyright (c) 2016-present, Gregory Szorc
4 4 #
5 5 # This software may be used and distributed according to the terms of the
6 6 # GNU General Public License version 2 or any later version.
7 7
8 8 from __future__ import absolute_import
9 9
10 10 import stat
11 11
12 12 from ..i18n import _
13 13 from ..pycompat import getattr
14 14 from .. import (
15 15 changelog,
16 16 error,
17 17 filelog,
18 18 manifest,
19 19 metadata,
20 20 pycompat,
21 21 requirements,
22 22 revlog,
23 23 scmutil,
24 24 util,
25 25 vfs as vfsmod,
26 26 )
27 27
28 28
29 29 def _revlogfrompath(repo, path):
30 30 """Obtain a revlog from a repo path.
31 31
32 32 An instance of the appropriate class is returned.
33 33 """
34 34 if path == b'00changelog.i':
35 35 return changelog.changelog(repo.svfs)
36 36 elif path.endswith(b'00manifest.i'):
37 37 mandir = path[: -len(b'00manifest.i')]
38 38 return manifest.manifestrevlog(repo.svfs, tree=mandir)
39 39 else:
40 40 # reverse of "/".join(("data", path + ".i"))
41 41 return filelog.filelog(repo.svfs, path[5:-2])
42 42
43 43
44 44 def _copyrevlog(tr, destrepo, oldrl, unencodedname):
45 45 """copy all relevant files for `oldrl` into `destrepo` store
46 46
47 47 Files are copied "as is" without any transformation. The copy is performed
48 48 without extra checks. Callers are responsible for making sure the copied
49 49 content is compatible with format of the destination repository.
50 50 """
51 51 oldrl = getattr(oldrl, '_revlog', oldrl)
52 52 newrl = _revlogfrompath(destrepo, unencodedname)
53 53 newrl = getattr(newrl, '_revlog', newrl)
54 54
55 55 oldvfs = oldrl.opener
56 56 newvfs = newrl.opener
57 57 oldindex = oldvfs.join(oldrl.indexfile)
58 58 newindex = newvfs.join(newrl.indexfile)
59 59 olddata = oldvfs.join(oldrl.datafile)
60 60 newdata = newvfs.join(newrl.datafile)
61 61
62 62 with newvfs(newrl.indexfile, b'w'):
63 63 pass # create all the directories
64 64
65 65 util.copyfile(oldindex, newindex)
66 66 copydata = oldrl.opener.exists(oldrl.datafile)
67 67 if copydata:
68 68 util.copyfile(olddata, newdata)
69 69
70 70 if not (
71 71 unencodedname.endswith(b'00changelog.i')
72 72 or unencodedname.endswith(b'00manifest.i')
73 73 ):
74 74 destrepo.svfs.fncache.add(unencodedname)
75 75 if copydata:
76 76 destrepo.svfs.fncache.add(unencodedname[:-2] + b'.d')
77 77
78 78
79 79 UPGRADE_CHANGELOG = b"changelog"
80 80 UPGRADE_MANIFEST = b"manifest"
81 81 UPGRADE_FILELOGS = b"all-filelogs"
82 82
83 83 UPGRADE_ALL_REVLOGS = frozenset(
84 84 [UPGRADE_CHANGELOG, UPGRADE_MANIFEST, UPGRADE_FILELOGS]
85 85 )
86 86
87 87
88 88 def getsidedatacompanion(srcrepo, dstrepo):
89 89 sidedatacompanion = None
90 90 removedreqs = srcrepo.requirements - dstrepo.requirements
91 91 addedreqs = dstrepo.requirements - srcrepo.requirements
92 92 if requirements.SIDEDATA_REQUIREMENT in removedreqs:
93 93
94 94 def sidedatacompanion(rl, rev):
95 95 rl = getattr(rl, '_revlog', rl)
96 96 if rl.flags(rev) & revlog.REVIDX_SIDEDATA:
97 97 return True, (), {}, 0, 0
98 98 return False, (), {}, 0, 0
99 99
100 100 elif requirements.COPIESSDC_REQUIREMENT in addedreqs:
101 101 sidedatacompanion = metadata.getsidedataadder(srcrepo, dstrepo)
102 102 elif requirements.COPIESSDC_REQUIREMENT in removedreqs:
103 103 sidedatacompanion = metadata.getsidedataremover(srcrepo, dstrepo)
104 104 return sidedatacompanion
105 105
106 106
107 107 def matchrevlog(revlogfilter, entry):
108 108 """check if a revlog is selected for cloning.
109 109
110 110 In other words, are there any updates which need to be done on revlog
111 111 or it can be blindly copied.
112 112
113 113 The store entry is checked against the passed filter"""
114 114 if entry.endswith(b'00changelog.i'):
115 115 return UPGRADE_CHANGELOG in revlogfilter
116 116 elif entry.endswith(b'00manifest.i'):
117 117 return UPGRADE_MANIFEST in revlogfilter
118 118 return UPGRADE_FILELOGS in revlogfilter
119 119
120 120
121 121 def _perform_clone(
122 122 ui,
123 123 dstrepo,
124 124 tr,
125 125 old_revlog,
126 126 unencoded,
127 127 upgrade_op,
128 128 sidedatacompanion,
129 129 oncopiedrevision,
130 130 ):
131 131 """ returns the new revlog object created"""
132 132 newrl = None
133 133 if matchrevlog(upgrade_op.revlogs_to_process, unencoded):
134 134 ui.note(
135 135 _(b'cloning %d revisions from %s\n') % (len(old_revlog), unencoded)
136 136 )
137 137 newrl = _revlogfrompath(dstrepo, unencoded)
138 138 old_revlog.clone(
139 139 tr,
140 140 newrl,
141 141 addrevisioncb=oncopiedrevision,
142 142 deltareuse=upgrade_op.delta_reuse_mode,
143 143 forcedeltabothparents=upgrade_op.force_re_delta_both_parents,
144 144 sidedatacompanion=sidedatacompanion,
145 145 )
146 146 else:
147 147 msg = _(b'blindly copying %s containing %i revisions\n')
148 148 ui.note(msg % (unencoded, len(old_revlog)))
149 149 _copyrevlog(tr, dstrepo, old_revlog, unencoded)
150 150
151 151 newrl = _revlogfrompath(dstrepo, unencoded)
152 152 return newrl
153 153
154 154
155 155 def _clonerevlogs(
156 156 ui,
157 157 srcrepo,
158 158 dstrepo,
159 159 tr,
160 160 upgrade_op,
161 161 ):
162 162 """Copy revlogs between 2 repos."""
163 163 revcount = 0
164 164 srcsize = 0
165 165 srcrawsize = 0
166 166 dstsize = 0
167 167 fcount = 0
168 168 frevcount = 0
169 169 fsrcsize = 0
170 170 frawsize = 0
171 171 fdstsize = 0
172 172 mcount = 0
173 173 mrevcount = 0
174 174 msrcsize = 0
175 175 mrawsize = 0
176 176 mdstsize = 0
177 177 crevcount = 0
178 178 csrcsize = 0
179 179 crawsize = 0
180 180 cdstsize = 0
181 181
182 182 alldatafiles = list(srcrepo.store.walk())
183 183 # mapping of data files which needs to be cloned
184 184 # key is unencoded filename
185 185 # value is revlog_object_from_srcrepo
186 186 manifests = {}
187 187 changelogs = {}
188 188 filelogs = {}
189 189
190 190 # Perform a pass to collect metadata. This validates we can open all
191 191 # source files and allows a unified progress bar to be displayed.
192 192 for unencoded, encoded, size in alldatafiles:
193 193 if not unencoded.endswith(b'.i'):
194 194 continue
195 195
196 196 rl = _revlogfrompath(srcrepo, unencoded)
197 197
198 198 info = rl.storageinfo(
199 199 exclusivefiles=True,
200 200 revisionscount=True,
201 201 trackedsize=True,
202 202 storedsize=True,
203 203 )
204 204
205 205 revcount += info[b'revisionscount'] or 0
206 206 datasize = info[b'storedsize'] or 0
207 207 rawsize = info[b'trackedsize'] or 0
208 208
209 209 srcsize += datasize
210 210 srcrawsize += rawsize
211 211
212 212 # This is for the separate progress bars.
213 213 if isinstance(rl, changelog.changelog):
214 214 changelogs[unencoded] = rl
215 215 crevcount += len(rl)
216 216 csrcsize += datasize
217 217 crawsize += rawsize
218 218 elif isinstance(rl, manifest.manifestrevlog):
219 219 manifests[unencoded] = rl
220 220 mcount += 1
221 221 mrevcount += len(rl)
222 222 msrcsize += datasize
223 223 mrawsize += rawsize
224 224 elif isinstance(rl, filelog.filelog):
225 225 filelogs[unencoded] = rl
226 226 fcount += 1
227 227 frevcount += len(rl)
228 228 fsrcsize += datasize
229 229 frawsize += rawsize
230 230 else:
231 231 error.ProgrammingError(b'unknown revlog type')
232 232
233 233 if not revcount:
234 234 return
235 235
236 236 ui.status(
237 237 _(
238 238 b'migrating %d total revisions (%d in filelogs, %d in manifests, '
239 239 b'%d in changelog)\n'
240 240 )
241 241 % (revcount, frevcount, mrevcount, crevcount)
242 242 )
243 243 ui.status(
244 244 _(b'migrating %s in store; %s tracked data\n')
245 245 % ((util.bytecount(srcsize), util.bytecount(srcrawsize)))
246 246 )
247 247
248 248 # Used to keep track of progress.
249 249 progress = None
250 250
251 251 def oncopiedrevision(rl, rev, node):
252 252 progress.increment()
253 253
254 254 sidedatacompanion = getsidedatacompanion(srcrepo, dstrepo)
255 255
256 256 # Migrating filelogs
257 257 ui.status(
258 258 _(
259 259 b'migrating %d filelogs containing %d revisions '
260 260 b'(%s in store; %s tracked data)\n'
261 261 )
262 262 % (
263 263 fcount,
264 264 frevcount,
265 265 util.bytecount(fsrcsize),
266 266 util.bytecount(frawsize),
267 267 )
268 268 )
269 269 progress = srcrepo.ui.makeprogress(_(b'file revisions'), total=frevcount)
270 270 for unencoded, oldrl in sorted(filelogs.items()):
271 271 newrl = _perform_clone(
272 272 ui,
273 273 dstrepo,
274 274 tr,
275 275 oldrl,
276 276 unencoded,
277 277 upgrade_op,
278 278 sidedatacompanion,
279 279 oncopiedrevision,
280 280 )
281 281 info = newrl.storageinfo(storedsize=True)
282 282 fdstsize += info[b'storedsize'] or 0
283 283 ui.status(
284 284 _(
285 285 b'finished migrating %d filelog revisions across %d '
286 286 b'filelogs; change in size: %s\n'
287 287 )
288 288 % (frevcount, fcount, util.bytecount(fdstsize - fsrcsize))
289 289 )
290 290
291 291 # Migrating manifests
292 292 ui.status(
293 293 _(
294 294 b'migrating %d manifests containing %d revisions '
295 295 b'(%s in store; %s tracked data)\n'
296 296 )
297 297 % (
298 298 mcount,
299 299 mrevcount,
300 300 util.bytecount(msrcsize),
301 301 util.bytecount(mrawsize),
302 302 )
303 303 )
304 304 if progress:
305 305 progress.complete()
306 306 progress = srcrepo.ui.makeprogress(
307 307 _(b'manifest revisions'), total=mrevcount
308 308 )
309 309 for unencoded, oldrl in sorted(manifests.items()):
310 310 newrl = _perform_clone(
311 311 ui,
312 312 dstrepo,
313 313 tr,
314 314 oldrl,
315 315 unencoded,
316 316 upgrade_op,
317 317 sidedatacompanion,
318 318 oncopiedrevision,
319 319 )
320 320 info = newrl.storageinfo(storedsize=True)
321 321 mdstsize += info[b'storedsize'] or 0
322 322 ui.status(
323 323 _(
324 324 b'finished migrating %d manifest revisions across %d '
325 325 b'manifests; change in size: %s\n'
326 326 )
327 327 % (mrevcount, mcount, util.bytecount(mdstsize - msrcsize))
328 328 )
329 329
330 330 # Migrating changelog
331 331 ui.status(
332 332 _(
333 333 b'migrating changelog containing %d revisions '
334 334 b'(%s in store; %s tracked data)\n'
335 335 )
336 336 % (
337 337 crevcount,
338 338 util.bytecount(csrcsize),
339 339 util.bytecount(crawsize),
340 340 )
341 341 )
342 342 if progress:
343 343 progress.complete()
344 344 progress = srcrepo.ui.makeprogress(
345 345 _(b'changelog revisions'), total=crevcount
346 346 )
347 347 for unencoded, oldrl in sorted(changelogs.items()):
348 348 newrl = _perform_clone(
349 349 ui,
350 350 dstrepo,
351 351 tr,
352 352 oldrl,
353 353 unencoded,
354 354 upgrade_op,
355 355 sidedatacompanion,
356 356 oncopiedrevision,
357 357 )
358 358 info = newrl.storageinfo(storedsize=True)
359 359 cdstsize += info[b'storedsize'] or 0
360 360 progress.complete()
361 361 ui.status(
362 362 _(
363 363 b'finished migrating %d changelog revisions; change in size: '
364 364 b'%s\n'
365 365 )
366 366 % (crevcount, util.bytecount(cdstsize - csrcsize))
367 367 )
368 368
369 369 dstsize = fdstsize + mdstsize + cdstsize
370 370 ui.status(
371 371 _(
372 372 b'finished migrating %d total revisions; total change in store '
373 373 b'size: %s\n'
374 374 )
375 375 % (revcount, util.bytecount(dstsize - srcsize))
376 376 )
377 377
378 378
379 379 def _files_to_copy_post_revlog_clone(srcrepo):
380 380 """yields files which should be copied to destination after revlogs
381 381 are cloned"""
382 382 for path, kind, st in sorted(srcrepo.store.vfs.readdir(b'', stat=True)):
383 383 # don't copy revlogs as they are already cloned
384 384 if path.endswith((b'.i', b'.d', b'.n', b'.nd')):
385 385 continue
386 386 # Skip transaction related files.
387 387 if path.startswith(b'undo'):
388 388 continue
389 389 # Only copy regular files.
390 390 if kind != stat.S_IFREG:
391 391 continue
392 392 # Skip other skipped files.
393 393 if path in (b'lock', b'fncache'):
394 394 continue
395 395 # TODO: should we skip cache too?
396 396
397 397 yield path
398 398
399 399
400 400 def _replacestores(currentrepo, upgradedrepo, backupvfs, upgrade_op):
401 401 """Replace the stores after current repository is upgraded
402 402
403 403 Creates a backup of current repository store at backup path
404 404 Replaces upgraded store files in current repo from upgraded one
405 405
406 406 Arguments:
407 407 currentrepo: repo object of current repository
408 408 upgradedrepo: repo object of the upgraded data
409 409 backupvfs: vfs object for the backup path
410 410 upgrade_op: upgrade operation object
411 411 to be used to decide what all is upgraded
412 412 """
413 413 # TODO: don't blindly rename everything in store
414 414 # There can be upgrades where store is not touched at all
415 415 if upgrade_op.backup_store:
416 416 util.rename(currentrepo.spath, backupvfs.join(b'store'))
417 417 else:
418 418 currentrepo.vfs.rmtree(b'store', forcibly=True)
419 419 util.rename(upgradedrepo.spath, currentrepo.spath)
420 420
421 421
422 422 def finishdatamigration(ui, srcrepo, dstrepo, requirements):
423 423 """Hook point for extensions to perform additional actions during upgrade.
424 424
425 425 This function is called after revlogs and store files have been copied but
426 426 before the new store is swapped into the original location.
427 427 """
428 428
429 429
430 430 def upgrade(ui, srcrepo, dstrepo, upgrade_op):
431 431 """Do the low-level work of upgrading a repository.
432 432
433 433 The upgrade is effectively performed as a copy between a source
434 434 repository and a temporary destination repository.
435 435
436 436 The source repository is unmodified for as long as possible so the
437 437 upgrade can abort at any time without causing loss of service for
438 438 readers and without corrupting the source repository.
439 439 """
440 440 assert srcrepo.currentwlock()
441 441 assert dstrepo.currentwlock()
442 442 backuppath = None
443 443 backupvfs = None
444 444
445 445 ui.status(
446 446 _(
447 447 b'(it is safe to interrupt this process any time before '
448 448 b'data migration completes)\n'
449 449 )
450 450 )
451 451
452 if not upgrade_op.requirements_only:
452 if upgrade_op.requirements_only:
453 ui.status(_(b'upgrading repository requirements\n'))
454 scmutil.writereporequirements(srcrepo, upgrade_op.new_requirements)
455 else:
453 456 with dstrepo.transaction(b'upgrade') as tr:
454 457 _clonerevlogs(
455 458 ui,
456 459 srcrepo,
457 460 dstrepo,
458 461 tr,
459 462 upgrade_op,
460 463 )
461 464
462 465 # Now copy other files in the store directory.
463 466 for p in _files_to_copy_post_revlog_clone(srcrepo):
464 467 srcrepo.ui.status(_(b'copying %s\n') % p)
465 468 src = srcrepo.store.rawvfs.join(p)
466 469 dst = dstrepo.store.rawvfs.join(p)
467 470 util.copyfile(src, dst, copystat=True)
468 471
469 472 finishdatamigration(ui, srcrepo, dstrepo, requirements)
470 473
471 474 ui.status(_(b'data fully upgraded in a temporary repository\n'))
472 475
473 476 if upgrade_op.backup_store:
474 477 backuppath = pycompat.mkdtemp(
475 478 prefix=b'upgradebackup.', dir=srcrepo.path
476 479 )
477 480 backupvfs = vfsmod.vfs(backuppath)
478 481
479 482 # Make a backup of requires file first, as it is the first to be modified.
480 483 util.copyfile(
481 484 srcrepo.vfs.join(b'requires'), backupvfs.join(b'requires')
482 485 )
483 486
484 487 # We install an arbitrary requirement that clients must not support
485 488 # as a mechanism to lock out new clients during the data swap. This is
486 489 # better than allowing a client to continue while the repository is in
487 490 # an inconsistent state.
488 491 ui.status(
489 492 _(
490 493 b'marking source repository as being upgraded; clients will be '
491 494 b'unable to read from repository\n'
492 495 )
493 496 )
494 497 scmutil.writereporequirements(
495 498 srcrepo, srcrepo.requirements | {b'upgradeinprogress'}
496 499 )
497 500
498 501 ui.status(_(b'starting in-place swap of repository data\n'))
499 502 if upgrade_op.backup_store:
500 503 ui.status(
501 504 _(b'replaced files will be backed up at %s\n') % backuppath
502 505 )
503 506
504 507 # Now swap in the new store directory. Doing it as a rename should make
505 508 # the operation nearly instantaneous and atomic (at least in well-behaved
506 509 # environments).
507 510 ui.status(_(b'replacing store...\n'))
508 511 tstart = util.timer()
509 512 _replacestores(srcrepo, dstrepo, backupvfs, upgrade_op)
510 513 elapsed = util.timer() - tstart
511 514 ui.status(
512 515 _(
513 516 b'store replacement complete; repository was inconsistent for '
514 517 b'%0.1fs\n'
515 518 )
516 519 % elapsed
517 520 )
518 521
519 522 # We first write the requirements file. Any new requirements will lock
520 523 # out legacy clients.
521 524 ui.status(
522 525 _(
523 526 b'finalizing requirements file and making repository readable '
524 527 b'again\n'
525 528 )
526 529 )
527 530 scmutil.writereporequirements(srcrepo, upgrade_op.new_requirements)
528 531
529 532 if upgrade_op.backup_store:
530 533 # The lock file from the old store won't be removed because nothing has a
531 534 # reference to its new location. So clean it up manually. Alternatively, we
532 535 # could update srcrepo.svfs and other variables to point to the new
533 536 # location. This is simpler.
534 537 backupvfs.unlink(b'store/lock')
535 else:
536 ui.status(_(b'upgrading repository requirements\n'))
537 scmutil.writereporequirements(srcrepo, upgrade_op.new_requirements)
538 538
539 539 return backuppath
General Comments 0
You need to be logged in to leave comments. Login now