##// END OF EJS Templates
engine: refactor actual cloning code into separate function...
Pulkit Goyal -
r46809:85f7cf31 default
parent child Browse files
Show More
@@ -1,500 +1,528 b''
1 1 # upgrade.py - functions for in place upgrade of Mercurial repository
2 2 #
3 3 # Copyright (c) 2016-present, Gregory Szorc
4 4 #
5 5 # This software may be used and distributed according to the terms of the
6 6 # GNU General Public License version 2 or any later version.
7 7
8 8 from __future__ import absolute_import
9 9
10 10 import stat
11 11
12 12 from ..i18n import _
13 13 from ..pycompat import getattr
14 14 from .. import (
15 15 changelog,
16 16 error,
17 17 filelog,
18 18 manifest,
19 19 metadata,
20 20 pycompat,
21 21 requirements,
22 22 revlog,
23 23 scmutil,
24 24 util,
25 25 vfs as vfsmod,
26 26 )
27 27
28 28
29 29 def _revlogfrompath(repo, path):
30 30 """Obtain a revlog from a repo path.
31 31
32 32 An instance of the appropriate class is returned.
33 33 """
34 34 if path == b'00changelog.i':
35 35 return changelog.changelog(repo.svfs)
36 36 elif path.endswith(b'00manifest.i'):
37 37 mandir = path[: -len(b'00manifest.i')]
38 38 return manifest.manifestrevlog(repo.svfs, tree=mandir)
39 39 else:
40 40 # reverse of "/".join(("data", path + ".i"))
41 41 return filelog.filelog(repo.svfs, path[5:-2])
42 42
43 43
44 44 def _copyrevlog(tr, destrepo, oldrl, unencodedname):
45 45 """copy all relevant files for `oldrl` into `destrepo` store
46 46
47 47 Files are copied "as is" without any transformation. The copy is performed
48 48 without extra checks. Callers are responsible for making sure the copied
49 49 content is compatible with format of the destination repository.
50 50 """
51 51 oldrl = getattr(oldrl, '_revlog', oldrl)
52 52 newrl = _revlogfrompath(destrepo, unencodedname)
53 53 newrl = getattr(newrl, '_revlog', newrl)
54 54
55 55 oldvfs = oldrl.opener
56 56 newvfs = newrl.opener
57 57 oldindex = oldvfs.join(oldrl.indexfile)
58 58 newindex = newvfs.join(newrl.indexfile)
59 59 olddata = oldvfs.join(oldrl.datafile)
60 60 newdata = newvfs.join(newrl.datafile)
61 61
62 62 with newvfs(newrl.indexfile, b'w'):
63 63 pass # create all the directories
64 64
65 65 util.copyfile(oldindex, newindex)
66 66 copydata = oldrl.opener.exists(oldrl.datafile)
67 67 if copydata:
68 68 util.copyfile(olddata, newdata)
69 69
70 70 if not (
71 71 unencodedname.endswith(b'00changelog.i')
72 72 or unencodedname.endswith(b'00manifest.i')
73 73 ):
74 74 destrepo.svfs.fncache.add(unencodedname)
75 75 if copydata:
76 76 destrepo.svfs.fncache.add(unencodedname[:-2] + b'.d')
77 77
78 78
79 79 UPGRADE_CHANGELOG = b"changelog"
80 80 UPGRADE_MANIFEST = b"manifest"
81 81 UPGRADE_FILELOGS = b"all-filelogs"
82 82
83 83 UPGRADE_ALL_REVLOGS = frozenset(
84 84 [UPGRADE_CHANGELOG, UPGRADE_MANIFEST, UPGRADE_FILELOGS]
85 85 )
86 86
87 87
88 88 def getsidedatacompanion(srcrepo, dstrepo):
89 89 sidedatacompanion = None
90 90 removedreqs = srcrepo.requirements - dstrepo.requirements
91 91 addedreqs = dstrepo.requirements - srcrepo.requirements
92 92 if requirements.SIDEDATA_REQUIREMENT in removedreqs:
93 93
94 94 def sidedatacompanion(rl, rev):
95 95 rl = getattr(rl, '_revlog', rl)
96 96 if rl.flags(rev) & revlog.REVIDX_SIDEDATA:
97 97 return True, (), {}, 0, 0
98 98 return False, (), {}, 0, 0
99 99
100 100 elif requirements.COPIESSDC_REQUIREMENT in addedreqs:
101 101 sidedatacompanion = metadata.getsidedataadder(srcrepo, dstrepo)
102 102 elif requirements.COPIESSDC_REQUIREMENT in removedreqs:
103 103 sidedatacompanion = metadata.getsidedataremover(srcrepo, dstrepo)
104 104 return sidedatacompanion
105 105
106 106
107 107 def matchrevlog(revlogfilter, entry):
108 108 """check if a revlog is selected for cloning.
109 109
110 110 In other words, are there any updates which need to be done on revlog
111 111 or it can be blindly copied.
112 112
113 113 The store entry is checked against the passed filter"""
114 114 if entry.endswith(b'00changelog.i'):
115 115 return UPGRADE_CHANGELOG in revlogfilter
116 116 elif entry.endswith(b'00manifest.i'):
117 117 return UPGRADE_MANIFEST in revlogfilter
118 118 return UPGRADE_FILELOGS in revlogfilter
119 119
120 120
121 def _perform_clone(
122 ui,
123 dstrepo,
124 tr,
125 old_revlog,
126 unencoded,
127 deltareuse,
128 forcedeltabothparents,
129 revlogs,
130 sidedatacompanion,
131 oncopiedrevision,
132 ):
133 """ returns the new revlog object created"""
134 newrl = None
135 if matchrevlog(revlogs, unencoded):
136 ui.note(
137 _(b'cloning %d revisions from %s\n') % (len(old_revlog), unencoded)
138 )
139 newrl = _revlogfrompath(dstrepo, unencoded)
140 old_revlog.clone(
141 tr,
142 newrl,
143 addrevisioncb=oncopiedrevision,
144 deltareuse=deltareuse,
145 forcedeltabothparents=forcedeltabothparents,
146 sidedatacompanion=sidedatacompanion,
147 )
148 else:
149 msg = _(b'blindly copying %s containing %i revisions\n')
150 ui.note(msg % (unencoded, len(old_revlog)))
151 _copyrevlog(tr, dstrepo, old_revlog, unencoded)
152
153 newrl = _revlogfrompath(dstrepo, unencoded)
154 return newrl
155
156
121 157 def _clonerevlogs(
122 158 ui,
123 159 srcrepo,
124 160 dstrepo,
125 161 tr,
126 162 deltareuse,
127 163 forcedeltabothparents,
128 164 revlogs=UPGRADE_ALL_REVLOGS,
129 165 ):
130 166 """Copy revlogs between 2 repos."""
131 167 revcount = 0
132 168 srcsize = 0
133 169 srcrawsize = 0
134 170 dstsize = 0
135 171 fcount = 0
136 172 frevcount = 0
137 173 fsrcsize = 0
138 174 frawsize = 0
139 175 fdstsize = 0
140 176 mcount = 0
141 177 mrevcount = 0
142 178 msrcsize = 0
143 179 mrawsize = 0
144 180 mdstsize = 0
145 181 crevcount = 0
146 182 csrcsize = 0
147 183 crawsize = 0
148 184 cdstsize = 0
149 185
150 186 alldatafiles = list(srcrepo.store.walk())
151 187
152 188 # Perform a pass to collect metadata. This validates we can open all
153 189 # source files and allows a unified progress bar to be displayed.
154 190 for unencoded, encoded, size in alldatafiles:
155 191 if unencoded.endswith(b'.d'):
156 192 continue
157 193
158 194 rl = _revlogfrompath(srcrepo, unencoded)
159 195
160 196 info = rl.storageinfo(
161 197 exclusivefiles=True,
162 198 revisionscount=True,
163 199 trackedsize=True,
164 200 storedsize=True,
165 201 )
166 202
167 203 revcount += info[b'revisionscount'] or 0
168 204 datasize = info[b'storedsize'] or 0
169 205 rawsize = info[b'trackedsize'] or 0
170 206
171 207 srcsize += datasize
172 208 srcrawsize += rawsize
173 209
174 210 # This is for the separate progress bars.
175 211 if isinstance(rl, changelog.changelog):
176 212 crevcount += len(rl)
177 213 csrcsize += datasize
178 214 crawsize += rawsize
179 215 elif isinstance(rl, manifest.manifestrevlog):
180 216 mcount += 1
181 217 mrevcount += len(rl)
182 218 msrcsize += datasize
183 219 mrawsize += rawsize
184 220 elif isinstance(rl, filelog.filelog):
185 221 fcount += 1
186 222 frevcount += len(rl)
187 223 fsrcsize += datasize
188 224 frawsize += rawsize
189 225 else:
190 226 error.ProgrammingError(b'unknown revlog type')
191 227
192 228 if not revcount:
193 229 return
194 230
195 231 ui.status(
196 232 _(
197 233 b'migrating %d total revisions (%d in filelogs, %d in manifests, '
198 234 b'%d in changelog)\n'
199 235 )
200 236 % (revcount, frevcount, mrevcount, crevcount)
201 237 )
202 238 ui.status(
203 239 _(b'migrating %s in store; %s tracked data\n')
204 240 % ((util.bytecount(srcsize), util.bytecount(srcrawsize)))
205 241 )
206 242
207 243 # Used to keep track of progress.
208 244 progress = None
209 245
210 246 def oncopiedrevision(rl, rev, node):
211 247 progress.increment()
212 248
213 249 sidedatacompanion = getsidedatacompanion(srcrepo, dstrepo)
214 250
215 251 # Do the actual copying.
216 252 # FUTURE this operation can be farmed off to worker processes.
217 253 seen = set()
218 254 for unencoded, encoded, size in alldatafiles:
219 255 if unencoded.endswith(b'.d'):
220 256 continue
221 257
222 258 oldrl = _revlogfrompath(srcrepo, unencoded)
223 259
224 260 if isinstance(oldrl, changelog.changelog) and b'c' not in seen:
225 261 ui.status(
226 262 _(
227 263 b'finished migrating %d manifest revisions across %d '
228 264 b'manifests; change in size: %s\n'
229 265 )
230 266 % (mrevcount, mcount, util.bytecount(mdstsize - msrcsize))
231 267 )
232 268
233 269 ui.status(
234 270 _(
235 271 b'migrating changelog containing %d revisions '
236 272 b'(%s in store; %s tracked data)\n'
237 273 )
238 274 % (
239 275 crevcount,
240 276 util.bytecount(csrcsize),
241 277 util.bytecount(crawsize),
242 278 )
243 279 )
244 280 seen.add(b'c')
245 281 progress = srcrepo.ui.makeprogress(
246 282 _(b'changelog revisions'), total=crevcount
247 283 )
248 284 elif isinstance(oldrl, manifest.manifestrevlog) and b'm' not in seen:
249 285 ui.status(
250 286 _(
251 287 b'finished migrating %d filelog revisions across %d '
252 288 b'filelogs; change in size: %s\n'
253 289 )
254 290 % (frevcount, fcount, util.bytecount(fdstsize - fsrcsize))
255 291 )
256 292
257 293 ui.status(
258 294 _(
259 295 b'migrating %d manifests containing %d revisions '
260 296 b'(%s in store; %s tracked data)\n'
261 297 )
262 298 % (
263 299 mcount,
264 300 mrevcount,
265 301 util.bytecount(msrcsize),
266 302 util.bytecount(mrawsize),
267 303 )
268 304 )
269 305 seen.add(b'm')
270 306 if progress:
271 307 progress.complete()
272 308 progress = srcrepo.ui.makeprogress(
273 309 _(b'manifest revisions'), total=mrevcount
274 310 )
275 311 elif b'f' not in seen:
276 312 ui.status(
277 313 _(
278 314 b'migrating %d filelogs containing %d revisions '
279 315 b'(%s in store; %s tracked data)\n'
280 316 )
281 317 % (
282 318 fcount,
283 319 frevcount,
284 320 util.bytecount(fsrcsize),
285 321 util.bytecount(frawsize),
286 322 )
287 323 )
288 324 seen.add(b'f')
289 325 if progress:
290 326 progress.complete()
291 327 progress = srcrepo.ui.makeprogress(
292 328 _(b'file revisions'), total=frevcount
293 329 )
294 330
295 if matchrevlog(revlogs, unencoded):
296 ui.note(
297 _(b'cloning %d revisions from %s\n') % (len(oldrl), unencoded)
298 )
299 newrl = _revlogfrompath(dstrepo, unencoded)
300 oldrl.clone(
301 tr,
302 newrl,
303 addrevisioncb=oncopiedrevision,
304 deltareuse=deltareuse,
305 forcedeltabothparents=forcedeltabothparents,
306 sidedatacompanion=sidedatacompanion,
307 )
308 else:
309 msg = _(b'blindly copying %s containing %i revisions\n')
310 ui.note(msg % (unencoded, len(oldrl)))
311 _copyrevlog(tr, dstrepo, oldrl, unencoded)
312
313 newrl = _revlogfrompath(dstrepo, unencoded)
314
331 newrl = _perform_clone(
332 ui,
333 dstrepo,
334 tr,
335 oldrl,
336 unencoded,
337 deltareuse,
338 forcedeltabothparents,
339 revlogs,
340 sidedatacompanion,
341 oncopiedrevision,
342 )
315 343 info = newrl.storageinfo(storedsize=True)
316 344 datasize = info[b'storedsize'] or 0
317 345
318 346 dstsize += datasize
319 347
320 348 if isinstance(newrl, changelog.changelog):
321 349 cdstsize += datasize
322 350 elif isinstance(newrl, manifest.manifestrevlog):
323 351 mdstsize += datasize
324 352 else:
325 353 fdstsize += datasize
326 354
327 355 progress.complete()
328 356
329 357 ui.status(
330 358 _(
331 359 b'finished migrating %d changelog revisions; change in size: '
332 360 b'%s\n'
333 361 )
334 362 % (crevcount, util.bytecount(cdstsize - csrcsize))
335 363 )
336 364
337 365 ui.status(
338 366 _(
339 367 b'finished migrating %d total revisions; total change in store '
340 368 b'size: %s\n'
341 369 )
342 370 % (revcount, util.bytecount(dstsize - srcsize))
343 371 )
344 372
345 373
346 374 def _filterstorefile(srcrepo, dstrepo, requirements, path, mode, st):
347 375 """Determine whether to copy a store file during upgrade.
348 376
349 377 This function is called when migrating store files from ``srcrepo`` to
350 378 ``dstrepo`` as part of upgrading a repository.
351 379
352 380 Args:
353 381 srcrepo: repo we are copying from
354 382 dstrepo: repo we are copying to
355 383 requirements: set of requirements for ``dstrepo``
356 384 path: store file being examined
357 385 mode: the ``ST_MODE`` file type of ``path``
358 386 st: ``stat`` data structure for ``path``
359 387
360 388 Function should return ``True`` if the file is to be copied.
361 389 """
362 390 # Skip revlogs.
363 391 if path.endswith((b'.i', b'.d', b'.n', b'.nd')):
364 392 return False
365 393 # Skip transaction related files.
366 394 if path.startswith(b'undo'):
367 395 return False
368 396 # Only copy regular files.
369 397 if mode != stat.S_IFREG:
370 398 return False
371 399 # Skip other skipped files.
372 400 if path in (b'lock', b'fncache'):
373 401 return False
374 402
375 403 return True
376 404
377 405
378 406 def _finishdatamigration(ui, srcrepo, dstrepo, requirements):
379 407 """Hook point for extensions to perform additional actions during upgrade.
380 408
381 409 This function is called after revlogs and store files have been copied but
382 410 before the new store is swapped into the original location.
383 411 """
384 412
385 413
386 414 def upgrade(ui, srcrepo, dstrepo, upgrade_op):
387 415 """Do the low-level work of upgrading a repository.
388 416
389 417 The upgrade is effectively performed as a copy between a source
390 418 repository and a temporary destination repository.
391 419
392 420 The source repository is unmodified for as long as possible so the
393 421 upgrade can abort at any time without causing loss of service for
394 422 readers and without corrupting the source repository.
395 423 """
396 424 assert srcrepo.currentwlock()
397 425 assert dstrepo.currentwlock()
398 426
399 427 ui.status(
400 428 _(
401 429 b'(it is safe to interrupt this process any time before '
402 430 b'data migration completes)\n'
403 431 )
404 432 )
405 433
406 434 if upgrade_op.has_action(b're-delta-all'):
407 435 deltareuse = revlog.revlog.DELTAREUSENEVER
408 436 elif upgrade_op.has_action(b're-delta-parent'):
409 437 deltareuse = revlog.revlog.DELTAREUSESAMEREVS
410 438 elif upgrade_op.has_action(b're-delta-multibase'):
411 439 deltareuse = revlog.revlog.DELTAREUSESAMEREVS
412 440 elif upgrade_op.has_action(b're-delta-fulladd'):
413 441 deltareuse = revlog.revlog.DELTAREUSEFULLADD
414 442 else:
415 443 deltareuse = revlog.revlog.DELTAREUSEALWAYS
416 444
417 445 with dstrepo.transaction(b'upgrade') as tr:
418 446 _clonerevlogs(
419 447 ui,
420 448 srcrepo,
421 449 dstrepo,
422 450 tr,
423 451 deltareuse,
424 452 upgrade_op.has_action(b're-delta-multibase'),
425 453 revlogs=upgrade_op.revlogs_to_process,
426 454 )
427 455
428 456 # Now copy other files in the store directory.
429 457 # The sorted() makes execution deterministic.
430 458 for p, kind, st in sorted(srcrepo.store.vfs.readdir(b'', stat=True)):
431 459 if not _filterstorefile(
432 460 srcrepo, dstrepo, upgrade_op.new_requirements, p, kind, st
433 461 ):
434 462 continue
435 463
436 464 srcrepo.ui.status(_(b'copying %s\n') % p)
437 465 src = srcrepo.store.rawvfs.join(p)
438 466 dst = dstrepo.store.rawvfs.join(p)
439 467 util.copyfile(src, dst, copystat=True)
440 468
441 469 _finishdatamigration(ui, srcrepo, dstrepo, requirements)
442 470
443 471 ui.status(_(b'data fully migrated to temporary repository\n'))
444 472
445 473 backuppath = pycompat.mkdtemp(prefix=b'upgradebackup.', dir=srcrepo.path)
446 474 backupvfs = vfsmod.vfs(backuppath)
447 475
448 476 # Make a backup of requires file first, as it is the first to be modified.
449 477 util.copyfile(srcrepo.vfs.join(b'requires'), backupvfs.join(b'requires'))
450 478
451 479 # We install an arbitrary requirement that clients must not support
452 480 # as a mechanism to lock out new clients during the data swap. This is
453 481 # better than allowing a client to continue while the repository is in
454 482 # an inconsistent state.
455 483 ui.status(
456 484 _(
457 485 b'marking source repository as being upgraded; clients will be '
458 486 b'unable to read from repository\n'
459 487 )
460 488 )
461 489 scmutil.writereporequirements(
462 490 srcrepo, srcrepo.requirements | {b'upgradeinprogress'}
463 491 )
464 492
465 493 ui.status(_(b'starting in-place swap of repository data\n'))
466 494 ui.status(_(b'replaced files will be backed up at %s\n') % backuppath)
467 495
468 496 # Now swap in the new store directory. Doing it as a rename should make
469 497 # the operation nearly instantaneous and atomic (at least in well-behaved
470 498 # environments).
471 499 ui.status(_(b'replacing store...\n'))
472 500 tstart = util.timer()
473 501 util.rename(srcrepo.spath, backupvfs.join(b'store'))
474 502 util.rename(dstrepo.spath, srcrepo.spath)
475 503 elapsed = util.timer() - tstart
476 504 ui.status(
477 505 _(
478 506 b'store replacement complete; repository was inconsistent for '
479 507 b'%0.1fs\n'
480 508 )
481 509 % elapsed
482 510 )
483 511
484 512 # We first write the requirements file. Any new requirements will lock
485 513 # out legacy clients.
486 514 ui.status(
487 515 _(
488 516 b'finalizing requirements file and making repository readable '
489 517 b'again\n'
490 518 )
491 519 )
492 520 scmutil.writereporequirements(srcrepo, upgrade_op.new_requirements)
493 521
494 522 # The lock file from the old store won't be removed because nothing has a
495 523 # reference to its new location. So clean it up manually. Alternatively, we
496 524 # could update srcrepo.svfs and other variables to point to the new
497 525 # location. This is simpler.
498 526 backupvfs.unlink(b'store/lock')
499 527
500 528 return backuppath
General Comments 0
You need to be logged in to leave comments. Login now