# HG changeset patch # User Pierre-Yves David # Date 2021-06-08 00:06:45 # Node ID 377d8fc20e348853caab2915fa4478b76daf7bc2 # Parent 09ff5d532a256e14f7f230162155d5d4ed97f452 clone: reuse the stream clone logic for local clone Streaming clone and local (non `--pull`) clone do mostly the same thing, however they were using different logic to do so. This means the logic frequently went out of sync and that new case had to be dealt with twice. This is fragile and anoying. So we replace this with a re-use of the logic we use for streaming clone. I can see various test changes: - a more precise progress output, - armless fncache loading during clone, - fncache is no longer hardlinked (since we write it by hand). I am not reinstalling the `reposimplestore` specific output, as far as I understand this variant have been broken for years. Differential Revision: https://phab.mercurial-scm.org/D10855 diff --git a/mercurial/hg.py b/mercurial/hg.py --- a/mercurial/hg.py +++ b/mercurial/hg.py @@ -24,7 +24,6 @@ from .pycompat import getattr from . import ( bookmarks, bundlerepo, - cacheutil, cmdutil, destutil, discovery, @@ -866,18 +865,10 @@ def clone( requirements=dest_reqs, ) destrepo = localrepo.makelocalrepository(ui, destrootpath) + destlock = destrepo.lock() + from . import streamclone # avoid cycle - destpath = destrepo.vfs.base - destlock = copystore(ui, srcrepo, destpath) - # copy bookmarks over - srcbookmarks = srcrepo.vfs.join(b'bookmarks') - dstbookmarks = os.path.join(destpath, b'bookmarks') - if os.path.exists(srcbookmarks): - util.copyfile(srcbookmarks, dstbookmarks) - - dstcachedir = os.path.join(destpath, b'cache') - for cache in cacheutil.cachetocopy(srcrepo): - _copycache(srcrepo, dstcachedir, cache) + streamclone.local_copy(srcrepo, destrepo) # we need to re-init the repo after manually copying the data # into it diff --git a/mercurial/streamclone.py b/mercurial/streamclone.py --- a/mercurial/streamclone.py +++ b/mercurial/streamclone.py @@ -8,6 +8,7 @@ from __future__ import absolute_import import contextlib +import errno import os import struct @@ -15,6 +16,7 @@ from .i18n import _ from .pycompat import open from .interfaces import repository from . import ( + bookmarks, cacheutil, error, narrowspec, @@ -25,6 +27,9 @@ from . import ( store, util, ) +from .utils import ( + stringutil, +) def canperformstreamclone(pullop, bundle2=False): @@ -771,3 +776,104 @@ def applybundlev2(repo, fp, filecount, f repo.ui, repo.requirements, repo.features ) scmutil.writereporequirements(repo) + + +def _copy_files(src_vfs_map, dst_vfs_map, entries, progress): + hardlink = [True] + + def copy_used(): + hardlink[0] = False + progress.topic = _(b'copying') + + for k, path, size in entries: + src_vfs = src_vfs_map[k] + dst_vfs = dst_vfs_map[k] + src_path = src_vfs.join(path) + dst_path = dst_vfs.join(path) + dirname = dst_vfs.dirname(path) + if not dst_vfs.exists(dirname): + dst_vfs.makedirs(dirname) + dst_vfs.register_file(path) + # XXX we could use the #nb_bytes argument. + util.copyfile( + src_path, + dst_path, + hardlink=hardlink[0], + no_hardlink_cb=copy_used, + check_fs_hardlink=False, + ) + progress.increment() + return hardlink[0] + + +def local_copy(src_repo, dest_repo): + """copy all content from one local repository to another + + This is useful for local clone""" + src_store_requirements = { + r + for r in src_repo.requirements + if r not in requirementsmod.WORKING_DIR_REQUIREMENTS + } + dest_store_requirements = { + r + for r in dest_repo.requirements + if r not in requirementsmod.WORKING_DIR_REQUIREMENTS + } + assert src_store_requirements == dest_store_requirements + + with dest_repo.lock(): + with src_repo.lock(): + entries, totalfilesize = _v2_walk( + src_repo, + includes=None, + excludes=None, + includeobsmarkers=True, + ) + src_vfs_map = _makemap(src_repo) + dest_vfs_map = _makemap(dest_repo) + progress = src_repo.ui.makeprogress( + topic=_(b'linking'), + total=len(entries), + unit=_(b'files'), + ) + # copy files + # + # We could copy the full file while the source repository is locked + # and the other one without the lock. However, in the linking case, + # this would also requires checks that nobody is appending any data + # to the files while we do the clone, so this is not done yet. We + # could do this blindly when copying files. + files = ((k, path, size) for k, path, ftype, size in entries) + hardlink = _copy_files(src_vfs_map, dest_vfs_map, files, progress) + + # copy bookmarks over + src_book_vfs = bookmarks.bookmarksvfs(src_repo) + srcbookmarks = src_book_vfs.join(b'bookmarks') + dst_book_vfs = bookmarks.bookmarksvfs(dest_repo) + dstbookmarks = dst_book_vfs.join(b'bookmarks') + if os.path.exists(srcbookmarks): + util.copyfile(srcbookmarks, dstbookmarks) + progress.complete() + if hardlink: + msg = b'linked %d files\n' + else: + msg = b'copied %d files\n' + src_repo.ui.debug(msg % len(entries)) + + with dest_repo.transaction(b"localclone") as tr: + dest_repo.store.write(tr) + + # clean up transaction file as they do not make sense + undo_files = [(dest_repo.svfs, b'undo.backupfiles')] + undo_files.extend(dest_repo.undofiles()) + for undovfs, undofile in undo_files: + try: + undovfs.unlink(undofile) + except OSError as e: + if e.errno != errno.ENOENT: + msg = _(b'error removing %s: %s\n') + path = undovfs.join(undofile) + e_msg = stringutil.forcebytestr(e) + msg %= (path, e_msg) + dest_repo.ui.warn(msg) diff --git a/tests/test-clone.t b/tests/test-clone.t --- a/tests/test-clone.t +++ b/tests/test-clone.t @@ -86,26 +86,22 @@ No update, with debug option: #if hardlink $ hg --debug clone -U . ../c --config progress.debug=true - linking: 1 files - linking: 2 files - linking: 3 files - linking: 4 files - linking: 5 files - linking: 6 files - linking: 7 files - linking: 8 files - linked 8 files (reporevlogstore !) - linking: 9 files (reposimplestore !) - linking: 10 files (reposimplestore !) - linking: 11 files (reposimplestore !) - linking: 12 files (reposimplestore !) - linking: 13 files (reposimplestore !) - linking: 14 files (reposimplestore !) - linking: 15 files (reposimplestore !) - linking: 16 files (reposimplestore !) - linking: 17 files (reposimplestore !) - linking: 18 files (reposimplestore !) - linked 18 files (reposimplestore !) + linking: 1/15 files (6.67%) + linking: 2/15 files (13.33%) + linking: 3/15 files (20.00%) + linking: 4/15 files (26.67%) + linking: 5/15 files (33.33%) + linking: 6/15 files (40.00%) + linking: 7/15 files (46.67%) + linking: 8/15 files (53.33%) + linking: 9/15 files (60.00%) + linking: 10/15 files (66.67%) + linking: 11/15 files (73.33%) + linking: 12/15 files (80.00%) + linking: 13/15 files (86.67%) + linking: 14/15 files (93.33%) + linking: 15/15 files (100.00%) + linked 15 files updating the branch cache #else $ hg --debug clone -U . ../c --config progress.debug=true @@ -117,18 +113,6 @@ No update, with debug option: copying: 6 files copying: 7 files copying: 8 files - copied 8 files (reporevlogstore !) - copying: 9 files (reposimplestore !) - copying: 10 files (reposimplestore !) - copying: 11 files (reposimplestore !) - copying: 12 files (reposimplestore !) - copying: 13 files (reposimplestore !) - copying: 14 files (reposimplestore !) - copying: 15 files (reposimplestore !) - copying: 16 files (reposimplestore !) - copying: 17 files (reposimplestore !) - copying: 18 files (reposimplestore !) - copied 18 files (reposimplestore !) #endif $ cd ../c diff --git a/tests/test-empty.t b/tests/test-empty.t --- a/tests/test-empty.t +++ b/tests/test-empty.t @@ -45,6 +45,7 @@ Poke at a clone: $ ls .hg 00changelog.i cache + dirstate hgrc requires store diff --git a/tests/test-fncache.t b/tests/test-fncache.t --- a/tests/test-fncache.t +++ b/tests/test-fncache.t @@ -528,7 +528,11 @@ a transaction, we simulate the situation Unbundling should follow the same rules; existing files should not cause a load: +(loading during the clone is expected) $ hg clone -q . tobundle + fncache load triggered! + fncache load triggered! + $ echo 'new line' > tobundle/bar $ hg -R tobundle ci -qm bar $ hg -R tobundle bundle -q barupdated.hg diff --git a/tests/test-hardlinks.t b/tests/test-hardlinks.t --- a/tests/test-hardlinks.t +++ b/tests/test-hardlinks.t @@ -61,13 +61,13 @@ Prepare repo r1: Create hardlinked clone r2: $ hg clone -U --debug r1 r2 --config progress.debug=true - linking: 1 files - linking: 2 files - linking: 3 files - linking: 4 files - linking: 5 files - linking: 6 files - linking: 7 files + linking: 1/7 files (14.29%) + linking: 2/7 files (28.57%) + linking: 3/7 files (42.86%) + linking: 4/7 files (57.14%) + linking: 5/7 files (71.43%) + linking: 6/7 files (85.71%) + linking: 7/7 files (100.00%) linked 7 files updating the branch cache @@ -91,7 +91,7 @@ Repos r1 and r2 should now contain hardl 2 r1/.hg/store/00manifest.i 2 r1/.hg/store/data/d1/f2.i 2 r1/.hg/store/data/f1.i - 2 r1/.hg/store/fncache (repofncache !) + 1 r1/.hg/store/fncache (repofncache !) 1 r1/.hg/store/phaseroots 1 r1/.hg/store/undo 1 r1/.hg/store/undo.backup.fncache (repofncache !) @@ -103,7 +103,7 @@ Repos r1 and r2 should now contain hardl 2 r2/.hg/store/00manifest.i 2 r2/.hg/store/data/d1/f2.i 2 r2/.hg/store/data/f1.i - 2 r2/.hg/store/fncache (repofncache !) + 1 r2/.hg/store/fncache (repofncache !) Repo r3 should not be hardlinked: @@ -175,7 +175,7 @@ Push to repo r1 should break up most har #if hardlink-whitelisted repofncache $ nlinksdir r2/.hg/store/fncache - 2 r2/.hg/store/fncache + 1 r2/.hg/store/fncache #endif $ hg -R r2 verify @@ -201,11 +201,11 @@ Committing a change to f1 in r1 must bre 1 r2/.hg/store/00manifest.i 1 r2/.hg/store/data/d1/f2.i 1 r2/.hg/store/data/f1.i - [12] r2/\.hg/store/fncache (re) (repofncache !) + 1 r2/.hg/store/fncache (repofncache !) #if hardlink-whitelisted repofncache $ nlinksdir r2/.hg/store/fncache - 2 r2/.hg/store/fncache + 1 r2/.hg/store/fncache #endif Create a file which exec permissions we will change diff --git a/tests/test-subrepo-deep-nested-change.t b/tests/test-subrepo-deep-nested-change.t --- a/tests/test-subrepo-deep-nested-change.t +++ b/tests/test-subrepo-deep-nested-change.t @@ -28,12 +28,12 @@ Preparing the 'sub1' repo which depends $ echo "sub2 = ../sub2" > sub1/.hgsub $ hg clone sub2 sub1/sub2 \r (no-eol) (esc) - linking [ <=> ] 1\r (no-eol) (esc) - linking [ <=> ] 2\r (no-eol) (esc) - linking [ <=> ] 3\r (no-eol) (esc) - linking [ <=> ] 4\r (no-eol) (esc) - linking [ <=> ] 5\r (no-eol) (esc) - linking [ <=> ] 6\r (no-eol) (esc) + linking [======> ] 1/6\r (no-eol) (esc) + linking [==============> ] 2/6\r (no-eol) (esc) + linking [=====================> ] 3/6\r (no-eol) (esc) + linking [=============================> ] 4/6\r (no-eol) (esc) + linking [====================================> ] 5/6\r (no-eol) (esc) + linking [============================================>] 6/6\r (no-eol) (esc) \r (no-eol) (esc) \r (no-eol) (esc) updating [===========================================>] 1/1\r (no-eol) (esc) @@ -52,27 +52,25 @@ Preparing the 'main' repo which depends $ echo "sub1 = ../sub1" > main/.hgsub $ hg clone sub1 main/sub1 \r (no-eol) (esc) - linking [ <=> ] 1\r (no-eol) (esc) - linking [ <=> ] 2\r (no-eol) (esc) - linking [ <=> ] 3\r (no-eol) (esc) - linking [ <=> ] 4\r (no-eol) (esc) - linking [ <=> ] 5\r (no-eol) (esc) - linking [ <=> ] 6\r (no-eol) (esc) - linking [ <=> ] 7\r (no-eol) (esc) - linking [ <=> ] 8\r (no-eol) (esc) - linking [ <=> ] 9\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 10\r (no-eol) (esc) (reposimplestore !) + linking [====> ] 1/8\r (no-eol) (esc) + linking [==========> ] 2/8\r (no-eol) (esc) + linking [===============> ] 3/8\r (no-eol) (esc) + linking [=====================> ] 4/8\r (no-eol) (esc) + linking [===========================> ] 5/8\r (no-eol) (esc) + linking [================================> ] 6/8\r (no-eol) (esc) + linking [======================================> ] 7/8\r (no-eol) (esc) + linking [============================================>] 8/8\r (no-eol) (esc) \r (no-eol) (esc) \r (no-eol) (esc) updating [===========================================>] 3/3\r (no-eol) (esc) \r (no-eol) (esc) \r (no-eol) (esc) - linking [ <=> ] 1\r (no-eol) (esc) - linking [ <=> ] 2\r (no-eol) (esc) - linking [ <=> ] 3\r (no-eol) (esc) - linking [ <=> ] 4\r (no-eol) (esc) - linking [ <=> ] 5\r (no-eol) (esc) - linking [ <=> ] 6\r (no-eol) (esc) + linking [======> ] 1/6\r (no-eol) (esc) + linking [==============> ] 2/6\r (no-eol) (esc) + linking [=====================> ] 3/6\r (no-eol) (esc) + linking [=============================> ] 4/6\r (no-eol) (esc) + linking [====================================> ] 5/6\r (no-eol) (esc) + linking [============================================>] 6/6\r (no-eol) (esc) updating [===========================================>] 1/1\r (no-eol) (esc) \r (no-eol) (esc) updating to branch default @@ -156,46 +154,36 @@ Clone main $ hg --config extensions.largefiles= clone main cloned \r (no-eol) (esc) - linking [ <=> ] 1\r (no-eol) (esc) - linking [ <=> ] 2\r (no-eol) (esc) - linking [ <=> ] 3\r (no-eol) (esc) - linking [ <=> ] 4\r (no-eol) (esc) - linking [ <=> ] 5\r (no-eol) (esc) - linking [ <=> ] 6\r (no-eol) (esc) - linking [ <=> ] 7\r (no-eol) (esc) - linking [ <=> ] 8\r (no-eol) (esc) - linking [ <=> ] 9\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 10\r (no-eol) (esc) (reposimplestore !) + linking [====> ] 1/8\r (no-eol) (esc) + linking [==========> ] 2/8\r (no-eol) (esc) + linking [===============> ] 3/8\r (no-eol) (esc) + linking [=====================> ] 4/8\r (no-eol) (esc) + linking [===========================> ] 5/8\r (no-eol) (esc) + linking [================================> ] 6/8\r (no-eol) (esc) + linking [======================================> ] 7/8\r (no-eol) (esc) + linking [============================================>] 8/8\r (no-eol) (esc) \r (no-eol) (esc) \r (no-eol) (esc) updating [===========================================>] 3/3\r (no-eol) (esc) \r (no-eol) (esc) \r (no-eol) (esc) - linking [ <=> ] 1\r (no-eol) (esc) - linking [ <=> ] 2\r (no-eol) (esc) - linking [ <=> ] 3\r (no-eol) (esc) - linking [ <=> ] 4\r (no-eol) (esc) - linking [ <=> ] 5\r (no-eol) (esc) - linking [ <=> ] 6\r (no-eol) (esc) - linking [ <=> ] 7\r (no-eol) (esc) - linking [ <=> ] 8\r (no-eol) (esc) - linking [ <=> ] 9\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 10\r (no-eol) (esc) (reposimplestore !) + linking [====> ] 1/8\r (no-eol) (esc) + linking [==========> ] 2/8\r (no-eol) (esc) + linking [===============> ] 3/8\r (no-eol) (esc) + linking [=====================> ] 4/8\r (no-eol) (esc) + linking [===========================> ] 5/8\r (no-eol) (esc) + linking [================================> ] 6/8\r (no-eol) (esc) + linking [======================================> ] 7/8\r (no-eol) (esc) + linking [============================================>] 8/8\r (no-eol) (esc) updating [===========================================>] 3/3\r (no-eol) (esc) \r (no-eol) (esc) \r (no-eol) (esc) - linking [ <=> ] 1\r (no-eol) (esc) (reporevlogstore !) - linking [ <=> ] 2\r (no-eol) (esc) (reporevlogstore !) - linking [ <=> ] 3\r (no-eol) (esc) (reporevlogstore !) - linking [ <=> ] 4\r (no-eol) (esc) (reporevlogstore !) - linking [ <=> ] 5\r (no-eol) (esc) (reporevlogstore !) - linking [ <=> ] 6\r (no-eol) (esc) (reporevlogstore !) - linking [ <=> ] 1\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 2\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 3\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 4\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 5\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 6\r (no-eol) (esc) (reposimplestore !) + linking [======> ] 1/6\r (no-eol) (esc) + linking [==============> ] 2/6\r (no-eol) (esc) + linking [=====================> ] 3/6\r (no-eol) (esc) + linking [=============================> ] 4/6\r (no-eol) (esc) + linking [====================================> ] 5/6\r (no-eol) (esc) + linking [============================================>] 6/6\r (no-eol) (esc) updating [===========================================>] 1/1\r (no-eol) (esc) \r (no-eol) (esc) updating to branch default diff --git a/tests/test-subrepo-recursion.t b/tests/test-subrepo-recursion.t --- a/tests/test-subrepo-recursion.t +++ b/tests/test-subrepo-recursion.t @@ -454,19 +454,15 @@ cloned: #if hardlink $ hg clone -U . ../empty \r (no-eol) (esc) - linking [ <=> ] 1\r (no-eol) (esc) - linking [ <=> ] 2\r (no-eol) (esc) - linking [ <=> ] 3\r (no-eol) (esc) - linking [ <=> ] 4\r (no-eol) (esc) - linking [ <=> ] 5\r (no-eol) (esc) - linking [ <=> ] 6\r (no-eol) (esc) - linking [ <=> ] 7\r (no-eol) (esc) - linking [ <=> ] 8\r (no-eol) (esc) - linking [ <=> ] 9\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 10\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 11\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 12\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 13\r (no-eol) (esc) (reposimplestore !) + linking [====> ] 1/9\r (no-eol) (esc) + linking [=========> ] 2/9\r (no-eol) (esc) + linking [==============> ] 3/9\r (no-eol) (esc) + linking [===================> ] 4/9\r (no-eol) (esc) + linking [========================> ] 5/9\r (no-eol) (esc) + linking [=============================> ] 6/9\r (no-eol) (esc) + linking [==================================> ] 7/9\r (no-eol) (esc) + linking [=======================================> ] 8/9\r (no-eol) (esc) + linking [============================================>] 9/9\r (no-eol) (esc) \r (no-eol) (esc) #else $ hg clone -U . ../empty @@ -484,22 +480,14 @@ cloned: archiving [==========================================>] 3/3\r (no-eol) (esc) \r (no-eol) (esc) \r (no-eol) (esc) - linking [ <=> ] 1\r (no-eol) (esc) - linking [ <=> ] 2\r (no-eol) (esc) - linking [ <=> ] 3\r (no-eol) (esc) - linking [ <=> ] 4\r (no-eol) (esc) - linking [ <=> ] 5\r (no-eol) (esc) - linking [ <=> ] 6\r (no-eol) (esc) - linking [ <=> ] 7\r (no-eol) (esc) - linking [ <=> ] 8\r (no-eol) (esc) - linking [ <=> ] 9\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 10\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 11\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 12\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 13\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 14\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 15\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 16\r (no-eol) (esc) (reposimplestore !) + linking [====> ] 1/8\r (no-eol) (esc) + linking [==========> ] 2/8\r (no-eol) (esc) + linking [===============> ] 3/8\r (no-eol) (esc) + linking [=====================> ] 4/8\r (no-eol) (esc) + linking [===========================> ] 5/8\r (no-eol) (esc) + linking [================================> ] 6/8\r (no-eol) (esc) + linking [======================================> ] 7/8\r (no-eol) (esc) + linking [============================================>] 8/8\r (no-eol) (esc) \r (no-eol) (esc) \r (no-eol) (esc) archiving (foo) [ ] 0/3\r (no-eol) (esc) @@ -508,15 +496,12 @@ cloned: archiving (foo) [====================================>] 3/3\r (no-eol) (esc) \r (no-eol) (esc) \r (no-eol) (esc) - linking [ <=> ] 1\r (no-eol) (esc) - linking [ <=> ] 2\r (no-eol) (esc) - linking [ <=> ] 3\r (no-eol) (esc) - linking [ <=> ] 4\r (no-eol) (esc) - linking [ <=> ] 5\r (no-eol) (esc) - linking [ <=> ] 6\r (no-eol) (esc) - linking [ <=> ] 7\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 8\r (no-eol) (esc) (reposimplestore !) - linking [ <=> ] 9\r (no-eol) (esc) (reposimplestore !) + linking [======> ] 1/6\r (no-eol) (esc) + linking [==============> ] 2/6\r (no-eol) (esc) + linking [=====================> ] 3/6\r (no-eol) (esc) + linking [=============================> ] 4/6\r (no-eol) (esc) + linking [====================================> ] 5/6\r (no-eol) (esc) + linking [============================================>] 6/6\r (no-eol) (esc) \r (no-eol) (esc) \r (no-eol) (esc) archiving (foo/bar) [ ] 0/1\r (no-eol) (esc)