diff --git a/hgext/lfs/__init__.py b/hgext/lfs/__init__.py --- a/hgext/lfs/__init__.py +++ b/hgext/lfs/__init__.py @@ -24,6 +24,10 @@ Configs:: # how many times to retry before giving up on transferring an object retry = 5 + + # the local directory to store lfs files for sharing across local clones. + # If not set, the cache is located in an OS specific cache location. + usercache = /path/to/global/cache """ from __future__ import absolute_import @@ -62,6 +66,9 @@ configitem = registrar.configitem(config configitem('lfs', 'url', default=configitem.dynamicdefault, ) +configitem('lfs', 'usercache', + default=None, +) configitem('lfs', 'threshold', default=None, ) diff --git a/hgext/lfs/blobstore.py b/hgext/lfs/blobstore.py --- a/hgext/lfs/blobstore.py +++ b/hgext/lfs/blobstore.py @@ -20,6 +20,8 @@ from mercurial import ( vfs as vfsmod, ) +from ..largefiles import lfutil + # 64 bytes for SHA256 _lfsre = re.compile(r'\A[a-f0-9]{64}\Z') @@ -68,20 +70,29 @@ class local(object): def __init__(self, repo): fullpath = repo.svfs.join('lfs/objects') self.vfs = lfsvfs(fullpath) + usercache = lfutil._usercachedir(repo.ui, 'lfs') + self.cachevfs = lfsvfs(usercache) def write(self, oid, data): """Write blob to local blobstore.""" with self.vfs(oid, 'wb', atomictemp=True) as fp: fp.write(data) + # XXX: should we verify the content of the cache, and hardlink back to + # the local store on success, but truncate, write and link on failure? + if not self.cachevfs.exists(oid): + lfutil.link(self.vfs.join(oid), self.cachevfs.join(oid)) + def read(self, oid): """Read blob from local blobstore.""" + if not self.vfs.exists(oid): + lfutil.link(self.cachevfs.join(oid), self.vfs.join(oid)) return self.vfs.read(oid) def has(self, oid): """Returns True if the local blobstore contains the requested blob, False otherwise.""" - return self.vfs.exists(oid) + return self.cachevfs.exists(oid) or self.vfs.exists(oid) class _gitlfsremote(object): diff --git a/tests/run-tests.py b/tests/run-tests.py --- a/tests/run-tests.py +++ b/tests/run-tests.py @@ -1102,6 +1102,9 @@ class Test(unittest.TestCase): hgrc.write(b'[largefiles]\n') hgrc.write(b'usercache = %s\n' % (os.path.join(self._testtmp, b'.cache/largefiles'))) + hgrc.write(b'[lfs]\n') + hgrc.write(b'usercache = %s\n' % + (os.path.join(self._testtmp, b'.cache/lfs'))) hgrc.write(b'[web]\n') hgrc.write(b'address = localhost\n') hgrc.write(b'ipv6 = %s\n' % str(self._useipv6).encode('ascii')) diff --git a/tests/test-basic.t b/tests/test-basic.t --- a/tests/test-basic.t +++ b/tests/test-basic.t @@ -5,6 +5,7 @@ Create a repository: devel.default-date=0 0 extensions.fsmonitor= (fsmonitor !) largefiles.usercache=$TESTTMP/.cache/largefiles (glob) + lfs.usercache=$TESTTMP/.cache/lfs (glob) ui.slash=True ui.interactive=False ui.mergemarkers=detailed diff --git a/tests/test-commandserver.t b/tests/test-commandserver.t --- a/tests/test-commandserver.t +++ b/tests/test-commandserver.t @@ -207,6 +207,7 @@ check that local configs for the cached devel.default-date=0 0 extensions.fsmonitor= (fsmonitor !) largefiles.usercache=$TESTTMP/.cache/largefiles + lfs.usercache=$TESTTMP/.cache/lfs ui.slash=True ui.interactive=False ui.mergemarkers=detailed diff --git a/tests/test-lfs-test-server.t b/tests/test-lfs-test-server.t --- a/tests/test-lfs-test-server.t +++ b/tests/test-lfs-test-server.t @@ -53,6 +53,8 @@ adding file changes added 1 changesets with 1 changes to 1 files +Clear the cache to force a download + $ rm -rf `hg config lfs.usercache` $ cd ../repo2 $ hg update tip -v resolving manifests @@ -79,6 +81,8 @@ When the server has some blobs already adding file changes added 1 changesets with 3 changes to 3 files +Clear the cache to force a download + $ rm -rf `hg config lfs.usercache` $ hg --repo ../repo1 update tip -v resolving manifests getting b @@ -95,6 +99,7 @@ Check error message when the remote miss $ echo FFFFF >> b $ hg commit -m b b $ rm -rf .hg/store/lfs + $ rm -rf `hg config lfs.usercache` $ hg update -C '.^' abort: LFS server claims required objects do not exist: 8e6ea5f6c066b44a0efa43bcce86aea73f17e6e23f0663df0251e7524e140a13! @@ -118,6 +123,7 @@ Check error message when object does not size 6 x-is-binary 0 $ cd .. + $ rm -rf `hg config lfs.usercache` $ hg --config 'lfs.url=https://dewey-lfs.vip.facebook.com/lfs' clone test test2 updating to branch default abort: LFS server error. Remote object for file data/a.i not found:(.*)! (re) diff --git a/tests/test-lfs.t b/tests/test-lfs.t --- a/tests/test-lfs.t +++ b/tests/test-lfs.t @@ -566,9 +566,9 @@ repo: repo9 repo: repo10 -TODO: repo12 doesn't have any cached lfs files. Figure out how to get the -unpushed files from repo12's source instead of the remote store, where they -don't exist. +repo12 doesn't have any cached lfs files and its source never pushed its +files. Therefore, the files don't exist in the remote store. Use the files in +the user cache. $ find $TESTTMP/repo12/.hg/store/lfs/objects -type f find: */repo12/.hg/store/lfs/objects': $ENOENT$ (glob) @@ -576,24 +576,28 @@ don't exist. $ hg --config extensions.share= share repo12 repo13 updating working directory - abort: $TESTTMP/dummy-remote/09/66faba9a01f6c78082aa45899a4fef732002d0b26404e90093adf1e876ab8d: $ENOTDIR$ (glob) - [255] + 1 files updated, 0 files merged, 0 files removed, 0 files unresolved + $ hg -R repo13 -q verify + $ hg clone repo12 repo14 updating to branch default - abort: $TESTTMP/dummy-remote/09/66faba9a01f6c78082aa45899a4fef732002d0b26404e90093adf1e876ab8d: $ENOTDIR$ (glob) - [255] + 1 files updated, 0 files merged, 0 files removed, 0 files unresolved + $ hg -R repo14 -q verify -TODO: If the source repo doesn't have the blob (maybe it was pulled or cloned -with --noupdate), the blob should be accessible via the global cache to send to -the remote store. +If the source repo doesn't have the blob (maybe it was pulled or cloned with +--noupdate), the blob is still accessible via the global cache to send to the +remote store. $ rm -rf $TESTTMP/repo14/.hg/store/lfs $ hg init repo15 $ hg -R repo14 push repo15 pushing to repo15 searching for changes - abort: $TESTTMP/repo14/.hg/store/lfs/objects/1c/896a0adcf9262119f4a98216aaa5ca00a58b9a0ce848914a02f9cd876f65a3: $ENOTDIR$ (glob) - [255] + adding changesets + adding manifests + adding file changes + added 3 changesets with 2 changes to 1 files + $ hg -R repo14 -q verify lfs -> normal -> lfs round trip conversions are possible. The threshold for the lfs destination is specified here because it was originally listed in the local