shallowrepo.py
351 lines
| 11.8 KiB
| text/x-python
|
PythonLexer
|
r40530 | # shallowrepo.py - shallow repository that uses remote filelogs | ||
# | ||||
# Copyright 2013 Facebook, Inc. | ||||
# | ||||
# This software may be used and distributed according to the terms of the | ||||
# GNU General Public License version 2 or any later version. | ||||
from __future__ import absolute_import | ||||
import os | ||||
from mercurial.i18n import _ | ||||
from mercurial.node import hex, nullid, nullrev | ||||
from mercurial import ( | ||||
encoding, | ||||
error, | ||||
localrepo, | ||||
match, | ||||
|
r43375 | pycompat, | ||
|
r40530 | scmutil, | ||
sparse, | ||||
util, | ||||
) | ||||
from mercurial.utils import procutil | ||||
from . import ( | ||||
connectionpool, | ||||
constants, | ||||
contentstore, | ||||
datapack, | ||||
fileserverclient, | ||||
historypack, | ||||
metadatastore, | ||||
remotefilectx, | ||||
remotefilelog, | ||||
shallowutil, | ||||
) | ||||
# These make*stores functions are global so that other extensions can replace | ||||
# them. | ||||
def makelocalstores(repo): | ||||
"""In-repo stores, like .hg/store/data; can not be discarded.""" | ||||
|
r43347 | localpath = os.path.join(repo.svfs.vfs.base, b'data') | ||
|
r40530 | if not os.path.exists(localpath): | ||
os.makedirs(localpath) | ||||
# Instantiate local data stores | ||||
localcontent = contentstore.remotefilelogcontentstore( | ||||
|
r43346 | repo, localpath, repo.name, shared=False | ||
) | ||||
|
r40530 | localmetadata = metadatastore.remotefilelogmetadatastore( | ||
|
r43346 | repo, localpath, repo.name, shared=False | ||
) | ||||
|
r40530 | return localcontent, localmetadata | ||
|
r43346 | |||
|
r40530 | def makecachestores(repo): | ||
"""Typically machine-wide, cache of remote data; can be discarded.""" | ||||
# Instantiate shared cache stores | ||||
cachepath = shallowutil.getcachepath(repo.ui) | ||||
cachecontent = contentstore.remotefilelogcontentstore( | ||||
|
r43346 | repo, cachepath, repo.name, shared=True | ||
) | ||||
|
r40530 | cachemetadata = metadatastore.remotefilelogmetadatastore( | ||
|
r43346 | repo, cachepath, repo.name, shared=True | ||
) | ||||
|
r40530 | |||
repo.sharedstore = cachecontent | ||||
repo.shareddatastores.append(cachecontent) | ||||
repo.sharedhistorystores.append(cachemetadata) | ||||
return cachecontent, cachemetadata | ||||
|
r43346 | |||
|
r40530 | def makeremotestores(repo, cachecontent, cachemetadata): | ||
"""These stores fetch data from a remote server.""" | ||||
# Instantiate remote stores | ||||
repo.fileservice = fileserverclient.fileserverclient(repo) | ||||
remotecontent = contentstore.remotecontentstore( | ||||
|
r43346 | repo.ui, repo.fileservice, cachecontent | ||
) | ||||
|
r40530 | remotemetadata = metadatastore.remotemetadatastore( | ||
|
r43346 | repo.ui, repo.fileservice, cachemetadata | ||
) | ||||
|
r40530 | return remotecontent, remotemetadata | ||
|
r43346 | |||
|
r40530 | def makepackstores(repo): | ||
"""Packs are more efficient (to read from) cache stores.""" | ||||
# Instantiate pack stores | ||||
|
r43346 | packpath = shallowutil.getcachepackpath(repo, constants.FILEPACK_CATEGORY) | ||
|
r40530 | packcontentstore = datapack.datapackstore(repo.ui, packpath) | ||
packmetadatastore = historypack.historypackstore(repo.ui, packpath) | ||||
repo.shareddatastores.append(packcontentstore) | ||||
repo.sharedhistorystores.append(packmetadatastore) | ||||
|
r43346 | shallowutil.reportpackmetrics( | ||
|
r43347 | repo.ui, b'filestore', packcontentstore, packmetadatastore | ||
|
r43346 | ) | ||
|
r40530 | return packcontentstore, packmetadatastore | ||
|
r43346 | |||
|
r40530 | def makeunionstores(repo): | ||
"""Union stores iterate the other stores and return the first result.""" | ||||
repo.shareddatastores = [] | ||||
repo.sharedhistorystores = [] | ||||
packcontentstore, packmetadatastore = makepackstores(repo) | ||||
cachecontent, cachemetadata = makecachestores(repo) | ||||
localcontent, localmetadata = makelocalstores(repo) | ||||
|
r43346 | remotecontent, remotemetadata = makeremotestores( | ||
repo, cachecontent, cachemetadata | ||||
) | ||||
|
r40530 | |||
# Instantiate union stores | ||||
repo.contentstore = contentstore.unioncontentstore( | ||||
|
r43346 | packcontentstore, | ||
cachecontent, | ||||
localcontent, | ||||
remotecontent, | ||||
writestore=localcontent, | ||||
) | ||||
|
r40530 | repo.metadatastore = metadatastore.unionmetadatastore( | ||
|
r43346 | packmetadatastore, | ||
cachemetadata, | ||||
localmetadata, | ||||
remotemetadata, | ||||
writestore=localmetadata, | ||||
) | ||||
|
r40530 | |||
fileservicedatawrite = cachecontent | ||||
fileservicehistorywrite = cachemetadata | ||||
|
r43346 | repo.fileservice.setstore( | ||
repo.contentstore, | ||||
repo.metadatastore, | ||||
fileservicedatawrite, | ||||
fileservicehistorywrite, | ||||
) | ||||
shallowutil.reportpackmetrics( | ||||
|
r43347 | repo.ui, b'filestore', packcontentstore, packmetadatastore | ||
|
r43346 | ) | ||
|
r40530 | |||
def wraprepo(repo): | ||||
class shallowrepository(repo.__class__): | ||||
@util.propertycache | ||||
def name(self): | ||||
|
r43347 | return self.ui.config(b'remotefilelog', b'reponame') | ||
|
r40530 | |||
@util.propertycache | ||||
def fallbackpath(self): | ||||
|
r43346 | path = repo.ui.config( | ||
|
r43347 | b"remotefilelog", | ||
b"fallbackpath", | ||||
repo.ui.config(b'paths', b'default'), | ||||
|
r43346 | ) | ||
|
r40530 | if not path: | ||
|
r43346 | raise error.Abort( | ||
|
r43347 | b"no remotefilelog server " | ||
b"configured - is your .hg/hgrc trusted?" | ||||
|
r43346 | ) | ||
|
r40530 | |||
return path | ||||
def maybesparsematch(self, *revs, **kwargs): | ||||
|
r46554 | """ | ||
|
r40530 | A wrapper that allows the remotefilelog to invoke sparsematch() if | ||
this is a sparse repository, or returns None if this is not a | ||||
sparse repository. | ||||
|
r46554 | """ | ||
|
r40530 | if revs: | ||
|
r41107 | ret = sparse.matcher(repo, revs=revs) | ||
else: | ||||
ret = sparse.matcher(repo) | ||||
if ret.always(): | ||||
return None | ||||
return ret | ||||
|
r40530 | |||
def file(self, f): | ||||
|
r43347 | if f[0] == b'/': | ||
|
r40530 | f = f[1:] | ||
if self.shallowmatch(f): | ||||
return remotefilelog.remotefilelog(self.svfs, f, self) | ||||
else: | ||||
return super(shallowrepository, self).file(f) | ||||
def filectx(self, path, *args, **kwargs): | ||||
if self.shallowmatch(path): | ||||
return remotefilectx.remotefilectx(self, path, *args, **kwargs) | ||||
else: | ||||
|
r43346 | return super(shallowrepository, self).filectx( | ||
path, *args, **kwargs | ||||
) | ||||
|
r40530 | |||
@localrepo.unfilteredmethod | ||||
|
r42839 | def commitctx(self, ctx, error=False, origctx=None): | ||
|
r40530 | """Add a new revision to current repository. | ||
Revision information is passed via the context argument. | ||||
""" | ||||
# some contexts already have manifest nodes, they don't need any | ||||
# prefetching (for example if we're just editing a commit message | ||||
# we can reuse manifest | ||||
if not ctx.manifestnode(): | ||||
# prefetch files that will likely be compared | ||||
m1 = ctx.p1().manifest() | ||||
files = [] | ||||
for f in ctx.modified() + ctx.added(): | ||||
fparent1 = m1.get(f, nullid) | ||||
if fparent1 != nullid: | ||||
files.append((f, hex(fparent1))) | ||||
self.fileservice.prefetch(files) | ||||
|
r43346 | return super(shallowrepository, self).commitctx( | ||
ctx, error=error, origctx=origctx | ||||
) | ||||
|
r40530 | |||
|
r43346 | def backgroundprefetch( | ||
r44303 | self, revs, base=None, repack=False, pats=None, opts=None | |||
|
r43346 | ): | ||
|
r46554 | """Runs prefetch in background with optional repack""" | ||
|
r43347 | cmd = [procutil.hgexecutable(), b'-R', repo.origroot, b'prefetch'] | ||
|
r40530 | if repack: | ||
|
r43347 | cmd.append(b'--repack') | ||
|
r40530 | if revs: | ||
|
r43347 | cmd += [b'-r', revs] | ||
|
r42697 | # We know this command will find a binary, so don't block | ||
# on it starting. | ||||
r44298 | kwargs = {} | |||
if repo.ui.configbool(b'devel', b'remotefilelog.bg-wait'): | ||||
kwargs['record_wait'] = repo.ui.atexit | ||||
|
r43346 | procutil.runbgcommand( | ||
r44303 | cmd, encoding.environ, ensurestart=False, **kwargs | |||
|
r43346 | ) | ||
|
r40530 | |||
def prefetch(self, revs, base=None, pats=None, opts=None): | ||||
"""Prefetches all the necessary file revisions for the given revs | ||||
Optionally runs repack in background | ||||
""" | ||||
|
r43346 | with repo._lock( | ||
repo.svfs, | ||||
|
r43347 | b'prefetchlock', | ||
|
r43346 | True, | ||
None, | ||||
None, | ||||
|
r43347 | _(b'prefetching in %s') % repo.origroot, | ||
|
r43346 | ): | ||
|
r40530 | self._prefetch(revs, base, pats, opts) | ||
def _prefetch(self, revs, base=None, pats=None, opts=None): | ||||
fallbackpath = self.fallbackpath | ||||
if fallbackpath: | ||||
# If we know a rev is on the server, we should fetch the server | ||||
# version of those files, since our local file versions might | ||||
# become obsolete if the local commits are stripped. | ||||
|
r43347 | localrevs = repo.revs(b'outgoing(%s)', fallbackpath) | ||
|
r40530 | if base is not None and base != nullrev: | ||
|
r43346 | serverbase = list( | ||
|
r43347 | repo.revs( | ||
b'first(reverse(::%s) - %ld)', base, localrevs | ||||
) | ||||
|
r43346 | ) | ||
|
r40530 | if serverbase: | ||
base = serverbase[0] | ||||
else: | ||||
localrevs = repo | ||||
mfl = repo.manifestlog | ||||
|
r43347 | mfrevlog = mfl.getstorage(b'') | ||
|
r40530 | if base is not None: | ||
mfdict = mfl[repo[base].manifestnode()].read() | ||||
|
r43375 | skip = set(pycompat.iteritems(mfdict)) | ||
|
r40530 | else: | ||
skip = set() | ||||
# Copy the skip set to start large and avoid constant resizing, | ||||
# and since it's likely to be very similar to the prefetch set. | ||||
files = skip.copy() | ||||
serverfiles = skip.copy() | ||||
visited = set() | ||||
visited.add(nullrev) | ||||
revcount = len(revs) | ||||
|
r43347 | progress = self.ui.makeprogress(_(b'prefetching'), total=revcount) | ||
|
r40880 | progress.update(0) | ||
|
r40530 | for rev in sorted(revs): | ||
ctx = repo[rev] | ||||
if pats: | ||||
m = scmutil.match(ctx, pats, opts) | ||||
sparsematch = repo.maybesparsematch(rev) | ||||
mfnode = ctx.manifestnode() | ||||
mfrev = mfrevlog.rev(mfnode) | ||||
# Decompressing manifests is expensive. | ||||
# When possible, only read the deltas. | ||||
p1, p2 = mfrevlog.parentrevs(mfrev) | ||||
if p1 in visited and p2 in visited: | ||||
mfdict = mfl[mfnode].readfast() | ||||
else: | ||||
mfdict = mfl[mfnode].read() | ||||
|
r43375 | diff = pycompat.iteritems(mfdict) | ||
|
r40530 | if pats: | ||
diff = (pf for pf in diff if m(pf[0])) | ||||
if sparsematch: | ||||
diff = (pf for pf in diff if sparsematch(pf[0])) | ||||
if rev not in localrevs: | ||||
serverfiles.update(diff) | ||||
else: | ||||
files.update(diff) | ||||
visited.add(mfrev) | ||||
|
r40880 | progress.increment() | ||
|
r40530 | |||
files.difference_update(skip) | ||||
serverfiles.difference_update(skip) | ||||
|
r40880 | progress.complete() | ||
|
r40530 | |||
# Fetch files known to be on the server | ||||
if serverfiles: | ||||
results = [(path, hex(fnode)) for (path, fnode) in serverfiles] | ||||
repo.fileservice.prefetch(results, force=True) | ||||
# Fetch files that may or may not be on the server | ||||
if files: | ||||
results = [(path, hex(fnode)) for (path, fnode) in files] | ||||
repo.fileservice.prefetch(results) | ||||
def close(self): | ||||
super(shallowrepository, self).close() | ||||
self.connectionpool.close() | ||||
repo.__class__ = shallowrepository | ||||
|
r41825 | repo.shallowmatch = match.always() | ||
|
r40530 | |||
makeunionstores(repo) | ||||
|
r43346 | repo.includepattern = repo.ui.configlist( | ||
|
r43347 | b"remotefilelog", b"includepattern", None | ||
|
r43346 | ) | ||
repo.excludepattern = repo.ui.configlist( | ||||
|
r43347 | b"remotefilelog", b"excludepattern", None | ||
|
r43346 | ) | ||
|
r43385 | if not util.safehasattr(repo, 'connectionpool'): | ||
|
r40530 | repo.connectionpool = connectionpool.connectionpool(repo) | ||
if repo.includepattern or repo.excludepattern: | ||||
|
r43346 | repo.shallowmatch = match.match( | ||
|
r43347 | repo.root, b'', None, repo.includepattern, repo.excludepattern | ||
|
r43346 | ) | ||