basestore.py
201 lines
| 7.2 KiB
| text/x-python
|
PythonLexer
|
r15168 | # Copyright 2009-2010 Gregory P. Ward | ||
# Copyright 2009-2010 Intelerad Medical Systems Incorporated | ||||
# Copyright 2010-2011 Fog Creek Software | ||||
# Copyright 2010-2011 Unity Technologies | ||||
# | ||||
# This software may be used and distributed according to the terms of the | ||||
# GNU General Public License version 2 or any later version. | ||||
|
r15252 | '''base class for store implementations and store-related utility code''' | ||
|
r15168 | |||
import binascii | ||||
import re | ||||
from mercurial import util, node, hg | ||||
from mercurial.i18n import _ | ||||
import lfutil | ||||
class StoreError(Exception): | ||||
'''Raised when there is a problem getting files from or putting | ||||
files to a central store.''' | ||||
def __init__(self, filename, hash, url, detail): | ||||
self.filename = filename | ||||
self.hash = hash | ||||
self.url = url | ||||
self.detail = detail | ||||
def longmessage(self): | ||||
|
r18461 | return (_("error getting id %s from url %s for file %s: %s\n") % | ||
|
r18155 | (self.hash, self.url, self.filename, self.detail)) | ||
|
r15168 | |||
def __str__(self): | ||||
return "%s: %s" % (self.url, self.detail) | ||||
class basestore(object): | ||||
def __init__(self, ui, repo, url): | ||||
self.ui = ui | ||||
self.repo = repo | ||||
self.url = url | ||||
def put(self, source, hash): | ||||
'''Put source file into the store under <filename>/<hash>.''' | ||||
raise NotImplementedError('abstract method') | ||||
|
r17127 | def exists(self, hashes): | ||
|
r18573 | '''Check to see if the store contains the given hashes. Given an | ||
iterable of hashes it returns a mapping from hash to bool.''' | ||||
|
r15168 | raise NotImplementedError('abstract method') | ||
def get(self, files): | ||||
'''Get the specified largefiles from the store and write to local | ||||
files under repo.root. files is a list of (filename, hash) | ||||
|
r17424 | tuples. Return (success, missing), lists of files successfully | ||
|
r15168 | downloaded and those not found in the store. success is a list | ||
of (filename, hash) tuples; missing is a list of filenames that | ||||
we could not get. (The detailed error message will already have | ||||
been presented to the user, so missing is just supplied as a | ||||
summary.)''' | ||||
success = [] | ||||
missing = [] | ||||
ui = self.ui | ||||
|
r18725 | util.makedirs(lfutil.storepath(self.repo, '')) | ||
|
r15168 | at = 0 | ||
for filename, hash in files: | ||||
ui.progress(_('getting largefiles'), at, unit='lfile', | ||||
total=len(files)) | ||||
at += 1 | ||||
ui.note(_('getting %s:%s\n') % (filename, hash)) | ||||
|
r15316 | storefilename = lfutil.storepath(self.repo, hash) | ||
|
r18483 | tmpfile = util.atomictempfile(storefilename + '.tmp', | ||
|
r16154 | createmode=self.repo.store.createmode) | ||
|
r15168 | |||
try: | ||||
hhash = binascii.hexlify(self._getfile(tmpfile, filename, hash)) | ||||
except StoreError, err: | ||||
ui.warn(err.longmessage()) | ||||
hhash = "" | ||||
|
r18483 | tmpfile.close() # has probably already been closed! | ||
|
r15168 | |||
if hhash != hash: | ||||
if hhash != "": | ||||
ui.warn(_('%s: data corruption (expected %s, got %s)\n') | ||||
% (filename, hash, hhash)) | ||||
|
r18483 | util.unlink(storefilename + '.tmp') | ||
|
r15168 | missing.append(filename) | ||
continue | ||||
|
r18483 | util.rename(storefilename + '.tmp', storefilename) | ||
|
r15316 | lfutil.linktousercache(self.repo, hash) | ||
|
r15168 | success.append((filename, hhash)) | ||
ui.progress(_('getting largefiles'), None) | ||||
return (success, missing) | ||||
def verify(self, revs, contents=False): | ||||
'''Verify the existence (and, optionally, contents) of every big | ||||
file revision referenced by every changeset in revs. | ||||
Return 0 if all is well, non-zero on any errors.''' | ||||
failed = False | ||||
|
r18546 | self.ui.status(_('searching %d changesets for largefiles\n') % | ||
len(revs)) | ||||
|
r15168 | verified = set() # set of (filename, filenode) tuples | ||
for rev in revs: | ||||
cctx = self.repo[rev] | ||||
cset = "%d:%s" % (cctx.rev(), node.short(cctx.node())) | ||||
|
r18486 | for standin in cctx: | ||
if self._verifyfile(cctx, cset, contents, standin, verified): | ||||
failed = True | ||||
|
r15168 | |||
|
r16247 | numrevs = len(verified) | ||
numlfiles = len(set([fname for (fname, fnode) in verified])) | ||||
|
r15168 | if contents: | ||
|
r18546 | self.ui.status( | ||
_('verified contents of %d revisions of %d largefiles\n') | ||||
% (numrevs, numlfiles)) | ||||
|
r15168 | else: | ||
|
r18546 | self.ui.status( | ||
_('verified existence of %d revisions of %d largefiles\n') | ||||
% (numrevs, numlfiles)) | ||||
|
r15168 | return int(failed) | ||
def _getfile(self, tmpfile, filename, hash): | ||||
'''Fetch one revision of one file from the store and write it | ||||
to tmpfile. Compute the hash of the file on-the-fly as it | ||||
downloads and return the binary hash. Close tmpfile. Raise | ||||
StoreError if unable to download the file (e.g. it does not | ||||
exist in the store).''' | ||||
raise NotImplementedError('abstract method') | ||||
def _verifyfile(self, cctx, cset, contents, standin, verified): | ||||
'''Perform the actual verification of a file in the store. | ||||
|
r18574 | 'cset' is only used in warnings. | ||
'contents' controls verification of content hash. | ||||
'standin' is the standin path of the largefile to verify. | ||||
'verified' is maintained as a set of already verified files. | ||||
Returns _true_ if it is a standin and any problems are found! | ||||
|
r15168 | ''' | ||
raise NotImplementedError('abstract method') | ||||
import localstore, wirestore | ||||
_storeprovider = { | ||||
'file': [localstore.localstore], | ||||
'http': [wirestore.wirestore], | ||||
'https': [wirestore.wirestore], | ||||
'ssh': [wirestore.wirestore], | ||||
} | ||||
_scheme_re = re.compile(r'^([a-zA-Z0-9+-.]+)://') | ||||
# During clone this function is passed the src's ui object | ||||
# but it needs the dest's ui object so it can read out of | ||||
# the config file. Use repo.ui instead. | ||||
def _openstore(repo, remote=None, put=False): | ||||
ui = repo.ui | ||||
if not remote: | ||||
|
r15943 | lfpullsource = getattr(repo, 'lfpullsource', None) | ||
if lfpullsource: | ||||
path = ui.expandpath(lfpullsource) | ||||
else: | ||||
path = ui.expandpath('default-push', 'default') | ||||
|
r15252 | |||
# ui.expandpath() leaves 'default-push' and 'default' alone if | ||||
# they cannot be expanded: fallback to the empty string, | ||||
# meaning the current directory. | ||||
|
r15168 | if path == 'default-push' or path == 'default': | ||
path = '' | ||||
remote = repo | ||||
else: | ||||
|
r18489 | path, _branches = hg.parseurl(path) | ||
|
r15168 | remote = hg.peer(repo, {}, path) | ||
# The path could be a scheme so use Mercurial's normal functionality | ||||
# to resolve the scheme to a repository and use its path | ||||
|
r15169 | path = util.safehasattr(remote, 'url') and remote.url() or remote.path | ||
|
r15168 | |||
match = _scheme_re.match(path) | ||||
if not match: # regular filesystem path | ||||
scheme = 'file' | ||||
else: | ||||
scheme = match.group(1) | ||||
try: | ||||
storeproviders = _storeprovider[scheme] | ||||
except KeyError: | ||||
raise util.Abort(_('unsupported URL scheme %r') % scheme) | ||||
|
r16247 | for classobj in storeproviders: | ||
|
r15168 | try: | ||
|
r16247 | return classobj(ui, repo, remote) | ||
|
r15168 | except lfutil.storeprotonotcapable: | ||
pass | ||||
|
r15302 | raise util.Abort(_('%s does not appear to be a largefile store') % path) | ||