basestore.py
195 lines
| 6.8 KiB
| text/x-python
|
PythonLexer
|
r15168 | # Copyright 2009-2010 Gregory P. Ward | ||
# Copyright 2009-2010 Intelerad Medical Systems Incorporated | ||||
# Copyright 2010-2011 Fog Creek Software | ||||
# Copyright 2010-2011 Unity Technologies | ||||
# | ||||
# This software may be used and distributed according to the terms of the | ||||
# GNU General Public License version 2 or any later version. | ||||
|
r15252 | '''base class for store implementations and store-related utility code''' | ||
|
r15168 | |||
import binascii | ||||
import re | ||||
from mercurial import util, node, hg | ||||
from mercurial.i18n import _ | ||||
import lfutil | ||||
class StoreError(Exception): | ||||
'''Raised when there is a problem getting files from or putting | ||||
files to a central store.''' | ||||
def __init__(self, filename, hash, url, detail): | ||||
self.filename = filename | ||||
self.hash = hash | ||||
self.url = url | ||||
self.detail = detail | ||||
def longmessage(self): | ||||
if self.url: | ||||
return ('%s: %s\n' | ||||
'(failed URL: %s)\n' | ||||
% (self.filename, self.detail, self.url)) | ||||
else: | ||||
return ('%s: %s\n' | ||||
'(no default or default-push path set in hgrc)\n' | ||||
% (self.filename, self.detail)) | ||||
def __str__(self): | ||||
return "%s: %s" % (self.url, self.detail) | ||||
class basestore(object): | ||||
def __init__(self, ui, repo, url): | ||||
self.ui = ui | ||||
self.repo = repo | ||||
self.url = url | ||||
def put(self, source, hash): | ||||
'''Put source file into the store under <filename>/<hash>.''' | ||||
raise NotImplementedError('abstract method') | ||||
def exists(self, hash): | ||||
'''Check to see if the store contains the given hash.''' | ||||
raise NotImplementedError('abstract method') | ||||
def get(self, files): | ||||
'''Get the specified largefiles from the store and write to local | ||||
files under repo.root. files is a list of (filename, hash) | ||||
tuples. Return (success, missing), lists of files successfuly | ||||
downloaded and those not found in the store. success is a list | ||||
of (filename, hash) tuples; missing is a list of filenames that | ||||
we could not get. (The detailed error message will already have | ||||
been presented to the user, so missing is just supplied as a | ||||
summary.)''' | ||||
success = [] | ||||
missing = [] | ||||
ui = self.ui | ||||
at = 0 | ||||
for filename, hash in files: | ||||
ui.progress(_('getting largefiles'), at, unit='lfile', | ||||
total=len(files)) | ||||
at += 1 | ||||
ui.note(_('getting %s:%s\n') % (filename, hash)) | ||||
|
r15316 | storefilename = lfutil.storepath(self.repo, hash) | ||
|
r16154 | tmpfile = util.atomictempfile(storefilename, | ||
createmode=self.repo.store.createmode) | ||||
|
r15168 | |||
try: | ||||
hhash = binascii.hexlify(self._getfile(tmpfile, filename, hash)) | ||||
except StoreError, err: | ||||
ui.warn(err.longmessage()) | ||||
hhash = "" | ||||
if hhash != hash: | ||||
if hhash != "": | ||||
ui.warn(_('%s: data corruption (expected %s, got %s)\n') | ||||
% (filename, hash, hhash)) | ||||
|
r16154 | tmpfile.discard() # no-op if it's already closed | ||
|
r15168 | missing.append(filename) | ||
continue | ||||
|
r16154 | tmpfile.close() | ||
|
r15316 | lfutil.linktousercache(self.repo, hash) | ||
|
r15168 | success.append((filename, hhash)) | ||
ui.progress(_('getting largefiles'), None) | ||||
return (success, missing) | ||||
def verify(self, revs, contents=False): | ||||
'''Verify the existence (and, optionally, contents) of every big | ||||
file revision referenced by every changeset in revs. | ||||
Return 0 if all is well, non-zero on any errors.''' | ||||
write = self.ui.write | ||||
failed = False | ||||
write(_('searching %d changesets for largefiles\n') % len(revs)) | ||||
verified = set() # set of (filename, filenode) tuples | ||||
for rev in revs: | ||||
cctx = self.repo[rev] | ||||
cset = "%d:%s" % (cctx.rev(), node.short(cctx.node())) | ||||
|
r15319 | failed = util.any(self._verifyfile( | ||
|
r15168 | cctx, cset, contents, standin, verified) for standin in cctx) | ||
|
r16247 | numrevs = len(verified) | ||
numlfiles = len(set([fname for (fname, fnode) in verified])) | ||||
|
r15168 | if contents: | ||
write(_('verified contents of %d revisions of %d largefiles\n') | ||||
|
r16247 | % (numrevs, numlfiles)) | ||
|
r15168 | else: | ||
write(_('verified existence of %d revisions of %d largefiles\n') | ||||
|
r16247 | % (numrevs, numlfiles)) | ||
|
r15168 | |||
return int(failed) | ||||
def _getfile(self, tmpfile, filename, hash): | ||||
'''Fetch one revision of one file from the store and write it | ||||
to tmpfile. Compute the hash of the file on-the-fly as it | ||||
downloads and return the binary hash. Close tmpfile. Raise | ||||
StoreError if unable to download the file (e.g. it does not | ||||
exist in the store).''' | ||||
raise NotImplementedError('abstract method') | ||||
def _verifyfile(self, cctx, cset, contents, standin, verified): | ||||
'''Perform the actual verification of a file in the store. | ||||
''' | ||||
raise NotImplementedError('abstract method') | ||||
import localstore, wirestore | ||||
_storeprovider = { | ||||
'file': [localstore.localstore], | ||||
'http': [wirestore.wirestore], | ||||
'https': [wirestore.wirestore], | ||||
'ssh': [wirestore.wirestore], | ||||
} | ||||
_scheme_re = re.compile(r'^([a-zA-Z0-9+-.]+)://') | ||||
# During clone this function is passed the src's ui object | ||||
# but it needs the dest's ui object so it can read out of | ||||
# the config file. Use repo.ui instead. | ||||
def _openstore(repo, remote=None, put=False): | ||||
ui = repo.ui | ||||
if not remote: | ||||
|
r15943 | lfpullsource = getattr(repo, 'lfpullsource', None) | ||
if lfpullsource: | ||||
path = ui.expandpath(lfpullsource) | ||||
else: | ||||
path = ui.expandpath('default-push', 'default') | ||||
|
r15252 | |||
# ui.expandpath() leaves 'default-push' and 'default' alone if | ||||
# they cannot be expanded: fallback to the empty string, | ||||
# meaning the current directory. | ||||
|
r15168 | if path == 'default-push' or path == 'default': | ||
path = '' | ||||
remote = repo | ||||
else: | ||||
remote = hg.peer(repo, {}, path) | ||||
# The path could be a scheme so use Mercurial's normal functionality | ||||
# to resolve the scheme to a repository and use its path | ||||
|
r15169 | path = util.safehasattr(remote, 'url') and remote.url() or remote.path | ||
|
r15168 | |||
match = _scheme_re.match(path) | ||||
if not match: # regular filesystem path | ||||
scheme = 'file' | ||||
else: | ||||
scheme = match.group(1) | ||||
try: | ||||
storeproviders = _storeprovider[scheme] | ||||
except KeyError: | ||||
raise util.Abort(_('unsupported URL scheme %r') % scheme) | ||||
|
r16247 | for classobj in storeproviders: | ||
|
r15168 | try: | ||
|
r16247 | return classobj(ui, repo, remote) | ||
|
r15168 | except lfutil.storeprotonotcapable: | ||
pass | ||||
|
r15302 | raise util.Abort(_('%s does not appear to be a largefile store') % path) | ||