basestore.py
182 lines
| 6.5 KiB
| text/x-python
|
PythonLexer
|
r15168 | # Copyright 2009-2010 Gregory P. Ward | ||
# Copyright 2009-2010 Intelerad Medical Systems Incorporated | ||||
# Copyright 2010-2011 Fog Creek Software | ||||
# Copyright 2010-2011 Unity Technologies | ||||
# | ||||
# This software may be used and distributed according to the terms of the | ||||
# GNU General Public License version 2 or any later version. | ||||
|
r15252 | '''base class for store implementations and store-related utility code''' | ||
|
r29307 | from __future__ import absolute_import | ||
|
r15168 | |||
from mercurial.i18n import _ | ||||
|
r29307 | from mercurial import node, util | ||
r47669 | from mercurial.utils import ( | |||
urlutil, | ||||
) | ||||
|
r29307 | |||
from . import lfutil | ||||
|
r15168 | |||
|
r43346 | |||
|
r15168 | class StoreError(Exception): | ||
|
r46554 | """Raised when there is a problem getting files from or putting | ||
files to a central store.""" | ||||
|
r43346 | |||
|
r15168 | def __init__(self, filename, hash, url, detail): | ||
self.filename = filename | ||||
self.hash = hash | ||||
self.url = url | ||||
self.detail = detail | ||||
def longmessage(self): | ||||
|
r43347 | return _(b"error getting id %s from url %s for file %s: %s\n") % ( | ||
|
r43346 | self.hash, | ||
r47669 | urlutil.hidepassword(self.url), | |||
|
r43346 | self.filename, | ||
self.detail, | ||||
) | ||||
|
r15168 | |||
def __str__(self): | ||||
r47669 | return b"%s: %s" % (urlutil.hidepassword(self.url), self.detail) | |||
|
r15168 | |||
|
r43346 | |||
|
r15168 | class basestore(object): | ||
def __init__(self, ui, repo, url): | ||||
self.ui = ui | ||||
self.repo = repo | ||||
self.url = url | ||||
def put(self, source, hash): | ||||
|
r19007 | '''Put source file into the store so it can be retrieved by hash.''' | ||
|
r43347 | raise NotImplementedError(b'abstract method') | ||
|
r15168 | |||
|
r17127 | def exists(self, hashes): | ||
|
r46554 | """Check to see if the store contains the given hashes. Given an | ||
iterable of hashes it returns a mapping from hash to bool.""" | ||||
|
r43347 | raise NotImplementedError(b'abstract method') | ||
|
r15168 | |||
def get(self, files): | ||||
|
r46554 | """Get the specified largefiles from the store and write to local | ||
|
r15168 | files under repo.root. files is a list of (filename, hash) | ||
|
r17424 | tuples. Return (success, missing), lists of files successfully | ||
|
r15168 | downloaded and those not found in the store. success is a list | ||
of (filename, hash) tuples; missing is a list of filenames that | ||||
we could not get. (The detailed error message will already have | ||||
been presented to the user, so missing is just supplied as a | ||||
|
r46554 | summary.)""" | ||
|
r15168 | success = [] | ||
missing = [] | ||||
ui = self.ui | ||||
at = 0 | ||||
|
r44937 | available = self.exists({hash for (_filename, hash) in files}) | ||
|
r43346 | with ui.makeprogress( | ||
|
r43347 | _(b'getting largefiles'), unit=_(b'files'), total=len(files) | ||
|
r43346 | ) as progress: | ||
|
r39427 | for filename, hash in files: | ||
progress.update(at) | ||||
at += 1 | ||||
|
r43347 | ui.note(_(b'getting %s:%s\n') % (filename, hash)) | ||
|
r15168 | |||
|
r39427 | if not available.get(hash): | ||
|
r43346 | ui.warn( | ||
|
r43347 | _(b'%s: largefile %s not available from %s\n') | ||
r47669 | % (filename, hash, urlutil.hidepassword(self.url)) | |||
|
r43346 | ) | ||
|
r39427 | missing.append(filename) | ||
continue | ||||
|
r19008 | |||
|
r39427 | if self._gethash(filename, hash): | ||
success.append((filename, hash)) | ||||
else: | ||||
missing.append(filename) | ||||
|
r15168 | |||
return (success, missing) | ||||
|
r19918 | def _gethash(self, filename, hash): | ||
"""Get file with the provided hash and store it in the local repo's | ||||
store and in the usercache. | ||||
filename is for informational messages only. | ||||
""" | ||||
|
r43347 | util.makedirs(lfutil.storepath(self.repo, b'')) | ||
|
r19918 | storefilename = lfutil.storepath(self.repo, hash) | ||
|
r43347 | tmpname = storefilename + b'.tmp' | ||
|
r43346 | with util.atomictempfile( | ||
tmpname, createmode=self.repo.store.createmode | ||||
) as tmpfile: | ||||
|
r30142 | try: | ||
gothash = self._getfile(tmpfile, filename, hash) | ||||
except StoreError as err: | ||||
self.ui.warn(err.longmessage()) | ||||
|
r43347 | gothash = b"" | ||
|
r19918 | |||
if gothash != hash: | ||||
|
r43347 | if gothash != b"": | ||
|
r43346 | self.ui.warn( | ||
|
r43347 | _(b'%s: data corruption (expected %s, got %s)\n') | ||
|
r43346 | % (filename, hash, gothash) | ||
) | ||||
|
r19918 | util.unlink(tmpname) | ||
return False | ||||
util.rename(tmpname, storefilename) | ||||
lfutil.linktousercache(self.repo, hash) | ||||
return True | ||||
|
r15168 | def verify(self, revs, contents=False): | ||
|
r46554 | """Verify the existence (and, optionally, contents) of every big | ||
|
r15168 | file revision referenced by every changeset in revs. | ||
|
r46554 | Return 0 if all is well, non-zero on any errors.""" | ||
|
r15168 | |||
|
r43346 | self.ui.status( | ||
|
r43347 | _(b'searching %d changesets for largefiles\n') % len(revs) | ||
|
r43346 | ) | ||
verified = set() # set of (filename, filenode) tuples | ||||
filestocheck = [] # list of (cset, filename, expectedhash) | ||||
|
r15168 | for rev in revs: | ||
cctx = self.repo[rev] | ||||
|
r43347 | cset = b"%d:%s" % (cctx.rev(), node.short(cctx.node())) | ||
|
r15168 | |||
|
r18486 | for standin in cctx: | ||
|
r29067 | filename = lfutil.splitstandin(standin) | ||
if filename: | ||||
fctx = cctx[standin] | ||||
key = (filename, fctx.filenode()) | ||||
if key not in verified: | ||||
verified.add(key) | ||||
|
r31740 | expectedhash = lfutil.readasstandin(fctx) | ||
|
r29067 | filestocheck.append((cset, filename, expectedhash)) | ||
failed = self._verifyfiles(contents, filestocheck) | ||||
|
r15168 | |||
|
r16247 | numrevs = len(verified) | ||
|
r42224 | numlfiles = len({fname for (fname, fnode) in verified}) | ||
|
r15168 | if contents: | ||
|
r18546 | self.ui.status( | ||
|
r43347 | _(b'verified contents of %d revisions of %d largefiles\n') | ||
|
r43346 | % (numrevs, numlfiles) | ||
) | ||||
|
r15168 | else: | ||
|
r18546 | self.ui.status( | ||
|
r43347 | _(b'verified existence of %d revisions of %d largefiles\n') | ||
|
r43346 | % (numrevs, numlfiles) | ||
) | ||||
|
r15168 | return int(failed) | ||
def _getfile(self, tmpfile, filename, hash): | ||||
|
r46554 | """Fetch one revision of one file from the store and write it | ||
|
r15168 | to tmpfile. Compute the hash of the file on-the-fly as it | ||
|
r18999 | downloads and return the hash. Close tmpfile. Raise | ||
|
r15168 | StoreError if unable to download the file (e.g. it does not | ||
|
r46554 | exist in the store).""" | ||
|
r43347 | raise NotImplementedError(b'abstract method') | ||
|
r15168 | |||
|
r29067 | def _verifyfiles(self, contents, filestocheck): | ||
|
r46554 | """Perform the actual verification of files in the store. | ||
|
r18574 | 'contents' controls verification of content hash. | ||
|
r29067 | 'filestocheck' is list of files to check. | ||
Returns _true_ if any problems are found! | ||||
|
r46554 | """ | ||
|
r43347 | raise NotImplementedError(b'abstract method') | ||