largefiles: move basestore._openstore into new module to remove cycle

hgext/largefiles/basestore.py

@@ -8,9 +8,7
 
 '''base class for store implementations and store-related utility code'''
 
-import re
-
-from mercurial import util, node, hg, error
+from mercurial import util, node
 from mercurial.i18n import _
 
 import lfutil
@@ -164,63 +162,3 class basestore(object):
         Returns _true_ if any problems are found!
         '''
         raise NotImplementedError('abstract method')
-
-import localstore, wirestore
-
-_storeprovider = {
-    'file':  [localstore.localstore],
-    'http':  [wirestore.wirestore],
-    'https': [wirestore.wirestore],
-    'ssh': [wirestore.wirestore],
-    }
-
-_scheme_re = re.compile(r'^([a-zA-Z0-9+-.]+)://')
-
-# During clone this function is passed the src's ui object
-# but it needs the dest's ui object so it can read out of
-# the config file. Use repo.ui instead.
-def _openstore(repo, remote=None, put=False):
-    ui = repo.ui
-
-    if not remote:
-        lfpullsource = getattr(repo, 'lfpullsource', None)
-        if lfpullsource:
-            path = ui.expandpath(lfpullsource)
-        elif put:
-            path = ui.expandpath('default-push', 'default')
-        else:
-            path = ui.expandpath('default')
-
-        # ui.expandpath() leaves 'default-push' and 'default' alone if
-        # they cannot be expanded: fallback to the empty string,
-        # meaning the current directory.
-        if path == 'default-push' or path == 'default':
-            path = ''
-            remote = repo
-        else:
-            path, _branches = hg.parseurl(path)
-            remote = hg.peer(repo, {}, path)
-
-    # The path could be a scheme so use Mercurial's normal functionality
-    # to resolve the scheme to a repository and use its path
-    path = util.safehasattr(remote, 'url') and remote.url() or remote.path
-
-    match = _scheme_re.match(path)
-    if not match:                       # regular filesystem path
-        scheme = 'file'
-    else:
-        scheme = match.group(1)
-
-    try:
-        storeproviders = _storeprovider[scheme]
-    except KeyError:
-        raise error.Abort(_('unsupported URL scheme %r') % scheme)
-
-    for classobj in storeproviders:
-        try:
-            return classobj(ui, repo, remote)
-        except lfutil.storeprotonotcapable:
-            pass
-
-    raise error.Abort(_('%s does not appear to be a largefile store') %
-                     util.hidepassword(path))

hgext/largefiles/lfcommands.py

@@ -20,7 +20,7 from hgext.convert import convcmd
 from hgext.convert import filemap
 
 import lfutil
-import basestore
+import storefactory
 
 # -- Commands ----------------------------------------------------------
 
@@ -337,7 +337,7 def uploadlfiles(ui, rsrc, rdst, files):
     if not files:
         return
 
-    store = basestore._openstore(rsrc, rdst, put=True)
+    store = storefactory._openstore(rsrc, rdst, put=True)
 
     at = 0
     ui.debug("sending statlfile command for %d largefiles\n" % len(files))
@@ -368,7 +368,7 def verifylfiles(ui, repo, all=False, co
     else:
         revs = ['.']
 
-    store = basestore._openstore(repo)
+    store = storefactory._openstore(repo)
     return store.verify(revs, contents=contents)
 
 def cachelfiles(ui, repo, node, filelist=None):
@@ -394,7 +394,7 def cachelfiles(ui, repo, node, filelist
             toget.append((lfile, expectedhash))
 
     if toget:
-        store = basestore._openstore(repo)
+        store = storefactory._openstore(repo)
         ret = store.get(toget)
         return ret
 

hgext/largefiles/overrides.py

@@ -17,7 +17,7 from mercurial.i18n import _
 
 import lfutil
 import lfcommands
-import basestore
+import storefactory
 
 # -- Utility functions: commonly/repeatedly needed functionality ---------------
 
@@ -1109,7 +1109,7 def _getoutgoings(repo, other, missing,
             lfhashes.add(lfhash)
     lfutil.getlfilestoupload(repo, missing, dedup)
     if lfhashes:
-        lfexists = basestore._openstore(repo, other).exists(lfhashes)
+        lfexists = storefactory._openstore(repo, other).exists(lfhashes)
         for fn, lfhash in knowns:
             if not lfexists[lfhash]: # lfhash doesn't exist on "other"
                 addfunc(fn, lfhash)
@@ -1338,7 +1338,7 def overridecat(orig, ui, repo, file1, *
         else:
             hash = lfutil.readstandin(repo, lf, ctx.rev())
             if not lfutil.inusercache(repo.ui, hash):
-                store = basestore._openstore(repo)
+                store = storefactory._openstore(repo)
                 success, missing = store.get([(lf, hash)])
                 if len(success) != 1:
                     raise error.Abort(

hgext/largefiles/storefactory.py

@@ -1,180 +1,23
-# Copyright 2009-2010 Gregory P. Ward
-# Copyright 2009-2010 Intelerad Medical Systems Incorporated
-# Copyright 2010-2011 Fog Creek Software
-# Copyright 2010-2011 Unity Technologies
-#
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
-'''base class for store implementations and store-related utility code'''
+from __future__ import absolute_import
 
 import re
 
-from mercurial import util, node, hg, error
 from mercurial.i18n import _
 
-import lfutil
-
-class StoreError(Exception):
-    '''Raised when there is a problem getting files from or putting
-    files to a central store.'''
-    def __init__(self, filename, hash, url, detail):
-        self.filename = filename
-        self.hash = hash
-        self.url = url
-        self.detail = detail
-
-    def longmessage(self):
-        return (_("error getting id %s from url %s for file %s: %s\n") %
-                 (self.hash, util.hidepassword(self.url), self.filename,
-                  self.detail))
-
-    def __str__(self):
-        return "%s: %s" % (util.hidepassword(self.url), self.detail)
-
-class basestore(object):
-    def __init__(self, ui, repo, url):
-        self.ui = ui
-        self.repo = repo
-        self.url = url
-
-    def put(self, source, hash):
-        '''Put source file into the store so it can be retrieved by hash.'''
-        raise NotImplementedError('abstract method')
-
-    def exists(self, hashes):
-        '''Check to see if the store contains the given hashes. Given an
-        iterable of hashes it returns a mapping from hash to bool.'''
-        raise NotImplementedError('abstract method')
-
-    def get(self, files):
-        '''Get the specified largefiles from the store and write to local
-        files under repo.root.  files is a list of (filename, hash)
-        tuples.  Return (success, missing), lists of files successfully
-        downloaded and those not found in the store.  success is a list
-        of (filename, hash) tuples; missing is a list of filenames that
-        we could not get.  (The detailed error message will already have
-        been presented to the user, so missing is just supplied as a
-        summary.)'''
-        success = []
-        missing = []
-        ui = self.ui
-
-        at = 0
-        available = self.exists(set(hash for (_filename, hash) in files))
-        for filename, hash in files:
-            ui.progress(_('getting largefiles'), at, unit=_('files'),
-                total=len(files))
-            at += 1
-            ui.note(_('getting %s:%s\n') % (filename, hash))
-
-            if not available.get(hash):
-                ui.warn(_('%s: largefile %s not available from %s\n')
-                        % (filename, hash, util.hidepassword(self.url)))
-                missing.append(filename)
-                continue
-
-            if self._gethash(filename, hash):
-                success.append((filename, hash))
-            else:
-                missing.append(filename)
-
-        ui.progress(_('getting largefiles'), None)
-        return (success, missing)
-
-    def _gethash(self, filename, hash):
-        """Get file with the provided hash and store it in the local repo's
-        store and in the usercache.
-        filename is for informational messages only.
-        """
-        util.makedirs(lfutil.storepath(self.repo, ''))
-        storefilename = lfutil.storepath(self.repo, hash)
-
-        tmpname = storefilename + '.tmp'
-        tmpfile = util.atomictempfile(tmpname,
-                                      createmode=self.repo.store.createmode)
+from mercurial import (
+    error,
+    hg,
+    util,
+)
 
-        try:
-            gothash = self._getfile(tmpfile, filename, hash)
-        except StoreError as err:
-            self.ui.warn(err.longmessage())
-            gothash = ""
-        tmpfile.close()
-
-        if gothash != hash:
-            if gothash != "":
-                self.ui.warn(_('%s: data corruption (expected %s, got %s)\n')
-                             % (filename, hash, gothash))
-            util.unlink(tmpname)
-            return False
-
-        util.rename(tmpname, storefilename)
-        lfutil.linktousercache(self.repo, hash)
-        return True
-
-    def verify(self, revs, contents=False):
-        '''Verify the existence (and, optionally, contents) of every big
-        file revision referenced by every changeset in revs.
-        Return 0 if all is well, non-zero on any errors.'''
-
-        self.ui.status(_('searching %d changesets for largefiles\n') %
-                       len(revs))
-        verified = set()                # set of (filename, filenode) tuples
-        filestocheck = []               # list of (cset, filename, expectedhash)
-        for rev in revs:
-            cctx = self.repo[rev]
-            cset = "%d:%s" % (cctx.rev(), node.short(cctx.node()))
-
-            for standin in cctx:
-                filename = lfutil.splitstandin(standin)
-                if filename:
-                    fctx = cctx[standin]
-                    key = (filename, fctx.filenode())
-                    if key not in verified:
-                        verified.add(key)
-                        expectedhash = fctx.data()[0:40]
-                        filestocheck.append((cset, filename, expectedhash))
-
-        failed = self._verifyfiles(contents, filestocheck)
-
-        numrevs = len(verified)
-        numlfiles = len(set([fname for (fname, fnode) in verified]))
-        if contents:
-            self.ui.status(
-                _('verified contents of %d revisions of %d largefiles\n')
-                % (numrevs, numlfiles))
-        else:
-            self.ui.status(
-                _('verified existence of %d revisions of %d largefiles\n')
-                % (numrevs, numlfiles))
-        return int(failed)
-
-    def _getfile(self, tmpfile, filename, hash):
-        '''Fetch one revision of one file from the store and write it
-        to tmpfile.  Compute the hash of the file on-the-fly as it
-        downloads and return the hash.  Close tmpfile.  Raise
-        StoreError if unable to download the file (e.g. it does not
-        exist in the store).'''
-        raise NotImplementedError('abstract method')
-
-    def _verifyfiles(self, contents, filestocheck):
-        '''Perform the actual verification of files in the store.
-        'contents' controls verification of content hash.
-        'filestocheck' is list of files to check.
-        Returns _true_ if any problems are found!
-        '''
-        raise NotImplementedError('abstract method')
-
-import localstore, wirestore
-
-_storeprovider = {
-    'file':  [localstore.localstore],
-    'http':  [wirestore.wirestore],
-    'https': [wirestore.wirestore],
-    'ssh': [wirestore.wirestore],
-    }
-
-_scheme_re = re.compile(r'^([a-zA-Z0-9+-.]+)://')
+from . import (
+    lfutil,
+    localstore,
+    wirestore,
+)
 
 # During clone this function is passed the src's ui object
 # but it needs the dest's ui object so it can read out of
@@ -224,3 +67,12 def _openstore(repo, remote=None, put=Fa
 
     raise error.Abort(_('%s does not appear to be a largefile store') %
                      util.hidepassword(path))
+
+_storeprovider = {
+    'file':  [localstore.localstore],
+    'http':  [wirestore.wirestore],
+    'https': [wirestore.wirestore],
+    'ssh': [wirestore.wirestore],
+    }
+
+_scheme_re = re.compile(r'^([a-zA-Z0-9+-.]+)://')

tests/test-check-module-imports.t

@@ -179,5 +179,3 outputs, which should be fixed later.
   > -X tests/test-hgweb-no-request-uri.t \
   > -X tests/test-hgweb-non-interactive.t \
   > | sed 's-\\-/-g' | python "$import_checker" -
-  Import cycle: hgext.largefiles.basestore -> hgext.largefiles.localstore -> hgext.largefiles.basestore
-  [1]