largefiles: don't close the fd passed to store._getfile

hgext/largefiles/basestore.py

@@ -1,200 +1,200 b''
 # Copyright 2009-2010 Gregory P. Ward
 # Copyright 2009-2010 Intelerad Medical Systems Incorporated
 # Copyright 2010-2011 Fog Creek Software
 # Copyright 2010-2011 Unity Technologies
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''base class for store implementations and store-related utility code'''
 
 import re
 
 from mercurial import util, node, hg
 from mercurial.i18n import _
 
 import lfutil
 
 class StoreError(Exception):
     '''Raised when there is a problem getting files from or putting
     files to a central store.'''
     def __init__(self, filename, hash, url, detail):
         self.filename = filename
         self.hash = hash
         self.url = url
         self.detail = detail
 
     def longmessage(self):
         return (_("error getting id %s from url %s for file %s: %s\n") %
                  (self.hash, self.url, self.filename, self.detail))
 
     def __str__(self):
         return "%s: %s" % (self.url, self.detail)
 
 class basestore(object):
     def __init__(self, ui, repo, url):
         self.ui = ui
         self.repo = repo
         self.url = url
 
     def put(self, source, hash):
         '''Put source file into the store under <filename>/<hash>.'''
         raise NotImplementedError('abstract method')
 
     def exists(self, hashes):
         '''Check to see if the store contains the given hashes. Given an
         iterable of hashes it returns a mapping from hash to bool.'''
         raise NotImplementedError('abstract method')
 
     def get(self, files):
         '''Get the specified largefiles from the store and write to local
         files under repo.root.  files is a list of (filename, hash)
         tuples.  Return (success, missing), lists of files successfully
         downloaded and those not found in the store.  success is a list
         of (filename, hash) tuples; missing is a list of filenames that
         we could not get.  (The detailed error message will already have
         been presented to the user, so missing is just supplied as a
         summary.)'''
         success = []
         missing = []
         ui = self.ui
 
         util.makedirs(lfutil.storepath(self.repo, ''))
 
         at = 0
         for filename, hash in files:
             ui.progress(_('getting largefiles'), at, unit='lfile',
                 total=len(files))
             at += 1
             ui.note(_('getting %s:%s\n') % (filename, hash))
 
             storefilename = lfutil.storepath(self.repo, hash)
             tmpfile = util.atomictempfile(storefilename + '.tmp',
                                           createmode=self.repo.store.createmode)
 
             try:
                 hhash = self._getfile(tmpfile, filename, hash)
             except StoreError, err:
                 ui.warn(err.longmessage())
                 hhash = ""
-            tmpfile.close() # has probably already been closed!
+            tmpfile.close()
 
             if hhash != hash:
                 if hhash != "":
                     ui.warn(_('%s: data corruption (expected %s, got %s)\n')
                             % (filename, hash, hhash))
                 util.unlink(storefilename + '.tmp')
                 missing.append(filename)
                 continue
 
             util.rename(storefilename + '.tmp', storefilename)
             lfutil.linktousercache(self.repo, hash)
             success.append((filename, hhash))
 
         ui.progress(_('getting largefiles'), None)
         return (success, missing)
 
     def verify(self, revs, contents=False):
         '''Verify the existence (and, optionally, contents) of every big
         file revision referenced by every changeset in revs.
         Return 0 if all is well, non-zero on any errors.'''
         failed = False
 
         self.ui.status(_('searching %d changesets for largefiles\n') %
                        len(revs))
         verified = set()                # set of (filename, filenode) tuples
 
         for rev in revs:
             cctx = self.repo[rev]
             cset = "%d:%s" % (cctx.rev(), node.short(cctx.node()))
 
             for standin in cctx:
                 if self._verifyfile(cctx, cset, contents, standin, verified):
                     failed = True
 
         numrevs = len(verified)
         numlfiles = len(set([fname for (fname, fnode) in verified]))
         if contents:
             self.ui.status(
                 _('verified contents of %d revisions of %d largefiles\n')
                 % (numrevs, numlfiles))
         else:
             self.ui.status(
                 _('verified existence of %d revisions of %d largefiles\n')
                 % (numrevs, numlfiles))
         return int(failed)
 
     def _getfile(self, tmpfile, filename, hash):
         '''Fetch one revision of one file from the store and write it
         to tmpfile.  Compute the hash of the file on-the-fly as it
         downloads and return the hash.  Close tmpfile.  Raise
         StoreError if unable to download the file (e.g. it does not
         exist in the store).'''
         raise NotImplementedError('abstract method')
 
     def _verifyfile(self, cctx, cset, contents, standin, verified):
         '''Perform the actual verification of a file in the store.
         'cset' is only used in warnings.
         'contents' controls verification of content hash.
         'standin' is the standin path of the largefile to verify.
         'verified' is maintained as a set of already verified files.
         Returns _true_ if it is a standin and any problems are found!
         '''
         raise NotImplementedError('abstract method')
 
 import localstore, wirestore
 
 _storeprovider = {
     'file':  [localstore.localstore],
     'http':  [wirestore.wirestore],
     'https': [wirestore.wirestore],
     'ssh': [wirestore.wirestore],
     }
 
 _scheme_re = re.compile(r'^([a-zA-Z0-9+-.]+)://')
 
 # During clone this function is passed the src's ui object
 # but it needs the dest's ui object so it can read out of
 # the config file. Use repo.ui instead.
 def _openstore(repo, remote=None, put=False):
     ui = repo.ui
 
     if not remote:
         lfpullsource = getattr(repo, 'lfpullsource', None)
         if lfpullsource:
             path = ui.expandpath(lfpullsource)
         else:
             path = ui.expandpath('default-push', 'default')
 
         # ui.expandpath() leaves 'default-push' and 'default' alone if
         # they cannot be expanded: fallback to the empty string,
         # meaning the current directory.
         if path == 'default-push' or path == 'default':
             path = ''
             remote = repo
         else:
             path, _branches = hg.parseurl(path)
             remote = hg.peer(repo, {}, path)
 
     # The path could be a scheme so use Mercurial's normal functionality
     # to resolve the scheme to a repository and use its path
     path = util.safehasattr(remote, 'url') and remote.url() or remote.path
 
     match = _scheme_re.match(path)
     if not match:                       # regular filesystem path
         scheme = 'file'
     else:
         scheme = match.group(1)
 
     try:
         storeproviders = _storeprovider[scheme]
     except KeyError:
         raise util.Abort(_('unsupported URL scheme %r') % scheme)
 
     for classobj in storeproviders:
         try:
             return classobj(ui, repo, remote)
         except lfutil.storeprotonotcapable:
             pass
 
     raise util.Abort(_('%s does not appear to be a largefile store') % path)

hgext/largefiles/localstore.py

@@ -1,75 +1,74 b''
 # Copyright 2009-2010 Gregory P. Ward
 # Copyright 2009-2010 Intelerad Medical Systems Incorporated
 # Copyright 2010-2011 Fog Creek Software
 # Copyright 2010-2011 Unity Technologies
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''store class for local filesystem'''
 
 from mercurial.i18n import _
 
 import lfutil
 import basestore
 
 class localstore(basestore.basestore):
     '''localstore first attempts to grab files out of the store in the remote
     Mercurial repository.  Failing that, it attempts to grab the files from
     the user cache.'''
 
     def __init__(self, ui, repo, remote):
         self.remote = remote.local()
         super(localstore, self).__init__(ui, repo, self.remote.url())
 
     def put(self, source, hash):
         if lfutil.instore(self.remote, hash):
             return
         lfutil.link(lfutil.storepath(self.repo, hash),
                 lfutil.storepath(self.remote, hash))
 
     def exists(self, hashes):
         retval = {}
         for hash in hashes:
             retval[hash] = lfutil.instore(self.remote, hash)
         return retval
 
 
     def _getfile(self, tmpfile, filename, hash):
         path = lfutil.findfile(self.remote, hash)
         if not path:
             raise basestore.StoreError(filename, hash, self.url,
                 _("can't get file locally"))
         fd = open(path, 'rb')
         try:
             return lfutil.copyandhash(fd, tmpfile)
         finally:
             fd.close()
-            tmpfile.close()
 
     def _verifyfile(self, cctx, cset, contents, standin, verified):
         filename = lfutil.splitstandin(standin)
         if not filename:
             return False
         fctx = cctx[standin]
         key = (filename, fctx.filenode())
         if key in verified:
             return False
 
         expecthash = fctx.data()[0:40]
         storepath = lfutil.storepath(self.remote, expecthash)
         verified.add(key)
         if not lfutil.instore(self.remote, expecthash):
             self.ui.warn(
                 _('changeset %s: %s references missing %s\n')
                 % (cset, filename, storepath))
             return True                 # failed
 
         if contents:
             actualhash = lfutil.hashfile(storepath)
             if actualhash != expecthash:
                 self.ui.warn(
                     _('changeset %s: %s references corrupted %s\n')
                     % (cset, filename, storepath))
                 return True             # failed
         return False

hgext/largefiles/remotestore.py

@@ -1,116 +1,115 b''
 # Copyright 2010-2011 Fog Creek Software
 # Copyright 2010-2011 Unity Technologies
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''remote largefile store; the base class for wirestore'''
 
 import urllib2
 
 from mercurial import util
 from mercurial.i18n import _
 from mercurial.wireproto import remotebatch
 
 import lfutil
 import basestore
 
 class remotestore(basestore.basestore):
     '''a largefile store accessed over a network'''
     def __init__(self, ui, repo, url):
         super(remotestore, self).__init__(ui, repo, url)
 
     def put(self, source, hash):
         if self.sendfile(source, hash):
             raise util.Abort(
                 _('remotestore: could not put %s to remote store %s')
                 % (source, self.url))
         self.ui.debug(
             _('remotestore: put %s to remote store %s') % (source, self.url))
 
     def exists(self, hashes):
         return dict((h, s == 0) for (h, s) in self._stat(hashes).iteritems())
 
     def sendfile(self, filename, hash):
         self.ui.debug('remotestore: sendfile(%s, %s)\n' % (filename, hash))
         fd = None
         try:
             try:
                 fd = lfutil.httpsendfile(self.ui, filename)
             except IOError, e:
                 raise util.Abort(
                     _('remotestore: could not open file %s: %s')
                     % (filename, str(e)))
             return self._put(hash, fd)
         finally:
             if fd:
                 fd.close()
 
     def _getfile(self, tmpfile, filename, hash):
         # quit if the largefile isn't there
         stat = self._stat([hash])[hash]
         if stat == 1:
             raise util.Abort(_('remotestore: largefile %s is invalid') % hash)
         elif stat == 2:
             raise util.Abort(_('remotestore: largefile %s is missing') % hash)
         elif stat != 0:
             raise RuntimeError('error getting file: unexpected response from '
                                'statlfile (%r)' % stat)
 
         try:
             length, infile = self._get(hash)
         except urllib2.HTTPError, e:
             # 401s get converted to util.Aborts; everything else is fine being
             # turned into a StoreError
             raise basestore.StoreError(filename, hash, self.url, str(e))
         except urllib2.URLError, e:
             # This usually indicates a connection problem, so don't
             # keep trying with the other files... they will probably
             # all fail too.
             raise util.Abort('%s: %s' % (self.url, e.reason))
         except IOError, e:
             raise basestore.StoreError(filename, hash, self.url, str(e))
 
         # Mercurial does not close its SSH connections after writing a stream
         if length is not None:
             infile = lfutil.limitreader(infile, length)
         try:
             return lfutil.copyandhash(util.filechunkiter(infile, 128 * 1024),
                                       tmpfile)
         finally:
             infile.close()
-            tmpfile.close()
 
     def _verifyfile(self, cctx, cset, contents, standin, verified):
         filename = lfutil.splitstandin(standin)
         if not filename:
             return False
         fctx = cctx[standin]
         key = (filename, fctx.filenode())
         if key in verified:
             return False
 
         verified.add(key)
 
         expecthash = fctx.data()[0:40]
         stat = self._stat([expecthash])[expecthash]
         if not stat:
             return False
         elif stat == 1:
             self.ui.warn(
                 _('changeset %s: %s: contents differ\n')
                 % (cset, filename))
             return True # failed
         elif stat == 2:
             self.ui.warn(
                 _('changeset %s: %s missing\n')
                 % (cset, filename))
             return True # failed
         else:
             raise RuntimeError('verify failed: unexpected response from '
                                'statlfile (%r)' % stat)
 
     def batch(self):
         '''Support for remote batching.'''
         return remotebatch(self)