remotefilelog: stop using RuntimeError for control flow

We introduce a new exception to handle the various failure categories,
rather than relying on RuntimeError.

Differential Revision: https://phab.mercurial-scm.org/D11006

hgext/remotefilelog/basestore.py

@@ -1,461 +1,461 b''
 from __future__ import absolute_import
 
 import errno
 import os
 import shutil
 import stat
 import time
 
 from mercurial.i18n import _
 from mercurial.node import bin, hex
 from mercurial.pycompat import open
 from mercurial import (
     error,
     pycompat,
     util,
 )
 from mercurial.utils import hashutil
 from . import (
     constants,
     shallowutil,
 )
 
 
 class basestore(object):
     def __init__(self, repo, path, reponame, shared=False):
         """Creates a remotefilelog store object for the given repo name.
 
         `path` - The file path where this store keeps its data
         `reponame` - The name of the repo. This is used to partition data from
         many repos.
         `shared` - True if this store is a shared cache of data from the central
         server, for many repos on this machine. False means this store is for
         the local data for one repo.
         """
         self.repo = repo
         self.ui = repo.ui
         self._path = path
         self._reponame = reponame
         self._shared = shared
         self._uid = os.getuid() if not pycompat.iswindows else None
 
         self._validatecachelog = self.ui.config(
             b"remotefilelog", b"validatecachelog"
         )
         self._validatecache = self.ui.config(
             b"remotefilelog", b"validatecache", b'on'
         )
         if self._validatecache not in (b'on', b'strict', b'off'):
             self._validatecache = b'on'
         if self._validatecache == b'off':
             self._validatecache = False
 
         if shared:
             shallowutil.mkstickygroupdir(self.ui, path)
 
     def getmissing(self, keys):
         missing = []
         for name, node in keys:
             filepath = self._getfilepath(name, node)
             exists = os.path.exists(filepath)
             if (
                 exists
                 and self._validatecache == b'strict'
                 and not self._validatekey(filepath, b'contains')
             ):
                 exists = False
             if not exists:
                 missing.append((name, node))
 
         return missing
 
     # BELOW THIS ARE IMPLEMENTATIONS OF REPACK SOURCE
 
     def markledger(self, ledger, options=None):
         if options and options.get(constants.OPTION_PACKSONLY):
             return
         if self._shared:
             for filename, nodes in self._getfiles():
                 for node in nodes:
                     ledger.markdataentry(self, filename, node)
                     ledger.markhistoryentry(self, filename, node)
 
     def cleanup(self, ledger):
         ui = self.ui
         entries = ledger.sources.get(self, [])
         count = 0
         progress = ui.makeprogress(
             _(b"cleaning up"), unit=b"files", total=len(entries)
         )
         for entry in entries:
             if entry.gced or (entry.datarepacked and entry.historyrepacked):
                 progress.update(count)
                 path = self._getfilepath(entry.filename, entry.node)
                 util.tryunlink(path)
             count += 1
         progress.complete()
 
         # Clean up the repo cache directory.
         self._cleanupdirectory(self._getrepocachepath())
 
     # BELOW THIS ARE NON-STANDARD APIS
 
     def _cleanupdirectory(self, rootdir):
         """Removes the empty directories and unnecessary files within the root
         directory recursively. Note that this method does not remove the root
         directory itself."""
 
         oldfiles = set()
         otherfiles = set()
         # osutil.listdir returns stat information which saves some rmdir/listdir
         # syscalls.
         for name, mode in util.osutil.listdir(rootdir):
             if stat.S_ISDIR(mode):
                 dirpath = os.path.join(rootdir, name)
                 self._cleanupdirectory(dirpath)
 
                 # Now that the directory specified by dirpath is potentially
                 # empty, try and remove it.
                 try:
                     os.rmdir(dirpath)
                 except OSError:
                     pass
 
             elif stat.S_ISREG(mode):
                 if name.endswith(b'_old'):
                     oldfiles.add(name[:-4])
                 else:
                     otherfiles.add(name)
 
         # Remove the files which end with suffix '_old' and have no
         # corresponding file without the suffix '_old'. See addremotefilelognode
         # method for the generation/purpose of files with '_old' suffix.
         for filename in oldfiles - otherfiles:
             filepath = os.path.join(rootdir, filename + b'_old')
             util.tryunlink(filepath)
 
     def _getfiles(self):
         """Return a list of (filename, [node,...]) for all the revisions that
         exist in the store.
 
         This is useful for obtaining a list of all the contents of the store
         when performing a repack to another store, since the store API requires
         name+node keys and not namehash+node keys.
         """
         existing = {}
         for filenamehash, node in self._listkeys():
             existing.setdefault(filenamehash, []).append(node)
 
         filenamemap = self._resolvefilenames(existing.keys())
 
         for filename, sha in pycompat.iteritems(filenamemap):
             yield (filename, existing[sha])
 
     def _resolvefilenames(self, hashes):
         """Given a list of filename hashes that are present in the
         remotefilelog store, return a mapping from filename->hash.
 
         This is useful when converting remotefilelog blobs into other storage
         formats.
         """
         if not hashes:
             return {}
 
         filenames = {}
         missingfilename = set(hashes)
 
         # Start with a full manifest, since it'll cover the majority of files
         for filename in self.repo[b'tip'].manifest():
             sha = hashutil.sha1(filename).digest()
             if sha in missingfilename:
                 filenames[filename] = sha
                 missingfilename.discard(sha)
 
         # Scan the changelog until we've found every file name
         cl = self.repo.unfiltered().changelog
         for rev in pycompat.xrange(len(cl) - 1, -1, -1):
             if not missingfilename:
                 break
             files = cl.readfiles(cl.node(rev))
             for filename in files:
                 sha = hashutil.sha1(filename).digest()
                 if sha in missingfilename:
                     filenames[filename] = sha
                     missingfilename.discard(sha)
 
         return filenames
 
     def _getrepocachepath(self):
         return (
             os.path.join(self._path, self._reponame)
             if self._shared
             else self._path
         )
 
     def _listkeys(self):
         """List all the remotefilelog keys that exist in the store.
 
         Returns a iterator of (filename hash, filecontent hash) tuples.
         """
 
         for root, dirs, files in os.walk(self._getrepocachepath()):
             for filename in files:
                 if len(filename) != 40:
                     continue
                 node = filename
                 if self._shared:
                     # .../1a/85ffda..be21
                     filenamehash = root[-41:-39] + root[-38:]
                 else:
                     filenamehash = root[-40:]
                 yield (bin(filenamehash), bin(node))
 
     def _getfilepath(self, name, node):
         node = hex(node)
         if self._shared:
             key = shallowutil.getcachekey(self._reponame, name, node)
         else:
             key = shallowutil.getlocalkey(name, node)
 
         return os.path.join(self._path, key)
 
     def _getdata(self, name, node):
         filepath = self._getfilepath(name, node)
         try:
             data = shallowutil.readfile(filepath)
             if self._validatecache and not self._validatedata(data, filepath):
                 if self._validatecachelog:
                     with open(self._validatecachelog, b'ab+') as f:
                         f.write(b"corrupt %s during read\n" % filepath)
                 os.rename(filepath, filepath + b".corrupt")
                 raise KeyError(b"corrupt local cache file %s" % filepath)
         except IOError:
             raise KeyError(
                 b"no file found at %s for %s:%s" % (filepath, name, hex(node))
             )
 
         return data
 
     def addremotefilelognode(self, name, node, data):
         filepath = self._getfilepath(name, node)
 
         oldumask = os.umask(0o002)
         try:
             # if this node already exists, save the old version for
             # recovery/debugging purposes.
             if os.path.exists(filepath):
                 newfilename = filepath + b'_old'
                 # newfilename can be read-only and shutil.copy will fail.
                 # Delete newfilename to avoid it
                 if os.path.exists(newfilename):
                     shallowutil.unlinkfile(newfilename)
                 shutil.copy(filepath, newfilename)
 
             shallowutil.mkstickygroupdir(self.ui, os.path.dirname(filepath))
             shallowutil.writefile(filepath, data, readonly=True)
 
             if self._validatecache:
                 if not self._validatekey(filepath, b'write'):
                     raise error.Abort(
                         _(b"local cache write was corrupted %s") % filepath
                     )
         finally:
             os.umask(oldumask)
 
     def markrepo(self, path):
         """Call this to add the given repo path to the store's list of
         repositories that are using it. This is useful later when doing garbage
         collection, since it allows us to insecpt the repos to see what nodes
         they want to be kept alive in the store.
         """
         repospath = os.path.join(self._path, b"repos")
         with open(repospath, b'ab') as reposfile:
             reposfile.write(os.path.dirname(path) + b"\n")
 
         repospathstat = os.stat(repospath)
         if repospathstat.st_uid == self._uid:
             os.chmod(repospath, 0o0664)
 
     def _validatekey(self, path, action):
         with open(path, b'rb') as f:
             data = f.read()
 
         if self._validatedata(data, path):
             return True
 
         if self._validatecachelog:
             with open(self._validatecachelog, b'ab+') as f:
                 f.write(b"corrupt %s during %s\n" % (path, action))
 
         os.rename(path, path + b".corrupt")
         return False
 
     def _validatedata(self, data, path):
         try:
             if len(data) > 0:
                 # see remotefilelogserver.createfileblob for the format
                 offset, size, flags = shallowutil.parsesizeflags(data)
                 if len(data) <= size:
                     # it is truncated
                     return False
 
                 # extract the node from the metadata
                 offset += size
                 datanode = data[offset : offset + 20]
 
                 # and compare against the path
                 if os.path.basename(path) == hex(datanode):
                     # Content matches the intended path
                     return True
                 return False
-        except (ValueError, RuntimeError):
+        except (ValueError, shallowutil.BadRemotefilelogHeader):
             pass
 
         return False
 
     def gc(self, keepkeys):
         ui = self.ui
         cachepath = self._path
 
         # prune cache
         queue = pycompat.queue.PriorityQueue()
         originalsize = 0
         size = 0
         count = 0
         removed = 0
 
         # keep files newer than a day even if they aren't needed
         limit = time.time() - (60 * 60 * 24)
 
         progress = ui.makeprogress(
             _(b"removing unnecessary files"), unit=b"files"
         )
         progress.update(0)
         for root, dirs, files in os.walk(cachepath):
             for file in files:
                 if file == b'repos':
                     continue
 
                 # Don't delete pack files
                 if b'/packs/' in root:
                     continue
 
                 progress.update(count)
                 path = os.path.join(root, file)
                 key = os.path.relpath(path, cachepath)
                 count += 1
                 try:
                     pathstat = os.stat(path)
                 except OSError as e:
                     # errno.ENOENT = no such file or directory
                     if e.errno != errno.ENOENT:
                         raise
                     msg = _(
                         b"warning: file %s was removed by another process\n"
                     )
                     ui.warn(msg % path)
                     continue
 
                 originalsize += pathstat.st_size
 
                 if key in keepkeys or pathstat.st_atime > limit:
                     queue.put((pathstat.st_atime, path, pathstat))
                     size += pathstat.st_size
                 else:
                     try:
                         shallowutil.unlinkfile(path)
                     except OSError as e:
                         # errno.ENOENT = no such file or directory
                         if e.errno != errno.ENOENT:
                             raise
                         msg = _(
                             b"warning: file %s was removed by another "
                             b"process\n"
                         )
                         ui.warn(msg % path)
                         continue
                     removed += 1
         progress.complete()
 
         # remove oldest files until under limit
         limit = ui.configbytes(b"remotefilelog", b"cachelimit")
         if size > limit:
             excess = size - limit
             progress = ui.makeprogress(
                 _(b"enforcing cache limit"), unit=b"bytes", total=excess
             )
             removedexcess = 0
             while queue and size > limit and size > 0:
                 progress.update(removedexcess)
                 atime, oldpath, oldpathstat = queue.get()
                 try:
                     shallowutil.unlinkfile(oldpath)
                 except OSError as e:
                     # errno.ENOENT = no such file or directory
                     if e.errno != errno.ENOENT:
                         raise
                     msg = _(
                         b"warning: file %s was removed by another process\n"
                     )
                     ui.warn(msg % oldpath)
                 size -= oldpathstat.st_size
                 removed += 1
                 removedexcess += oldpathstat.st_size
             progress.complete()
 
         ui.status(
             _(b"finished: removed %d of %d files (%0.2f GB to %0.2f GB)\n")
             % (
                 removed,
                 count,
                 float(originalsize) / 1024.0 / 1024.0 / 1024.0,
                 float(size) / 1024.0 / 1024.0 / 1024.0,
             )
         )
 
 
 class baseunionstore(object):
     def __init__(self, *args, **kwargs):
         # If one of the functions that iterates all of the stores is about to
         # throw a KeyError, try this many times with a full refresh between
         # attempts. A repack operation may have moved data from one store to
         # another while we were running.
         self.numattempts = kwargs.get('numretries', 0) + 1
         # If not-None, call this function on every retry and if the attempts are
         # exhausted.
         self.retrylog = kwargs.get('retrylog', None)
 
     def markforrefresh(self):
         for store in self.stores:
             if util.safehasattr(store, b'markforrefresh'):
                 store.markforrefresh()
 
     @staticmethod
     def retriable(fn):
         def noop(*args):
             pass
 
         def wrapped(self, *args, **kwargs):
             retrylog = self.retrylog or noop
             funcname = fn.__name__
             i = 0
             while i < self.numattempts:
                 if i > 0:
                     retrylog(
                         b're-attempting (n=%d) %s\n'
                         % (i, pycompat.sysbytes(funcname))
                     )
                     self.markforrefresh()
                 i += 1
                 try:
                     return fn(self, *args, **kwargs)
                 except KeyError:
                     if i == self.numattempts:
                         # retries exhausted
                         retrylog(
                             b'retries exhausted in %s, raising KeyError\n'
                             % pycompat.sysbytes(funcname)
                         )
                         raise
 
         return wrapped

hgext/remotefilelog/shallowutil.py

@@ -1,536 +1,544 b''
 # shallowutil.py -- remotefilelog utilities
 #
 # Copyright 2014 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 from __future__ import absolute_import
 
 import collections
 import errno
 import os
 import stat
 import struct
 import tempfile
 
 from mercurial.i18n import _
 from mercurial.pycompat import open
 from mercurial.node import hex
 from mercurial import (
     error,
     pycompat,
     revlog,
     util,
 )
 from mercurial.utils import (
     hashutil,
     storageutil,
     stringutil,
 )
 from . import constants
 
 if not pycompat.iswindows:
     import grp
 
 
 def isenabled(repo):
     """returns whether the repository is remotefilelog enabled or not"""
     return constants.SHALLOWREPO_REQUIREMENT in repo.requirements
 
 
 def getcachekey(reponame, file, id):
     pathhash = hex(hashutil.sha1(file).digest())
     return os.path.join(reponame, pathhash[:2], pathhash[2:], id)
 
 
 def getlocalkey(file, id):
     pathhash = hex(hashutil.sha1(file).digest())
     return os.path.join(pathhash, id)
 
 
 def getcachepath(ui, allowempty=False):
     cachepath = ui.config(b"remotefilelog", b"cachepath")
     if not cachepath:
         if allowempty:
             return None
         else:
             raise error.Abort(
                 _(b"could not find config option remotefilelog.cachepath")
             )
     return util.expandpath(cachepath)
 
 
 def getcachepackpath(repo, category):
     cachepath = getcachepath(repo.ui)
     if category != constants.FILEPACK_CATEGORY:
         return os.path.join(cachepath, repo.name, b'packs', category)
     else:
         return os.path.join(cachepath, repo.name, b'packs')
 
 
 def getlocalpackpath(base, category):
     return os.path.join(base, b'packs', category)
 
 
 def createrevlogtext(text, copyfrom=None, copyrev=None):
     """returns a string that matches the revlog contents in a
     traditional revlog
     """
     meta = {}
     if copyfrom or text.startswith(b'\1\n'):
         if copyfrom:
             meta[b'copy'] = copyfrom
             meta[b'copyrev'] = copyrev
         text = storageutil.packmeta(meta, text)
 
     return text
 
 
 def parsemeta(text):
     """parse mercurial filelog metadata"""
     meta, size = storageutil.parsemeta(text)
     if text.startswith(b'\1\n'):
         s = text.index(b'\1\n', 2)
         text = text[s + 2 :]
     return meta or {}, text
 
 
 def sumdicts(*dicts):
     """Adds all the values of *dicts together into one dictionary. This assumes
     the values in *dicts are all summable.
 
     e.g. [{'a': 4', 'b': 2}, {'b': 3, 'c': 1}] -> {'a': 4, 'b': 5, 'c': 1}
     """
     result = collections.defaultdict(lambda: 0)
     for dict in dicts:
         for k, v in pycompat.iteritems(dict):
             result[k] += v
     return result
 
 
 def prefixkeys(dict, prefix):
     """Returns ``dict`` with ``prefix`` prepended to all its keys."""
     result = {}
     for k, v in pycompat.iteritems(dict):
         result[prefix + k] = v
     return result
 
 
 def reportpackmetrics(ui, prefix, *stores):
     dicts = [s.getmetrics() for s in stores]
     dict = prefixkeys(sumdicts(*dicts), prefix + b'_')
     ui.log(prefix + b"_packsizes", b"\n", **pycompat.strkwargs(dict))
 
 
 def _parsepackmeta(metabuf):
     """parse datapack meta, bytes (<metadata-list>) -> dict
 
     The dict contains raw content - both keys and values are strings.
     Upper-level business may want to convert some of them to other types like
     integers, on their own.
 
     raise ValueError if the data is corrupted
     """
     metadict = {}
     offset = 0
     buflen = len(metabuf)
     while buflen - offset >= 3:
         key = metabuf[offset : offset + 1]
         offset += 1
         metalen = struct.unpack_from(b'!H', metabuf, offset)[0]
         offset += 2
         if offset + metalen > buflen:
             raise ValueError(b'corrupted metadata: incomplete buffer')
         value = metabuf[offset : offset + metalen]
         metadict[key] = value
         offset += metalen
     if offset != buflen:
         raise ValueError(b'corrupted metadata: redundant data')
     return metadict
 
 
 def _buildpackmeta(metadict):
     """reverse of _parsepackmeta, dict -> bytes (<metadata-list>)
 
     The dict contains raw content - both keys and values are strings.
     Upper-level business may want to serialize some of other types (like
     integers) to strings before calling this function.
 
     raise ProgrammingError when metadata key is illegal, or ValueError if
     length limit is exceeded
     """
     metabuf = b''
     for k, v in sorted(pycompat.iteritems((metadict or {}))):
         if len(k) != 1:
             raise error.ProgrammingError(b'packmeta: illegal key: %s' % k)
         if len(v) > 0xFFFE:
             raise ValueError(
                 b'metadata value is too long: 0x%x > 0xfffe' % len(v)
             )
         metabuf += k
         metabuf += struct.pack(b'!H', len(v))
         metabuf += v
     # len(metabuf) is guaranteed representable in 4 bytes, because there are
     # only 256 keys, and for each value, len(value) <= 0xfffe.
     return metabuf
 
 
 _metaitemtypes = {
     constants.METAKEYFLAG: (int, pycompat.long),
     constants.METAKEYSIZE: (int, pycompat.long),
 }
 
 
 def buildpackmeta(metadict):
     """like _buildpackmeta, but typechecks metadict and normalize it.
 
     This means, METAKEYSIZE and METAKEYSIZE should have integers as values,
     and METAKEYFLAG will be dropped if its value is 0.
     """
     newmeta = {}
     for k, v in pycompat.iteritems(metadict or {}):
         expectedtype = _metaitemtypes.get(k, (bytes,))
         if not isinstance(v, expectedtype):
             raise error.ProgrammingError(b'packmeta: wrong type of key %s' % k)
         # normalize int to binary buffer
         if int in expectedtype:
             # optimization: remove flag if it's 0 to save space
             if k == constants.METAKEYFLAG and v == 0:
                 continue
             v = int2bin(v)
         newmeta[k] = v
     return _buildpackmeta(newmeta)
 
 
 def parsepackmeta(metabuf):
     """like _parsepackmeta, but convert fields to desired types automatically.
 
     This means, METAKEYFLAG and METAKEYSIZE fields will be converted to
     integers.
     """
     metadict = _parsepackmeta(metabuf)
     for k, v in pycompat.iteritems(metadict):
         if k in _metaitemtypes and int in _metaitemtypes[k]:
             metadict[k] = bin2int(v)
     return metadict
 
 
 def int2bin(n):
     """convert a non-negative integer to raw binary buffer"""
     buf = bytearray()
     while n > 0:
         buf.insert(0, n & 0xFF)
         n >>= 8
     return bytes(buf)
 
 
 def bin2int(buf):
     """the reverse of int2bin, convert a binary buffer to an integer"""
     x = 0
     for b in bytearray(buf):
         x <<= 8
         x |= b
     return x
 
 
+class BadRemotefilelogHeader(error.StorageError):
+    """Exception raised when parsing a remotefilelog blob header fails."""
+
+
 def parsesizeflags(raw):
     """given a remotefilelog blob, return (headersize, rawtextsize, flags)
 
     see remotefilelogserver.createfileblob for the format.
     raise RuntimeError if the content is illformed.
     """
     flags = revlog.REVIDX_DEFAULT_FLAGS
     size = None
     try:
         index = raw.index(b'\0')
         header = raw[:index]
         if header.startswith(b'v'):
             # v1 and above, header starts with 'v'
             if header.startswith(b'v1\n'):
                 for s in header.split(b'\n'):
                     if s.startswith(constants.METAKEYSIZE):
                         size = int(s[len(constants.METAKEYSIZE) :])
                     elif s.startswith(constants.METAKEYFLAG):
                         flags = int(s[len(constants.METAKEYFLAG) :])
             else:
-                raise RuntimeError(
+                raise BadRemotefilelogHeader(
                     b'unsupported remotefilelog header: %s' % header
                 )
         else:
             # v0, str(int(size)) is the header
             size = int(header)
     except ValueError:
-        raise RuntimeError("unexpected remotefilelog header: illegal format")
+        raise BadRemotefilelogHeader(
+            "unexpected remotefilelog header: illegal format"
+        )
     if size is None:
-        raise RuntimeError("unexpected remotefilelog header: no size found")
+        raise BadRemotefilelogHeader(
+            "unexpected remotefilelog header: no size found"
+        )
     return index + 1, size, flags
 
 
 def buildfileblobheader(size, flags, version=None):
     """return the header of a remotefilelog blob.
 
     see remotefilelogserver.createfileblob for the format.
     approximately the reverse of parsesizeflags.
 
     version could be 0 or 1, or None (auto decide).
     """
     # choose v0 if flags is empty, otherwise v1
     if version is None:
         version = int(bool(flags))
     if version == 1:
         header = b'v1\n%s%d\n%s%d' % (
             constants.METAKEYSIZE,
             size,
             constants.METAKEYFLAG,
             flags,
         )
     elif version == 0:
         if flags:
             raise error.ProgrammingError(b'fileblob v0 does not support flag')
         header = b'%d' % size
     else:
         raise error.ProgrammingError(b'unknown fileblob version %d' % version)
     return header
 
 
 def ancestormap(raw):
     offset, size, flags = parsesizeflags(raw)
     start = offset + size
 
     mapping = {}
     while start < len(raw):
         divider = raw.index(b'\0', start + 80)
 
         currentnode = raw[start : (start + 20)]
         p1 = raw[(start + 20) : (start + 40)]
         p2 = raw[(start + 40) : (start + 60)]
         linknode = raw[(start + 60) : (start + 80)]
         copyfrom = raw[(start + 80) : divider]
 
         mapping[currentnode] = (p1, p2, linknode, copyfrom)
         start = divider + 1
 
     return mapping
 
 
 def readfile(path):
     f = open(path, b'rb')
     try:
         result = f.read()
 
         # we should never have empty files
         if not result:
             os.remove(path)
             raise IOError(b"empty file: %s" % path)
 
         return result
     finally:
         f.close()
 
 
 def unlinkfile(filepath):
     if pycompat.iswindows:
         # On Windows, os.unlink cannnot delete readonly files
         os.chmod(filepath, stat.S_IWUSR)
     os.unlink(filepath)
 
 
 def renamefile(source, destination):
     if pycompat.iswindows:
         # On Windows, os.rename cannot rename readonly files
         # and cannot overwrite destination if it exists
         os.chmod(source, stat.S_IWUSR)
         if os.path.isfile(destination):
             os.chmod(destination, stat.S_IWUSR)
             os.unlink(destination)
 
     os.rename(source, destination)
 
 
 def writefile(path, content, readonly=False):
     dirname, filename = os.path.split(path)
     if not os.path.exists(dirname):
         try:
             os.makedirs(dirname)
         except OSError as ex:
             if ex.errno != errno.EEXIST:
                 raise
 
     fd, temp = tempfile.mkstemp(prefix=b'.%s-' % filename, dir=dirname)
     os.close(fd)
 
     try:
         f = util.posixfile(temp, b'wb')
         f.write(content)
         f.close()
 
         if readonly:
             mode = 0o444
         else:
             # tempfiles are created with 0o600, so we need to manually set the
             # mode.
             oldumask = os.umask(0)
             # there's no way to get the umask without modifying it, so set it
             # back
             os.umask(oldumask)
             mode = ~oldumask
 
         renamefile(temp, path)
         os.chmod(path, mode)
     except Exception:
         try:
             unlinkfile(temp)
         except OSError:
             pass
         raise
 
 
 def sortnodes(nodes, parentfunc):
     """Topologically sorts the nodes, using the parentfunc to find
     the parents of nodes."""
     nodes = set(nodes)
     childmap = {}
     parentmap = {}
     roots = []
 
     # Build a child and parent map
     for n in nodes:
         parents = [p for p in parentfunc(n) if p in nodes]
         parentmap[n] = set(parents)
         for p in parents:
             childmap.setdefault(p, set()).add(n)
         if not parents:
             roots.append(n)
 
     roots.sort()
     # Process roots, adding children to the queue as they become roots
     results = []
     while roots:
         n = roots.pop(0)
         results.append(n)
         if n in childmap:
             children = childmap[n]
             for c in children:
                 childparents = parentmap[c]
                 childparents.remove(n)
                 if len(childparents) == 0:
                     # insert at the beginning, that way child nodes
                     # are likely to be output immediately after their
                     # parents.  This gives better compression results.
                     roots.insert(0, c)
 
     return results
 
 
 def readexactly(stream, n):
     '''read n bytes from stream.read and abort if less was available'''
     s = stream.read(n)
     if len(s) < n:
         raise error.Abort(
             _(b"stream ended unexpectedly (got %d bytes, expected %d)")
             % (len(s), n)
         )
     return s
 
 
 def readunpack(stream, fmt):
     data = readexactly(stream, struct.calcsize(fmt))
     return struct.unpack(fmt, data)
 
 
 def readpath(stream):
     rawlen = readexactly(stream, constants.FILENAMESIZE)
     pathlen = struct.unpack(constants.FILENAMESTRUCT, rawlen)[0]
     return readexactly(stream, pathlen)
 
 
 def readnodelist(stream):
     rawlen = readexactly(stream, constants.NODECOUNTSIZE)
     nodecount = struct.unpack(constants.NODECOUNTSTRUCT, rawlen)[0]
     for i in pycompat.xrange(nodecount):
         yield readexactly(stream, constants.NODESIZE)
 
 
 def readpathlist(stream):
     rawlen = readexactly(stream, constants.PATHCOUNTSIZE)
     pathcount = struct.unpack(constants.PATHCOUNTSTRUCT, rawlen)[0]
     for i in pycompat.xrange(pathcount):
         yield readpath(stream)
 
 
 def getgid(groupname):
     try:
         gid = grp.getgrnam(pycompat.fsdecode(groupname)).gr_gid
         return gid
     except KeyError:
         return None
 
 
 def setstickygroupdir(path, gid, warn=None):
     if gid is None:
         return
     try:
         os.chown(path, -1, gid)
         os.chmod(path, 0o2775)
     except (IOError, OSError) as ex:
         if warn:
             warn(_(b'unable to chown/chmod on %s: %s\n') % (path, ex))
 
 
 def mkstickygroupdir(ui, path):
     """Creates the given directory (if it doesn't exist) and give it a
     particular group with setgid enabled."""
     gid = None
     groupname = ui.config(b"remotefilelog", b"cachegroup")
     if groupname:
         gid = getgid(groupname)
         if gid is None:
             ui.warn(_(b'unable to resolve group name: %s\n') % groupname)
 
     # we use a single stat syscall to test the existence and mode / group bit
     st = None
     try:
         st = os.stat(path)
     except OSError:
         pass
 
     if st:
         # exists
         if (st.st_mode & 0o2775) != 0o2775 or st.st_gid != gid:
             # permission needs to be fixed
             setstickygroupdir(path, gid, ui.warn)
         return
 
     oldumask = os.umask(0o002)
     try:
         missingdirs = [path]
         path = os.path.dirname(path)
         while path and not os.path.exists(path):
             missingdirs.append(path)
             path = os.path.dirname(path)
 
         for path in reversed(missingdirs):
             try:
                 os.mkdir(path)
             except OSError as ex:
                 if ex.errno != errno.EEXIST:
                     raise
 
         for path in missingdirs:
             setstickygroupdir(path, gid, ui.warn)
     finally:
         os.umask(oldumask)
 
 
 def getusername(ui):
     try:
         return stringutil.shortuser(ui.username())
     except Exception:
         return b'unknown'
 
 
 def getreponame(ui):
     reponame = ui.config(b'paths', b'default')
     if reponame:
         return os.path.basename(reponame)
     return b"unknown"

tests/test-remotefilelog-corrupt-cache.t

@@ -1,72 +1,73 b''
 #require no-windows
 
   $ . "$TESTDIR/remotefilelog-library.sh"
 
   $ hg init master
   $ cd master
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > server=True
   > EOF
   $ echo x > x
   $ echo y > y
   $ echo z > z
   $ hg commit -qAm xy
 
   $ cd ..
 
   $ hgcloneshallow ssh://user@dummy/master shallow -q
   3 files fetched over 1 fetches - (3 misses, 0.00% hit ratio) over *s (glob)
   $ cd shallow
 
 Verify corrupt cache handling repairs by default
 
   $ hg up -q null
   $ chmod u+w $CACHEDIR/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/1406e74118627694268417491f018a4a883152f0
   $ echo x > $CACHEDIR/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/1406e74118627694268417491f018a4a883152f0
   $ hg up tip
   3 files updated, 0 files merged, 0 files removed, 0 files unresolved
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over *s (glob)
 
 Verify corrupt cache error message
 
   $ hg up -q null
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > validatecache=off
   > EOF
   $ chmod u+w $CACHEDIR/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/1406e74118627694268417491f018a4a883152f0
   $ echo x > $CACHEDIR/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/1406e74118627694268417491f018a4a883152f0
   $ hg up tip 2>&1 | egrep "^[^ ].*unexpected remotefilelog"
-  RuntimeError: unexpected remotefilelog header: illegal format
+  abort: unexpected remotefilelog header: illegal format (no-py3 !)
+  hgext.remotefilelog.shallowutil.BadRemotefilelogHeader: unexpected remotefilelog header: illegal format (py3 !)
 
 Verify detection and remediation when remotefilelog.validatecachelog is set
 
   $ cat >> .hg/hgrc <<EOF
   > [remotefilelog]
   > validatecachelog=$PWD/.hg/remotefilelog_cache.log
   > validatecache=strict
   > EOF
   $ chmod u+w $CACHEDIR/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/1406e74118627694268417491f018a4a883152f0
   $ echo x > $CACHEDIR/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/1406e74118627694268417491f018a4a883152f0
   $ hg up tip
   3 files updated, 0 files merged, 0 files removed, 0 files unresolved
   1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over *s (glob)
   $ cat .hg/remotefilelog_cache.log
   corrupt $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/1406e74118627694268417491f018a4a883152f0 during contains
 
 Verify handling of corrupt server cache
 
   $ rm -f ../master/.hg/remotefilelogcache/y/076f5e2225b3ff0400b98c92aa6cdf403ee24cca
   $ touch ../master/.hg/remotefilelogcache/y/076f5e2225b3ff0400b98c92aa6cdf403ee24cca
   $ clearcache
   $ hg prefetch -r .
   3 files fetched over 1 fetches - (3 misses, 0.00% hit ratio) over *s (glob)
   $ test -s ../master/.hg/remotefilelogcache/y/076f5e2225b3ff0400b98c92aa6cdf403ee24cca
   $ hg debugremotefilelog $CACHEDIR/master/95/cb0bfd2977c761298d9624e4b4d4c72a39974a/076f5e2225b3ff0400b98c92aa6cdf403ee24cca
   size: 2 bytes
   path: $TESTTMP/hgcache/master/95/cb0bfd2977c761298d9624e4b4d4c72a39974a/076f5e2225b3ff0400b98c92aa6cdf403ee24cca 
   key: 076f5e2225b3 
   
           node =>           p1            p2      linknode     copyfrom
   076f5e2225b3 => 000000000000  000000000000  f3d0bb0d1e48