remotefilelog: add newlines to ui.log() invocations

The comment at the top of ui.log() says that the message should be a
newline-terminated string. When using the blackbox logger, if the string does
not end in a newline, the appearance in the file gets quite messy.

This sometimes leaves a string with just the newline as the message, these are
logged by the blackbox logger just fine. I don't know what other loggers do when
logging structured data and a message that is just the newline.

Differential Revision: https://phab.mercurial-scm.org/D5533

hgext/remotefilelog/remotefilectx.py

@@ -1,491 +1,491 b''
 # remotefilectx.py - filectx/workingfilectx implementations for remotefilelog
 #
 # Copyright 2013 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 from __future__ import absolute_import
 
 import collections
 import time
 
 from mercurial.node import bin, hex, nullid, nullrev
 from mercurial import (
     ancestor,
     context,
     error,
     phases,
     pycompat,
     util,
 )
 from . import shallowutil
 
 propertycache = util.propertycache
 FASTLOG_TIMEOUT_IN_SECS = 0.5
 
 class remotefilectx(context.filectx):
     def __init__(self, repo, path, changeid=None, fileid=None,
                  filelog=None, changectx=None, ancestormap=None):
         if fileid == nullrev:
             fileid = nullid
         if fileid and len(fileid) == 40:
             fileid = bin(fileid)
         super(remotefilectx, self).__init__(repo, path, changeid,
             fileid, filelog, changectx)
         self._ancestormap = ancestormap
 
     def size(self):
         return self._filelog.size(self._filenode)
 
     @propertycache
     def _changeid(self):
         if '_changeid' in self.__dict__:
             return self._changeid
         elif '_changectx' in self.__dict__:
             return self._changectx.rev()
         elif '_descendantrev' in self.__dict__:
             # this file context was created from a revision with a known
             # descendant, we can (lazily) correct for linkrev aliases
             linknode = self._adjustlinknode(self._path, self._filelog,
                                             self._filenode, self._descendantrev)
             return self._repo.unfiltered().changelog.rev(linknode)
         else:
             return self.linkrev()
 
     def filectx(self, fileid, changeid=None):
         '''opens an arbitrary revision of the file without
         opening a new filelog'''
         return remotefilectx(self._repo, self._path, fileid=fileid,
                              filelog=self._filelog, changeid=changeid)
 
     def linkrev(self):
         return self._linkrev
 
     @propertycache
     def _linkrev(self):
         if self._filenode == nullid:
             return nullrev
 
         ancestormap = self.ancestormap()
         p1, p2, linknode, copyfrom = ancestormap[self._filenode]
         rev = self._repo.changelog.nodemap.get(linknode)
         if rev is not None:
             return rev
 
         # Search all commits for the appropriate linkrev (slow, but uncommon)
         path = self._path
         fileid = self._filenode
         cl = self._repo.unfiltered().changelog
         mfl = self._repo.manifestlog
 
         for rev in range(len(cl) - 1, 0, -1):
             node = cl.node(rev)
             data = cl.read(node) # get changeset data (we avoid object creation)
             if path in data[3]: # checking the 'files' field.
                 # The file has been touched, check if the hash is what we're
                 # looking for.
                 if fileid == mfl[data[0]].readfast().get(path):
                     return rev
 
         # Couldn't find the linkrev. This should generally not happen, and will
         # likely cause a crash.
         return None
 
     def introrev(self):
         """return the rev of the changeset which introduced this file revision
 
         This method is different from linkrev because it take into account the
         changeset the filectx was created from. It ensures the returned
         revision is one of its ancestors. This prevents bugs from
         'linkrev-shadowing' when a file revision is used by multiple
         changesets.
         """
         lkr = self.linkrev()
         attrs = vars(self)
         noctx = not ('_changeid' in attrs or '_changectx' in attrs)
         if noctx or self.rev() == lkr:
             return lkr
         linknode = self._adjustlinknode(self._path, self._filelog,
                                         self._filenode, self.rev(),
                                         inclusive=True)
         return self._repo.changelog.rev(linknode)
 
     def renamed(self):
         """check if file was actually renamed in this changeset revision
 
         If rename logged in file revision, we report copy for changeset only
         if file revisions linkrev points back to the changeset in question
         or both changeset parents contain different file revisions.
         """
         ancestormap = self.ancestormap()
 
         p1, p2, linknode, copyfrom = ancestormap[self._filenode]
         if not copyfrom:
             return None
 
         renamed = (copyfrom, p1)
         if self.rev() == self.linkrev():
             return renamed
 
         name = self.path()
         fnode = self._filenode
         for p in self._changectx.parents():
             try:
                 if fnode == p.filenode(name):
                     return None
             except error.LookupError:
                 pass
         return renamed
 
     def ancestormap(self):
         if not self._ancestormap:
             self._ancestormap = self.filelog().ancestormap(self._filenode)
 
         return self._ancestormap
 
     def parents(self):
         repo = self._repo
         ancestormap = self.ancestormap()
 
         p1, p2, linknode, copyfrom = ancestormap[self._filenode]
         results = []
         if p1 != nullid:
             path = copyfrom or self._path
             flog = repo.file(path)
             p1ctx = remotefilectx(repo, path, fileid=p1, filelog=flog,
                                   ancestormap=ancestormap)
             p1ctx._descendantrev = self.rev()
             results.append(p1ctx)
 
         if p2 != nullid:
             path = self._path
             flog = repo.file(path)
             p2ctx = remotefilectx(repo, path, fileid=p2, filelog=flog,
                                   ancestormap=ancestormap)
             p2ctx._descendantrev = self.rev()
             results.append(p2ctx)
 
         return results
 
     def _nodefromancrev(self, ancrev, cl, mfl, path, fnode):
         """returns the node for <path> in <ancrev> if content matches <fnode>"""
         ancctx = cl.read(ancrev) # This avoids object creation.
         manifestnode, files = ancctx[0], ancctx[3]
         # If the file was touched in this ancestor, and the content is similar
         # to the one we are searching for.
         if path in files and fnode == mfl[manifestnode].readfast().get(path):
             return cl.node(ancrev)
         return None
 
     def _adjustlinknode(self, path, filelog, fnode, srcrev, inclusive=False):
         """return the first ancestor of <srcrev> introducing <fnode>
 
         If the linkrev of the file revision does not point to an ancestor of
         srcrev, we'll walk down the ancestors until we find one introducing
         this file revision.
 
         :repo: a localrepository object (used to access changelog and manifest)
         :path: the file path
         :fnode: the nodeid of the file revision
         :filelog: the filelog of this path
         :srcrev: the changeset revision we search ancestors from
         :inclusive: if true, the src revision will also be checked
 
         Note: This is based on adjustlinkrev in core, but it's quite different.
 
         adjustlinkrev depends on the fact that the linkrev is the bottom most
         node, and uses that as a stopping point for the ancestor traversal. We
         can't do that here because the linknode is not guaranteed to be the
         bottom most one.
 
         In our code here, we actually know what a bunch of potential ancestor
         linknodes are, so instead of stopping the cheap-ancestor-traversal when
         we get to a linkrev, we stop when we see any of the known linknodes.
         """
         repo = self._repo
         cl = repo.unfiltered().changelog
         mfl = repo.manifestlog
         ancestormap = self.ancestormap()
         linknode = ancestormap[fnode][2]
 
         if srcrev is None:
             # wctx case, used by workingfilectx during mergecopy
             revs = [p.rev() for p in self._repo[None].parents()]
             inclusive = True # we skipped the real (revless) source
         else:
             revs = [srcrev]
 
         if self._verifylinknode(revs, linknode):
             return linknode
 
         commonlogkwargs = {
             r'revs': ' '.join([hex(cl.node(rev)) for rev in revs]),
             r'fnode': hex(fnode),
             r'filepath': path,
             r'user': shallowutil.getusername(repo.ui),
             r'reponame': shallowutil.getreponame(repo.ui),
         }
 
-        repo.ui.log('linkrevfixup', 'adjusting linknode', **commonlogkwargs)
+        repo.ui.log('linkrevfixup', 'adjusting linknode\n', **commonlogkwargs)
 
         pc = repo._phasecache
         seenpublic = False
         iteranc = cl.ancestors(revs, inclusive=inclusive)
         for ancrev in iteranc:
             # First, check locally-available history.
             lnode = self._nodefromancrev(ancrev, cl, mfl, path, fnode)
             if lnode is not None:
                 return lnode
 
             # adjusting linknode can be super-slow. To mitigate the issue
             # we use two heuristics: calling fastlog and forcing remotefilelog
             # prefetch
             if not seenpublic and pc.phase(repo, ancrev) == phases.public:
                 # TODO: there used to be a codepath to fetch linknodes
                 # from a server as a fast path, but it appeared to
                 # depend on an API FB added to their phabricator.
                 lnode = self._forceprefetch(repo, path, fnode, revs,
                                             commonlogkwargs)
                 if lnode:
                     return lnode
                 seenpublic = True
 
         return linknode
 
     def _forceprefetch(self, repo, path, fnode, revs,
                        commonlogkwargs):
         # This next part is super non-obvious, so big comment block time!
         #
         # It is possible to get extremely bad performance here when a fairly
         # common set of circumstances occur when this extension is combined
         # with a server-side commit rewriting extension like pushrebase.
         #
         # First, an engineer creates Commit A and pushes it to the server.
         # While the server's data structure will have the correct linkrev
         # for the files touched in Commit A, the client will have the
         # linkrev of the local commit, which is "invalid" because it's not
         # an ancestor of the main line of development.
         #
         # The client will never download the remotefilelog with the correct
         # linkrev as long as nobody else touches that file, since the file
         # data and history hasn't changed since Commit A.
         #
         # After a long time (or a short time in a heavily used repo), if the
         # same engineer returns to change the same file, some commands --
         # such as amends of commits with file moves, logs, diffs, etc  --
         # can trigger this _adjustlinknode code. In those cases, finding
         # the correct rev can become quite expensive, as the correct
         # revision is far back in history and we need to walk back through
         # history to find it.
         #
         # In order to improve this situation, we force a prefetch of the
         # remotefilelog data blob for the file we were called on. We do this
         # at most once, when we first see a public commit in the history we
         # are traversing.
         #
         # Forcing the prefetch means we will download the remote blob even
         # if we have the "correct" blob in the local store. Since the union
         # store checks the remote store first, this means we are much more
         # likely to get the correct linkrev at this point.
         #
         # In rare circumstances (such as the server having a suboptimal
         # linkrev for our use case), we will fall back to the old slow path.
         #
         # We may want to add additional heuristics here in the future if
         # the slow path is used too much. One promising possibility is using
         # obsolescence markers to find a more-likely-correct linkrev.
 
         logmsg = ''
         start = time.time()
         try:
             repo.fileservice.prefetch([(path, hex(fnode))], force=True)
 
             # Now that we've downloaded a new blob from the server,
             # we need to rebuild the ancestor map to recompute the
             # linknodes.
             self._ancestormap = None
             linknode = self.ancestormap()[fnode][2] # 2 is linknode
             if self._verifylinknode(revs, linknode):
                 logmsg = 'remotefilelog prefetching succeeded'
                 return linknode
             logmsg = 'remotefilelog prefetching not found'
             return None
         except Exception as e:
             logmsg = 'remotefilelog prefetching failed (%s)' % e
             return None
         finally:
             elapsed = time.time() - start
-            repo.ui.log('linkrevfixup', logmsg, elapsed=elapsed * 1000,
+            repo.ui.log('linkrevfixup', logmsg + '\n', elapsed=elapsed * 1000,
                         **pycompat.strkwargs(commonlogkwargs))
 
     def _verifylinknode(self, revs, linknode):
         """
         Check if a linknode is correct one for the current history.
 
         That is, return True if the linkrev is the ancestor of any of the
         passed in revs, otherwise return False.
 
         `revs` is a list that usually has one element -- usually the wdir parent
         or the user-passed rev we're looking back from. It may contain two revs
         when there is a merge going on, or zero revs when a root node with no
         parents is being created.
         """
         if not revs:
             return False
         try:
             # Use the C fastpath to check if the given linknode is correct.
             cl = self._repo.unfiltered().changelog
             return any(cl.isancestor(linknode, cl.node(r)) for r in revs)
         except error.LookupError:
             # The linknode read from the blob may have been stripped or
             # otherwise not present in the repository anymore. Do not fail hard
             # in this case. Instead, return false and continue the search for
             # the correct linknode.
             return False
 
     def ancestors(self, followfirst=False):
         ancestors = []
         queue = collections.deque((self,))
         seen = set()
         while queue:
             current = queue.pop()
             if current.filenode() in seen:
                 continue
             seen.add(current.filenode())
 
             ancestors.append(current)
 
             parents = current.parents()
             first = True
             for p in parents:
                 if first or not followfirst:
                     queue.append(p)
                 first = False
 
         # Remove self
         ancestors.pop(0)
 
         # Sort by linkrev
         # The copy tracing algorithm depends on these coming out in order
         ancestors = sorted(ancestors, reverse=True, key=lambda x:x.linkrev())
 
         for ancestor in ancestors:
             yield ancestor
 
     def ancestor(self, fc2, actx):
         # the easy case: no (relevant) renames
         if fc2.path() == self.path() and self.path() in actx:
             return actx[self.path()]
 
         # the next easiest cases: unambiguous predecessor (name trumps
         # history)
         if self.path() in actx and fc2.path() not in actx:
             return actx[self.path()]
         if fc2.path() in actx and self.path() not in actx:
             return actx[fc2.path()]
 
         # do a full traversal
         amap = self.ancestormap()
         bmap = fc2.ancestormap()
 
         def parents(x):
             f, n = x
             p = amap.get(n) or bmap.get(n)
             if not p:
                 return []
 
             return [(p[3] or f, p[0]), (f, p[1])]
 
         a = (self.path(), self.filenode())
         b = (fc2.path(), fc2.filenode())
         result = ancestor.genericancestor(a, b, parents)
         if result:
             f, n = result
             r = remotefilectx(self._repo, f, fileid=n,
                                  ancestormap=amap)
             return r
 
         return None
 
     def annotate(self, *args, **kwargs):
         introctx = self
         prefetchskip = kwargs.pop(r'prefetchskip', None)
         if prefetchskip:
             # use introrev so prefetchskip can be accurately tested
             introrev = self.introrev()
             if self.rev() != introrev:
                 introctx = remotefilectx(self._repo, self._path,
                                          changeid=introrev,
                                          fileid=self._filenode,
                                          filelog=self._filelog,
                                          ancestormap=self._ancestormap)
 
         # like self.ancestors, but append to "fetch" and skip visiting parents
         # of nodes in "prefetchskip".
         fetch = []
         seen = set()
         queue = collections.deque((introctx,))
         seen.add(introctx.node())
         while queue:
             current = queue.pop()
             if current.filenode() != self.filenode():
                 # this is a "joint point". fastannotate needs contents of
                 # "joint point"s to calculate diffs for side branches.
                 fetch.append((current.path(), hex(current.filenode())))
             if prefetchskip and current in prefetchskip:
                 continue
             for parent in current.parents():
                 if parent.node() not in seen:
                     seen.add(parent.node())
                     queue.append(parent)
 
         self._repo.ui.debug('remotefilelog: prefetching %d files '
                             'for annotate\n' % len(fetch))
         if fetch:
             self._repo.fileservice.prefetch(fetch)
         return super(remotefilectx, self).annotate(*args, **kwargs)
 
     # Return empty set so that the hg serve and thg don't stack trace
     def children(self):
         return []
 
 class remoteworkingfilectx(context.workingfilectx, remotefilectx):
     def __init__(self, repo, path, filelog=None, workingctx=None):
         self._ancestormap = None
         return super(remoteworkingfilectx, self).__init__(repo, path,
             filelog, workingctx)
 
     def parents(self):
         return remotefilectx.parents(self)
 
     def ancestormap(self):
         if not self._ancestormap:
             path = self._path
             pcl = self._changectx._parents
             renamed = self.renamed()
 
             if renamed:
                 p1 = renamed
             else:
                 p1 = (path, pcl[0]._manifest.get(path, nullid))
 
             p2 = (path, nullid)
             if len(pcl) > 1:
                 p2 = (path, pcl[1]._manifest.get(path, nullid))
 
             m = {}
             if p1[1] != nullid:
                 p1ctx = self._repo.filectx(p1[0], fileid=p1[1])
                 m.update(p1ctx.filelog().ancestormap(p1[1]))
 
             if p2[1] != nullid:
                 p2ctx = self._repo.filectx(p2[0], fileid=p2[1])
                 m.update(p2ctx.filelog().ancestormap(p2[1]))
 
             copyfrom = ''
             if renamed:
                 copyfrom = renamed[0]
             m[None] = (p1[1], p2[1], nullid, copyfrom)
             self._ancestormap = m
 
         return self._ancestormap

hgext/remotefilelog/shallowutil.py

@@ -1,492 +1,492 b''
 # shallowutil.py -- remotefilelog utilities
 #
 # Copyright 2014 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 from __future__ import absolute_import
 
 import collections
 import errno
 import hashlib
 import os
 import stat
 import struct
 import tempfile
 
 from mercurial.i18n import _
 from mercurial import (
     error,
     node,
     pycompat,
     revlog,
     util,
 )
 from mercurial.utils import (
     storageutil,
     stringutil,
 )
 from . import constants
 
 if not pycompat.iswindows:
     import grp
 
 def isenabled(repo):
     """returns whether the repository is remotefilelog enabled or not"""
     return constants.SHALLOWREPO_REQUIREMENT in repo.requirements
 
 def getcachekey(reponame, file, id):
     pathhash = node.hex(hashlib.sha1(file).digest())
     return os.path.join(reponame, pathhash[:2], pathhash[2:], id)
 
 def getlocalkey(file, id):
     pathhash = node.hex(hashlib.sha1(file).digest())
     return os.path.join(pathhash, id)
 
 def getcachepath(ui, allowempty=False):
     cachepath = ui.config("remotefilelog", "cachepath")
     if not cachepath:
         if allowempty:
             return None
         else:
             raise error.Abort(_("could not find config option "
                                 "remotefilelog.cachepath"))
     return util.expandpath(cachepath)
 
 def getcachepackpath(repo, category):
     cachepath = getcachepath(repo.ui)
     if category != constants.FILEPACK_CATEGORY:
         return os.path.join(cachepath, repo.name, 'packs', category)
     else:
         return os.path.join(cachepath, repo.name, 'packs')
 
 def getlocalpackpath(base, category):
     return os.path.join(base, 'packs', category)
 
 def createrevlogtext(text, copyfrom=None, copyrev=None):
     """returns a string that matches the revlog contents in a
     traditional revlog
     """
     meta = {}
     if copyfrom or text.startswith('\1\n'):
         if copyfrom:
             meta['copy'] = copyfrom
             meta['copyrev'] = copyrev
         text = storageutil.packmeta(meta, text)
 
     return text
 
 def parsemeta(text):
     """parse mercurial filelog metadata"""
     meta, size = storageutil.parsemeta(text)
     if text.startswith('\1\n'):
         s = text.index('\1\n', 2)
         text = text[s + 2:]
     return meta or {}, text
 
 def sumdicts(*dicts):
     """Adds all the values of *dicts together into one dictionary. This assumes
     the values in *dicts are all summable.
 
     e.g. [{'a': 4', 'b': 2}, {'b': 3, 'c': 1}] -> {'a': 4, 'b': 5, 'c': 1}
     """
     result = collections.defaultdict(lambda: 0)
     for dict in dicts:
         for k, v in dict.iteritems():
             result[k] += v
     return result
 
 def prefixkeys(dict, prefix):
     """Returns ``dict`` with ``prefix`` prepended to all its keys."""
     result = {}
     for k, v in dict.iteritems():
         result[prefix + k] = v
     return result
 
 def reportpackmetrics(ui, prefix, *stores):
     dicts = [s.getmetrics() for s in stores]
     dict = prefixkeys(sumdicts(*dicts), prefix + '_')
-    ui.log(prefix + "_packsizes", "", **pycompat.strkwargs(dict))
+    ui.log(prefix + "_packsizes", "\n", **pycompat.strkwargs(dict))
 
 def _parsepackmeta(metabuf):
     """parse datapack meta, bytes (<metadata-list>) -> dict
 
     The dict contains raw content - both keys and values are strings.
     Upper-level business may want to convert some of them to other types like
     integers, on their own.
 
     raise ValueError if the data is corrupted
     """
     metadict = {}
     offset = 0
     buflen = len(metabuf)
     while buflen - offset >= 3:
         key = metabuf[offset]
         offset += 1
         metalen = struct.unpack_from('!H', metabuf, offset)[0]
         offset += 2
         if offset + metalen > buflen:
             raise ValueError('corrupted metadata: incomplete buffer')
         value = metabuf[offset:offset + metalen]
         metadict[key] = value
         offset += metalen
     if offset != buflen:
         raise ValueError('corrupted metadata: redundant data')
     return metadict
 
 def _buildpackmeta(metadict):
     """reverse of _parsepackmeta, dict -> bytes (<metadata-list>)
 
     The dict contains raw content - both keys and values are strings.
     Upper-level business may want to serialize some of other types (like
     integers) to strings before calling this function.
 
     raise ProgrammingError when metadata key is illegal, or ValueError if
     length limit is exceeded
     """
     metabuf = ''
     for k, v in sorted((metadict or {}).iteritems()):
         if len(k) != 1:
             raise error.ProgrammingError('packmeta: illegal key: %s' % k)
         if len(v) > 0xfffe:
             raise ValueError('metadata value is too long: 0x%x > 0xfffe'
                              % len(v))
         metabuf += k
         metabuf += struct.pack('!H', len(v))
         metabuf += v
     # len(metabuf) is guaranteed representable in 4 bytes, because there are
     # only 256 keys, and for each value, len(value) <= 0xfffe.
     return metabuf
 
 _metaitemtypes = {
     constants.METAKEYFLAG: (int, pycompat.long),
     constants.METAKEYSIZE: (int, pycompat.long),
 }
 
 def buildpackmeta(metadict):
     """like _buildpackmeta, but typechecks metadict and normalize it.
 
     This means, METAKEYSIZE and METAKEYSIZE should have integers as values,
     and METAKEYFLAG will be dropped if its value is 0.
     """
     newmeta = {}
     for k, v in (metadict or {}).iteritems():
         expectedtype = _metaitemtypes.get(k, (bytes,))
         if not isinstance(v, expectedtype):
             raise error.ProgrammingError('packmeta: wrong type of key %s' % k)
         # normalize int to binary buffer
         if int in expectedtype:
             # optimization: remove flag if it's 0 to save space
             if k == constants.METAKEYFLAG and v == 0:
                 continue
             v = int2bin(v)
         newmeta[k] = v
     return _buildpackmeta(newmeta)
 
 def parsepackmeta(metabuf):
     """like _parsepackmeta, but convert fields to desired types automatically.
 
     This means, METAKEYFLAG and METAKEYSIZE fields will be converted to
     integers.
     """
     metadict = _parsepackmeta(metabuf)
     for k, v in metadict.iteritems():
         if k in _metaitemtypes and int in _metaitemtypes[k]:
             metadict[k] = bin2int(v)
     return metadict
 
 def int2bin(n):
     """convert a non-negative integer to raw binary buffer"""
     buf = bytearray()
     while n > 0:
         buf.insert(0, n & 0xff)
         n >>= 8
     return bytes(buf)
 
 def bin2int(buf):
     """the reverse of int2bin, convert a binary buffer to an integer"""
     x = 0
     for b in bytearray(buf):
         x <<= 8
         x |= b
     return x
 
 def parsesizeflags(raw):
     """given a remotefilelog blob, return (headersize, rawtextsize, flags)
 
     see remotefilelogserver.createfileblob for the format.
     raise RuntimeError if the content is illformed.
     """
     flags = revlog.REVIDX_DEFAULT_FLAGS
     size = None
     try:
         index = raw.index('\0')
         header = raw[:index]
         if header.startswith('v'):
             # v1 and above, header starts with 'v'
             if header.startswith('v1\n'):
                 for s in header.split('\n'):
                     if s.startswith(constants.METAKEYSIZE):
                         size = int(s[len(constants.METAKEYSIZE):])
                     elif s.startswith(constants.METAKEYFLAG):
                         flags = int(s[len(constants.METAKEYFLAG):])
             else:
                 raise RuntimeError('unsupported remotefilelog header: %s'
                                    % header)
         else:
             # v0, str(int(size)) is the header
             size = int(header)
     except ValueError:
         raise RuntimeError("unexpected remotefilelog header: illegal format")
     if size is None:
         raise RuntimeError("unexpected remotefilelog header: no size found")
     return index + 1, size, flags
 
 def buildfileblobheader(size, flags, version=None):
     """return the header of a remotefilelog blob.
 
     see remotefilelogserver.createfileblob for the format.
     approximately the reverse of parsesizeflags.
 
     version could be 0 or 1, or None (auto decide).
     """
     # choose v0 if flags is empty, otherwise v1
     if version is None:
         version = int(bool(flags))
     if version == 1:
         header = ('v1\n%s%d\n%s%d'
                   % (constants.METAKEYSIZE, size,
                      constants.METAKEYFLAG, flags))
     elif version == 0:
         if flags:
             raise error.ProgrammingError('fileblob v0 does not support flag')
         header = '%d' % size
     else:
         raise error.ProgrammingError('unknown fileblob version %d' % version)
     return header
 
 def ancestormap(raw):
     offset, size, flags = parsesizeflags(raw)
     start = offset + size
 
     mapping = {}
     while start < len(raw):
         divider = raw.index('\0', start + 80)
 
         currentnode = raw[start:(start + 20)]
         p1 = raw[(start + 20):(start + 40)]
         p2 = raw[(start + 40):(start + 60)]
         linknode = raw[(start + 60):(start + 80)]
         copyfrom = raw[(start + 80):divider]
 
         mapping[currentnode] = (p1, p2, linknode, copyfrom)
         start = divider + 1
 
     return mapping
 
 def readfile(path):
     f = open(path, 'rb')
     try:
         result = f.read()
 
         # we should never have empty files
         if not result:
             os.remove(path)
             raise IOError("empty file: %s" % path)
 
         return result
     finally:
         f.close()
 
 def unlinkfile(filepath):
     if pycompat.iswindows:
         # On Windows, os.unlink cannnot delete readonly files
         os.chmod(filepath, stat.S_IWUSR)
     os.unlink(filepath)
 
 def renamefile(source, destination):
     if pycompat.iswindows:
         # On Windows, os.rename cannot rename readonly files
         # and cannot overwrite destination if it exists
         os.chmod(source, stat.S_IWUSR)
         if os.path.isfile(destination):
             os.chmod(destination, stat.S_IWUSR)
             os.unlink(destination)
 
     os.rename(source, destination)
 
 def writefile(path, content, readonly=False):
     dirname, filename = os.path.split(path)
     if not os.path.exists(dirname):
         try:
             os.makedirs(dirname)
         except OSError as ex:
             if ex.errno != errno.EEXIST:
                 raise
 
     fd, temp = tempfile.mkstemp(prefix='.%s-' % filename, dir=dirname)
     os.close(fd)
 
     try:
         f = util.posixfile(temp, 'wb')
         f.write(content)
         f.close()
 
         if readonly:
             mode = 0o444
         else:
             # tempfiles are created with 0o600, so we need to manually set the
             # mode.
             oldumask = os.umask(0)
             # there's no way to get the umask without modifying it, so set it
             # back
             os.umask(oldumask)
             mode = ~oldumask
 
         renamefile(temp, path)
         os.chmod(path, mode)
     except Exception:
         try:
             unlinkfile(temp)
         except OSError:
             pass
         raise
 
 def sortnodes(nodes, parentfunc):
     """Topologically sorts the nodes, using the parentfunc to find
     the parents of nodes."""
     nodes = set(nodes)
     childmap = {}
     parentmap = {}
     roots = []
 
     # Build a child and parent map
     for n in nodes:
         parents = [p for p in parentfunc(n) if p in nodes]
         parentmap[n] = set(parents)
         for p in parents:
             childmap.setdefault(p, set()).add(n)
         if not parents:
             roots.append(n)
 
     roots.sort()
     # Process roots, adding children to the queue as they become roots
     results = []
     while roots:
         n = roots.pop(0)
         results.append(n)
         if n in childmap:
             children = childmap[n]
             for c in children:
                 childparents = parentmap[c]
                 childparents.remove(n)
                 if len(childparents) == 0:
                     # insert at the beginning, that way child nodes
                     # are likely to be output immediately after their
                     # parents.  This gives better compression results.
                     roots.insert(0, c)
 
     return results
 
 def readexactly(stream, n):
     '''read n bytes from stream.read and abort if less was available'''
     s = stream.read(n)
     if len(s) < n:
         raise error.Abort(_("stream ended unexpectedly"
                            " (got %d bytes, expected %d)")
                           % (len(s), n))
     return s
 
 def readunpack(stream, fmt):
     data = readexactly(stream, struct.calcsize(fmt))
     return struct.unpack(fmt, data)
 
 def readpath(stream):
     rawlen = readexactly(stream, constants.FILENAMESIZE)
     pathlen = struct.unpack(constants.FILENAMESTRUCT, rawlen)[0]
     return readexactly(stream, pathlen)
 
 def readnodelist(stream):
     rawlen = readexactly(stream, constants.NODECOUNTSIZE)
     nodecount = struct.unpack(constants.NODECOUNTSTRUCT, rawlen)[0]
     for i in pycompat.xrange(nodecount):
         yield readexactly(stream, constants.NODESIZE)
 
 def readpathlist(stream):
     rawlen = readexactly(stream, constants.PATHCOUNTSIZE)
     pathcount = struct.unpack(constants.PATHCOUNTSTRUCT, rawlen)[0]
     for i in pycompat.xrange(pathcount):
         yield readpath(stream)
 
 def getgid(groupname):
     try:
         gid = grp.getgrnam(groupname).gr_gid
         return gid
     except KeyError:
         return None
 
 def setstickygroupdir(path, gid, warn=None):
     if gid is None:
         return
     try:
         os.chown(path, -1, gid)
         os.chmod(path, 0o2775)
     except (IOError, OSError) as ex:
         if warn:
             warn(_('unable to chown/chmod on %s: %s\n') % (path, ex))
 
 def mkstickygroupdir(ui, path):
     """Creates the given directory (if it doesn't exist) and give it a
     particular group with setgid enabled."""
     gid = None
     groupname = ui.config("remotefilelog", "cachegroup")
     if groupname:
         gid = getgid(groupname)
         if gid is None:
             ui.warn(_('unable to resolve group name: %s\n') % groupname)
 
     # we use a single stat syscall to test the existence and mode / group bit
     st = None
     try:
         st = os.stat(path)
     except OSError:
         pass
 
     if st:
         # exists
         if (st.st_mode & 0o2775) != 0o2775 or st.st_gid != gid:
             # permission needs to be fixed
             setstickygroupdir(path, gid, ui.warn)
         return
 
     oldumask = os.umask(0o002)
     try:
         missingdirs = [path]
         path = os.path.dirname(path)
         while path and not os.path.exists(path):
             missingdirs.append(path)
             path = os.path.dirname(path)
 
         for path in reversed(missingdirs):
             try:
                 os.mkdir(path)
             except OSError as ex:
                 if ex.errno != errno.EEXIST:
                     raise
 
         for path in missingdirs:
             setstickygroupdir(path, gid, ui.warn)
     finally:
         os.umask(oldumask)
 
 def getusername(ui):
     try:
         return stringutil.shortuser(ui.username())
     except Exception:
         return 'unknown'
 
 def getreponame(ui):
     reponame = ui.config('paths', 'default')
     if reponame:
         return os.path.basename(reponame)
     return "unknown"