osutil: proxy through util (and platform) modules (API)

See the previous commit for why. Marked as API change since osutil.listdir()
seems widely used in third-party extensions.

The win32mbcs extension is updated to wrap both util. and windows. aliases.

hgext/win32mbcs.py

@@ -1,194 +1,195
 # win32mbcs.py -- MBCS filename support for Mercurial
 #
 # Copyright (c) 2008 Shun-ichi Goto <[e-mail]>
 #
 # Version: 0.3
 # Author:  Shun-ichi Goto <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 #
 
 '''allow the use of MBCS paths with problematic encodings
 
 Some MBCS encodings are not good for some path operations (i.e.
 splitting path, case conversion, etc.) with its encoded bytes. We call
 such a encoding (i.e. shift_jis and big5) as "problematic encoding".
 This extension can be used to fix the issue with those encodings by
 wrapping some functions to convert to Unicode string before path
 operation.
 
 This extension is useful for:
 
 - Japanese Windows users using shift_jis encoding.
 - Chinese Windows users using big5 encoding.
 - All users who use a repository with one of problematic encodings on
   case-insensitive file system.
 
 This extension is not needed for:
 
 - Any user who use only ASCII chars in path.
 - Any user who do not use any of problematic encodings.
 
 Note that there are some limitations on using this extension:
 
 - You should use single encoding in one repository.
 - If the repository path ends with 0x5c, .hg/hgrc cannot be read.
 - win32mbcs is not compatible with fixutf8 extension.
 
 By default, win32mbcs uses encoding.encoding decided by Mercurial.
 You can specify the encoding by config option::
 
  [win32mbcs]
  encoding = sjis
 
 It is useful for the users who want to commit with UTF-8 log message.
 '''
 from __future__ import absolute_import
 
 import os
 import sys
 
 from mercurial.i18n import _
 from mercurial import (
     encoding,
     error,
     pycompat,
 )
 
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = 'ships-with-hg-core'
 
 _encoding = None                                # see extsetup
 
 def decode(arg):
     if isinstance(arg, str):
         uarg = arg.decode(_encoding)
         if arg == uarg.encode(_encoding):
             return uarg
         raise UnicodeError("Not local encoding")
     elif isinstance(arg, tuple):
         return tuple(map(decode, arg))
     elif isinstance(arg, list):
         return map(decode, arg)
     elif isinstance(arg, dict):
         for k, v in arg.items():
             arg[k] = decode(v)
     return arg
 
 def encode(arg):
     if isinstance(arg, unicode):
         return arg.encode(_encoding)
     elif isinstance(arg, tuple):
         return tuple(map(encode, arg))
     elif isinstance(arg, list):
         return map(encode, arg)
     elif isinstance(arg, dict):
         for k, v in arg.items():
             arg[k] = encode(v)
     return arg
 
 def appendsep(s):
     # ensure the path ends with os.sep, appending it if necessary.
     try:
         us = decode(s)
     except UnicodeError:
         us = s
     if us and us[-1] not in ':/\\':
         s += pycompat.ossep
     return s
 
 
 def basewrapper(func, argtype, enc, dec, args, kwds):
     # check check already converted, then call original
     for arg in args:
         if isinstance(arg, argtype):
             return func(*args, **kwds)
 
     try:
         # convert string arguments, call func, then convert back the
         # return value.
         return enc(func(*dec(args), **dec(kwds)))
     except UnicodeError:
         raise error.Abort(_("[win32mbcs] filename conversion failed with"
                          " %s encoding\n") % (_encoding))
 
 def wrapper(func, args, kwds):
     return basewrapper(func, unicode, encode, decode, args, kwds)
 
 
 def reversewrapper(func, args, kwds):
     return basewrapper(func, str, decode, encode, args, kwds)
 
 def wrapperforlistdir(func, args, kwds):
     # Ensure 'path' argument ends with os.sep to avoids
     # misinterpreting last 0x5c of MBCS 2nd byte as path separator.
     if args:
         args = list(args)
         args[0] = appendsep(args[0])
     if 'path' in kwds:
         kwds['path'] = appendsep(kwds['path'])
     return func(*args, **kwds)
 
 def wrapname(name, wrapper):
     module, name = name.rsplit('.', 1)
     module = sys.modules[module]
     func = getattr(module, name)
     def f(*args, **kwds):
         return wrapper(func, args, kwds)
     f.__name__ = func.__name__
     setattr(module, name, f)
 
 # List of functions to be wrapped.
 # NOTE: os.path.dirname() and os.path.basename() are safe because
 #       they use result of os.path.split()
 funcs = '''os.path.join os.path.split os.path.splitext
  os.path.normpath os.makedirs mercurial.util.endswithsep
  mercurial.util.splitpath mercurial.util.fscasesensitive
  mercurial.util.fspath mercurial.util.pconvert mercurial.util.normpath
  mercurial.util.checkwinfilename mercurial.util.checkosfilename
  mercurial.util.split'''
 
 # These functions are required to be called with local encoded string
 # because they expects argument is local encoded string and cause
 # problem with unicode string.
 rfuncs = '''mercurial.encoding.upper mercurial.encoding.lower'''
 
 # List of Windows specific functions to be wrapped.
 winfuncs = '''os.path.splitunc'''
 
 # codec and alias names of sjis and big5 to be faked.
 problematic_encodings = '''big5 big5-tw csbig5 big5hkscs big5-hkscs
  hkscs cp932 932 ms932 mskanji ms-kanji shift_jis csshiftjis shiftjis
  sjis s_jis shift_jis_2004 shiftjis2004 sjis_2004 sjis2004
  shift_jisx0213 shiftjisx0213 sjisx0213 s_jisx0213 950 cp950 ms950 '''
 
 def extsetup(ui):
     # TODO: decide use of config section for this extension
     if ((not os.path.supports_unicode_filenames) and
         (pycompat.sysplatform != 'cygwin')):
         ui.warn(_("[win32mbcs] cannot activate on this platform.\n"))
         return
     # determine encoding for filename
     global _encoding
     _encoding = ui.config('win32mbcs', 'encoding', encoding.encoding)
     # fake is only for relevant environment.
     if _encoding.lower() in problematic_encodings.split():
         for f in funcs.split():
             wrapname(f, wrapper)
         if pycompat.osname == 'nt':
             for f in winfuncs.split():
                 wrapname(f, wrapper)
-        wrapname("mercurial.osutil.listdir", wrapperforlistdir)
+        wrapname("mercurial.util.listdir", wrapperforlistdir)
+        wrapname("mercurial.windows.listdir", wrapperforlistdir)
         # wrap functions to be called with local byte string arguments
         for f in rfuncs.split():
             wrapname(f, reversewrapper)
         # Check sys.args manually instead of using ui.debug() because
         # command line options is not yet applied when
         # extensions.loadall() is called.
         if '--debug' in sys.argv:
             ui.write(("[win32mbcs] activated with encoding: %s\n")
                      % _encoding)

mercurial/chgserver.py

@@ -1,578 +1,577
 # chgserver.py - command server extension for cHg
 #
 # Copyright 2011 Yuya Nishihara <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """command server extension for cHg
 
 'S' channel (read/write)
     propagate ui.system() request to client
 
 'attachio' command
     attach client's stdio passed by sendmsg()
 
 'chdir' command
     change current directory
 
 'setenv' command
     replace os.environ completely
 
 'setumask' command
     set umask
 
 'validate' command
     reload the config and check if the server is up to date
 
 Config
 ------
 
 ::
 
   [chgserver]
   # how long (in seconds) should an idle chg server exit
   idletimeout = 3600
 
   # whether to skip config or env change checks
   skiphash = False
 """
 
 from __future__ import absolute_import
 
 import hashlib
 import inspect
 import os
 import re
 import struct
 import time
 
 from .i18n import _
 
 from . import (
     commandserver,
     encoding,
     error,
     extensions,
-    osutil,
     pycompat,
     util,
 )
 
 _log = commandserver.log
 
 def _hashlist(items):
     """return sha1 hexdigest for a list"""
     return hashlib.sha1(str(items)).hexdigest()
 
 # sensitive config sections affecting confighash
 _configsections = [
     'alias',  # affects global state commands.table
     'extdiff',  # uisetup will register new commands
     'extensions',
 ]
 
 # sensitive environment variables affecting confighash
 _envre = re.compile(r'''\A(?:
                     CHGHG
                     |HG(?:[A-Z].*)?
                     |LANG(?:UAGE)?
                     |LC_.*
                     |LD_.*
                     |PATH
                     |PYTHON.*
                     |TERM(?:INFO)?
                     |TZ
                     )\Z''', re.X)
 
 def _confighash(ui):
     """return a quick hash for detecting config/env changes
 
     confighash is the hash of sensitive config items and environment variables.
 
     for chgserver, it is designed that once confighash changes, the server is
     not qualified to serve its client and should redirect the client to a new
     server. different from mtimehash, confighash change will not mark the
     server outdated and exit since the user can have different configs at the
     same time.
     """
     sectionitems = []
     for section in _configsections:
         sectionitems.append(ui.configitems(section))
     sectionhash = _hashlist(sectionitems)
     envitems = [(k, v) for k, v in encoding.environ.iteritems()
                 if _envre.match(k)]
     envhash = _hashlist(sorted(envitems))
     return sectionhash[:6] + envhash[:6]
 
 def _getmtimepaths(ui):
     """get a list of paths that should be checked to detect change
 
     The list will include:
     - extensions (will not cover all files for complex extensions)
     - mercurial/__version__.py
     - python binary
     """
     modules = [m for n, m in extensions.extensions(ui)]
     try:
         from . import __version__
         modules.append(__version__)
     except ImportError:
         pass
     files = [pycompat.sysexecutable]
     for m in modules:
         try:
             files.append(inspect.getabsfile(m))
         except TypeError:
             pass
     return sorted(set(files))
 
 def _mtimehash(paths):
     """return a quick hash for detecting file changes
 
     mtimehash calls stat on given paths and calculate a hash based on size and
     mtime of each file. mtimehash does not read file content because reading is
     expensive. therefore it's not 100% reliable for detecting content changes.
     it's possible to return different hashes for same file contents.
     it's also possible to return a same hash for different file contents for
     some carefully crafted situation.
 
     for chgserver, it is designed that once mtimehash changes, the server is
     considered outdated immediately and should no longer provide service.
 
     mtimehash is not included in confighash because we only know the paths of
     extensions after importing them (there is imp.find_module but that faces
     race conditions). We need to calculate confighash without importing.
     """
     def trystat(path):
         try:
             st = os.stat(path)
             return (st.st_mtime, st.st_size)
         except OSError:
             # could be ENOENT, EPERM etc. not fatal in any case
             pass
     return _hashlist(map(trystat, paths))[:12]
 
 class hashstate(object):
     """a structure storing confighash, mtimehash, paths used for mtimehash"""
     def __init__(self, confighash, mtimehash, mtimepaths):
         self.confighash = confighash
         self.mtimehash = mtimehash
         self.mtimepaths = mtimepaths
 
     @staticmethod
     def fromui(ui, mtimepaths=None):
         if mtimepaths is None:
             mtimepaths = _getmtimepaths(ui)
         confighash = _confighash(ui)
         mtimehash = _mtimehash(mtimepaths)
         _log('confighash = %s mtimehash = %s\n' % (confighash, mtimehash))
         return hashstate(confighash, mtimehash, mtimepaths)
 
 def _newchgui(srcui, csystem, attachio):
     class chgui(srcui.__class__):
         def __init__(self, src=None):
             super(chgui, self).__init__(src)
             if src:
                 self._csystem = getattr(src, '_csystem', csystem)
             else:
                 self._csystem = csystem
 
         def _runsystem(self, cmd, environ, cwd, out):
             # fallback to the original system method if the output needs to be
             # captured (to self._buffers), or the output stream is not stdout
             # (e.g. stderr, cStringIO), because the chg client is not aware of
             # these situations and will behave differently (write to stdout).
             if (out is not self.fout
                 or not util.safehasattr(self.fout, 'fileno')
                 or self.fout.fileno() != util.stdout.fileno()):
                 return util.system(cmd, environ=environ, cwd=cwd, out=out)
             self.flush()
             return self._csystem(cmd, util.shellenviron(environ), cwd)
 
         def _runpager(self, cmd, env=None):
             self._csystem(cmd, util.shellenviron(env), type='pager',
                           cmdtable={'attachio': attachio})
             return True
 
     return chgui(srcui)
 
 def _loadnewui(srcui, args):
     from . import dispatch  # avoid cycle
 
     newui = srcui.__class__.load()
     for a in ['fin', 'fout', 'ferr', 'environ']:
         setattr(newui, a, getattr(srcui, a))
     if util.safehasattr(srcui, '_csystem'):
         newui._csystem = srcui._csystem
 
     # command line args
     args = args[:]
     dispatch._parseconfig(newui, dispatch._earlygetopt(['--config'], args))
 
     # stolen from tortoisehg.util.copydynamicconfig()
     for section, name, value in srcui.walkconfig():
         source = srcui.configsource(section, name)
         if ':' in source or source == '--config' or source.startswith('$'):
             # path:line or command line, or environ
             continue
         newui.setconfig(section, name, value, source)
 
     # load wd and repo config, copied from dispatch.py
     cwds = dispatch._earlygetopt(['--cwd'], args)
     cwd = cwds and os.path.realpath(cwds[-1]) or None
     rpath = dispatch._earlygetopt(["-R", "--repository", "--repo"], args)
     path, newlui = dispatch._getlocal(newui, rpath, wd=cwd)
 
     return (newui, newlui)
 
 class channeledsystem(object):
     """Propagate ui.system() request in the following format:
 
     payload length (unsigned int),
     type, '\0',
     cmd, '\0',
     cwd, '\0',
     envkey, '=', val, '\0',
     ...
     envkey, '=', val
 
     if type == 'system', waits for:
 
     exitcode length (unsigned int),
     exitcode (int)
 
     if type == 'pager', repetitively waits for a command name ending with '\n'
     and executes it defined by cmdtable, or exits the loop if the command name
     is empty.
     """
     def __init__(self, in_, out, channel):
         self.in_ = in_
         self.out = out
         self.channel = channel
 
     def __call__(self, cmd, environ, cwd=None, type='system', cmdtable=None):
         args = [type, util.quotecommand(cmd), os.path.abspath(cwd or '.')]
         args.extend('%s=%s' % (k, v) for k, v in environ.iteritems())
         data = '\0'.join(args)
         self.out.write(struct.pack('>cI', self.channel, len(data)))
         self.out.write(data)
         self.out.flush()
 
         if type == 'system':
             length = self.in_.read(4)
             length, = struct.unpack('>I', length)
             if length != 4:
                 raise error.Abort(_('invalid response'))
             rc, = struct.unpack('>i', self.in_.read(4))
             return rc
         elif type == 'pager':
             while True:
                 cmd = self.in_.readline()[:-1]
                 if not cmd:
                     break
                 if cmdtable and cmd in cmdtable:
                     _log('pager subcommand: %s' % cmd)
                     cmdtable[cmd]()
                 else:
                     raise error.Abort(_('unexpected command: %s') % cmd)
         else:
             raise error.ProgrammingError('invalid S channel type: %s' % type)
 
 _iochannels = [
     # server.ch, ui.fp, mode
     ('cin', 'fin', pycompat.sysstr('rb')),
     ('cout', 'fout', pycompat.sysstr('wb')),
     ('cerr', 'ferr', pycompat.sysstr('wb')),
 ]
 
 class chgcmdserver(commandserver.server):
     def __init__(self, ui, repo, fin, fout, sock, hashstate, baseaddress):
         super(chgcmdserver, self).__init__(
             _newchgui(ui, channeledsystem(fin, fout, 'S'), self.attachio),
             repo, fin, fout)
         self.clientsock = sock
         self._oldios = []  # original (self.ch, ui.fp, fd) before "attachio"
         self.hashstate = hashstate
         self.baseaddress = baseaddress
         if hashstate is not None:
             self.capabilities = self.capabilities.copy()
             self.capabilities['validate'] = chgcmdserver.validate
 
     def cleanup(self):
         super(chgcmdserver, self).cleanup()
         # dispatch._runcatch() does not flush outputs if exception is not
         # handled by dispatch._dispatch()
         self.ui.flush()
         self._restoreio()
 
     def attachio(self):
         """Attach to client's stdio passed via unix domain socket; all
         channels except cresult will no longer be used
         """
         # tell client to sendmsg() with 1-byte payload, which makes it
         # distinctive from "attachio\n" command consumed by client.read()
         self.clientsock.sendall(struct.pack('>cI', 'I', 1))
-        clientfds = osutil.recvfds(self.clientsock.fileno())
+        clientfds = util.recvfds(self.clientsock.fileno())
         _log('received fds: %r\n' % clientfds)
 
         ui = self.ui
         ui.flush()
         first = self._saveio()
         for fd, (cn, fn, mode) in zip(clientfds, _iochannels):
             assert fd > 0
             fp = getattr(ui, fn)
             os.dup2(fd, fp.fileno())
             os.close(fd)
             if not first:
                 continue
             # reset buffering mode when client is first attached. as we want
             # to see output immediately on pager, the mode stays unchanged
             # when client re-attached. ferr is unchanged because it should
             # be unbuffered no matter if it is a tty or not.
             if fn == 'ferr':
                 newfp = fp
             else:
                 # make it line buffered explicitly because the default is
                 # decided on first write(), where fout could be a pager.
                 if fp.isatty():
                     bufsize = 1  # line buffered
                 else:
                     bufsize = -1  # system default
                 newfp = os.fdopen(fp.fileno(), mode, bufsize)
                 setattr(ui, fn, newfp)
             setattr(self, cn, newfp)
 
         self.cresult.write(struct.pack('>i', len(clientfds)))
 
     def _saveio(self):
         if self._oldios:
             return False
         ui = self.ui
         for cn, fn, _mode in _iochannels:
             ch = getattr(self, cn)
             fp = getattr(ui, fn)
             fd = os.dup(fp.fileno())
             self._oldios.append((ch, fp, fd))
         return True
 
     def _restoreio(self):
         ui = self.ui
         for (ch, fp, fd), (cn, fn, _mode) in zip(self._oldios, _iochannels):
             newfp = getattr(ui, fn)
             # close newfp while it's associated with client; otherwise it
             # would be closed when newfp is deleted
             if newfp is not fp:
                 newfp.close()
             # restore original fd: fp is open again
             os.dup2(fd, fp.fileno())
             os.close(fd)
             setattr(self, cn, ch)
             setattr(ui, fn, fp)
         del self._oldios[:]
 
     def validate(self):
         """Reload the config and check if the server is up to date
 
         Read a list of '\0' separated arguments.
         Write a non-empty list of '\0' separated instruction strings or '\0'
         if the list is empty.
         An instruction string could be either:
             - "unlink $path", the client should unlink the path to stop the
               outdated server.
             - "redirect $path", the client should attempt to connect to $path
               first. If it does not work, start a new server. It implies
               "reconnect".
             - "exit $n", the client should exit directly with code n.
               This may happen if we cannot parse the config.
             - "reconnect", the client should close the connection and
               reconnect.
         If neither "reconnect" nor "redirect" is included in the instruction
         list, the client can continue with this server after completing all
         the instructions.
         """
         from . import dispatch  # avoid cycle
 
         args = self._readlist()
         try:
             self.ui, lui = _loadnewui(self.ui, args)
         except error.ParseError as inst:
             dispatch._formatparse(self.ui.warn, inst)
             self.ui.flush()
             self.cresult.write('exit 255')
             return
         newhash = hashstate.fromui(lui, self.hashstate.mtimepaths)
         insts = []
         if newhash.mtimehash != self.hashstate.mtimehash:
             addr = _hashaddress(self.baseaddress, self.hashstate.confighash)
             insts.append('unlink %s' % addr)
             # mtimehash is empty if one or more extensions fail to load.
             # to be compatible with hg, still serve the client this time.
             if self.hashstate.mtimehash:
                 insts.append('reconnect')
         if newhash.confighash != self.hashstate.confighash:
             addr = _hashaddress(self.baseaddress, newhash.confighash)
             insts.append('redirect %s' % addr)
         _log('validate: %s\n' % insts)
         self.cresult.write('\0'.join(insts) or '\0')
 
     def chdir(self):
         """Change current directory
 
         Note that the behavior of --cwd option is bit different from this.
         It does not affect --config parameter.
         """
         path = self._readstr()
         if not path:
             return
         _log('chdir to %r\n' % path)
         os.chdir(path)
 
     def setumask(self):
         """Change umask"""
         mask = struct.unpack('>I', self._read(4))[0]
         _log('setumask %r\n' % mask)
         os.umask(mask)
 
     def runcommand(self):
         return super(chgcmdserver, self).runcommand()
 
     def setenv(self):
         """Clear and update os.environ
 
         Note that not all variables can make an effect on the running process.
         """
         l = self._readlist()
         try:
             newenv = dict(s.split('=', 1) for s in l)
         except ValueError:
             raise ValueError('unexpected value in setenv request')
         _log('setenv: %r\n' % sorted(newenv.keys()))
         encoding.environ.clear()
         encoding.environ.update(newenv)
 
     capabilities = commandserver.server.capabilities.copy()
     capabilities.update({'attachio': attachio,
                          'chdir': chdir,
                          'runcommand': runcommand,
                          'setenv': setenv,
                          'setumask': setumask})
 
-    if util.safehasattr(osutil, 'setprocname'):
+    if util.safehasattr(util, 'setprocname'):
         def setprocname(self):
             """Change process title"""
             name = self._readstr()
             _log('setprocname: %r\n' % name)
-            osutil.setprocname(name)
+            util.setprocname(name)
         capabilities['setprocname'] = setprocname
 
 def _tempaddress(address):
     return '%s.%d.tmp' % (address, os.getpid())
 
 def _hashaddress(address, hashstr):
     # if the basename of address contains '.', use only the left part. this
     # makes it possible for the client to pass 'server.tmp$PID' and follow by
     # an atomic rename to avoid locking when spawning new servers.
     dirname, basename = os.path.split(address)
     basename = basename.split('.', 1)[0]
     return '%s-%s' % (os.path.join(dirname, basename), hashstr)
 
 class chgunixservicehandler(object):
     """Set of operations for chg services"""
 
     pollinterval = 1  # [sec]
 
     def __init__(self, ui):
         self.ui = ui
         self._idletimeout = ui.configint('chgserver', 'idletimeout', 3600)
         self._lastactive = time.time()
 
     def bindsocket(self, sock, address):
         self._inithashstate(address)
         self._checkextensions()
         self._bind(sock)
         self._createsymlink()
 
     def _inithashstate(self, address):
         self._baseaddress = address
         if self.ui.configbool('chgserver', 'skiphash', False):
             self._hashstate = None
             self._realaddress = address
             return
         self._hashstate = hashstate.fromui(self.ui)
         self._realaddress = _hashaddress(address, self._hashstate.confighash)
 
     def _checkextensions(self):
         if not self._hashstate:
             return
         if extensions.notloaded():
             # one or more extensions failed to load. mtimehash becomes
             # meaningless because we do not know the paths of those extensions.
             # set mtimehash to an illegal hash value to invalidate the server.
             self._hashstate.mtimehash = ''
 
     def _bind(self, sock):
         # use a unique temp address so we can stat the file and do ownership
         # check later
         tempaddress = _tempaddress(self._realaddress)
         util.bindunixsocket(sock, tempaddress)
         self._socketstat = os.stat(tempaddress)
         # rename will replace the old socket file if exists atomically. the
         # old server will detect ownership change and exit.
         util.rename(tempaddress, self._realaddress)
 
     def _createsymlink(self):
         if self._baseaddress == self._realaddress:
             return
         tempaddress = _tempaddress(self._baseaddress)
         os.symlink(os.path.basename(self._realaddress), tempaddress)
         util.rename(tempaddress, self._baseaddress)
 
     def _issocketowner(self):
         try:
             stat = os.stat(self._realaddress)
             return (stat.st_ino == self._socketstat.st_ino and
                     stat.st_mtime == self._socketstat.st_mtime)
         except OSError:
             return False
 
     def unlinksocket(self, address):
         if not self._issocketowner():
             return
         # it is possible to have a race condition here that we may
         # remove another server's socket file. but that's okay
         # since that server will detect and exit automatically and
         # the client will start a new server on demand.
         util.tryunlink(self._realaddress)
 
     def printbanner(self, address):
         # no "listening at" message should be printed to simulate hg behavior
         pass
 
     def shouldexit(self):
         if not self._issocketowner():
             self.ui.debug('%s is not owned, exiting.\n' % self._realaddress)
             return True
         if time.time() - self._lastactive > self._idletimeout:
             self.ui.debug('being idle too long. exiting.\n')
             return True
         return False
 
     def newconnection(self):
         self._lastactive = time.time()
 
     def createcmdserver(self, repo, conn, fin, fout):
         return chgcmdserver(self.ui, repo, fin, fout, conn,
                             self._hashstate, self._baseaddress)
 
 def chgunixservice(ui, repo, opts):
     # CHGINTERNALMARK is temporarily set by chg client to detect if chg will
     # start another chg. drop it to avoid possible side effects.
     if 'CHGINTERNALMARK' in encoding.environ:
         del encoding.environ['CHGINTERNALMARK']
 
     if repo:
         # one chgserver can serve multiple repos. drop repo information
         ui.setconfig('bundle', 'mainreporoot', '', 'repo')
     h = chgunixservicehandler(ui)
     return commandserver.unixforkingservice(ui, repo=None, opts=opts, handler=h)

mercurial/dirstate.py

@@ -1,1288 +1,1287
 # dirstate.py - working directory tracking for mercurial
 #
 # Copyright 2005-2007 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import collections
 import errno
 import os
 import stat
 
 from .i18n import _
 from .node import nullid
 from . import (
     encoding,
     error,
     match as matchmod,
-    osutil,
     parsers,
     pathutil,
     pycompat,
     scmutil,
     txnutil,
     util,
 )
 
 propertycache = util.propertycache
 filecache = scmutil.filecache
 _rangemask = 0x7fffffff
 
 dirstatetuple = parsers.dirstatetuple
 
 class repocache(filecache):
     """filecache for files in .hg/"""
     def join(self, obj, fname):
         return obj._opener.join(fname)
 
 class rootcache(filecache):
     """filecache for files in the repository root"""
     def join(self, obj, fname):
         return obj._join(fname)
 
 def _getfsnow(vfs):
     '''Get "now" timestamp on filesystem'''
     tmpfd, tmpname = vfs.mkstemp()
     try:
         return os.fstat(tmpfd).st_mtime
     finally:
         os.close(tmpfd)
         vfs.unlink(tmpname)
 
 def nonnormalentries(dmap):
     '''Compute the nonnormal dirstate entries from the dmap'''
     try:
         return parsers.nonnormalotherparententries(dmap)
     except AttributeError:
         nonnorm = set()
         otherparent = set()
         for fname, e in dmap.iteritems():
             if e[0] != 'n' or e[3] == -1:
                 nonnorm.add(fname)
             if e[0] == 'n' and e[2] == -2:
                 otherparent.add(fname)
         return nonnorm, otherparent
 
 class dirstate(object):
 
     def __init__(self, opener, ui, root, validate):
         '''Create a new dirstate object.
 
         opener is an open()-like callable that can be used to open the
         dirstate file; root is the root of the directory tracked by
         the dirstate.
         '''
         self._opener = opener
         self._validate = validate
         self._root = root
         # ntpath.join(root, '') of Python 2.7.9 does not add sep if root is
         # UNC path pointing to root share (issue4557)
         self._rootdir = pathutil.normasprefix(root)
         # internal config: ui.forcecwd
         forcecwd = ui.config('ui', 'forcecwd')
         if forcecwd:
             self._cwd = forcecwd
         self._dirty = False
         self._dirtypl = False
         self._lastnormaltime = 0
         self._ui = ui
         self._filecache = {}
         self._parentwriters = 0
         self._filename = 'dirstate'
         self._pendingfilename = '%s.pending' % self._filename
         self._plchangecallbacks = {}
         self._origpl = None
         self._updatedfiles = set()
 
         # for consistent view between _pl() and _read() invocations
         self._pendingmode = None
 
     def beginparentchange(self):
         '''Marks the beginning of a set of changes that involve changing
         the dirstate parents. If there is an exception during this time,
         the dirstate will not be written when the wlock is released. This
         prevents writing an incoherent dirstate where the parent doesn't
         match the contents.
         '''
         self._parentwriters += 1
 
     def endparentchange(self):
         '''Marks the end of a set of changes that involve changing the
         dirstate parents. Once all parent changes have been marked done,
         the wlock will be free to write the dirstate on release.
         '''
         if self._parentwriters > 0:
             self._parentwriters -= 1
 
     def pendingparentchange(self):
         '''Returns true if the dirstate is in the middle of a set of changes
         that modify the dirstate parent.
         '''
         return self._parentwriters > 0
 
     @propertycache
     def _map(self):
         '''Return the dirstate contents as a map from filename to
         (state, mode, size, time).'''
         self._read()
         return self._map
 
     @propertycache
     def _copymap(self):
         self._read()
         return self._copymap
 
     @propertycache
     def _nonnormalset(self):
         nonnorm, otherparents = nonnormalentries(self._map)
         self._otherparentset = otherparents
         return nonnorm
 
     @propertycache
     def _otherparentset(self):
         nonnorm, otherparents = nonnormalentries(self._map)
         self._nonnormalset = nonnorm
         return otherparents
 
     @propertycache
     def _filefoldmap(self):
         try:
             makefilefoldmap = parsers.make_file_foldmap
         except AttributeError:
             pass
         else:
             return makefilefoldmap(self._map, util.normcasespec,
                                    util.normcasefallback)
 
         f = {}
         normcase = util.normcase
         for name, s in self._map.iteritems():
             if s[0] != 'r':
                 f[normcase(name)] = name
         f['.'] = '.' # prevents useless util.fspath() invocation
         return f
 
     @propertycache
     def _dirfoldmap(self):
         f = {}
         normcase = util.normcase
         for name in self._dirs:
             f[normcase(name)] = name
         return f
 
     @repocache('branch')
     def _branch(self):
         try:
             return self._opener.read("branch").strip() or "default"
         except IOError as inst:
             if inst.errno != errno.ENOENT:
                 raise
             return "default"
 
     @propertycache
     def _pl(self):
         try:
             fp = self._opendirstatefile()
             st = fp.read(40)
             fp.close()
             l = len(st)
             if l == 40:
                 return st[:20], st[20:40]
             elif l > 0 and l < 40:
                 raise error.Abort(_('working directory state appears damaged!'))
         except IOError as err:
             if err.errno != errno.ENOENT:
                 raise
         return [nullid, nullid]
 
     @propertycache
     def _dirs(self):
         return util.dirs(self._map, 'r')
 
     def dirs(self):
         return self._dirs
 
     @rootcache('.hgignore')
     def _ignore(self):
         files = self._ignorefiles()
         if not files:
             return util.never
 
         pats = ['include:%s' % f for f in files]
         return matchmod.match(self._root, '', [], pats, warn=self._ui.warn)
 
     @propertycache
     def _slash(self):
         return self._ui.configbool('ui', 'slash') and pycompat.ossep != '/'
 
     @propertycache
     def _checklink(self):
         return util.checklink(self._root)
 
     @propertycache
     def _checkexec(self):
         return util.checkexec(self._root)
 
     @propertycache
     def _checkcase(self):
         return not util.fscasesensitive(self._join('.hg'))
 
     def _join(self, f):
         # much faster than os.path.join()
         # it's safe because f is always a relative path
         return self._rootdir + f
 
     def flagfunc(self, buildfallback):
         if self._checklink and self._checkexec:
             def f(x):
                 try:
                     st = os.lstat(self._join(x))
                     if util.statislink(st):
                         return 'l'
                     if util.statisexec(st):
                         return 'x'
                 except OSError:
                     pass
                 return ''
             return f
 
         fallback = buildfallback()
         if self._checklink:
             def f(x):
                 if os.path.islink(self._join(x)):
                     return 'l'
                 if 'x' in fallback(x):
                     return 'x'
                 return ''
             return f
         if self._checkexec:
             def f(x):
                 if 'l' in fallback(x):
                     return 'l'
                 if util.isexec(self._join(x)):
                     return 'x'
                 return ''
             return f
         else:
             return fallback
 
     @propertycache
     def _cwd(self):
         return pycompat.getcwd()
 
     def getcwd(self):
         '''Return the path from which a canonical path is calculated.
 
         This path should be used to resolve file patterns or to convert
         canonical paths back to file paths for display. It shouldn't be
         used to get real file paths. Use vfs functions instead.
         '''
         cwd = self._cwd
         if cwd == self._root:
             return ''
         # self._root ends with a path separator if self._root is '/' or 'C:\'
         rootsep = self._root
         if not util.endswithsep(rootsep):
             rootsep += pycompat.ossep
         if cwd.startswith(rootsep):
             return cwd[len(rootsep):]
         else:
             # we're outside the repo. return an absolute path.
             return cwd
 
     def pathto(self, f, cwd=None):
         if cwd is None:
             cwd = self.getcwd()
         path = util.pathto(self._root, cwd, f)
         if self._slash:
             return util.pconvert(path)
         return path
 
     def __getitem__(self, key):
         '''Return the current state of key (a filename) in the dirstate.
 
         States are:
           n  normal
           m  needs merging
           r  marked for removal
           a  marked for addition
           ?  not tracked
         '''
         return self._map.get(key, ("?",))[0]
 
     def __contains__(self, key):
         return key in self._map
 
     def __iter__(self):
         for x in sorted(self._map):
             yield x
 
     def iteritems(self):
         return self._map.iteritems()
 
     def parents(self):
         return [self._validate(p) for p in self._pl]
 
     def p1(self):
         return self._validate(self._pl[0])
 
     def p2(self):
         return self._validate(self._pl[1])
 
     def branch(self):
         return encoding.tolocal(self._branch)
 
     def setparents(self, p1, p2=nullid):
         """Set dirstate parents to p1 and p2.
 
         When moving from two parents to one, 'm' merged entries a
         adjusted to normal and previous copy records discarded and
         returned by the call.
 
         See localrepo.setparents()
         """
         if self._parentwriters == 0:
             raise ValueError("cannot set dirstate parent without "
                              "calling dirstate.beginparentchange")
 
         self._dirty = self._dirtypl = True
         oldp2 = self._pl[1]
         if self._origpl is None:
             self._origpl = self._pl
         self._pl = p1, p2
         copies = {}
         if oldp2 != nullid and p2 == nullid:
             candidatefiles = self._nonnormalset.union(self._otherparentset)
             for f in candidatefiles:
                 s = self._map.get(f)
                 if s is None:
                     continue
 
                 # Discard 'm' markers when moving away from a merge state
                 if s[0] == 'm':
                     if f in self._copymap:
                         copies[f] = self._copymap[f]
                     self.normallookup(f)
                 # Also fix up otherparent markers
                 elif s[0] == 'n' and s[2] == -2:
                     if f in self._copymap:
                         copies[f] = self._copymap[f]
                     self.add(f)
         return copies
 
     def setbranch(self, branch):
         self._branch = encoding.fromlocal(branch)
         f = self._opener('branch', 'w', atomictemp=True, checkambig=True)
         try:
             f.write(self._branch + '\n')
             f.close()
 
             # make sure filecache has the correct stat info for _branch after
             # replacing the underlying file
             ce = self._filecache['_branch']
             if ce:
                 ce.refresh()
         except: # re-raises
             f.discard()
             raise
 
     def _opendirstatefile(self):
         fp, mode = txnutil.trypending(self._root, self._opener, self._filename)
         if self._pendingmode is not None and self._pendingmode != mode:
             fp.close()
             raise error.Abort(_('working directory state may be '
                                 'changed parallelly'))
         self._pendingmode = mode
         return fp
 
     def _read(self):
         self._map = {}
         self._copymap = {}
         try:
             fp = self._opendirstatefile()
             try:
                 st = fp.read()
             finally:
                 fp.close()
         except IOError as err:
             if err.errno != errno.ENOENT:
                 raise
             return
         if not st:
             return
 
         if util.safehasattr(parsers, 'dict_new_presized'):
             # Make an estimate of the number of files in the dirstate based on
             # its size. From a linear regression on a set of real-world repos,
             # all over 10,000 files, the size of a dirstate entry is 85
             # bytes. The cost of resizing is significantly higher than the cost
             # of filling in a larger presized dict, so subtract 20% from the
             # size.
             #
             # This heuristic is imperfect in many ways, so in a future dirstate
             # format update it makes sense to just record the number of entries
             # on write.
             self._map = parsers.dict_new_presized(len(st) / 71)
 
         # Python's garbage collector triggers a GC each time a certain number
         # of container objects (the number being defined by
         # gc.get_threshold()) are allocated. parse_dirstate creates a tuple
         # for each file in the dirstate. The C version then immediately marks
         # them as not to be tracked by the collector. However, this has no
         # effect on when GCs are triggered, only on what objects the GC looks
         # into. This means that O(number of files) GCs are unavoidable.
         # Depending on when in the process's lifetime the dirstate is parsed,
         # this can get very expensive. As a workaround, disable GC while
         # parsing the dirstate.
         #
         # (we cannot decorate the function directly since it is in a C module)
         parse_dirstate = util.nogc(parsers.parse_dirstate)
         p = parse_dirstate(self._map, self._copymap, st)
         if not self._dirtypl:
             self._pl = p
 
     def invalidate(self):
         for a in ("_map", "_copymap", "_filefoldmap", "_dirfoldmap", "_branch",
                   "_pl", "_dirs", "_ignore", "_nonnormalset",
                   "_otherparentset"):
             if a in self.__dict__:
                 delattr(self, a)
         self._lastnormaltime = 0
         self._dirty = False
         self._updatedfiles.clear()
         self._parentwriters = 0
         self._origpl = None
 
     def copy(self, source, dest):
         """Mark dest as a copy of source. Unmark dest if source is None."""
         if source == dest:
             return
         self._dirty = True
         if source is not None:
             self._copymap[dest] = source
             self._updatedfiles.add(source)
             self._updatedfiles.add(dest)
         elif dest in self._copymap:
             del self._copymap[dest]
             self._updatedfiles.add(dest)
 
     def copied(self, file):
         return self._copymap.get(file, None)
 
     def copies(self):
         return self._copymap
 
     def _droppath(self, f):
         if self[f] not in "?r" and "_dirs" in self.__dict__:
             self._dirs.delpath(f)
 
         if "_filefoldmap" in self.__dict__:
             normed = util.normcase(f)
             if normed in self._filefoldmap:
                 del self._filefoldmap[normed]
 
         self._updatedfiles.add(f)
 
     def _addpath(self, f, state, mode, size, mtime):
         oldstate = self[f]
         if state == 'a' or oldstate == 'r':
             scmutil.checkfilename(f)
             if f in self._dirs:
                 raise error.Abort(_('directory %r already in dirstate') % f)
             # shadows
             for d in util.finddirs(f):
                 if d in self._dirs:
                     break
                 if d in self._map and self[d] != 'r':
                     raise error.Abort(
                         _('file %r in dirstate clashes with %r') % (d, f))
         if oldstate in "?r" and "_dirs" in self.__dict__:
             self._dirs.addpath(f)
         self._dirty = True
         self._updatedfiles.add(f)
         self._map[f] = dirstatetuple(state, mode, size, mtime)
         if state != 'n' or mtime == -1:
             self._nonnormalset.add(f)
         if size == -2:
             self._otherparentset.add(f)
 
     def normal(self, f):
         '''Mark a file normal and clean.'''
         s = os.lstat(self._join(f))
         mtime = s.st_mtime
         self._addpath(f, 'n', s.st_mode,
                       s.st_size & _rangemask, mtime & _rangemask)
         if f in self._copymap:
             del self._copymap[f]
         if f in self._nonnormalset:
             self._nonnormalset.remove(f)
         if mtime > self._lastnormaltime:
             # Remember the most recent modification timeslot for status(),
             # to make sure we won't miss future size-preserving file content
             # modifications that happen within the same timeslot.
             self._lastnormaltime = mtime
 
     def normallookup(self, f):
         '''Mark a file normal, but possibly dirty.'''
         if self._pl[1] != nullid and f in self._map:
             # if there is a merge going on and the file was either
             # in state 'm' (-1) or coming from other parent (-2) before
             # being removed, restore that state.
             entry = self._map[f]
             if entry[0] == 'r' and entry[2] in (-1, -2):
                 source = self._copymap.get(f)
                 if entry[2] == -1:
                     self.merge(f)
                 elif entry[2] == -2:
                     self.otherparent(f)
                 if source:
                     self.copy(source, f)
                 return
             if entry[0] == 'm' or entry[0] == 'n' and entry[2] == -2:
                 return
         self._addpath(f, 'n', 0, -1, -1)
         if f in self._copymap:
             del self._copymap[f]
         if f in self._nonnormalset:
             self._nonnormalset.remove(f)
 
     def otherparent(self, f):
         '''Mark as coming from the other parent, always dirty.'''
         if self._pl[1] == nullid:
             raise error.Abort(_("setting %r to other parent "
                                "only allowed in merges") % f)
         if f in self and self[f] == 'n':
             # merge-like
             self._addpath(f, 'm', 0, -2, -1)
         else:
             # add-like
             self._addpath(f, 'n', 0, -2, -1)
 
         if f in self._copymap:
             del self._copymap[f]
 
     def add(self, f):
         '''Mark a file added.'''
         self._addpath(f, 'a', 0, -1, -1)
         if f in self._copymap:
             del self._copymap[f]
 
     def remove(self, f):
         '''Mark a file removed.'''
         self._dirty = True
         self._droppath(f)
         size = 0
         if self._pl[1] != nullid and f in self._map:
             # backup the previous state
             entry = self._map[f]
             if entry[0] == 'm': # merge
                 size = -1
             elif entry[0] == 'n' and entry[2] == -2: # other parent
                 size = -2
                 self._otherparentset.add(f)
         self._map[f] = dirstatetuple('r', 0, size, 0)
         self._nonnormalset.add(f)
         if size == 0 and f in self._copymap:
             del self._copymap[f]
 
     def merge(self, f):
         '''Mark a file merged.'''
         if self._pl[1] == nullid:
             return self.normallookup(f)
         return self.otherparent(f)
 
     def drop(self, f):
         '''Drop a file from the dirstate'''
         if f in self._map:
             self._dirty = True
             self._droppath(f)
             del self._map[f]
             if f in self._nonnormalset:
                 self._nonnormalset.remove(f)
             if f in self._copymap:
                 del self._copymap[f]
 
     def _discoverpath(self, path, normed, ignoremissing, exists, storemap):
         if exists is None:
             exists = os.path.lexists(os.path.join(self._root, path))
         if not exists:
             # Maybe a path component exists
             if not ignoremissing and '/' in path:
                 d, f = path.rsplit('/', 1)
                 d = self._normalize(d, False, ignoremissing, None)
                 folded = d + "/" + f
             else:
                 # No path components, preserve original case
                 folded = path
         else:
             # recursively normalize leading directory components
             # against dirstate
             if '/' in normed:
                 d, f = normed.rsplit('/', 1)
                 d = self._normalize(d, False, ignoremissing, True)
                 r = self._root + "/" + d
                 folded = d + "/" + util.fspath(f, r)
             else:
                 folded = util.fspath(normed, self._root)
             storemap[normed] = folded
 
         return folded
 
     def _normalizefile(self, path, isknown, ignoremissing=False, exists=None):
         normed = util.normcase(path)
         folded = self._filefoldmap.get(normed, None)
         if folded is None:
             if isknown:
                 folded = path
             else:
                 folded = self._discoverpath(path, normed, ignoremissing, exists,
                                             self._filefoldmap)
         return folded
 
     def _normalize(self, path, isknown, ignoremissing=False, exists=None):
         normed = util.normcase(path)
         folded = self._filefoldmap.get(normed, None)
         if folded is None:
             folded = self._dirfoldmap.get(normed, None)
         if folded is None:
             if isknown:
                 folded = path
             else:
                 # store discovered result in dirfoldmap so that future
                 # normalizefile calls don't start matching directories
                 folded = self._discoverpath(path, normed, ignoremissing, exists,
                                             self._dirfoldmap)
         return folded
 
     def normalize(self, path, isknown=False, ignoremissing=False):
         '''
         normalize the case of a pathname when on a casefolding filesystem
 
         isknown specifies whether the filename came from walking the
         disk, to avoid extra filesystem access.
 
         If ignoremissing is True, missing path are returned
         unchanged. Otherwise, we try harder to normalize possibly
         existing path components.
 
         The normalized case is determined based on the following precedence:
 
         - version of name already stored in the dirstate
         - version of name stored on disk
         - version provided via command arguments
         '''
 
         if self._checkcase:
             return self._normalize(path, isknown, ignoremissing)
         return path
 
     def clear(self):
         self._map = {}
         self._nonnormalset = set()
         self._otherparentset = set()
         if "_dirs" in self.__dict__:
             delattr(self, "_dirs")
         self._copymap = {}
         self._pl = [nullid, nullid]
         self._lastnormaltime = 0
         self._updatedfiles.clear()
         self._dirty = True
 
     def rebuild(self, parent, allfiles, changedfiles=None):
         if changedfiles is None:
             # Rebuild entire dirstate
             changedfiles = allfiles
             lastnormaltime = self._lastnormaltime
             self.clear()
             self._lastnormaltime = lastnormaltime
 
         if self._origpl is None:
             self._origpl = self._pl
         self._pl = (parent, nullid)
         for f in changedfiles:
             if f in allfiles:
                 self.normallookup(f)
             else:
                 self.drop(f)
 
         self._dirty = True
 
     def write(self, tr):
         if not self._dirty:
             return
 
         filename = self._filename
         if tr:
             # 'dirstate.write()' is not only for writing in-memory
             # changes out, but also for dropping ambiguous timestamp.
             # delayed writing re-raise "ambiguous timestamp issue".
             # See also the wiki page below for detail:
             # https://www.mercurial-scm.org/wiki/DirstateTransactionPlan
 
             # emulate dropping timestamp in 'parsers.pack_dirstate'
             now = _getfsnow(self._opener)
             dmap = self._map
             for f in self._updatedfiles:
                 e = dmap.get(f)
                 if e is not None and e[0] == 'n' and e[3] == now:
                     dmap[f] = dirstatetuple(e[0], e[1], e[2], -1)
                     self._nonnormalset.add(f)
 
             # emulate that all 'dirstate.normal' results are written out
             self._lastnormaltime = 0
             self._updatedfiles.clear()
 
             # delay writing in-memory changes out
             tr.addfilegenerator('dirstate', (self._filename,),
                                 self._writedirstate, location='plain')
             return
 
         st = self._opener(filename, "w", atomictemp=True, checkambig=True)
         self._writedirstate(st)
 
     def addparentchangecallback(self, category, callback):
         """add a callback to be called when the wd parents are changed
 
         Callback will be called with the following arguments:
             dirstate, (oldp1, oldp2), (newp1, newp2)
 
         Category is a unique identifier to allow overwriting an old callback
         with a newer callback.
         """
         self._plchangecallbacks[category] = callback
 
     def _writedirstate(self, st):
         # notify callbacks about parents change
         if self._origpl is not None and self._origpl != self._pl:
             for c, callback in sorted(self._plchangecallbacks.iteritems()):
                 callback(self, self._origpl, self._pl)
             self._origpl = None
         # use the modification time of the newly created temporary file as the
         # filesystem's notion of 'now'
         now = util.fstat(st).st_mtime & _rangemask
 
         # enough 'delaywrite' prevents 'pack_dirstate' from dropping
         # timestamp of each entries in dirstate, because of 'now > mtime'
         delaywrite = self._ui.configint('debug', 'dirstate.delaywrite', 0)
         if delaywrite > 0:
             # do we have any files to delay for?
             for f, e in self._map.iteritems():
                 if e[0] == 'n' and e[3] == now:
                     import time # to avoid useless import
                     # rather than sleep n seconds, sleep until the next
                     # multiple of n seconds
                     clock = time.time()
                     start = int(clock) - (int(clock) % delaywrite)
                     end = start + delaywrite
                     time.sleep(end - clock)
                     now = end # trust our estimate that the end is near now
                     break
 
         st.write(parsers.pack_dirstate(self._map, self._copymap, self._pl, now))
         self._nonnormalset, self._otherparentset = nonnormalentries(self._map)
         st.close()
         self._lastnormaltime = 0
         self._dirty = self._dirtypl = False
 
     def _dirignore(self, f):
         if f == '.':
             return False
         if self._ignore(f):
             return True
         for p in util.finddirs(f):
             if self._ignore(p):
                 return True
         return False
 
     def _ignorefiles(self):
         files = []
         if os.path.exists(self._join('.hgignore')):
             files.append(self._join('.hgignore'))
         for name, path in self._ui.configitems("ui"):
             if name == 'ignore' or name.startswith('ignore.'):
                 # we need to use os.path.join here rather than self._join
                 # because path is arbitrary and user-specified
                 files.append(os.path.join(self._rootdir, util.expandpath(path)))
         return files
 
     def _ignorefileandline(self, f):
         files = collections.deque(self._ignorefiles())
         visited = set()
         while files:
             i = files.popleft()
             patterns = matchmod.readpatternfile(i, self._ui.warn,
                                                 sourceinfo=True)
             for pattern, lineno, line in patterns:
                 kind, p = matchmod._patsplit(pattern, 'glob')
                 if kind == "subinclude":
                     if p not in visited:
                         files.append(p)
                     continue
                 m = matchmod.match(self._root, '', [], [pattern],
                                    warn=self._ui.warn)
                 if m(f):
                     return (i, lineno, line)
             visited.add(i)
         return (None, -1, "")
 
     def _walkexplicit(self, match, subrepos):
         '''Get stat data about the files explicitly specified by match.
 
         Return a triple (results, dirsfound, dirsnotfound).
         - results is a mapping from filename to stat result. It also contains
           listings mapping subrepos and .hg to None.
         - dirsfound is a list of files found to be directories.
         - dirsnotfound is a list of files that the dirstate thinks are
           directories and that were not found.'''
 
         def badtype(mode):
             kind = _('unknown')
             if stat.S_ISCHR(mode):
                 kind = _('character device')
             elif stat.S_ISBLK(mode):
                 kind = _('block device')
             elif stat.S_ISFIFO(mode):
                 kind = _('fifo')
             elif stat.S_ISSOCK(mode):
                 kind = _('socket')
             elif stat.S_ISDIR(mode):
                 kind = _('directory')
             return _('unsupported file type (type is %s)') % kind
 
         matchedir = match.explicitdir
         badfn = match.bad
         dmap = self._map
         lstat = os.lstat
         getkind = stat.S_IFMT
         dirkind = stat.S_IFDIR
         regkind = stat.S_IFREG
         lnkkind = stat.S_IFLNK
         join = self._join
         dirsfound = []
         foundadd = dirsfound.append
         dirsnotfound = []
         notfoundadd = dirsnotfound.append
 
         if not match.isexact() and self._checkcase:
             normalize = self._normalize
         else:
             normalize = None
 
         files = sorted(match.files())
         subrepos.sort()
         i, j = 0, 0
         while i < len(files) and j < len(subrepos):
             subpath = subrepos[j] + "/"
             if files[i] < subpath:
                 i += 1
                 continue
             while i < len(files) and files[i].startswith(subpath):
                 del files[i]
             j += 1
 
         if not files or '.' in files:
             files = ['.']
         results = dict.fromkeys(subrepos)
         results['.hg'] = None
 
         alldirs = None
         for ff in files:
             # constructing the foldmap is expensive, so don't do it for the
             # common case where files is ['.']
             if normalize and ff != '.':
                 nf = normalize(ff, False, True)
             else:
                 nf = ff
             if nf in results:
                 continue
 
             try:
                 st = lstat(join(nf))
                 kind = getkind(st.st_mode)
                 if kind == dirkind:
                     if nf in dmap:
                         # file replaced by dir on disk but still in dirstate
                         results[nf] = None
                     if matchedir:
                         matchedir(nf)
                     foundadd((nf, ff))
                 elif kind == regkind or kind == lnkkind:
                     results[nf] = st
                 else:
                     badfn(ff, badtype(kind))
                     if nf in dmap:
                         results[nf] = None
             except OSError as inst: # nf not found on disk - it is dirstate only
                 if nf in dmap: # does it exactly match a missing file?
                     results[nf] = None
                 else: # does it match a missing directory?
                     if alldirs is None:
                         alldirs = util.dirs(dmap)
                     if nf in alldirs:
                         if matchedir:
                             matchedir(nf)
                         notfoundadd(nf)
                     else:
                         badfn(ff, inst.strerror)
 
         # Case insensitive filesystems cannot rely on lstat() failing to detect
         # a case-only rename.  Prune the stat object for any file that does not
         # match the case in the filesystem, if there are multiple files that
         # normalize to the same path.
         if match.isexact() and self._checkcase:
             normed = {}
 
             for f, st in results.iteritems():
                 if st is None:
                     continue
 
                 nc = util.normcase(f)
                 paths = normed.get(nc)
 
                 if paths is None:
                     paths = set()
                     normed[nc] = paths
 
                 paths.add(f)
 
             for norm, paths in normed.iteritems():
                 if len(paths) > 1:
                     for path in paths:
                         folded = self._discoverpath(path, norm, True, None,
                                                     self._dirfoldmap)
                         if path != folded:
                             results[path] = None
 
         return results, dirsfound, dirsnotfound
 
     def walk(self, match, subrepos, unknown, ignored, full=True):
         '''
         Walk recursively through the directory tree, finding all files
         matched by match.
 
         If full is False, maybe skip some known-clean files.
 
         Return a dict mapping filename to stat-like object (either
         mercurial.osutil.stat instance or return value of os.stat()).
 
         '''
         # full is a flag that extensions that hook into walk can use -- this
         # implementation doesn't use it at all. This satisfies the contract
         # because we only guarantee a "maybe".
 
         if ignored:
             ignore = util.never
             dirignore = util.never
         elif unknown:
             ignore = self._ignore
             dirignore = self._dirignore
         else:
             # if not unknown and not ignored, drop dir recursion and step 2
             ignore = util.always
             dirignore = util.always
 
         matchfn = match.matchfn
         matchalways = match.always()
         matchtdir = match.traversedir
         dmap = self._map
-        listdir = osutil.listdir
+        listdir = util.listdir
         lstat = os.lstat
         dirkind = stat.S_IFDIR
         regkind = stat.S_IFREG
         lnkkind = stat.S_IFLNK
         join = self._join
 
         exact = skipstep3 = False
         if match.isexact(): # match.exact
             exact = True
             dirignore = util.always # skip step 2
         elif match.prefix(): # match.match, no patterns
             skipstep3 = True
 
         if not exact and self._checkcase:
             normalize = self._normalize
             normalizefile = self._normalizefile
             skipstep3 = False
         else:
             normalize = self._normalize
             normalizefile = None
 
         # step 1: find all explicit files
         results, work, dirsnotfound = self._walkexplicit(match, subrepos)
 
         skipstep3 = skipstep3 and not (work or dirsnotfound)
         work = [d for d in work if not dirignore(d[0])]
 
         # step 2: visit subdirectories
         def traverse(work, alreadynormed):
             wadd = work.append
             while work:
                 nd = work.pop()
                 if not match.visitdir(nd):
                     continue
                 skip = None
                 if nd == '.':
                     nd = ''
                 else:
                     skip = '.hg'
                 try:
                     entries = listdir(join(nd), stat=True, skip=skip)
                 except OSError as inst:
                     if inst.errno in (errno.EACCES, errno.ENOENT):
                         match.bad(self.pathto(nd), inst.strerror)
                         continue
                     raise
                 for f, kind, st in entries:
                     if normalizefile:
                         # even though f might be a directory, we're only
                         # interested in comparing it to files currently in the
                         # dmap -- therefore normalizefile is enough
                         nf = normalizefile(nd and (nd + "/" + f) or f, True,
                                            True)
                     else:
                         nf = nd and (nd + "/" + f) or f
                     if nf not in results:
                         if kind == dirkind:
                             if not ignore(nf):
                                 if matchtdir:
                                     matchtdir(nf)
                                 wadd(nf)
                             if nf in dmap and (matchalways or matchfn(nf)):
                                 results[nf] = None
                         elif kind == regkind or kind == lnkkind:
                             if nf in dmap:
                                 if matchalways or matchfn(nf):
                                     results[nf] = st
                             elif ((matchalways or matchfn(nf))
                                   and not ignore(nf)):
                                 # unknown file -- normalize if necessary
                                 if not alreadynormed:
                                     nf = normalize(nf, False, True)
                                 results[nf] = st
                         elif nf in dmap and (matchalways or matchfn(nf)):
                             results[nf] = None
 
         for nd, d in work:
             # alreadynormed means that processwork doesn't have to do any
             # expensive directory normalization
             alreadynormed = not normalize or nd == d
             traverse([d], alreadynormed)
 
         for s in subrepos:
             del results[s]
         del results['.hg']
 
         # step 3: visit remaining files from dmap
         if not skipstep3 and not exact:
             # If a dmap file is not in results yet, it was either
             # a) not matching matchfn b) ignored, c) missing, or d) under a
             # symlink directory.
             if not results and matchalways:
                 visit = [f for f in dmap]
             else:
                 visit = [f for f in dmap if f not in results and matchfn(f)]
             visit.sort()
 
             if unknown:
                 # unknown == True means we walked all dirs under the roots
                 # that wasn't ignored, and everything that matched was stat'ed
                 # and is already in results.
                 # The rest must thus be ignored or under a symlink.
                 audit_path = pathutil.pathauditor(self._root)
 
                 for nf in iter(visit):
                     # If a stat for the same file was already added with a
                     # different case, don't add one for this, since that would
                     # make it appear as if the file exists under both names
                     # on disk.
                     if (normalizefile and
                         normalizefile(nf, True, True) in results):
                         results[nf] = None
                     # Report ignored items in the dmap as long as they are not
                     # under a symlink directory.
                     elif audit_path.check(nf):
                         try:
                             results[nf] = lstat(join(nf))
                             # file was just ignored, no links, and exists
                         except OSError:
                             # file doesn't exist
                             results[nf] = None
                     else:
                         # It's either missing or under a symlink directory
                         # which we in this case report as missing
                         results[nf] = None
             else:
                 # We may not have walked the full directory tree above,
                 # so stat and check everything we missed.
                 iv = iter(visit)
                 for st in util.statfiles([join(i) for i in visit]):
                     results[next(iv)] = st
         return results
 
     def status(self, match, subrepos, ignored, clean, unknown):
         '''Determine the status of the working copy relative to the
         dirstate and return a pair of (unsure, status), where status is of type
         scmutil.status and:
 
           unsure:
             files that might have been modified since the dirstate was
             written, but need to be read to be sure (size is the same
             but mtime differs)
           status.modified:
             files that have definitely been modified since the dirstate
             was written (different size or mode)
           status.clean:
             files that have definitely not been modified since the
             dirstate was written
         '''
         listignored, listclean, listunknown = ignored, clean, unknown
         lookup, modified, added, unknown, ignored = [], [], [], [], []
         removed, deleted, clean = [], [], []
 
         dmap = self._map
         ladd = lookup.append            # aka "unsure"
         madd = modified.append
         aadd = added.append
         uadd = unknown.append
         iadd = ignored.append
         radd = removed.append
         dadd = deleted.append
         cadd = clean.append
         mexact = match.exact
         dirignore = self._dirignore
         checkexec = self._checkexec
         copymap = self._copymap
         lastnormaltime = self._lastnormaltime
 
         # We need to do full walks when either
         # - we're listing all clean files, or
         # - match.traversedir does something, because match.traversedir should
         #   be called for every dir in the working dir
         full = listclean or match.traversedir is not None
         for fn, st in self.walk(match, subrepos, listunknown, listignored,
                                 full=full).iteritems():
             if fn not in dmap:
                 if (listignored or mexact(fn)) and dirignore(fn):
                     if listignored:
                         iadd(fn)
                 else:
                     uadd(fn)
                 continue
 
             # This is equivalent to 'state, mode, size, time = dmap[fn]' but not
             # written like that for performance reasons. dmap[fn] is not a
             # Python tuple in compiled builds. The CPython UNPACK_SEQUENCE
             # opcode has fast paths when the value to be unpacked is a tuple or
             # a list, but falls back to creating a full-fledged iterator in
             # general. That is much slower than simply accessing and storing the
             # tuple members one by one.
             t = dmap[fn]
             state = t[0]
             mode = t[1]
             size = t[2]
             time = t[3]
 
             if not st and state in "nma":
                 dadd(fn)
             elif state == 'n':
                 if (size >= 0 and
                     ((size != st.st_size and size != st.st_size & _rangemask)
                      or ((mode ^ st.st_mode) & 0o100 and checkexec))
                     or size == -2 # other parent
                     or fn in copymap):
                     madd(fn)
                 elif time != st.st_mtime and time != st.st_mtime & _rangemask:
                     ladd(fn)
                 elif st.st_mtime == lastnormaltime:
                     # fn may have just been marked as normal and it may have
                     # changed in the same second without changing its size.
                     # This can happen if we quickly do multiple commits.
                     # Force lookup, so we don't miss such a racy file change.
                     ladd(fn)
                 elif listclean:
                     cadd(fn)
             elif state == 'm':
                 madd(fn)
             elif state == 'a':
                 aadd(fn)
             elif state == 'r':
                 radd(fn)
 
         return (lookup, scmutil.status(modified, added, removed, deleted,
                                        unknown, ignored, clean))
 
     def matches(self, match):
         '''
         return files in the dirstate (in whatever state) filtered by match
         '''
         dmap = self._map
         if match.always():
             return dmap.keys()
         files = match.files()
         if match.isexact():
             # fast path -- filter the other way around, since typically files is
             # much smaller than dmap
             return [f for f in files if f in dmap]
         if match.prefix() and all(fn in dmap for fn in files):
             # fast path -- all the values are known to be files, so just return
             # that
             return list(files)
         return [f for f in dmap if match(f)]
 
     def _actualfilename(self, tr):
         if tr:
             return self._pendingfilename
         else:
             return self._filename
 
     def savebackup(self, tr, suffix='', prefix=''):
         '''Save current dirstate into backup file with suffix'''
         assert len(suffix) > 0 or len(prefix) > 0
         filename = self._actualfilename(tr)
 
         # use '_writedirstate' instead of 'write' to write changes certainly,
         # because the latter omits writing out if transaction is running.
         # output file will be used to create backup of dirstate at this point.
         if self._dirty or not self._opener.exists(filename):
             self._writedirstate(self._opener(filename, "w", atomictemp=True,
                                              checkambig=True))
 
         if tr:
             # ensure that subsequent tr.writepending returns True for
             # changes written out above, even if dirstate is never
             # changed after this
             tr.addfilegenerator('dirstate', (self._filename,),
                                 self._writedirstate, location='plain')
 
             # ensure that pending file written above is unlinked at
             # failure, even if tr.writepending isn't invoked until the
             # end of this transaction
             tr.registertmp(filename, location='plain')
 
         backupname = prefix + self._filename + suffix
         assert backupname != filename
         self._opener.tryunlink(backupname)
         # hardlink backup is okay because _writedirstate is always called
         # with an "atomictemp=True" file.
         util.copyfile(self._opener.join(filename),
                       self._opener.join(backupname), hardlink=True)
 
     def restorebackup(self, tr, suffix='', prefix=''):
         '''Restore dirstate by backup file with suffix'''
         assert len(suffix) > 0 or len(prefix) > 0
         # this "invalidate()" prevents "wlock.release()" from writing
         # changes of dirstate out after restoring from backup file
         self.invalidate()
         filename = self._actualfilename(tr)
         # using self._filename to avoid having "pending" in the backup filename
         self._opener.rename(prefix + self._filename + suffix, filename,
                             checkambig=True)
 
     def clearbackup(self, tr, suffix='', prefix=''):
         '''Clear backup file with suffix'''
         assert len(suffix) > 0 or len(prefix) > 0
         # using self._filename to avoid having "pending" in the backup filename
         self._opener.unlink(prefix + self._filename + suffix)

mercurial/rcutil.py

@@ -1,99 +1,98
 # rcutil.py - utilities about config paths, special config sections etc.
 #
 #  Copyright Mercurial Contributors
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import os
 
 from . import (
     encoding,
-    osutil,
     pycompat,
     util,
 )
 
 if pycompat.osname == 'nt':
     from . import scmwindows as scmplatform
 else:
     from . import scmposix as scmplatform
 
 fallbackpager = scmplatform.fallbackpager
 systemrcpath = scmplatform.systemrcpath
 userrcpath = scmplatform.userrcpath
 
 def _expandrcpath(path):
     '''path could be a file or a directory. return a list of file paths'''
     p = util.expandpath(path)
     if os.path.isdir(p):
         join = os.path.join
-        return [join(p, f) for f, k in osutil.listdir(p) if f.endswith('.rc')]
+        return [join(p, f) for f, k in util.listdir(p) if f.endswith('.rc')]
     return [p]
 
 def envrcitems(env=None):
     '''Return [(section, name, value, source)] config items.
 
     The config items are extracted from environment variables specified by env,
     used to override systemrc, but not userrc.
 
     If env is not provided, encoding.environ will be used.
     '''
     if env is None:
         env = encoding.environ
     checklist = [
         ('EDITOR', 'ui', 'editor'),
         ('VISUAL', 'ui', 'editor'),
         ('PAGER', 'pager', 'pager'),
     ]
     result = []
     for envname, section, configname in checklist:
         if envname not in env:
             continue
         result.append((section, configname, env[envname], '$%s' % envname))
     return result
 
 def defaultrcpath():
     '''return rc paths in default.d'''
     path = []
     defaultpath = os.path.join(util.datapath, 'default.d')
     if os.path.isdir(defaultpath):
         path = _expandrcpath(defaultpath)
     return path
 
 def rccomponents():
     '''return an ordered [(type, obj)] about where to load configs.
 
     respect $HGRCPATH. if $HGRCPATH is empty, only .hg/hgrc of current repo is
     used. if $HGRCPATH is not set, the platform default will be used.
 
     if a directory is provided, *.rc files under it will be used.
 
     type could be either 'path' or 'items', if type is 'path', obj is a string,
     and is the config file path. if type is 'items', obj is a list of (section,
     name, value, source) that should fill the config directly.
     '''
     envrc = ('items', envrcitems())
 
     if 'HGRCPATH' in encoding.environ:
         # assume HGRCPATH is all about user configs so environments can be
         # overridden.
         _rccomponents = [envrc]
         for p in encoding.environ['HGRCPATH'].split(pycompat.ospathsep):
             if not p:
                 continue
             _rccomponents.extend(('path', p) for p in _expandrcpath(p))
     else:
         normpaths = lambda paths: [('path', os.path.normpath(p)) for p in paths]
         _rccomponents = normpaths(defaultrcpath() + systemrcpath())
         _rccomponents.append(envrc)
         _rccomponents.extend(normpaths(userrcpath()))
     return _rccomponents
 
 def defaultpagerenv():
     '''return a dict of default environment variables and their values,
     intended to be set before starting a pager.
     '''
     return {'LESS': 'FRX', 'LV': '-c'}

mercurial/scmposix.py

@@ -1,85 +1,85
 from __future__ import absolute_import
 
 import array
 import errno
 import fcntl
 import os
 import sys
 
 from . import (
     encoding,
-    osutil,
     pycompat,
+    util,
 )
 
 # BSD 'more' escapes ANSI color sequences by default. This can be disabled by
 # $MORE variable, but there's no compatible option with Linux 'more'. Given
 # OS X is widely used and most modern Unix systems would have 'less', setting
 # 'less' as the default seems reasonable.
 fallbackpager = 'less'
 
 def _rcfiles(path):
     rcs = [os.path.join(path, 'hgrc')]
     rcdir = os.path.join(path, 'hgrc.d')
     try:
         rcs.extend([os.path.join(rcdir, f)
-                    for f, kind in osutil.listdir(rcdir)
+                    for f, kind in util.listdir(rcdir)
                     if f.endswith(".rc")])
     except OSError:
         pass
     return rcs
 
 def systemrcpath():
     path = []
     if pycompat.sysplatform == 'plan9':
         root = 'lib/mercurial'
     else:
         root = 'etc/mercurial'
     # old mod_python does not set sys.argv
     if len(getattr(sys, 'argv', [])) > 0:
         p = os.path.dirname(os.path.dirname(pycompat.sysargv[0]))
         if p != '/':
             path.extend(_rcfiles(os.path.join(p, root)))
     path.extend(_rcfiles('/' + root))
     return path
 
 def userrcpath():
     if pycompat.sysplatform == 'plan9':
         return [encoding.environ['home'] + '/lib/hgrc']
     elif pycompat.sysplatform == 'darwin':
         return [os.path.expanduser('~/.hgrc')]
     else:
         confighome = encoding.environ.get('XDG_CONFIG_HOME')
         if confighome is None or not os.path.isabs(confighome):
             confighome = os.path.expanduser('~/.config')
 
         return [os.path.expanduser('~/.hgrc'),
                 os.path.join(confighome, 'hg', 'hgrc')]
 
 def termsize(ui):
     try:
         import termios
         TIOCGWINSZ = termios.TIOCGWINSZ  # unavailable on IRIX (issue3449)
     except (AttributeError, ImportError):
         return 80, 24
 
     for dev in (ui.ferr, ui.fout, ui.fin):
         try:
             try:
                 fd = dev.fileno()
             except AttributeError:
                 continue
             if not os.isatty(fd):
                 continue
             arri = fcntl.ioctl(fd, TIOCGWINSZ, '\0' * 8)
             height, width = array.array(r'h', arri)[:2]
             if width > 0 and height > 0:
                 return width, height
         except ValueError:
             pass
         except IOError as e:
             if e[0] == errno.EINVAL:
                 pass
             else:
                 raise
     return 80, 24

mercurial/scmwindows.py

@@ -1,62 +1,61
 from __future__ import absolute_import
 
 import os
 
 from . import (
     encoding,
-    osutil,
     pycompat,
     util,
     win32,
 )
 
 try:
     import _winreg as winreg
     winreg.CloseKey
 except ImportError:
     import winreg
 
 # MS-DOS 'more' is the only pager available by default on Windows.
 fallbackpager = 'more'
 
 def systemrcpath():
     '''return default os-specific hgrc search path'''
     rcpath = []
     filename = util.executablepath()
     # Use mercurial.ini found in directory with hg.exe
     progrc = os.path.join(os.path.dirname(filename), 'mercurial.ini')
     rcpath.append(progrc)
     # Use hgrc.d found in directory with hg.exe
     progrcd = os.path.join(os.path.dirname(filename), 'hgrc.d')
     if os.path.isdir(progrcd):
-        for f, kind in osutil.listdir(progrcd):
+        for f, kind in util.listdir(progrcd):
             if f.endswith('.rc'):
                 rcpath.append(os.path.join(progrcd, f))
     # else look for a system rcpath in the registry
     value = util.lookupreg('SOFTWARE\\Mercurial', None,
                            winreg.HKEY_LOCAL_MACHINE)
     if not isinstance(value, str) or not value:
         return rcpath
     value = util.localpath(value)
     for p in value.split(pycompat.ospathsep):
         if p.lower().endswith('mercurial.ini'):
             rcpath.append(p)
         elif os.path.isdir(p):
-            for f, kind in osutil.listdir(p):
+            for f, kind in util.listdir(p):
                 if f.endswith('.rc'):
                     rcpath.append(os.path.join(p, f))
     return rcpath
 
 def userrcpath():
     '''return os-specific hgrc search path to the user dir'''
     home = os.path.expanduser('~')
     path = [os.path.join(home, 'mercurial.ini'),
             os.path.join(home, '.hgrc')]
     userprofile = encoding.environ.get('USERPROFILE')
     if userprofile and userprofile != home:
         path.append(os.path.join(userprofile, 'mercurial.ini'))
         path.append(os.path.join(userprofile, '.hgrc'))
     return path
 
 def termsize(ui):
     return win32.termsize()

mercurial/util.py

@@ -1,3750 +1,3760
 # util.py - Mercurial utility functions and platform specific implementations
 #
 #  Copyright 2005 K. Thananchayan <[e-mail]>
 #  Copyright 2005-2007 Matt Mackall <[e-mail]>
 #  Copyright 2006 Vadim Gelfer <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """Mercurial utility functions and platform specific implementations.
 
 This contains helper routines that are independent of the SCM core and
 hide platform-specific details from the core.
 """
 
 from __future__ import absolute_import
 
 import bz2
 import calendar
 import codecs
 import collections
 import datetime
 import errno
 import gc
 import hashlib
 import imp
 import os
 import platform as pyplatform
 import re as remod
 import shutil
 import signal
 import socket
 import stat
 import string
 import subprocess
 import sys
 import tempfile
 import textwrap
 import time
 import traceback
 import warnings
 import zlib
 
 from . import (
     base85,
     encoding,
     error,
     i18n,
     osutil,
     parsers,
     pycompat,
 )
 
 b85decode = base85.b85decode
 b85encode = base85.b85encode
 
 cookielib = pycompat.cookielib
 empty = pycompat.empty
 httplib = pycompat.httplib
 httpserver = pycompat.httpserver
 pickle = pycompat.pickle
 queue = pycompat.queue
 socketserver = pycompat.socketserver
 stderr = pycompat.stderr
 stdin = pycompat.stdin
 stdout = pycompat.stdout
 stringio = pycompat.stringio
 urlerr = pycompat.urlerr
 urlreq = pycompat.urlreq
 xmlrpclib = pycompat.xmlrpclib
 
 def isatty(fp):
     try:
         return fp.isatty()
     except AttributeError:
         return False
 
 # glibc determines buffering on first write to stdout - if we replace a TTY
 # destined stdout with a pipe destined stdout (e.g. pager), we want line
 # buffering
 if isatty(stdout):
     stdout = os.fdopen(stdout.fileno(), pycompat.sysstr('wb'), 1)
 
 if pycompat.osname == 'nt':
     from . import windows as platform
     stdout = platform.winstdout(stdout)
 else:
     from . import posix as platform
 
 _ = i18n._
 
 bindunixsocket = platform.bindunixsocket
 cachestat = platform.cachestat
 checkexec = platform.checkexec
 checklink = platform.checklink
 copymode = platform.copymode
 executablepath = platform.executablepath
 expandglobs = platform.expandglobs
 explainexit = platform.explainexit
 findexe = platform.findexe
 gethgcmd = platform.gethgcmd
 getuser = platform.getuser
 getpid = os.getpid
 groupmembers = platform.groupmembers
 groupname = platform.groupname
 hidewindow = platform.hidewindow
 isexec = platform.isexec
 isowner = platform.isowner
+listdir = osutil.listdir
 localpath = platform.localpath
 lookupreg = platform.lookupreg
 makedir = platform.makedir
 nlinks = platform.nlinks
 normpath = platform.normpath
 normcase = platform.normcase
 normcasespec = platform.normcasespec
 normcasefallback = platform.normcasefallback
 openhardlinks = platform.openhardlinks
 oslink = platform.oslink
 parsepatchoutput = platform.parsepatchoutput
 pconvert = platform.pconvert
 poll = platform.poll
 popen = platform.popen
 posixfile = platform.posixfile
 quotecommand = platform.quotecommand
 readpipe = platform.readpipe
 rename = platform.rename
 removedirs = platform.removedirs
 samedevice = platform.samedevice
 samefile = platform.samefile
 samestat = platform.samestat
 setbinary = platform.setbinary
 setflags = platform.setflags
 setsignalhandler = platform.setsignalhandler
 shellquote = platform.shellquote
 spawndetached = platform.spawndetached
 split = platform.split
 sshargs = platform.sshargs
 statfiles = getattr(osutil, 'statfiles', platform.statfiles)
 statisexec = platform.statisexec
 statislink = platform.statislink
 testpid = platform.testpid
 umask = platform.umask
 unlink = platform.unlink
 username = platform.username
 
+try:
+    recvfds = osutil.recvfds
+except AttributeError:
+    pass
+try:
+    setprocname = osutil.setprocname
+except AttributeError:
+    pass
+
 # Python compatibility
 
 _notset = object()
 
 # disable Python's problematic floating point timestamps (issue4836)
 # (Python hypocritically says you shouldn't change this behavior in
 # libraries, and sure enough Mercurial is not a library.)
 os.stat_float_times(False)
 
 def safehasattr(thing, attr):
     return getattr(thing, attr, _notset) is not _notset
 
 def bitsfrom(container):
     bits = 0
     for bit in container:
         bits |= bit
     return bits
 
 # python 2.6 still have deprecation warning enabled by default. We do not want
 # to display anything to standard user so detect if we are running test and
 # only use python deprecation warning in this case.
 _dowarn = bool(encoding.environ.get('HGEMITWARNINGS'))
 if _dowarn:
     # explicitly unfilter our warning for python 2.7
     #
     # The option of setting PYTHONWARNINGS in the test runner was investigated.
     # However, module name set through PYTHONWARNINGS was exactly matched, so
     # we cannot set 'mercurial' and have it match eg: 'mercurial.scmutil'. This
     # makes the whole PYTHONWARNINGS thing useless for our usecase.
     warnings.filterwarnings(r'default', r'', DeprecationWarning, r'mercurial')
     warnings.filterwarnings(r'default', r'', DeprecationWarning, r'hgext')
     warnings.filterwarnings(r'default', r'', DeprecationWarning, r'hgext3rd')
 
 def nouideprecwarn(msg, version, stacklevel=1):
     """Issue an python native deprecation warning
 
     This is a noop outside of tests, use 'ui.deprecwarn' when possible.
     """
     if _dowarn:
         msg += ("\n(compatibility will be dropped after Mercurial-%s,"
                 " update your code.)") % version
         warnings.warn(msg, DeprecationWarning, stacklevel + 1)
 
 DIGESTS = {
     'md5': hashlib.md5,
     'sha1': hashlib.sha1,
     'sha512': hashlib.sha512,
 }
 # List of digest types from strongest to weakest
 DIGESTS_BY_STRENGTH = ['sha512', 'sha1', 'md5']
 
 for k in DIGESTS_BY_STRENGTH:
     assert k in DIGESTS
 
 class digester(object):
     """helper to compute digests.
 
     This helper can be used to compute one or more digests given their name.
 
     >>> d = digester(['md5', 'sha1'])
     >>> d.update('foo')
     >>> [k for k in sorted(d)]
     ['md5', 'sha1']
     >>> d['md5']
     'acbd18db4cc2f85cedef654fccc4a4d8'
     >>> d['sha1']
     '0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33'
     >>> digester.preferred(['md5', 'sha1'])
     'sha1'
     """
 
     def __init__(self, digests, s=''):
         self._hashes = {}
         for k in digests:
             if k not in DIGESTS:
                 raise Abort(_('unknown digest type: %s') % k)
             self._hashes[k] = DIGESTS[k]()
         if s:
             self.update(s)
 
     def update(self, data):
         for h in self._hashes.values():
             h.update(data)
 
     def __getitem__(self, key):
         if key not in DIGESTS:
             raise Abort(_('unknown digest type: %s') % k)
         return self._hashes[key].hexdigest()
 
     def __iter__(self):
         return iter(self._hashes)
 
     @staticmethod
     def preferred(supported):
         """returns the strongest digest type in both supported and DIGESTS."""
 
         for k in DIGESTS_BY_STRENGTH:
             if k in supported:
                 return k
         return None
 
 class digestchecker(object):
     """file handle wrapper that additionally checks content against a given
     size and digests.
 
         d = digestchecker(fh, size, {'md5': '...'})
 
     When multiple digests are given, all of them are validated.
     """
 
     def __init__(self, fh, size, digests):
         self._fh = fh
         self._size = size
         self._got = 0
         self._digests = dict(digests)
         self._digester = digester(self._digests.keys())
 
     def read(self, length=-1):
         content = self._fh.read(length)
         self._digester.update(content)
         self._got += len(content)
         return content
 
     def validate(self):
         if self._size != self._got:
             raise Abort(_('size mismatch: expected %d, got %d') %
                 (self._size, self._got))
         for k, v in self._digests.items():
             if v != self._digester[k]:
                 # i18n: first parameter is a digest name
                 raise Abort(_('%s mismatch: expected %s, got %s') %
                     (k, v, self._digester[k]))
 
 try:
     buffer = buffer
 except NameError:
     if not pycompat.ispy3:
         def buffer(sliceable, offset=0, length=None):
             if length is not None:
                 return sliceable[offset:offset + length]
             return sliceable[offset:]
     else:
         def buffer(sliceable, offset=0, length=None):
             if length is not None:
                 return memoryview(sliceable)[offset:offset + length]
             return memoryview(sliceable)[offset:]
 
 closefds = pycompat.osname == 'posix'
 
 _chunksize = 4096
 
 class bufferedinputpipe(object):
     """a manually buffered input pipe
 
     Python will not let us use buffered IO and lazy reading with 'polling' at
     the same time. We cannot probe the buffer state and select will not detect
     that data are ready to read if they are already buffered.
 
     This class let us work around that by implementing its own buffering
     (allowing efficient readline) while offering a way to know if the buffer is
     empty from the output (allowing collaboration of the buffer with polling).
 
     This class lives in the 'util' module because it makes use of the 'os'
     module from the python stdlib.
     """
 
     def __init__(self, input):
         self._input = input
         self._buffer = []
         self._eof = False
         self._lenbuf = 0
 
     @property
     def hasbuffer(self):
         """True is any data is currently buffered
 
         This will be used externally a pre-step for polling IO. If there is
         already data then no polling should be set in place."""
         return bool(self._buffer)
 
     @property
     def closed(self):
         return self._input.closed
 
     def fileno(self):
         return self._input.fileno()
 
     def close(self):
         return self._input.close()
 
     def read(self, size):
         while (not self._eof) and (self._lenbuf < size):
             self._fillbuffer()
         return self._frombuffer(size)
 
     def readline(self, *args, **kwargs):
         if 1 < len(self._buffer):
             # this should not happen because both read and readline end with a
             # _frombuffer call that collapse it.
             self._buffer = [''.join(self._buffer)]
             self._lenbuf = len(self._buffer[0])
         lfi = -1
         if self._buffer:
             lfi = self._buffer[-1].find('\n')
         while (not self._eof) and lfi < 0:
             self._fillbuffer()
             if self._buffer:
                 lfi = self._buffer[-1].find('\n')
         size = lfi + 1
         if lfi < 0: # end of file
             size = self._lenbuf
         elif 1 < len(self._buffer):
             # we need to take previous chunks into account
             size += self._lenbuf - len(self._buffer[-1])
         return self._frombuffer(size)
 
     def _frombuffer(self, size):
         """return at most 'size' data from the buffer
 
         The data are removed from the buffer."""
         if size == 0 or not self._buffer:
             return ''
         buf = self._buffer[0]
         if 1 < len(self._buffer):
             buf = ''.join(self._buffer)
 
         data = buf[:size]
         buf = buf[len(data):]
         if buf:
             self._buffer = [buf]
             self._lenbuf = len(buf)
         else:
             self._buffer = []
             self._lenbuf = 0
         return data
 
     def _fillbuffer(self):
         """read data to the buffer"""
         data = os.read(self._input.fileno(), _chunksize)
         if not data:
             self._eof = True
         else:
             self._lenbuf += len(data)
             self._buffer.append(data)
 
 def popen2(cmd, env=None, newlines=False):
     # Setting bufsize to -1 lets the system decide the buffer size.
     # The default for bufsize is 0, meaning unbuffered. This leads to
     # poor performance on Mac OS X: http://bugs.python.org/issue4194
     p = subprocess.Popen(cmd, shell=True, bufsize=-1,
                          close_fds=closefds,
                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
                          universal_newlines=newlines,
                          env=env)
     return p.stdin, p.stdout
 
 def popen3(cmd, env=None, newlines=False):
     stdin, stdout, stderr, p = popen4(cmd, env, newlines)
     return stdin, stdout, stderr
 
 def popen4(cmd, env=None, newlines=False, bufsize=-1):
     p = subprocess.Popen(cmd, shell=True, bufsize=bufsize,
                          close_fds=closefds,
                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
                          stderr=subprocess.PIPE,
                          universal_newlines=newlines,
                          env=env)
     return p.stdin, p.stdout, p.stderr, p
 
 def version():
     """Return version information if available."""
     try:
         from . import __version__
         return __version__.version
     except ImportError:
         return 'unknown'
 
 def versiontuple(v=None, n=4):
     """Parses a Mercurial version string into an N-tuple.
 
     The version string to be parsed is specified with the ``v`` argument.
     If it isn't defined, the current Mercurial version string will be parsed.
 
     ``n`` can be 2, 3, or 4. Here is how some version strings map to
     returned values:
 
     >>> v = '3.6.1+190-df9b73d2d444'
     >>> versiontuple(v, 2)
     (3, 6)
     >>> versiontuple(v, 3)
     (3, 6, 1)
     >>> versiontuple(v, 4)
     (3, 6, 1, '190-df9b73d2d444')
 
     >>> versiontuple('3.6.1+190-df9b73d2d444+20151118')
     (3, 6, 1, '190-df9b73d2d444+20151118')
 
     >>> v = '3.6'
     >>> versiontuple(v, 2)
     (3, 6)
     >>> versiontuple(v, 3)
     (3, 6, None)
     >>> versiontuple(v, 4)
     (3, 6, None, None)
 
     >>> v = '3.9-rc'
     >>> versiontuple(v, 2)
     (3, 9)
     >>> versiontuple(v, 3)
     (3, 9, None)
     >>> versiontuple(v, 4)
     (3, 9, None, 'rc')
 
     >>> v = '3.9-rc+2-02a8fea4289b'
     >>> versiontuple(v, 2)
     (3, 9)
     >>> versiontuple(v, 3)
     (3, 9, None)
     >>> versiontuple(v, 4)
     (3, 9, None, 'rc+2-02a8fea4289b')
     """
     if not v:
         v = version()
     parts = remod.split('[\+-]', v, 1)
     if len(parts) == 1:
         vparts, extra = parts[0], None
     else:
         vparts, extra = parts
 
     vints = []
     for i in vparts.split('.'):
         try:
             vints.append(int(i))
         except ValueError:
             break
     # (3, 6) -> (3, 6, None)
     while len(vints) < 3:
         vints.append(None)
 
     if n == 2:
         return (vints[0], vints[1])
     if n == 3:
         return (vints[0], vints[1], vints[2])
     if n == 4:
         return (vints[0], vints[1], vints[2], extra)
 
 # used by parsedate
 defaultdateformats = (
     '%Y-%m-%dT%H:%M:%S', # the 'real' ISO8601
     '%Y-%m-%dT%H:%M',    #   without seconds
     '%Y-%m-%dT%H%M%S',   # another awful but legal variant without :
     '%Y-%m-%dT%H%M',     #   without seconds
     '%Y-%m-%d %H:%M:%S', # our common legal variant
     '%Y-%m-%d %H:%M',    #   without seconds
     '%Y-%m-%d %H%M%S',   # without :
     '%Y-%m-%d %H%M',     #   without seconds
     '%Y-%m-%d %I:%M:%S%p',
     '%Y-%m-%d %H:%M',
     '%Y-%m-%d %I:%M%p',
     '%Y-%m-%d',
     '%m-%d',
     '%m/%d',
     '%m/%d/%y',
     '%m/%d/%Y',
     '%a %b %d %H:%M:%S %Y',
     '%a %b %d %I:%M:%S%p %Y',
     '%a, %d %b %Y %H:%M:%S',        #  GNU coreutils "/bin/date --rfc-2822"
     '%b %d %H:%M:%S %Y',
     '%b %d %I:%M:%S%p %Y',
     '%b %d %H:%M:%S',
     '%b %d %I:%M:%S%p',
     '%b %d %H:%M',
     '%b %d %I:%M%p',
     '%b %d %Y',
     '%b %d',
     '%H:%M:%S',
     '%I:%M:%S%p',
     '%H:%M',
     '%I:%M%p',
 )
 
 extendeddateformats = defaultdateformats + (
     "%Y",
     "%Y-%m",
     "%b",
     "%b %Y",
     )
 
 def cachefunc(func):
     '''cache the result of function calls'''
     # XXX doesn't handle keywords args
     if func.__code__.co_argcount == 0:
         cache = []
         def f():
             if len(cache) == 0:
                 cache.append(func())
             return cache[0]
         return f
     cache = {}
     if func.__code__.co_argcount == 1:
         # we gain a small amount of time because
         # we don't need to pack/unpack the list
         def f(arg):
             if arg not in cache:
                 cache[arg] = func(arg)
             return cache[arg]
     else:
         def f(*args):
             if args not in cache:
                 cache[args] = func(*args)
             return cache[args]
 
     return f
 
 class sortdict(dict):
     '''a simple sorted dictionary'''
     def __init__(self, data=None):
         self._list = []
         if data:
             self.update(data)
     def copy(self):
         return sortdict(self)
     def __setitem__(self, key, val):
         if key in self:
             self._list.remove(key)
         self._list.append(key)
         dict.__setitem__(self, key, val)
     def __iter__(self):
         return self._list.__iter__()
     def update(self, src):
         if isinstance(src, dict):
             src = src.iteritems()
         for k, v in src:
             self[k] = v
     def clear(self):
         dict.clear(self)
         self._list = []
     def items(self):
         return [(k, self[k]) for k in self._list]
     def __delitem__(self, key):
         dict.__delitem__(self, key)
         self._list.remove(key)
     def pop(self, key, *args, **kwargs):
         try:
             self._list.remove(key)
         except ValueError:
             pass
         return dict.pop(self, key, *args, **kwargs)
     def keys(self):
         return self._list[:]
     def iterkeys(self):
         return self._list.__iter__()
     def iteritems(self):
         for k in self._list:
             yield k, self[k]
     def insert(self, index, key, val):
         self._list.insert(index, key)
         dict.__setitem__(self, key, val)
     def __repr__(self):
         if not self:
             return '%s()' % self.__class__.__name__
         return '%s(%r)' % (self.__class__.__name__, self.items())
 
 class _lrucachenode(object):
     """A node in a doubly linked list.
 
     Holds a reference to nodes on either side as well as a key-value
     pair for the dictionary entry.
     """
     __slots__ = (u'next', u'prev', u'key', u'value')
 
     def __init__(self):
         self.next = None
         self.prev = None
 
         self.key = _notset
         self.value = None
 
     def markempty(self):
         """Mark the node as emptied."""
         self.key = _notset
 
 class lrucachedict(object):
     """Dict that caches most recent accesses and sets.
 
     The dict consists of an actual backing dict - indexed by original
     key - and a doubly linked circular list defining the order of entries in
     the cache.
 
     The head node is the newest entry in the cache. If the cache is full,
     we recycle head.prev and make it the new head. Cache accesses result in
     the node being moved to before the existing head and being marked as the
     new head node.
     """
     def __init__(self, max):
         self._cache = {}
 
         self._head = head = _lrucachenode()
         head.prev = head
         head.next = head
         self._size = 1
         self._capacity = max
 
     def __len__(self):
         return len(self._cache)
 
     def __contains__(self, k):
         return k in self._cache
 
     def __iter__(self):
         # We don't have to iterate in cache order, but why not.
         n = self._head
         for i in range(len(self._cache)):
             yield n.key
             n = n.next
 
     def __getitem__(self, k):
         node = self._cache[k]
         self._movetohead(node)
         return node.value
 
     def __setitem__(self, k, v):
         node = self._cache.get(k)
         # Replace existing value and mark as newest.
         if node is not None:
             node.value = v
             self._movetohead(node)
             return
 
         if self._size < self._capacity:
             node = self._addcapacity()
         else:
             # Grab the last/oldest item.
             node = self._head.prev
 
         # At capacity. Kill the old entry.
         if node.key is not _notset:
             del self._cache[node.key]
 
         node.key = k
         node.value = v
         self._cache[k] = node
         # And mark it as newest entry. No need to adjust order since it
         # is already self._head.prev.
         self._head = node
 
     def __delitem__(self, k):
         node = self._cache.pop(k)
         node.markempty()
 
         # Temporarily mark as newest item before re-adjusting head to make
         # this node the oldest item.
         self._movetohead(node)
         self._head = node.next
 
     # Additional dict methods.
 
     def get(self, k, default=None):
         try:
             return self._cache[k].value
         except KeyError:
             return default
 
     def clear(self):
         n = self._head
         while n.key is not _notset:
             n.markempty()
             n = n.next
 
         self._cache.clear()
 
     def copy(self):
         result = lrucachedict(self._capacity)
         n = self._head.prev
         # Iterate in oldest-to-newest order, so the copy has the right ordering
         for i in range(len(self._cache)):
             result[n.key] = n.value
             n = n.prev
         return result
 
     def _movetohead(self, node):
         """Mark a node as the newest, making it the new head.
 
         When a node is accessed, it becomes the freshest entry in the LRU
         list, which is denoted by self._head.
 
         Visually, let's make ``N`` the new head node (* denotes head):
 
             previous/oldest <-> head <-> next/next newest
 
             ----<->--- A* ---<->-----
             |                       |
             E <-> D <-> N <-> C <-> B
 
         To:
 
             ----<->--- N* ---<->-----
             |                       |
             E <-> D <-> C <-> B <-> A
 
         This requires the following moves:
 
            C.next = D  (node.prev.next = node.next)
            D.prev = C  (node.next.prev = node.prev)
            E.next = N  (head.prev.next = node)
            N.prev = E  (node.prev = head.prev)
            N.next = A  (node.next = head)
            A.prev = N  (head.prev = node)
         """
         head = self._head
         # C.next = D
         node.prev.next = node.next
         # D.prev = C
         node.next.prev = node.prev
         # N.prev = E
         node.prev = head.prev
         # N.next = A
         # It is tempting to do just "head" here, however if node is
         # adjacent to head, this will do bad things.
         node.next = head.prev.next
         # E.next = N
         node.next.prev = node
         # A.prev = N
         node.prev.next = node
 
         self._head = node
 
     def _addcapacity(self):
         """Add a node to the circular linked list.
 
         The new node is inserted before the head node.
         """
         head = self._head
         node = _lrucachenode()
         head.prev.next = node
         node.prev = head.prev
         node.next = head
         head.prev = node
         self._size += 1
         return node
 
 def lrucachefunc(func):
     '''cache most recent results of function calls'''
     cache = {}
     order = collections.deque()
     if func.__code__.co_argcount == 1:
         def f(arg):
             if arg not in cache:
                 if len(cache) > 20:
                     del cache[order.popleft()]
                 cache[arg] = func(arg)
             else:
                 order.remove(arg)
             order.append(arg)
             return cache[arg]
     else:
         def f(*args):
             if args not in cache:
                 if len(cache) > 20:
                     del cache[order.popleft()]
                 cache[args] = func(*args)
             else:
                 order.remove(args)
             order.append(args)
             return cache[args]
 
     return f
 
 class propertycache(object):
     def __init__(self, func):
         self.func = func
         self.name = func.__name__
     def __get__(self, obj, type=None):
         result = self.func(obj)
         self.cachevalue(obj, result)
         return result
 
     def cachevalue(self, obj, value):
         # __dict__ assignment required to bypass __setattr__ (eg: repoview)
         obj.__dict__[self.name] = value
 
 def pipefilter(s, cmd):
     '''filter string S through command CMD, returning its output'''
     p = subprocess.Popen(cmd, shell=True, close_fds=closefds,
                          stdin=subprocess.PIPE, stdout=subprocess.PIPE)
     pout, perr = p.communicate(s)
     return pout
 
 def tempfilter(s, cmd):
     '''filter string S through a pair of temporary files with CMD.
     CMD is used as a template to create the real command to be run,
     with the strings INFILE and OUTFILE replaced by the real names of
     the temporary files generated.'''
     inname, outname = None, None
     try:
         infd, inname = tempfile.mkstemp(prefix='hg-filter-in-')
         fp = os.fdopen(infd, pycompat.sysstr('wb'))
         fp.write(s)
         fp.close()
         outfd, outname = tempfile.mkstemp(prefix='hg-filter-out-')
         os.close(outfd)
         cmd = cmd.replace('INFILE', inname)
         cmd = cmd.replace('OUTFILE', outname)
         code = os.system(cmd)
         if pycompat.sysplatform == 'OpenVMS' and code & 1:
             code = 0
         if code:
             raise Abort(_("command '%s' failed: %s") %
                         (cmd, explainexit(code)))
         return readfile(outname)
     finally:
         try:
             if inname:
                 os.unlink(inname)
         except OSError:
             pass
         try:
             if outname:
                 os.unlink(outname)
         except OSError:
             pass
 
 filtertable = {
     'tempfile:': tempfilter,
     'pipe:': pipefilter,
     }
 
 def filter(s, cmd):
     "filter a string through a command that transforms its input to its output"
     for name, fn in filtertable.iteritems():
         if cmd.startswith(name):
             return fn(s, cmd[len(name):].lstrip())
     return pipefilter(s, cmd)
 
 def binary(s):
     """return true if a string is binary data"""
     return bool(s and '\0' in s)
 
 def increasingchunks(source, min=1024, max=65536):
     '''return no less than min bytes per chunk while data remains,
     doubling min after each chunk until it reaches max'''
     def log2(x):
         if not x:
             return 0
         i = 0
         while x:
             x >>= 1
             i += 1
         return i - 1
 
     buf = []
     blen = 0
     for chunk in source:
         buf.append(chunk)
         blen += len(chunk)
         if blen >= min:
             if min < max:
                 min = min << 1
                 nmin = 1 << log2(blen)
                 if nmin > min:
                     min = nmin
                 if min > max:
                     min = max
             yield ''.join(buf)
             blen = 0
             buf = []
     if buf:
         yield ''.join(buf)
 
 Abort = error.Abort
 
 def always(fn):
     return True
 
 def never(fn):
     return False
 
 def nogc(func):
     """disable garbage collector
 
     Python's garbage collector triggers a GC each time a certain number of
     container objects (the number being defined by gc.get_threshold()) are
     allocated even when marked not to be tracked by the collector. Tracking has
     no effect on when GCs are triggered, only on what objects the GC looks
     into. As a workaround, disable GC while building complex (huge)
     containers.
 
     This garbage collector issue have been fixed in 2.7.
     """
     if sys.version_info >= (2, 7):
         return func
     def wrapper(*args, **kwargs):
         gcenabled = gc.isenabled()
         gc.disable()
         try:
             return func(*args, **kwargs)
         finally:
             if gcenabled:
                 gc.enable()
     return wrapper
 
 def pathto(root, n1, n2):
     '''return the relative path from one place to another.
     root should use os.sep to separate directories
     n1 should use os.sep to separate directories
     n2 should use "/" to separate directories
     returns an os.sep-separated path.
 
     If n1 is a relative path, it's assumed it's
     relative to root.
     n2 should always be relative to root.
     '''
     if not n1:
         return localpath(n2)
     if os.path.isabs(n1):
         if os.path.splitdrive(root)[0] != os.path.splitdrive(n1)[0]:
             return os.path.join(root, localpath(n2))
         n2 = '/'.join((pconvert(root), n2))
     a, b = splitpath(n1), n2.split('/')
     a.reverse()
     b.reverse()
     while a and b and a[-1] == b[-1]:
         a.pop()
         b.pop()
     b.reverse()
     return pycompat.ossep.join((['..'] * len(a)) + b) or '.'
 
 def mainfrozen():
     """return True if we are a frozen executable.
 
     The code supports py2exe (most common, Windows only) and tools/freeze
     (portable, not much used).
     """
     return (safehasattr(sys, "frozen") or # new py2exe
             safehasattr(sys, "importers") or # old py2exe
             imp.is_frozen(u"__main__")) # tools/freeze
 
 # the location of data files matching the source code
 if mainfrozen() and getattr(sys, 'frozen', None) != 'macosx_app':
     # executable version (py2exe) doesn't support __file__
     datapath = os.path.dirname(pycompat.sysexecutable)
 else:
     datapath = os.path.dirname(pycompat.fsencode(__file__))
 
 i18n.setdatapath(datapath)
 
 _hgexecutable = None
 
 def hgexecutable():
     """return location of the 'hg' executable.
 
     Defaults to $HG or 'hg' in the search path.
     """
     if _hgexecutable is None:
         hg = encoding.environ.get('HG')
         mainmod = sys.modules[pycompat.sysstr('__main__')]
         if hg:
             _sethgexecutable(hg)
         elif mainfrozen():
             if getattr(sys, 'frozen', None) == 'macosx_app':
                 # Env variable set by py2app
                 _sethgexecutable(encoding.environ['EXECUTABLEPATH'])
             else:
                 _sethgexecutable(pycompat.sysexecutable)
         elif (os.path.basename(
             pycompat.fsencode(getattr(mainmod, '__file__', ''))) == 'hg'):
             _sethgexecutable(pycompat.fsencode(mainmod.__file__))
         else:
             exe = findexe('hg') or os.path.basename(sys.argv[0])
             _sethgexecutable(exe)
     return _hgexecutable
 
 def _sethgexecutable(path):
     """set location of the 'hg' executable"""
     global _hgexecutable
     _hgexecutable = path
 
 def _isstdout(f):
     fileno = getattr(f, 'fileno', None)
     return fileno and fileno() == sys.__stdout__.fileno()
 
 def shellenviron(environ=None):
     """return environ with optional override, useful for shelling out"""
     def py2shell(val):
         'convert python object into string that is useful to shell'
         if val is None or val is False:
             return '0'
         if val is True:
             return '1'
         return str(val)
     env = dict(encoding.environ)
     if environ:
         env.update((k, py2shell(v)) for k, v in environ.iteritems())
     env['HG'] = hgexecutable()
     return env
 
 def system(cmd, environ=None, cwd=None, out=None):
     '''enhanced shell command execution.
     run with environment maybe modified, maybe in different dir.
 
     if out is specified, it is assumed to be a file-like object that has a
     write() method. stdout and stderr will be redirected to out.'''
     try:
         stdout.flush()
     except Exception:
         pass
     cmd = quotecommand(cmd)
     if pycompat.sysplatform == 'plan9' and (sys.version_info[0] == 2
                                     and sys.version_info[1] < 7):
         # subprocess kludge to work around issues in half-baked Python
         # ports, notably bichued/python:
         if not cwd is None:
             os.chdir(cwd)
         rc = os.system(cmd)
     else:
         env = shellenviron(environ)
         if out is None or _isstdout(out):
             rc = subprocess.call(cmd, shell=True, close_fds=closefds,
                                  env=env, cwd=cwd)
         else:
             proc = subprocess.Popen(cmd, shell=True, close_fds=closefds,
                                     env=env, cwd=cwd, stdout=subprocess.PIPE,
                                     stderr=subprocess.STDOUT)
             for line in iter(proc.stdout.readline, ''):
                 out.write(line)
             proc.wait()
             rc = proc.returncode
         if pycompat.sysplatform == 'OpenVMS' and rc & 1:
             rc = 0
     return rc
 
 def checksignature(func):
     '''wrap a function with code to check for calling errors'''
     def check(*args, **kwargs):
         try:
             return func(*args, **kwargs)
         except TypeError:
             if len(traceback.extract_tb(sys.exc_info()[2])) == 1:
                 raise error.SignatureError
             raise
 
     return check
 
 # a whilelist of known filesystems where hardlink works reliably
 _hardlinkfswhitelist = set([
     'btrfs',
     'ext2',
     'ext3',
     'ext4',
     'hfs',
     'jfs',
     'reiserfs',
     'tmpfs',
     'ufs',
     'xfs',
     'zfs',
 ])
 
 def copyfile(src, dest, hardlink=False, copystat=False, checkambig=False):
     '''copy a file, preserving mode and optionally other stat info like
     atime/mtime
 
     checkambig argument is used with filestat, and is useful only if
     destination file is guarded by any lock (e.g. repo.lock or
     repo.wlock).
 
     copystat and checkambig should be exclusive.
     '''
     assert not (copystat and checkambig)
     oldstat = None
     if os.path.lexists(dest):
         if checkambig:
             oldstat = checkambig and filestat(dest)
         unlink(dest)
     if hardlink:
         # Hardlinks are problematic on CIFS (issue4546), do not allow hardlinks
         # unless we are confident that dest is on a whitelisted filesystem.
         try:
             fstype = getfstype(os.path.dirname(dest))
         except OSError:
             fstype = None
         if fstype not in _hardlinkfswhitelist:
             hardlink = False
     if hardlink:
         try:
             oslink(src, dest)
             return
         except (IOError, OSError):
             pass # fall back to normal copy
     if os.path.islink(src):
         os.symlink(os.readlink(src), dest)
         # copytime is ignored for symlinks, but in general copytime isn't needed
         # for them anyway
     else:
         try:
             shutil.copyfile(src, dest)
             if copystat:
                 # copystat also copies mode
                 shutil.copystat(src, dest)
             else:
                 shutil.copymode(src, dest)
                 if oldstat and oldstat.stat:
                     newstat = filestat(dest)
                     if newstat.isambig(oldstat):
                         # stat of copied file is ambiguous to original one
                         advanced = (oldstat.stat.st_mtime + 1) & 0x7fffffff
                         os.utime(dest, (advanced, advanced))
         except shutil.Error as inst:
             raise Abort(str(inst))
 
 def copyfiles(src, dst, hardlink=None, progress=lambda t, pos: None):
     """Copy a directory tree using hardlinks if possible."""
     num = 0
 
     gettopic = lambda: hardlink and _('linking') or _('copying')
 
     if os.path.isdir(src):
         if hardlink is None:
             hardlink = (os.stat(src).st_dev ==
                         os.stat(os.path.dirname(dst)).st_dev)
         topic = gettopic()
         os.mkdir(dst)
-        for name, kind in osutil.listdir(src):
+        for name, kind in listdir(src):
             srcname = os.path.join(src, name)
             dstname = os.path.join(dst, name)
             def nprog(t, pos):
                 if pos is not None:
                     return progress(t, pos + num)
             hardlink, n = copyfiles(srcname, dstname, hardlink, progress=nprog)
             num += n
     else:
         if hardlink is None:
             hardlink = (os.stat(os.path.dirname(src)).st_dev ==
                         os.stat(os.path.dirname(dst)).st_dev)
         topic = gettopic()
 
         if hardlink:
             try:
                 oslink(src, dst)
             except (IOError, OSError):
                 hardlink = False
                 shutil.copy(src, dst)
         else:
             shutil.copy(src, dst)
         num += 1
         progress(topic, num)
     progress(topic, None)
 
     return hardlink, num
 
 _winreservednames = '''con prn aux nul
     com1 com2 com3 com4 com5 com6 com7 com8 com9
     lpt1 lpt2 lpt3 lpt4 lpt5 lpt6 lpt7 lpt8 lpt9'''.split()
 _winreservedchars = ':*?"<>|'
 def checkwinfilename(path):
     r'''Check that the base-relative path is a valid filename on Windows.
     Returns None if the path is ok, or a UI string describing the problem.
 
     >>> checkwinfilename("just/a/normal/path")
     >>> checkwinfilename("foo/bar/con.xml")
     "filename contains 'con', which is reserved on Windows"
     >>> checkwinfilename("foo/con.xml/bar")
     "filename contains 'con', which is reserved on Windows"
     >>> checkwinfilename("foo/bar/xml.con")
     >>> checkwinfilename("foo/bar/AUX/bla.txt")
     "filename contains 'AUX', which is reserved on Windows"
     >>> checkwinfilename("foo/bar/bla:.txt")
     "filename contains ':', which is reserved on Windows"
     >>> checkwinfilename("foo/bar/b\07la.txt")
     "filename contains '\\x07', which is invalid on Windows"
     >>> checkwinfilename("foo/bar/bla ")
     "filename ends with ' ', which is not allowed on Windows"
     >>> checkwinfilename("../bar")
     >>> checkwinfilename("foo\\")
     "filename ends with '\\', which is invalid on Windows"
     >>> checkwinfilename("foo\\/bar")
     "directory name ends with '\\', which is invalid on Windows"
     '''
     if path.endswith('\\'):
         return _("filename ends with '\\', which is invalid on Windows")
     if '\\/' in path:
         return _("directory name ends with '\\', which is invalid on Windows")
     for n in path.replace('\\', '/').split('/'):
         if not n:
             continue
         for c in pycompat.bytestr(n):
             if c in _winreservedchars:
                 return _("filename contains '%s', which is reserved "
                          "on Windows") % c
             if ord(c) <= 31:
                 return _("filename contains %r, which is invalid "
                          "on Windows") % c
         base = n.split('.')[0]
         if base and base.lower() in _winreservednames:
             return _("filename contains '%s', which is reserved "
                      "on Windows") % base
         t = n[-1]
         if t in '. ' and n not in '..':
             return _("filename ends with '%s', which is not allowed "
                      "on Windows") % t
 
 if pycompat.osname == 'nt':
     checkosfilename = checkwinfilename
     timer = time.clock
 else:
     checkosfilename = platform.checkosfilename
     timer = time.time
 
 if safehasattr(time, "perf_counter"):
     timer = time.perf_counter
 
 def makelock(info, pathname):
     try:
         return os.symlink(info, pathname)
     except OSError as why:
         if why.errno == errno.EEXIST:
             raise
     except AttributeError: # no symlink in os
         pass
 
     ld = os.open(pathname, os.O_CREAT | os.O_WRONLY | os.O_EXCL)
     os.write(ld, info)
     os.close(ld)
 
 def readlock(pathname):
     try:
         return os.readlink(pathname)
     except OSError as why:
         if why.errno not in (errno.EINVAL, errno.ENOSYS):
             raise
     except AttributeError: # no symlink in os
         pass
     fp = posixfile(pathname)
     r = fp.read()
     fp.close()
     return r
 
 def fstat(fp):
     '''stat file object that may not have fileno method.'''
     try:
         return os.fstat(fp.fileno())
     except AttributeError:
         return os.stat(fp.name)
 
 # File system features
 
 def fscasesensitive(path):
     """
     Return true if the given path is on a case-sensitive filesystem
 
     Requires a path (like /foo/.hg) ending with a foldable final
     directory component.
     """
     s1 = os.lstat(path)
     d, b = os.path.split(path)
     b2 = b.upper()
     if b == b2:
         b2 = b.lower()
         if b == b2:
             return True # no evidence against case sensitivity
     p2 = os.path.join(d, b2)
     try:
         s2 = os.lstat(p2)
         if s2 == s1:
             return False
         return True
     except OSError:
         return True
 
 try:
     import re2
     _re2 = None
 except ImportError:
     _re2 = False
 
 class _re(object):
     def _checkre2(self):
         global _re2
         try:
             # check if match works, see issue3964
             _re2 = bool(re2.match(r'\[([^\[]+)\]', '[ui]'))
         except ImportError:
             _re2 = False
 
     def compile(self, pat, flags=0):
         '''Compile a regular expression, using re2 if possible
 
         For best performance, use only re2-compatible regexp features. The
         only flags from the re module that are re2-compatible are
         IGNORECASE and MULTILINE.'''
         if _re2 is None:
             self._checkre2()
         if _re2 and (flags & ~(remod.IGNORECASE | remod.MULTILINE)) == 0:
             if flags & remod.IGNORECASE:
                 pat = '(?i)' + pat
             if flags & remod.MULTILINE:
                 pat = '(?m)' + pat
             try:
                 return re2.compile(pat)
             except re2.error:
                 pass
         return remod.compile(pat, flags)
 
     @propertycache
     def escape(self):
         '''Return the version of escape corresponding to self.compile.
 
         This is imperfect because whether re2 or re is used for a particular
         function depends on the flags, etc, but it's the best we can do.
         '''
         global _re2
         if _re2 is None:
             self._checkre2()
         if _re2:
             return re2.escape
         else:
             return remod.escape
 
 re = _re()
 
 _fspathcache = {}
 def fspath(name, root):
     '''Get name in the case stored in the filesystem
 
     The name should be relative to root, and be normcase-ed for efficiency.
 
     Note that this function is unnecessary, and should not be
     called, for case-sensitive filesystems (simply because it's expensive).
 
     The root should be normcase-ed, too.
     '''
     def _makefspathcacheentry(dir):
         return dict((normcase(n), n) for n in os.listdir(dir))
 
     seps = pycompat.ossep
     if pycompat.osaltsep:
         seps = seps + pycompat.osaltsep
     # Protect backslashes. This gets silly very quickly.
     seps.replace('\\','\\\\')
     pattern = remod.compile(br'([^%s]+)|([%s]+)' % (seps, seps))
     dir = os.path.normpath(root)
     result = []
     for part, sep in pattern.findall(name):
         if sep:
             result.append(sep)
             continue
 
         if dir not in _fspathcache:
             _fspathcache[dir] = _makefspathcacheentry(dir)
         contents = _fspathcache[dir]
 
         found = contents.get(part)
         if not found:
             # retry "once per directory" per "dirstate.walk" which
             # may take place for each patches of "hg qpush", for example
             _fspathcache[dir] = contents = _makefspathcacheentry(dir)
             found = contents.get(part)
 
         result.append(found or part)
         dir = os.path.join(dir, part)
 
     return ''.join(result)
 
 def getfstype(dirpath):
     '''Get the filesystem type name from a directory (best-effort)
 
     Returns None if we are unsure. Raises OSError on ENOENT, EPERM, etc.
     '''
     return getattr(osutil, 'getfstype', lambda x: None)(dirpath)
 
 def checknlink(testfile):
     '''check whether hardlink count reporting works properly'''
 
     # testfile may be open, so we need a separate file for checking to
     # work around issue2543 (or testfile may get lost on Samba shares)
     f1 = testfile + ".hgtmp1"
     if os.path.lexists(f1):
         return False
     try:
         posixfile(f1, 'w').close()
     except IOError:
         try:
             os.unlink(f1)
         except OSError:
             pass
         return False
 
     f2 = testfile + ".hgtmp2"
     fd = None
     try:
         oslink(f1, f2)
         # nlinks() may behave differently for files on Windows shares if
         # the file is open.
         fd = posixfile(f2)
         return nlinks(f2) > 1
     except OSError:
         return False
     finally:
         if fd is not None:
             fd.close()
         for f in (f1, f2):
             try:
                 os.unlink(f)
             except OSError:
                 pass
 
 def endswithsep(path):
     '''Check path ends with os.sep or os.altsep.'''
     return (path.endswith(pycompat.ossep)
             or pycompat.osaltsep and path.endswith(pycompat.osaltsep))
 
 def splitpath(path):
     '''Split path by os.sep.
     Note that this function does not use os.altsep because this is
     an alternative of simple "xxx.split(os.sep)".
     It is recommended to use os.path.normpath() before using this
     function if need.'''
     return path.split(pycompat.ossep)
 
 def gui():
     '''Are we running in a GUI?'''
     if pycompat.sysplatform == 'darwin':
         if 'SSH_CONNECTION' in encoding.environ:
             # handle SSH access to a box where the user is logged in
             return False
         elif getattr(osutil, 'isgui', None):
             # check if a CoreGraphics session is available
             return osutil.isgui()
         else:
             # pure build; use a safe default
             return True
     else:
         return pycompat.osname == "nt" or encoding.environ.get("DISPLAY")
 
 def mktempcopy(name, emptyok=False, createmode=None):
     """Create a temporary file with the same contents from name
 
     The permission bits are copied from the original file.
 
     If the temporary file is going to be truncated immediately, you
     can use emptyok=True as an optimization.
 
     Returns the name of the temporary file.
     """
     d, fn = os.path.split(name)
     fd, temp = tempfile.mkstemp(prefix='.%s-' % fn, dir=d)
     os.close(fd)
     # Temporary files are created with mode 0600, which is usually not
     # what we want.  If the original file already exists, just copy
     # its mode.  Otherwise, manually obey umask.
     copymode(name, temp, createmode)
     if emptyok:
         return temp
     try:
         try:
             ifp = posixfile(name, "rb")
         except IOError as inst:
             if inst.errno == errno.ENOENT:
                 return temp
             if not getattr(inst, 'filename', None):
                 inst.filename = name
             raise
         ofp = posixfile(temp, "wb")
         for chunk in filechunkiter(ifp):
             ofp.write(chunk)
         ifp.close()
         ofp.close()
     except: # re-raises
         try: os.unlink(temp)
         except OSError: pass
         raise
     return temp
 
 class filestat(object):
     """help to exactly detect change of a file
 
     'stat' attribute is result of 'os.stat()' if specified 'path'
     exists. Otherwise, it is None. This can avoid preparative
     'exists()' examination on client side of this class.
     """
     def __init__(self, path):
         try:
             self.stat = os.stat(path)
         except OSError as err:
             if err.errno != errno.ENOENT:
                 raise
             self.stat = None
 
     __hash__ = object.__hash__
 
     def __eq__(self, old):
         try:
             # if ambiguity between stat of new and old file is
             # avoided, comparison of size, ctime and mtime is enough
             # to exactly detect change of a file regardless of platform
             return (self.stat.st_size == old.stat.st_size and
                     self.stat.st_ctime == old.stat.st_ctime and
                     self.stat.st_mtime == old.stat.st_mtime)
         except AttributeError:
             return False
 
     def isambig(self, old):
         """Examine whether new (= self) stat is ambiguous against old one
 
         "S[N]" below means stat of a file at N-th change:
 
         - S[n-1].ctime  < S[n].ctime: can detect change of a file
         - S[n-1].ctime == S[n].ctime
           - S[n-1].ctime  < S[n].mtime: means natural advancing (*1)
           - S[n-1].ctime == S[n].mtime: is ambiguous (*2)
           - S[n-1].ctime  > S[n].mtime: never occurs naturally (don't care)
         - S[n-1].ctime  > S[n].ctime: never occurs naturally (don't care)
 
         Case (*2) above means that a file was changed twice or more at
         same time in sec (= S[n-1].ctime), and comparison of timestamp
         is ambiguous.
 
         Base idea to avoid such ambiguity is "advance mtime 1 sec, if
         timestamp is ambiguous".
 
         But advancing mtime only in case (*2) doesn't work as
         expected, because naturally advanced S[n].mtime in case (*1)
         might be equal to manually advanced S[n-1 or earlier].mtime.
 
         Therefore, all "S[n-1].ctime == S[n].ctime" cases should be
         treated as ambiguous regardless of mtime, to avoid overlooking
         by confliction between such mtime.
 
         Advancing mtime "if isambig(oldstat)" ensures "S[n-1].mtime !=
         S[n].mtime", even if size of a file isn't changed.
         """
         try:
             return (self.stat.st_ctime == old.stat.st_ctime)
         except AttributeError:
             return False
 
     def avoidambig(self, path, old):
         """Change file stat of specified path to avoid ambiguity
 
         'old' should be previous filestat of 'path'.
 
         This skips avoiding ambiguity, if a process doesn't have
         appropriate privileges for 'path'.
         """
         advanced = (old.stat.st_mtime + 1) & 0x7fffffff
         try:
             os.utime(path, (advanced, advanced))
         except OSError as inst:
             if inst.errno == errno.EPERM:
                 # utime() on the file created by another user causes EPERM,
                 # if a process doesn't have appropriate privileges
                 return
             raise
 
     def __ne__(self, other):
         return not self == other
 
 class atomictempfile(object):
     '''writable file object that atomically updates a file
 
     All writes will go to a temporary copy of the original file. Call
     close() when you are done writing, and atomictempfile will rename
     the temporary copy to the original name, making the changes
     visible. If the object is destroyed without being closed, all your
     writes are discarded.
 
     checkambig argument of constructor is used with filestat, and is
     useful only if target file is guarded by any lock (e.g. repo.lock
     or repo.wlock).
     '''
     def __init__(self, name, mode='w+b', createmode=None, checkambig=False):
         self.__name = name      # permanent name
         self._tempname = mktempcopy(name, emptyok=('w' in mode),
                                     createmode=createmode)
         self._fp = posixfile(self._tempname, mode)
         self._checkambig = checkambig
 
         # delegated methods
         self.read = self._fp.read
         self.write = self._fp.write
         self.seek = self._fp.seek
         self.tell = self._fp.tell
         self.fileno = self._fp.fileno
 
     def close(self):
         if not self._fp.closed:
             self._fp.close()
             filename = localpath(self.__name)
             oldstat = self._checkambig and filestat(filename)
             if oldstat and oldstat.stat:
                 rename(self._tempname, filename)
                 newstat = filestat(filename)
                 if newstat.isambig(oldstat):
                     # stat of changed file is ambiguous to original one
                     advanced = (oldstat.stat.st_mtime + 1) & 0x7fffffff
                     os.utime(filename, (advanced, advanced))
             else:
                 rename(self._tempname, filename)
 
     def discard(self):
         if not self._fp.closed:
             try:
                 os.unlink(self._tempname)
             except OSError:
                 pass
             self._fp.close()
 
     def __del__(self):
         if safehasattr(self, '_fp'): # constructor actually did something
             self.discard()
 
     def __enter__(self):
         return self
 
     def __exit__(self, exctype, excvalue, traceback):
         if exctype is not None:
             self.discard()
         else:
             self.close()
 
 def unlinkpath(f, ignoremissing=False):
     """unlink and remove the directory if it is empty"""
     if ignoremissing:
         tryunlink(f)
     else:
         unlink(f)
     # try removing directories that might now be empty
     try:
         removedirs(os.path.dirname(f))
     except OSError:
         pass
 
 def tryunlink(f):
     """Attempt to remove a file, ignoring ENOENT errors."""
     try:
         unlink(f)
     except OSError as e:
         if e.errno != errno.ENOENT:
             raise
 
 def makedirs(name, mode=None, notindexed=False):
     """recursive directory creation with parent mode inheritance
 
     Newly created directories are marked as "not to be indexed by
     the content indexing service", if ``notindexed`` is specified
     for "write" mode access.
     """
     try:
         makedir(name, notindexed)
     except OSError as err:
         if err.errno == errno.EEXIST:
             return
         if err.errno != errno.ENOENT or not name:
             raise
         parent = os.path.dirname(os.path.abspath(name))
         if parent == name:
             raise
         makedirs(parent, mode, notindexed)
         try:
             makedir(name, notindexed)
         except OSError as err:
             # Catch EEXIST to handle races
             if err.errno == errno.EEXIST:
                 return
             raise
     if mode is not None:
         os.chmod(name, mode)
 
 def readfile(path):
     with open(path, 'rb') as fp:
         return fp.read()
 
 def writefile(path, text):
     with open(path, 'wb') as fp:
         fp.write(text)
 
 def appendfile(path, text):
     with open(path, 'ab') as fp:
         fp.write(text)
 
 class chunkbuffer(object):
     """Allow arbitrary sized chunks of data to be efficiently read from an
     iterator over chunks of arbitrary size."""
 
     def __init__(self, in_iter):
         """in_iter is the iterator that's iterating over the input chunks."""
         def splitbig(chunks):
             for chunk in chunks:
                 if len(chunk) > 2**20:
                     pos = 0
                     while pos < len(chunk):
                         end = pos + 2 ** 18
                         yield chunk[pos:end]
                         pos = end
                 else:
                     yield chunk
         self.iter = splitbig(in_iter)
         self._queue = collections.deque()
         self._chunkoffset = 0
 
     def read(self, l=None):
         """Read L bytes of data from the iterator of chunks of data.
         Returns less than L bytes if the iterator runs dry.
 
         If size parameter is omitted, read everything"""
         if l is None:
             return ''.join(self.iter)
 
         left = l
         buf = []
         queue = self._queue
         while left > 0:
             # refill the queue
             if not queue:
                 target = 2**18
                 for chunk in self.iter:
                     queue.append(chunk)
                     target -= len(chunk)
                     if target <= 0:
                         break
                 if not queue:
                     break
 
             # The easy way to do this would be to queue.popleft(), modify the
             # chunk (if necessary), then queue.appendleft(). However, for cases
             # where we read partial chunk content, this incurs 2 dequeue
             # mutations and creates a new str for the remaining chunk in the
             # queue. Our code below avoids this overhead.
 
             chunk = queue[0]
             chunkl = len(chunk)
             offset = self._chunkoffset
 
             # Use full chunk.
             if offset == 0 and left >= chunkl:
                 left -= chunkl
                 queue.popleft()
                 buf.append(chunk)
                 # self._chunkoffset remains at 0.
                 continue
 
             chunkremaining = chunkl - offset
 
             # Use all of unconsumed part of chunk.
             if left >= chunkremaining:
                 left -= chunkremaining
                 queue.popleft()
                 # offset == 0 is enabled by block above, so this won't merely
                 # copy via ``chunk[0:]``.
                 buf.append(chunk[offset:])
                 self._chunkoffset = 0
 
             # Partial chunk needed.
             else:
                 buf.append(chunk[offset:offset + left])
                 self._chunkoffset += left
                 left -= chunkremaining
 
         return ''.join(buf)
 
 def filechunkiter(f, size=131072, limit=None):
     """Create a generator that produces the data in the file size
     (default 131072) bytes at a time, up to optional limit (default is
     to read all data).  Chunks may be less than size bytes if the
     chunk is the last chunk in the file, or the file is a socket or
     some other type of file that sometimes reads less data than is
     requested."""
     assert size >= 0
     assert limit is None or limit >= 0
     while True:
         if limit is None:
             nbytes = size
         else:
             nbytes = min(limit, size)
         s = nbytes and f.read(nbytes)
         if not s:
             break
         if limit:
             limit -= len(s)
         yield s
 
 def makedate(timestamp=None):
     '''Return a unix timestamp (or the current time) as a (unixtime,
     offset) tuple based off the local timezone.'''
     if timestamp is None:
         timestamp = time.time()
     if timestamp < 0:
         hint = _("check your clock")
         raise Abort(_("negative timestamp: %d") % timestamp, hint=hint)
     delta = (datetime.datetime.utcfromtimestamp(timestamp) -
              datetime.datetime.fromtimestamp(timestamp))
     tz = delta.days * 86400 + delta.seconds
     return timestamp, tz
 
 def datestr(date=None, format='%a %b %d %H:%M:%S %Y %1%2'):
     """represent a (unixtime, offset) tuple as a localized time.
     unixtime is seconds since the epoch, and offset is the time zone's
     number of seconds away from UTC.
 
     >>> datestr((0, 0))
     'Thu Jan 01 00:00:00 1970 +0000'
     >>> datestr((42, 0))
     'Thu Jan 01 00:00:42 1970 +0000'
     >>> datestr((-42, 0))
     'Wed Dec 31 23:59:18 1969 +0000'
     >>> datestr((0x7fffffff, 0))
     'Tue Jan 19 03:14:07 2038 +0000'
     >>> datestr((-0x80000000, 0))
     'Fri Dec 13 20:45:52 1901 +0000'
     """
     t, tz = date or makedate()
     if "%1" in format or "%2" in format or "%z" in format:
         sign = (tz > 0) and "-" or "+"
         minutes = abs(tz) // 60
         q, r = divmod(minutes, 60)
         format = format.replace("%z", "%1%2")
         format = format.replace("%1", "%c%02d" % (sign, q))
         format = format.replace("%2", "%02d" % r)
     d = t - tz
     if d > 0x7fffffff:
         d = 0x7fffffff
     elif d < -0x80000000:
         d = -0x80000000
     # Never use time.gmtime() and datetime.datetime.fromtimestamp()
     # because they use the gmtime() system call which is buggy on Windows
     # for negative values.
     t = datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=d)
     s = encoding.strtolocal(t.strftime(encoding.strfromlocal(format)))
     return s
 
 def shortdate(date=None):
     """turn (timestamp, tzoff) tuple into iso 8631 date."""
     return datestr(date, format='%Y-%m-%d')
 
 def parsetimezone(s):
     """find a trailing timezone, if any, in string, and return a
        (offset, remainder) pair"""
 
     if s.endswith("GMT") or s.endswith("UTC"):
         return 0, s[:-3].rstrip()
 
     # Unix-style timezones [+-]hhmm
     if len(s) >= 5 and s[-5] in "+-" and s[-4:].isdigit():
         sign = (s[-5] == "+") and 1 or -1
         hours = int(s[-4:-2])
         minutes = int(s[-2:])
         return -sign * (hours * 60 + minutes) * 60, s[:-5].rstrip()
 
     # ISO8601 trailing Z
     if s.endswith("Z") and s[-2:-1].isdigit():
         return 0, s[:-1]
 
     # ISO8601-style [+-]hh:mm
     if (len(s) >= 6 and s[-6] in "+-" and s[-3] == ":" and
         s[-5:-3].isdigit() and s[-2:].isdigit()):
         sign = (s[-6] == "+") and 1 or -1
         hours = int(s[-5:-3])
         minutes = int(s[-2:])
         return -sign * (hours * 60 + minutes) * 60, s[:-6]
 
     return None, s
 
 def strdate(string, format, defaults=None):
     """parse a localized time string and return a (unixtime, offset) tuple.
     if the string cannot be parsed, ValueError is raised."""
     if defaults is None:
         defaults = {}
 
     # NOTE: unixtime = localunixtime + offset
     offset, date = parsetimezone(string)
 
     # add missing elements from defaults
     usenow = False # default to using biased defaults
     for part in ("S", "M", "HI", "d", "mb", "yY"): # decreasing specificity
         part = pycompat.bytestr(part)
         found = [True for p in part if ("%"+p) in format]
         if not found:
             date += "@" + defaults[part][usenow]
             format += "@%" + part[0]
         else:
             # We've found a specific time element, less specific time
             # elements are relative to today
             usenow = True
 
     timetuple = time.strptime(date, format)
     localunixtime = int(calendar.timegm(timetuple))
     if offset is None:
         # local timezone
         unixtime = int(time.mktime(timetuple))
         offset = unixtime - localunixtime
     else:
         unixtime = localunixtime + offset
     return unixtime, offset
 
 def parsedate(date, formats=None, bias=None):
     """parse a localized date/time and return a (unixtime, offset) tuple.
 
     The date may be a "unixtime offset" string or in one of the specified
     formats. If the date already is a (unixtime, offset) tuple, it is returned.
 
     >>> parsedate(' today ') == parsedate(\
                                   datetime.date.today().strftime('%b %d'))
     True
     >>> parsedate( 'yesterday ') == parsedate((datetime.date.today() -\
                                                datetime.timedelta(days=1)\
                                               ).strftime('%b %d'))
     True
     >>> now, tz = makedate()
     >>> strnow, strtz = parsedate('now')
     >>> (strnow - now) < 1
     True
     >>> tz == strtz
     True
     """
     if bias is None:
         bias = {}
     if not date:
         return 0, 0
     if isinstance(date, tuple) and len(date) == 2:
         return date
     if not formats:
         formats = defaultdateformats
     date = date.strip()
 
     if date == 'now' or date == _('now'):
         return makedate()
     if date == 'today' or date == _('today'):
         date = datetime.date.today().strftime('%b %d')
     elif date == 'yesterday' or date == _('yesterday'):
         date = (datetime.date.today() -
                 datetime.timedelta(days=1)).strftime('%b %d')
 
     try:
         when, offset = map(int, date.split(' '))
     except ValueError:
         # fill out defaults
         now = makedate()
         defaults = {}
         for part in ("d", "mb", "yY", "HI", "M", "S"):
             # this piece is for rounding the specific end of unknowns
             b = bias.get(part)
             if b is None:
                 if part[0:1] in "HMS":
                     b = "00"
                 else:
                     b = "0"
 
             # this piece is for matching the generic end to today's date
             n = datestr(now, "%" + part[0:1])
 
             defaults[part] = (b, n)
 
         for format in formats:
             try:
                 when, offset = strdate(date, format, defaults)
             except (ValueError, OverflowError):
                 pass
             else:
                 break
         else:
             raise Abort(_('invalid date: %r') % date)
     # validate explicit (probably user-specified) date and
     # time zone offset. values must fit in signed 32 bits for
     # current 32-bit linux runtimes. timezones go from UTC-12
     # to UTC+14
     if when < -0x80000000 or when > 0x7fffffff:
         raise Abort(_('date exceeds 32 bits: %d') % when)
     if offset < -50400 or offset > 43200:
         raise Abort(_('impossible time zone offset: %d') % offset)
     return when, offset
 
 def matchdate(date):
     """Return a function that matches a given date match specifier
 
     Formats include:
 
     '{date}' match a given date to the accuracy provided
 
     '<{date}' on or before a given date
 
     '>{date}' on or after a given date
 
     >>> p1 = parsedate("10:29:59")
     >>> p2 = parsedate("10:30:00")
     >>> p3 = parsedate("10:30:59")
     >>> p4 = parsedate("10:31:00")
     >>> p5 = parsedate("Sep 15 10:30:00 1999")
     >>> f = matchdate("10:30")
     >>> f(p1[0])
     False
     >>> f(p2[0])
     True
     >>> f(p3[0])
     True
     >>> f(p4[0])
     False
     >>> f(p5[0])
     False
     """
 
     def lower(date):
         d = {'mb': "1", 'd': "1"}
         return parsedate(date, extendeddateformats, d)[0]
 
     def upper(date):
         d = {'mb': "12", 'HI': "23", 'M': "59", 'S': "59"}
         for days in ("31", "30", "29"):
             try:
                 d["d"] = days
                 return parsedate(date, extendeddateformats, d)[0]
             except Abort:
                 pass
         d["d"] = "28"
         return parsedate(date, extendeddateformats, d)[0]
 
     date = date.strip()
 
     if not date:
         raise Abort(_("dates cannot consist entirely of whitespace"))
     elif date[0] == "<":
         if not date[1:]:
             raise Abort(_("invalid day spec, use '<DATE'"))
         when = upper(date[1:])
         return lambda x: x <= when
     elif date[0] == ">":
         if not date[1:]:
             raise Abort(_("invalid day spec, use '>DATE'"))
         when = lower(date[1:])
         return lambda x: x >= when
     elif date[0] == "-":
         try:
             days = int(date[1:])
         except ValueError:
             raise Abort(_("invalid day spec: %s") % date[1:])
         if days < 0:
             raise Abort(_("%s must be nonnegative (see 'hg help dates')")
                 % date[1:])
         when = makedate()[0] - days * 3600 * 24
         return lambda x: x >= when
     elif " to " in date:
         a, b = date.split(" to ")
         start, stop = lower(a), upper(b)
         return lambda x: x >= start and x <= stop
     else:
         start, stop = lower(date), upper(date)
         return lambda x: x >= start and x <= stop
 
 def stringmatcher(pattern, casesensitive=True):
     """
     accepts a string, possibly starting with 're:' or 'literal:' prefix.
     returns the matcher name, pattern, and matcher function.
     missing or unknown prefixes are treated as literal matches.
 
     helper for tests:
     >>> def test(pattern, *tests):
     ...     kind, pattern, matcher = stringmatcher(pattern)
     ...     return (kind, pattern, [bool(matcher(t)) for t in tests])
     >>> def itest(pattern, *tests):
     ...     kind, pattern, matcher = stringmatcher(pattern, casesensitive=False)
     ...     return (kind, pattern, [bool(matcher(t)) for t in tests])
 
     exact matching (no prefix):
     >>> test('abcdefg', 'abc', 'def', 'abcdefg')
     ('literal', 'abcdefg', [False, False, True])
 
     regex matching ('re:' prefix)
     >>> test('re:a.+b', 'nomatch', 'fooadef', 'fooadefbar')
     ('re', 'a.+b', [False, False, True])
 
     force exact matches ('literal:' prefix)
     >>> test('literal:re:foobar', 'foobar', 're:foobar')
     ('literal', 're:foobar', [False, True])
 
     unknown prefixes are ignored and treated as literals
     >>> test('foo:bar', 'foo', 'bar', 'foo:bar')
     ('literal', 'foo:bar', [False, False, True])
 
     case insensitive regex matches
     >>> itest('re:A.+b', 'nomatch', 'fooadef', 'fooadefBar')
     ('re', 'A.+b', [False, False, True])
 
     case insensitive literal matches
     >>> itest('ABCDEFG', 'abc', 'def', 'abcdefg')
     ('literal', 'ABCDEFG', [False, False, True])
     """
     if pattern.startswith('re:'):
         pattern = pattern[3:]
         try:
             flags = 0
             if not casesensitive:
                 flags = remod.I
             regex = remod.compile(pattern, flags)
         except remod.error as e:
             raise error.ParseError(_('invalid regular expression: %s')
                                    % e)
         return 're', pattern, regex.search
     elif pattern.startswith('literal:'):
         pattern = pattern[8:]
 
     match = pattern.__eq__
 
     if not casesensitive:
         ipat = encoding.lower(pattern)
         match = lambda s: ipat == encoding.lower(s)
     return 'literal', pattern, match
 
 def shortuser(user):
     """Return a short representation of a user name or email address."""
     f = user.find('@')
     if f >= 0:
         user = user[:f]
     f = user.find('<')
     if f >= 0:
         user = user[f + 1:]
     f = user.find(' ')
     if f >= 0:
         user = user[:f]
     f = user.find('.')
     if f >= 0:
         user = user[:f]
     return user
 
 def emailuser(user):
     """Return the user portion of an email address."""
     f = user.find('@')
     if f >= 0:
         user = user[:f]
     f = user.find('<')
     if f >= 0:
         user = user[f + 1:]
     return user
 
 def email(author):
     '''get email of author.'''
     r = author.find('>')
     if r == -1:
         r = None
     return author[author.find('<') + 1:r]
 
 def ellipsis(text, maxlength=400):
     """Trim string to at most maxlength (default: 400) columns in display."""
     return encoding.trim(text, maxlength, ellipsis='...')
 
 def unitcountfn(*unittable):
     '''return a function that renders a readable count of some quantity'''
 
     def go(count):
         for multiplier, divisor, format in unittable:
             if abs(count) >= divisor * multiplier:
                 return format % (count / float(divisor))
         return unittable[-1][2] % count
 
     return go
 
 def processlinerange(fromline, toline):
     """Check that linerange <fromline>:<toline> makes sense and return a
     0-based range.
 
     >>> processlinerange(10, 20)
     (9, 20)
     >>> processlinerange(2, 1)
     Traceback (most recent call last):
         ...
     ParseError: line range must be positive
     >>> processlinerange(0, 5)
     Traceback (most recent call last):
         ...
     ParseError: fromline must be strictly positive
     """
     if toline - fromline < 0:
         raise error.ParseError(_("line range must be positive"))
     if fromline < 1:
         raise error.ParseError(_("fromline must be strictly positive"))
     return fromline - 1, toline
 
 bytecount = unitcountfn(
     (100, 1 << 30, _('%.0f GB')),
     (10, 1 << 30, _('%.1f GB')),
     (1, 1 << 30, _('%.2f GB')),
     (100, 1 << 20, _('%.0f MB')),
     (10, 1 << 20, _('%.1f MB')),
     (1, 1 << 20, _('%.2f MB')),
     (100, 1 << 10, _('%.0f KB')),
     (10, 1 << 10, _('%.1f KB')),
     (1, 1 << 10, _('%.2f KB')),
     (1, 1, _('%.0f bytes')),
     )
 
 # Matches a single EOL which can either be a CRLF where repeated CR
 # are removed or a LF. We do not care about old Macintosh files, so a
 # stray CR is an error.
 _eolre = remod.compile(br'\r*\n')
 
 def tolf(s):
     return _eolre.sub('\n', s)
 
 def tocrlf(s):
     return _eolre.sub('\r\n', s)
 
 if pycompat.oslinesep == '\r\n':
     tonativeeol = tocrlf
     fromnativeeol = tolf
 else:
     tonativeeol = pycompat.identity
     fromnativeeol = pycompat.identity
 
 def escapestr(s):
     # call underlying function of s.encode('string_escape') directly for
     # Python 3 compatibility
     return codecs.escape_encode(s)[0]
 
 def unescapestr(s):
     return codecs.escape_decode(s)[0]
 
 def uirepr(s):
     # Avoid double backslash in Windows path repr()
     return repr(s).replace('\\\\', '\\')
 
 # delay import of textwrap
 def MBTextWrapper(**kwargs):
     class tw(textwrap.TextWrapper):
         """
         Extend TextWrapper for width-awareness.
 
         Neither number of 'bytes' in any encoding nor 'characters' is
         appropriate to calculate terminal columns for specified string.
 
         Original TextWrapper implementation uses built-in 'len()' directly,
         so overriding is needed to use width information of each characters.
 
         In addition, characters classified into 'ambiguous' width are
         treated as wide in East Asian area, but as narrow in other.
 
         This requires use decision to determine width of such characters.
         """
         def _cutdown(self, ucstr, space_left):
             l = 0
             colwidth = encoding.ucolwidth
             for i in xrange(len(ucstr)):
                 l += colwidth(ucstr[i])
                 if space_left < l:
                     return (ucstr[:i], ucstr[i:])
             return ucstr, ''
 
         # overriding of base class
         def _handle_long_word(self, reversed_chunks, cur_line, cur_len, width):
             space_left = max(width - cur_len, 1)
 
             if self.break_long_words:
                 cut, res = self._cutdown(reversed_chunks[-1], space_left)
                 cur_line.append(cut)
                 reversed_chunks[-1] = res
             elif not cur_line:
                 cur_line.append(reversed_chunks.pop())
 
         # this overriding code is imported from TextWrapper of Python 2.6
         # to calculate columns of string by 'encoding.ucolwidth()'
         def _wrap_chunks(self, chunks):
             colwidth = encoding.ucolwidth
 
             lines = []
             if self.width <= 0:
                 raise ValueError("invalid width %r (must be > 0)" % self.width)
 
             # Arrange in reverse order so items can be efficiently popped
             # from a stack of chucks.
             chunks.reverse()
 
             while chunks:
 
                 # Start the list of chunks that will make up the current line.
                 # cur_len is just the length of all the chunks in cur_line.
                 cur_line = []
                 cur_len = 0
 
                 # Figure out which static string will prefix this line.
                 if lines:
                     indent = self.subsequent_indent
                 else:
                     indent = self.initial_indent
 
                 # Maximum width for this line.
                 width = self.width - len(indent)
 
                 # First chunk on line is whitespace -- drop it, unless this
                 # is the very beginning of the text (i.e. no lines started yet).
                 if self.drop_whitespace and chunks[-1].strip() == '' and lines:
                     del chunks[-1]
 
                 while chunks:
                     l = colwidth(chunks[-1])
 
                     # Can at least squeeze this chunk onto the current line.
                     if cur_len + l <= width:
                         cur_line.append(chunks.pop())
                         cur_len += l
 
                     # Nope, this line is full.
                     else:
                         break
 
                 # The current line is full, and the next chunk is too big to
                 # fit on *any* line (not just this one).
                 if chunks and colwidth(chunks[-1]) > width:
                     self._handle_long_word(chunks, cur_line, cur_len, width)
 
                 # If the last chunk on this line is all whitespace, drop it.
                 if (self.drop_whitespace and
                     cur_line and cur_line[-1].strip() == ''):
                     del cur_line[-1]
 
                 # Convert current line back to a string and store it in list
                 # of all lines (return value).
                 if cur_line:
                     lines.append(indent + ''.join(cur_line))
 
             return lines
 
     global MBTextWrapper
     MBTextWrapper = tw
     return tw(**kwargs)
 
 def wrap(line, width, initindent='', hangindent=''):
     maxindent = max(len(hangindent), len(initindent))
     if width <= maxindent:
         # adjust for weird terminal size
         width = max(78, maxindent + 1)
     line = line.decode(pycompat.sysstr(encoding.encoding),
                     pycompat.sysstr(encoding.encodingmode))
     initindent = initindent.decode(pycompat.sysstr(encoding.encoding),
                     pycompat.sysstr(encoding.encodingmode))
     hangindent = hangindent.decode(pycompat.sysstr(encoding.encoding),
                     pycompat.sysstr(encoding.encodingmode))
     wrapper = MBTextWrapper(width=width,
                             initial_indent=initindent,
                             subsequent_indent=hangindent)
     return wrapper.fill(line).encode(pycompat.sysstr(encoding.encoding))
 
 if (pyplatform.python_implementation() == 'CPython' and
     sys.version_info < (3, 0)):
     # There is an issue in CPython that some IO methods do not handle EINTR
     # correctly. The following table shows what CPython version (and functions)
     # are affected (buggy: has the EINTR bug, okay: otherwise):
     #
     #                | < 2.7.4 | 2.7.4 to 2.7.12 | >= 3.0
     #   --------------------------------------------------
     #    fp.__iter__ | buggy   | buggy           | okay
     #    fp.read*    | buggy   | okay [1]        | okay
     #
     # [1]: fixed by changeset 67dc99a989cd in the cpython hg repo.
     #
     # Here we workaround the EINTR issue for fileobj.__iter__. Other methods
     # like "read*" are ignored for now, as Python < 2.7.4 is a minority.
     #
     # Although we can workaround the EINTR issue for fp.__iter__, it is slower:
     # "for x in fp" is 4x faster than "for x in iter(fp.readline, '')" in
     # CPython 2, because CPython 2 maintains an internal readahead buffer for
     # fp.__iter__ but not other fp.read* methods.
     #
     # On modern systems like Linux, the "read" syscall cannot be interrupted
     # when reading "fast" files like on-disk files. So the EINTR issue only
     # affects things like pipes, sockets, ttys etc. We treat "normal" (S_ISREG)
     # files approximately as "fast" files and use the fast (unsafe) code path,
     # to minimize the performance impact.
     if sys.version_info >= (2, 7, 4):
         # fp.readline deals with EINTR correctly, use it as a workaround.
         def _safeiterfile(fp):
             return iter(fp.readline, '')
     else:
         # fp.read* are broken too, manually deal with EINTR in a stupid way.
         # note: this may block longer than necessary because of bufsize.
         def _safeiterfile(fp, bufsize=4096):
             fd = fp.fileno()
             line = ''
             while True:
                 try:
                     buf = os.read(fd, bufsize)
                 except OSError as ex:
                     # os.read only raises EINTR before any data is read
                     if ex.errno == errno.EINTR:
                         continue
                     else:
                         raise
                 line += buf
                 if '\n' in buf:
                     splitted = line.splitlines(True)
                     line = ''
                     for l in splitted:
                         if l[-1] == '\n':
                             yield l
                         else:
                             line = l
                 if not buf:
                     break
             if line:
                 yield line
 
     def iterfile(fp):
         fastpath = True
         if type(fp) is file:
             fastpath = stat.S_ISREG(os.fstat(fp.fileno()).st_mode)
         if fastpath:
             return fp
         else:
             return _safeiterfile(fp)
 else:
     # PyPy and CPython 3 do not have the EINTR issue thus no workaround needed.
     def iterfile(fp):
         return fp
 
 def iterlines(iterator):
     for chunk in iterator:
         for line in chunk.splitlines():
             yield line
 
 def expandpath(path):
     return os.path.expanduser(os.path.expandvars(path))
 
 def hgcmd():
     """Return the command used to execute current hg
 
     This is different from hgexecutable() because on Windows we want
     to avoid things opening new shell windows like batch files, so we
     get either the python call or current executable.
     """
     if mainfrozen():
         if getattr(sys, 'frozen', None) == 'macosx_app':
             # Env variable set by py2app
             return [encoding.environ['EXECUTABLEPATH']]
         else:
             return [pycompat.sysexecutable]
     return gethgcmd()
 
 def rundetached(args, condfn):
     """Execute the argument list in a detached process.
 
     condfn is a callable which is called repeatedly and should return
     True once the child process is known to have started successfully.
     At this point, the child process PID is returned. If the child
     process fails to start or finishes before condfn() evaluates to
     True, return -1.
     """
     # Windows case is easier because the child process is either
     # successfully starting and validating the condition or exiting
     # on failure. We just poll on its PID. On Unix, if the child
     # process fails to start, it will be left in a zombie state until
     # the parent wait on it, which we cannot do since we expect a long
     # running process on success. Instead we listen for SIGCHLD telling
     # us our child process terminated.
     terminated = set()
     def handler(signum, frame):
         terminated.add(os.wait())
     prevhandler = None
     SIGCHLD = getattr(signal, 'SIGCHLD', None)
     if SIGCHLD is not None:
         prevhandler = signal.signal(SIGCHLD, handler)
     try:
         pid = spawndetached(args)
         while not condfn():
             if ((pid in terminated or not testpid(pid))
                 and not condfn()):
                 return -1
             time.sleep(0.1)
         return pid
     finally:
         if prevhandler is not None:
             signal.signal(signal.SIGCHLD, prevhandler)
 
 def interpolate(prefix, mapping, s, fn=None, escape_prefix=False):
     """Return the result of interpolating items in the mapping into string s.
 
     prefix is a single character string, or a two character string with
     a backslash as the first character if the prefix needs to be escaped in
     a regular expression.
 
     fn is an optional function that will be applied to the replacement text
     just before replacement.
 
     escape_prefix is an optional flag that allows using doubled prefix for
     its escaping.
     """
     fn = fn or (lambda s: s)
     patterns = '|'.join(mapping.keys())
     if escape_prefix:
         patterns += '|' + prefix
         if len(prefix) > 1:
             prefix_char = prefix[1:]
         else:
             prefix_char = prefix
         mapping[prefix_char] = prefix_char
     r = remod.compile(r'%s(%s)' % (prefix, patterns))
     return r.sub(lambda x: fn(mapping[x.group()[1:]]), s)
 
 def getport(port):
     """Return the port for a given network service.
 
     If port is an integer, it's returned as is. If it's a string, it's
     looked up using socket.getservbyname(). If there's no matching
     service, error.Abort is raised.
     """
     try:
         return int(port)
     except ValueError:
         pass
 
     try:
         return socket.getservbyname(port)
     except socket.error:
         raise Abort(_("no port number associated with service '%s'") % port)
 
 _booleans = {'1': True, 'yes': True, 'true': True, 'on': True, 'always': True,
              '0': False, 'no': False, 'false': False, 'off': False,
              'never': False}
 
 def parsebool(s):
     """Parse s into a boolean.
 
     If s is not a valid boolean, returns None.
     """
     return _booleans.get(s.lower(), None)
 
 _hextochr = dict((a + b, chr(int(a + b, 16)))
                  for a in string.hexdigits for b in string.hexdigits)
 
 class url(object):
     r"""Reliable URL parser.
 
     This parses URLs and provides attributes for the following
     components:
 
     <scheme>://<user>:<passwd>@<host>:<port>/<path>?<query>#<fragment>
 
     Missing components are set to None. The only exception is
     fragment, which is set to '' if present but empty.
 
     If parsefragment is False, fragment is included in query. If
     parsequery is False, query is included in path. If both are
     False, both fragment and query are included in path.
 
     See http://www.ietf.org/rfc/rfc2396.txt for more information.
 
     Note that for backward compatibility reasons, bundle URLs do not
     take host names. That means 'bundle://../' has a path of '../'.
 
     Examples:
 
     >>> url('http://www.ietf.org/rfc/rfc2396.txt')
     <url scheme: 'http', host: 'www.ietf.org', path: 'rfc/rfc2396.txt'>
     >>> url('ssh://[::1]:2200//home/joe/repo')
     <url scheme: 'ssh', host: '[::1]', port: '2200', path: '/home/joe/repo'>
     >>> url('file:///home/joe/repo')
     <url scheme: 'file', path: '/home/joe/repo'>
     >>> url('file:///c:/temp/foo/')
     <url scheme: 'file', path: 'c:/temp/foo/'>
     >>> url('bundle:foo')
     <url scheme: 'bundle', path: 'foo'>
     >>> url('bundle://../foo')
     <url scheme: 'bundle', path: '../foo'>
     >>> url(r'c:\foo\bar')
     <url path: 'c:\\foo\\bar'>
     >>> url(r'\\blah\blah\blah')
     <url path: '\\\\blah\\blah\\blah'>
     >>> url(r'\\blah\blah\blah#baz')
     <url path: '\\\\blah\\blah\\blah', fragment: 'baz'>
     >>> url(r'file:///C:\users\me')
     <url scheme: 'file', path: 'C:\\users\\me'>
 
     Authentication credentials:
 
     >>> url('ssh://joe:xyz@x/repo')
     <url scheme: 'ssh', user: 'joe', passwd: 'xyz', host: 'x', path: 'repo'>
     >>> url('ssh://joe@x/repo')
     <url scheme: 'ssh', user: 'joe', host: 'x', path: 'repo'>
 
     Query strings and fragments:
 
     >>> url('http://host/a?b#c')
     <url scheme: 'http', host: 'host', path: 'a', query: 'b', fragment: 'c'>
     >>> url('http://host/a?b#c', parsequery=False, parsefragment=False)
     <url scheme: 'http', host: 'host', path: 'a?b#c'>
 
     Empty path:
 
     >>> url('')
     <url path: ''>
     >>> url('#a')
     <url path: '', fragment: 'a'>
     >>> url('http://host/')
     <url scheme: 'http', host: 'host', path: ''>
     >>> url('http://host/#a')
     <url scheme: 'http', host: 'host', path: '', fragment: 'a'>
 
     Only scheme:
 
     >>> url('http:')
     <url scheme: 'http'>
     """
 
     _safechars = "!~*'()+"
     _safepchars = "/!~*'()+:\\"
     _matchscheme = remod.compile('^[a-zA-Z0-9+.\\-]+:').match
 
     def __init__(self, path, parsequery=True, parsefragment=True):
         # We slowly chomp away at path until we have only the path left
         self.scheme = self.user = self.passwd = self.host = None
         self.port = self.path = self.query = self.fragment = None
         self._localpath = True
         self._hostport = ''
         self._origpath = path
 
         if parsefragment and '#' in path:
             path, self.fragment = path.split('#', 1)
 
         # special case for Windows drive letters and UNC paths
         if hasdriveletter(path) or path.startswith('\\\\'):
             self.path = path
             return
 
         # For compatibility reasons, we can't handle bundle paths as
         # normal URLS
         if path.startswith('bundle:'):
             self.scheme = 'bundle'
             path = path[7:]
             if path.startswith('//'):
                 path = path[2:]
             self.path = path
             return
 
         if self._matchscheme(path):
             parts = path.split(':', 1)
             if parts[0]:
                 self.scheme, path = parts
                 self._localpath = False
 
         if not path:
             path = None
             if self._localpath:
                 self.path = ''
                 return
         else:
             if self._localpath:
                 self.path = path
                 return
 
             if parsequery and '?' in path:
                 path, self.query = path.split('?', 1)
                 if not path:
                     path = None
                 if not self.query:
                     self.query = None
 
             # // is required to specify a host/authority
             if path and path.startswith('//'):
                 parts = path[2:].split('/', 1)
                 if len(parts) > 1:
                     self.host, path = parts
                 else:
                     self.host = parts[0]
                     path = None
                 if not self.host:
                     self.host = None
                     # path of file:///d is /d
                     # path of file:///d:/ is d:/, not /d:/
                     if path and not hasdriveletter(path):
                         path = '/' + path
 
             if self.host and '@' in self.host:
                 self.user, self.host = self.host.rsplit('@', 1)
                 if ':' in self.user:
                     self.user, self.passwd = self.user.split(':', 1)
                 if not self.host:
                     self.host = None
 
             # Don't split on colons in IPv6 addresses without ports
             if (self.host and ':' in self.host and
                 not (self.host.startswith('[') and self.host.endswith(']'))):
                 self._hostport = self.host
                 self.host, self.port = self.host.rsplit(':', 1)
                 if not self.host:
                     self.host = None
 
             if (self.host and self.scheme == 'file' and
                 self.host not in ('localhost', '127.0.0.1', '[::1]')):
                 raise Abort(_('file:// URLs can only refer to localhost'))
 
         self.path = path
 
         # leave the query string escaped
         for a in ('user', 'passwd', 'host', 'port',
                   'path', 'fragment'):
             v = getattr(self, a)
             if v is not None:
                 setattr(self, a, urlreq.unquote(v))
 
     def __repr__(self):
         attrs = []
         for a in ('scheme', 'user', 'passwd', 'host', 'port', 'path',
                   'query', 'fragment'):
             v = getattr(self, a)
             if v is not None:
                 attrs.append('%s: %r' % (a, v))
         return '<url %s>' % ', '.join(attrs)
 
     def __str__(self):
         r"""Join the URL's components back into a URL string.
 
         Examples:
 
         >>> str(url('http://user:pw@host:80/c:/bob?fo:oo#ba:ar'))
         'http://user:pw@host:80/c:/bob?fo:oo#ba:ar'
         >>> str(url('http://user:pw@host:80/?foo=bar&baz=42'))
         'http://user:pw@host:80/?foo=bar&baz=42'
         >>> str(url('http://user:pw@host:80/?foo=bar%3dbaz'))
         'http://user:pw@host:80/?foo=bar%3dbaz'
         >>> str(url('ssh://user:pw@[::1]:2200//home/joe#'))
         'ssh://user:pw@[::1]:2200//home/joe#'
         >>> str(url('http://localhost:80//'))
         'http://localhost:80//'
         >>> str(url('http://localhost:80/'))
         'http://localhost:80/'
         >>> str(url('http://localhost:80'))
         'http://localhost:80/'
         >>> str(url('bundle:foo'))
         'bundle:foo'
         >>> str(url('bundle://../foo'))
         'bundle:../foo'
         >>> str(url('path'))
         'path'
         >>> str(url('file:///tmp/foo/bar'))
         'file:///tmp/foo/bar'
         >>> str(url('file:///c:/tmp/foo/bar'))
         'file:///c:/tmp/foo/bar'
         >>> print url(r'bundle:foo\bar')
         bundle:foo\bar
         >>> print url(r'file:///D:\data\hg')
         file:///D:\data\hg
         """
         return encoding.strfromlocal(self.__bytes__())
 
     def __bytes__(self):
         if self._localpath:
             s = self.path
             if self.scheme == 'bundle':
                 s = 'bundle:' + s
             if self.fragment:
                 s += '#' + self.fragment
             return s
 
         s = self.scheme + ':'
         if self.user or self.passwd or self.host:
             s += '//'
         elif self.scheme and (not self.path or self.path.startswith('/')
                               or hasdriveletter(self.path)):
             s += '//'
             if hasdriveletter(self.path):
                 s += '/'
         if self.user:
             s += urlreq.quote(self.user, safe=self._safechars)
         if self.passwd:
             s += ':' + urlreq.quote(self.passwd, safe=self._safechars)
         if self.user or self.passwd:
             s += '@'
         if self.host:
             if not (self.host.startswith('[') and self.host.endswith(']')):
                 s += urlreq.quote(self.host)
             else:
                 s += self.host
         if self.port:
             s += ':' + urlreq.quote(self.port)
         if self.host:
             s += '/'
         if self.path:
             # TODO: similar to the query string, we should not unescape the
             # path when we store it, the path might contain '%2f' = '/',
             # which we should *not* escape.
             s += urlreq.quote(self.path, safe=self._safepchars)
         if self.query:
             # we store the query in escaped form.
             s += '?' + self.query
         if self.fragment is not None:
             s += '#' + urlreq.quote(self.fragment, safe=self._safepchars)
         return s
 
     def authinfo(self):
         user, passwd = self.user, self.passwd
         try:
             self.user, self.passwd = None, None
             s = bytes(self)
         finally:
             self.user, self.passwd = user, passwd
         if not self.user:
             return (s, None)
         # authinfo[1] is passed to urllib2 password manager, and its
         # URIs must not contain credentials. The host is passed in the
         # URIs list because Python < 2.4.3 uses only that to search for
         # a password.
         return (s, (None, (s, self.host),
                     self.user, self.passwd or ''))
 
     def isabs(self):
         if self.scheme and self.scheme != 'file':
             return True # remote URL
         if hasdriveletter(self.path):
             return True # absolute for our purposes - can't be joined()
         if self.path.startswith(r'\\'):
             return True # Windows UNC path
         if self.path.startswith('/'):
             return True # POSIX-style
         return False
 
     def localpath(self):
         if self.scheme == 'file' or self.scheme == 'bundle':
             path = self.path or '/'
             # For Windows, we need to promote hosts containing drive
             # letters to paths with drive letters.
             if hasdriveletter(self._hostport):
                 path = self._hostport + '/' + self.path
             elif (self.host is not None and self.path
                   and not hasdriveletter(path)):
                 path = '/' + path
             return path
         return self._origpath
 
     def islocal(self):
         '''whether localpath will return something that posixfile can open'''
         return (not self.scheme or self.scheme == 'file'
                 or self.scheme == 'bundle')
 
 def hasscheme(path):
     return bool(url(path).scheme)
 
 def hasdriveletter(path):
     return path and path[1:2] == ':' and path[0:1].isalpha()
 
 def urllocalpath(path):
     return url(path, parsequery=False, parsefragment=False).localpath()
 
 def hidepassword(u):
     '''hide user credential in a url string'''
     u = url(u)
     if u.passwd:
         u.passwd = '***'
     return bytes(u)
 
 def removeauth(u):
     '''remove all authentication information from a url string'''
     u = url(u)
     u.user = u.passwd = None
     return str(u)
 
 timecount = unitcountfn(
     (1, 1e3, _('%.0f s')),
     (100, 1, _('%.1f s')),
     (10, 1, _('%.2f s')),
     (1, 1, _('%.3f s')),
     (100, 0.001, _('%.1f ms')),
     (10, 0.001, _('%.2f ms')),
     (1, 0.001, _('%.3f ms')),
     (100, 0.000001, _('%.1f us')),
     (10, 0.000001, _('%.2f us')),
     (1, 0.000001, _('%.3f us')),
     (100, 0.000000001, _('%.1f ns')),
     (10, 0.000000001, _('%.2f ns')),
     (1, 0.000000001, _('%.3f ns')),
     )
 
 _timenesting = [0]
 
 def timed(func):
     '''Report the execution time of a function call to stderr.
 
     During development, use as a decorator when you need to measure
     the cost of a function, e.g. as follows:
 
     @util.timed
     def foo(a, b, c):
         pass
     '''
 
     def wrapper(*args, **kwargs):
         start = timer()
         indent = 2
         _timenesting[0] += indent
         try:
             return func(*args, **kwargs)
         finally:
             elapsed = timer() - start
             _timenesting[0] -= indent
             stderr.write('%s%s: %s\n' %
                          (' ' * _timenesting[0], func.__name__,
                           timecount(elapsed)))
     return wrapper
 
 _sizeunits = (('m', 2**20), ('k', 2**10), ('g', 2**30),
               ('kb', 2**10), ('mb', 2**20), ('gb', 2**30), ('b', 1))
 
 def sizetoint(s):
     '''Convert a space specifier to a byte count.
 
     >>> sizetoint('30')
     30
     >>> sizetoint('2.2kb')
     2252
     >>> sizetoint('6M')
     6291456
     '''
     t = s.strip().lower()
     try:
         for k, u in _sizeunits:
             if t.endswith(k):
                 return int(float(t[:-len(k)]) * u)
         return int(t)
     except ValueError:
         raise error.ParseError(_("couldn't parse size: %s") % s)
 
 class hooks(object):
     '''A collection of hook functions that can be used to extend a
     function's behavior. Hooks are called in lexicographic order,
     based on the names of their sources.'''
 
     def __init__(self):
         self._hooks = []
 
     def add(self, source, hook):
         self._hooks.append((source, hook))
 
     def __call__(self, *args):
         self._hooks.sort(key=lambda x: x[0])
         results = []
         for source, hook in self._hooks:
             results.append(hook(*args))
         return results
 
 def getstackframes(skip=0, line=' %-*s in %s\n', fileline='%s:%s', depth=0):
     '''Yields lines for a nicely formatted stacktrace.
     Skips the 'skip' last entries, then return the last 'depth' entries.
     Each file+linenumber is formatted according to fileline.
     Each line is formatted according to line.
     If line is None, it yields:
       length of longest filepath+line number,
       filepath+linenumber,
       function
 
     Not be used in production code but very convenient while developing.
     '''
     entries = [(fileline % (fn, ln), func)
         for fn, ln, func, _text in traceback.extract_stack()[:-skip - 1]
         ][-depth:]
     if entries:
         fnmax = max(len(entry[0]) for entry in entries)
         for fnln, func in entries:
             if line is None:
                 yield (fnmax, fnln, func)
             else:
                 yield line % (fnmax, fnln, func)
 
 def debugstacktrace(msg='stacktrace', skip=0,
                     f=stderr, otherf=stdout, depth=0):
     '''Writes a message to f (stderr) with a nicely formatted stacktrace.
     Skips the 'skip' entries closest to the call, then show 'depth' entries.
     By default it will flush stdout first.
     It can be used everywhere and intentionally does not require an ui object.
     Not be used in production code but very convenient while developing.
     '''
     if otherf:
         otherf.flush()
     f.write('%s at:\n' % msg.rstrip())
     for line in getstackframes(skip + 1, depth=depth):
         f.write(line)
     f.flush()
 
 class dirs(object):
     '''a multiset of directory names from a dirstate or manifest'''
 
     def __init__(self, map, skip=None):
         self._dirs = {}
         addpath = self.addpath
         if safehasattr(map, 'iteritems') and skip is not None:
             for f, s in map.iteritems():
                 if s[0] != skip:
                     addpath(f)
         else:
             for f in map:
                 addpath(f)
 
     def addpath(self, path):
         dirs = self._dirs
         for base in finddirs(path):
             if base in dirs:
                 dirs[base] += 1
                 return
             dirs[base] = 1
 
     def delpath(self, path):
         dirs = self._dirs
         for base in finddirs(path):
             if dirs[base] > 1:
                 dirs[base] -= 1
                 return
             del dirs[base]
 
     def __iter__(self):
         return iter(self._dirs)
 
     def __contains__(self, d):
         return d in self._dirs
 
 if safehasattr(parsers, 'dirs'):
     dirs = parsers.dirs
 
 def finddirs(path):
     pos = path.rfind('/')
     while pos != -1:
         yield path[:pos]
         pos = path.rfind('/', 0, pos)
 
 class ctxmanager(object):
     '''A context manager for use in 'with' blocks to allow multiple
     contexts to be entered at once.  This is both safer and more
     flexible than contextlib.nested.
 
     Once Mercurial supports Python 2.7+, this will become mostly
     unnecessary.
     '''
 
     def __init__(self, *args):
         '''Accepts a list of no-argument functions that return context
         managers.  These will be invoked at __call__ time.'''
         self._pending = args
         self._atexit = []
 
     def __enter__(self):
         return self
 
     def enter(self):
         '''Create and enter context managers in the order in which they were
         passed to the constructor.'''
         values = []
         for func in self._pending:
             obj = func()
             values.append(obj.__enter__())
             self._atexit.append(obj.__exit__)
         del self._pending
         return values
 
     def atexit(self, func, *args, **kwargs):
         '''Add a function to call when this context manager exits.  The
         ordering of multiple atexit calls is unspecified, save that
         they will happen before any __exit__ functions.'''
         def wrapper(exc_type, exc_val, exc_tb):
             func(*args, **kwargs)
         self._atexit.append(wrapper)
         return func
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         '''Context managers are exited in the reverse order from which
         they were created.'''
         received = exc_type is not None
         suppressed = False
         pending = None
         self._atexit.reverse()
         for exitfunc in self._atexit:
             try:
                 if exitfunc(exc_type, exc_val, exc_tb):
                     suppressed = True
                     exc_type = None
                     exc_val = None
                     exc_tb = None
             except BaseException:
                 pending = sys.exc_info()
                 exc_type, exc_val, exc_tb = pending = sys.exc_info()
         del self._atexit
         if pending:
             raise exc_val
         return received and suppressed
 
 # compression code
 
 SERVERROLE = 'server'
 CLIENTROLE = 'client'
 
 compewireprotosupport = collections.namedtuple(u'compenginewireprotosupport',
                                                (u'name', u'serverpriority',
                                                 u'clientpriority'))
 
 class compressormanager(object):
     """Holds registrations of various compression engines.
 
     This class essentially abstracts the differences between compression
     engines to allow new compression formats to be added easily, possibly from
     extensions.
 
     Compressors are registered against the global instance by calling its
     ``register()`` method.
     """
     def __init__(self):
         self._engines = {}
         # Bundle spec human name to engine name.
         self._bundlenames = {}
         # Internal bundle identifier to engine name.
         self._bundletypes = {}
         # Revlog header to engine name.
         self._revlogheaders = {}
         # Wire proto identifier to engine name.
         self._wiretypes = {}
 
     def __getitem__(self, key):
         return self._engines[key]
 
     def __contains__(self, key):
         return key in self._engines
 
     def __iter__(self):
         return iter(self._engines.keys())
 
     def register(self, engine):
         """Register a compression engine with the manager.
 
         The argument must be a ``compressionengine`` instance.
         """
         if not isinstance(engine, compressionengine):
             raise ValueError(_('argument must be a compressionengine'))
 
         name = engine.name()
 
         if name in self._engines:
             raise error.Abort(_('compression engine %s already registered') %
                               name)
 
         bundleinfo = engine.bundletype()
         if bundleinfo:
             bundlename, bundletype = bundleinfo
 
             if bundlename in self._bundlenames:
                 raise error.Abort(_('bundle name %s already registered') %
                                   bundlename)
             if bundletype in self._bundletypes:
                 raise error.Abort(_('bundle type %s already registered by %s') %
                                   (bundletype, self._bundletypes[bundletype]))
 
             # No external facing name declared.
             if bundlename:
                 self._bundlenames[bundlename] = name
 
             self._bundletypes[bundletype] = name
 
         wiresupport = engine.wireprotosupport()
         if wiresupport:
             wiretype = wiresupport.name
             if wiretype in self._wiretypes:
                 raise error.Abort(_('wire protocol compression %s already '
                                     'registered by %s') %
                                   (wiretype, self._wiretypes[wiretype]))
 
             self._wiretypes[wiretype] = name
 
         revlogheader = engine.revlogheader()
         if revlogheader and revlogheader in self._revlogheaders:
             raise error.Abort(_('revlog header %s already registered by %s') %
                               (revlogheader, self._revlogheaders[revlogheader]))
 
         if revlogheader:
             self._revlogheaders[revlogheader] = name
 
         self._engines[name] = engine
 
     @property
     def supportedbundlenames(self):
         return set(self._bundlenames.keys())
 
     @property
     def supportedbundletypes(self):
         return set(self._bundletypes.keys())
 
     def forbundlename(self, bundlename):
         """Obtain a compression engine registered to a bundle name.
 
         Will raise KeyError if the bundle type isn't registered.
 
         Will abort if the engine is known but not available.
         """
         engine = self._engines[self._bundlenames[bundlename]]
         if not engine.available():
             raise error.Abort(_('compression engine %s could not be loaded') %
                               engine.name())
         return engine
 
     def forbundletype(self, bundletype):
         """Obtain a compression engine registered to a bundle type.
 
         Will raise KeyError if the bundle type isn't registered.
 
         Will abort if the engine is known but not available.
         """
         engine = self._engines[self._bundletypes[bundletype]]
         if not engine.available():
             raise error.Abort(_('compression engine %s could not be loaded') %
                               engine.name())
         return engine
 
     def supportedwireengines(self, role, onlyavailable=True):
         """Obtain compression engines that support the wire protocol.
 
         Returns a list of engines in prioritized order, most desired first.
 
         If ``onlyavailable`` is set, filter out engines that can't be
         loaded.
         """
         assert role in (SERVERROLE, CLIENTROLE)
 
         attr = 'serverpriority' if role == SERVERROLE else 'clientpriority'
 
         engines = [self._engines[e] for e in self._wiretypes.values()]
         if onlyavailable:
             engines = [e for e in engines if e.available()]
 
         def getkey(e):
             # Sort first by priority, highest first. In case of tie, sort
             # alphabetically. This is arbitrary, but ensures output is
             # stable.
             w = e.wireprotosupport()
             return -1 * getattr(w, attr), w.name
 
         return list(sorted(engines, key=getkey))
 
     def forwiretype(self, wiretype):
         engine = self._engines[self._wiretypes[wiretype]]
         if not engine.available():
             raise error.Abort(_('compression engine %s could not be loaded') %
                               engine.name())
         return engine
 
     def forrevlogheader(self, header):
         """Obtain a compression engine registered to a revlog header.
 
         Will raise KeyError if the revlog header value isn't registered.
         """
         return self._engines[self._revlogheaders[header]]
 
 compengines = compressormanager()
 
 class compressionengine(object):
     """Base class for compression engines.
 
     Compression engines must implement the interface defined by this class.
     """
     def name(self):
         """Returns the name of the compression engine.
 
         This is the key the engine is registered under.
 
         This method must be implemented.
         """
         raise NotImplementedError()
 
     def available(self):
         """Whether the compression engine is available.
 
         The intent of this method is to allow optional compression engines
         that may not be available in all installations (such as engines relying
         on C extensions that may not be present).
         """
         return True
 
     def bundletype(self):
         """Describes bundle identifiers for this engine.
 
         If this compression engine isn't supported for bundles, returns None.
 
         If this engine can be used for bundles, returns a 2-tuple of strings of
         the user-facing "bundle spec" compression name and an internal
         identifier used to denote the compression format within bundles. To
         exclude the name from external usage, set the first element to ``None``.
 
         If bundle compression is supported, the class must also implement
         ``compressstream`` and `decompressorreader``.
 
         The docstring of this method is used in the help system to tell users
         about this engine.
         """
         return None
 
     def wireprotosupport(self):
         """Declare support for this compression format on the wire protocol.
 
         If this compression engine isn't supported for compressing wire
         protocol payloads, returns None.
 
         Otherwise, returns ``compenginewireprotosupport`` with the following
         fields:
 
         * String format identifier
         * Integer priority for the server
         * Integer priority for the client
 
         The integer priorities are used to order the advertisement of format
         support by server and client. The highest integer is advertised
         first. Integers with non-positive values aren't advertised.
 
         The priority values are somewhat arbitrary and only used for default
         ordering. The relative order can be changed via config options.
 
         If wire protocol compression is supported, the class must also implement
         ``compressstream`` and ``decompressorreader``.
         """
         return None
 
     def revlogheader(self):
         """Header added to revlog chunks that identifies this engine.
 
         If this engine can be used to compress revlogs, this method should
         return the bytes used to identify chunks compressed with this engine.
         Else, the method should return ``None`` to indicate it does not
         participate in revlog compression.
         """
         return None
 
     def compressstream(self, it, opts=None):
         """Compress an iterator of chunks.
 
         The method receives an iterator (ideally a generator) of chunks of
         bytes to be compressed. It returns an iterator (ideally a generator)
         of bytes of chunks representing the compressed output.
 
         Optionally accepts an argument defining how to perform compression.
         Each engine treats this argument differently.
         """
         raise NotImplementedError()
 
     def decompressorreader(self, fh):
         """Perform decompression on a file object.
 
         Argument is an object with a ``read(size)`` method that returns
         compressed data. Return value is an object with a ``read(size)`` that
         returns uncompressed data.
         """
         raise NotImplementedError()
 
     def revlogcompressor(self, opts=None):
         """Obtain an object that can be used to compress revlog entries.
 
         The object has a ``compress(data)`` method that compresses binary
         data. This method returns compressed binary data or ``None`` if
         the data could not be compressed (too small, not compressible, etc).
         The returned data should have a header uniquely identifying this
         compression format so decompression can be routed to this engine.
         This header should be identified by the ``revlogheader()`` return
         value.
 
         The object has a ``decompress(data)`` method that decompresses
         data. The method will only be called if ``data`` begins with
         ``revlogheader()``. The method should return the raw, uncompressed
         data or raise a ``RevlogError``.
 
         The object is reusable but is not thread safe.
         """
         raise NotImplementedError()
 
 class _zlibengine(compressionengine):
     def name(self):
         return 'zlib'
 
     def bundletype(self):
         """zlib compression using the DEFLATE algorithm.
 
         All Mercurial clients should support this format. The compression
         algorithm strikes a reasonable balance between compression ratio
         and size.
         """
         return 'gzip', 'GZ'
 
     def wireprotosupport(self):
         return compewireprotosupport('zlib', 20, 20)
 
     def revlogheader(self):
         return 'x'
 
     def compressstream(self, it, opts=None):
         opts = opts or {}
 
         z = zlib.compressobj(opts.get('level', -1))
         for chunk in it:
             data = z.compress(chunk)
             # Not all calls to compress emit data. It is cheaper to inspect
             # here than to feed empty chunks through generator.
             if data:
                 yield data
 
         yield z.flush()
 
     def decompressorreader(self, fh):
         def gen():
             d = zlib.decompressobj()
             for chunk in filechunkiter(fh):
                 while chunk:
                     # Limit output size to limit memory.
                     yield d.decompress(chunk, 2 ** 18)
                     chunk = d.unconsumed_tail
 
         return chunkbuffer(gen())
 
     class zlibrevlogcompressor(object):
         def compress(self, data):
             insize = len(data)
             # Caller handles empty input case.
             assert insize > 0
 
             if insize < 44:
                 return None
 
             elif insize <= 1000000:
                 compressed = zlib.compress(data)
                 if len(compressed) < insize:
                     return compressed
                 return None
 
             # zlib makes an internal copy of the input buffer, doubling
             # memory usage for large inputs. So do streaming compression
             # on large inputs.
             else:
                 z = zlib.compressobj()
                 parts = []
                 pos = 0
                 while pos < insize:
                     pos2 = pos + 2**20
                     parts.append(z.compress(data[pos:pos2]))
                     pos = pos2
                 parts.append(z.flush())
 
                 if sum(map(len, parts)) < insize:
                     return ''.join(parts)
                 return None
 
         def decompress(self, data):
             try:
                 return zlib.decompress(data)
             except zlib.error as e:
                 raise error.RevlogError(_('revlog decompress error: %s') %
                                         str(e))
 
     def revlogcompressor(self, opts=None):
         return self.zlibrevlogcompressor()
 
 compengines.register(_zlibengine())
 
 class _bz2engine(compressionengine):
     def name(self):
         return 'bz2'
 
     def bundletype(self):
         """An algorithm that produces smaller bundles than ``gzip``.
 
         All Mercurial clients should support this format.
 
         This engine will likely produce smaller bundles than ``gzip`` but
         will be significantly slower, both during compression and
         decompression.
 
         If available, the ``zstd`` engine can yield similar or better
         compression at much higher speeds.
         """
         return 'bzip2', 'BZ'
 
     # We declare a protocol name but don't advertise by default because
     # it is slow.
     def wireprotosupport(self):
         return compewireprotosupport('bzip2', 0, 0)
 
     def compressstream(self, it, opts=None):
         opts = opts or {}
         z = bz2.BZ2Compressor(opts.get('level', 9))
         for chunk in it:
             data = z.compress(chunk)
             if data:
                 yield data
 
         yield z.flush()
 
     def decompressorreader(self, fh):
         def gen():
             d = bz2.BZ2Decompressor()
             for chunk in filechunkiter(fh):
                 yield d.decompress(chunk)
 
         return chunkbuffer(gen())
 
 compengines.register(_bz2engine())
 
 class _truncatedbz2engine(compressionengine):
     def name(self):
         return 'bz2truncated'
 
     def bundletype(self):
         return None, '_truncatedBZ'
 
     # We don't implement compressstream because it is hackily handled elsewhere.
 
     def decompressorreader(self, fh):
         def gen():
             # The input stream doesn't have the 'BZ' header. So add it back.
             d = bz2.BZ2Decompressor()
             d.decompress('BZ')
             for chunk in filechunkiter(fh):
                 yield d.decompress(chunk)
 
         return chunkbuffer(gen())
 
 compengines.register(_truncatedbz2engine())
 
 class _noopengine(compressionengine):
     def name(self):
         return 'none'
 
     def bundletype(self):
         """No compression is performed.
 
         Use this compression engine to explicitly disable compression.
         """
         return 'none', 'UN'
 
     # Clients always support uncompressed payloads. Servers don't because
     # unless you are on a fast network, uncompressed payloads can easily
     # saturate your network pipe.
     def wireprotosupport(self):
         return compewireprotosupport('none', 0, 10)
 
     # We don't implement revlogheader because it is handled specially
     # in the revlog class.
 
     def compressstream(self, it, opts=None):
         return it
 
     def decompressorreader(self, fh):
         return fh
 
     class nooprevlogcompressor(object):
         def compress(self, data):
             return None
 
     def revlogcompressor(self, opts=None):
         return self.nooprevlogcompressor()
 
 compengines.register(_noopengine())
 
 class _zstdengine(compressionengine):
     def name(self):
         return 'zstd'
 
     @propertycache
     def _module(self):
         # Not all installs have the zstd module available. So defer importing
         # until first access.
         try:
             from . import zstd
             # Force delayed import.
             zstd.__version__
             return zstd
         except ImportError:
             return None
 
     def available(self):
         return bool(self._module)
 
     def bundletype(self):
         """A modern compression algorithm that is fast and highly flexible.
 
         Only supported by Mercurial 4.1 and newer clients.
 
         With the default settings, zstd compression is both faster and yields
         better compression than ``gzip``. It also frequently yields better
         compression than ``bzip2`` while operating at much higher speeds.
 
         If this engine is available and backwards compatibility is not a
         concern, it is likely the best available engine.
         """
         return 'zstd', 'ZS'
 
     def wireprotosupport(self):
         return compewireprotosupport('zstd', 50, 50)
 
     def revlogheader(self):
         return '\x28'
 
     def compressstream(self, it, opts=None):
         opts = opts or {}
         # zstd level 3 is almost always significantly faster than zlib
         # while providing no worse compression. It strikes a good balance
         # between speed and compression.
         level = opts.get('level', 3)
 
         zstd = self._module
         z = zstd.ZstdCompressor(level=level).compressobj()
         for chunk in it:
             data = z.compress(chunk)
             if data:
                 yield data
 
         yield z.flush()
 
     def decompressorreader(self, fh):
         zstd = self._module
         dctx = zstd.ZstdDecompressor()
         return chunkbuffer(dctx.read_from(fh))
 
     class zstdrevlogcompressor(object):
         def __init__(self, zstd, level=3):
             # Writing the content size adds a few bytes to the output. However,
             # it allows decompression to be more optimal since we can
             # pre-allocate a buffer to hold the result.
             self._cctx = zstd.ZstdCompressor(level=level,
                                              write_content_size=True)
             self._dctx = zstd.ZstdDecompressor()
             self._compinsize = zstd.COMPRESSION_RECOMMENDED_INPUT_SIZE
             self._decompinsize = zstd.DECOMPRESSION_RECOMMENDED_INPUT_SIZE
 
         def compress(self, data):
             insize = len(data)
             # Caller handles empty input case.
             assert insize > 0
 
             if insize < 50:
                 return None
 
             elif insize <= 1000000:
                 compressed = self._cctx.compress(data)
                 if len(compressed) < insize:
                     return compressed
                 return None
             else:
                 z = self._cctx.compressobj()
                 chunks = []
                 pos = 0
                 while pos < insize:
                     pos2 = pos + self._compinsize
                     chunk = z.compress(data[pos:pos2])
                     if chunk:
                         chunks.append(chunk)
                     pos = pos2
                 chunks.append(z.flush())
 
                 if sum(map(len, chunks)) < insize:
                     return ''.join(chunks)
                 return None
 
         def decompress(self, data):
             insize = len(data)
 
             try:
                 # This was measured to be faster than other streaming
                 # decompressors.
                 dobj = self._dctx.decompressobj()
                 chunks = []
                 pos = 0
                 while pos < insize:
                     pos2 = pos + self._decompinsize
                     chunk = dobj.decompress(data[pos:pos2])
                     if chunk:
                         chunks.append(chunk)
                     pos = pos2
                 # Frame should be exhausted, so no finish() API.
 
                 return ''.join(chunks)
             except Exception as e:
                 raise error.RevlogError(_('revlog decompress error: %s') %
                                         str(e))
 
     def revlogcompressor(self, opts=None):
         opts = opts or {}
         return self.zstdrevlogcompressor(self._module,
                                          level=opts.get('level', 3))
 
 compengines.register(_zstdengine())
 
 def bundlecompressiontopics():
     """Obtains a list of available bundle compressions for use in help."""
     # help.makeitemsdocs() expects a dict of names to items with a .__doc__.
     items = {}
 
     # We need to format the docstring. So use a dummy object/type to hold it
     # rather than mutating the original.
     class docobject(object):
         pass
 
     for name in compengines:
         engine = compengines[name]
 
         if not engine.available():
             continue
 
         bt = engine.bundletype()
         if not bt or not bt[0]:
             continue
 
         doc = pycompat.sysstr('``%s``\n    %s') % (
             bt[0], engine.bundletype.__doc__)
 
         value = docobject()
         value.__doc__ = doc
 
         items[bt[0]] = value
 
     return items
 
 # convenient shortcut
 dst = debugstacktrace

mercurial/vfs.py

@@ -1,637 +1,636
 # vfs.py - Mercurial 'vfs' classes
 #
 #  Copyright Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 from __future__ import absolute_import
 
 import contextlib
 import errno
 import os
 import shutil
 import stat
 import tempfile
 import threading
 
 from .i18n import _
 from . import (
     error,
-    osutil,
     pathutil,
     pycompat,
     util,
 )
 
 class abstractvfs(object):
     """Abstract base class; cannot be instantiated"""
 
     def __init__(self, *args, **kwargs):
         '''Prevent instantiation; don't call this from subclasses.'''
         raise NotImplementedError('attempted instantiating ' + str(type(self)))
 
     def tryread(self, path):
         '''gracefully return an empty string for missing files'''
         try:
             return self.read(path)
         except IOError as inst:
             if inst.errno != errno.ENOENT:
                 raise
         return ""
 
     def tryreadlines(self, path, mode='rb'):
         '''gracefully return an empty array for missing files'''
         try:
             return self.readlines(path, mode=mode)
         except IOError as inst:
             if inst.errno != errno.ENOENT:
                 raise
         return []
 
     @util.propertycache
     def open(self):
         '''Open ``path`` file, which is relative to vfs root.
 
         Newly created directories are marked as "not to be indexed by
         the content indexing service", if ``notindexed`` is specified
         for "write" mode access.
         '''
         return self.__call__
 
     def read(self, path):
         with self(path, 'rb') as fp:
             return fp.read()
 
     def readlines(self, path, mode='rb'):
         with self(path, mode=mode) as fp:
             return fp.readlines()
 
     def write(self, path, data, backgroundclose=False):
         with self(path, 'wb', backgroundclose=backgroundclose) as fp:
             return fp.write(data)
 
     def writelines(self, path, data, mode='wb', notindexed=False):
         with self(path, mode=mode, notindexed=notindexed) as fp:
             return fp.writelines(data)
 
     def append(self, path, data):
         with self(path, 'ab') as fp:
             return fp.write(data)
 
     def basename(self, path):
         """return base element of a path (as os.path.basename would do)
 
         This exists to allow handling of strange encoding if needed."""
         return os.path.basename(path)
 
     def chmod(self, path, mode):
         return os.chmod(self.join(path), mode)
 
     def dirname(self, path):
         """return dirname element of a path (as os.path.dirname would do)
 
         This exists to allow handling of strange encoding if needed."""
         return os.path.dirname(path)
 
     def exists(self, path=None):
         return os.path.exists(self.join(path))
 
     def fstat(self, fp):
         return util.fstat(fp)
 
     def isdir(self, path=None):
         return os.path.isdir(self.join(path))
 
     def isfile(self, path=None):
         return os.path.isfile(self.join(path))
 
     def islink(self, path=None):
         return os.path.islink(self.join(path))
 
     def isfileorlink(self, path=None):
         '''return whether path is a regular file or a symlink
 
         Unlike isfile, this doesn't follow symlinks.'''
         try:
             st = self.lstat(path)
         except OSError:
             return False
         mode = st.st_mode
         return stat.S_ISREG(mode) or stat.S_ISLNK(mode)
 
     def reljoin(self, *paths):
         """join various elements of a path together (as os.path.join would do)
 
         The vfs base is not injected so that path stay relative. This exists
         to allow handling of strange encoding if needed."""
         return os.path.join(*paths)
 
     def split(self, path):
         """split top-most element of a path (as os.path.split would do)
 
         This exists to allow handling of strange encoding if needed."""
         return os.path.split(path)
 
     def lexists(self, path=None):
         return os.path.lexists(self.join(path))
 
     def lstat(self, path=None):
         return os.lstat(self.join(path))
 
     def listdir(self, path=None):
         return os.listdir(self.join(path))
 
     def makedir(self, path=None, notindexed=True):
         return util.makedir(self.join(path), notindexed)
 
     def makedirs(self, path=None, mode=None):
         return util.makedirs(self.join(path), mode)
 
     def makelock(self, info, path):
         return util.makelock(info, self.join(path))
 
     def mkdir(self, path=None):
         return os.mkdir(self.join(path))
 
     def mkstemp(self, suffix='', prefix='tmp', dir=None, text=False):
         fd, name = tempfile.mkstemp(suffix=suffix, prefix=prefix,
                                     dir=self.join(dir), text=text)
         dname, fname = util.split(name)
         if dir:
             return fd, os.path.join(dir, fname)
         else:
             return fd, fname
 
     def readdir(self, path=None, stat=None, skip=None):
-        return osutil.listdir(self.join(path), stat, skip)
+        return util.listdir(self.join(path), stat, skip)
 
     def readlock(self, path):
         return util.readlock(self.join(path))
 
     def rename(self, src, dst, checkambig=False):
         """Rename from src to dst
 
         checkambig argument is used with util.filestat, and is useful
         only if destination file is guarded by any lock
         (e.g. repo.lock or repo.wlock).
         """
         dstpath = self.join(dst)
         oldstat = checkambig and util.filestat(dstpath)
         if oldstat and oldstat.stat:
             ret = util.rename(self.join(src), dstpath)
             newstat = util.filestat(dstpath)
             if newstat.isambig(oldstat):
                 # stat of renamed file is ambiguous to original one
                 newstat.avoidambig(dstpath, oldstat)
             return ret
         return util.rename(self.join(src), dstpath)
 
     def readlink(self, path):
         return os.readlink(self.join(path))
 
     def removedirs(self, path=None):
         """Remove a leaf directory and all empty intermediate ones
         """
         return util.removedirs(self.join(path))
 
     def rmtree(self, path=None, ignore_errors=False, forcibly=False):
         """Remove a directory tree recursively
 
         If ``forcibly``, this tries to remove READ-ONLY files, too.
         """
         if forcibly:
             def onerror(function, path, excinfo):
                 if function is not os.remove:
                     raise
                 # read-only files cannot be unlinked under Windows
                 s = os.stat(path)
                 if (s.st_mode & stat.S_IWRITE) != 0:
                     raise
                 os.chmod(path, stat.S_IMODE(s.st_mode) | stat.S_IWRITE)
                 os.remove(path)
         else:
             onerror = None
         return shutil.rmtree(self.join(path),
                              ignore_errors=ignore_errors, onerror=onerror)
 
     def setflags(self, path, l, x):
         return util.setflags(self.join(path), l, x)
 
     def stat(self, path=None):
         return os.stat(self.join(path))
 
     def unlink(self, path=None):
         return util.unlink(self.join(path))
 
     def tryunlink(self, path=None):
         """Attempt to remove a file, ignoring missing file errors."""
         util.tryunlink(self.join(path))
 
     def unlinkpath(self, path=None, ignoremissing=False):
         return util.unlinkpath(self.join(path), ignoremissing=ignoremissing)
 
     def utime(self, path=None, t=None):
         return os.utime(self.join(path), t)
 
     def walk(self, path=None, onerror=None):
         """Yield (dirpath, dirs, files) tuple for each directories under path
 
         ``dirpath`` is relative one from the root of this vfs. This
         uses ``os.sep`` as path separator, even you specify POSIX
         style ``path``.
 
         "The root of this vfs" is represented as empty ``dirpath``.
         """
         root = os.path.normpath(self.join(None))
         # when dirpath == root, dirpath[prefixlen:] becomes empty
         # because len(dirpath) < prefixlen.
         prefixlen = len(pathutil.normasprefix(root))
         for dirpath, dirs, files in os.walk(self.join(path), onerror=onerror):
             yield (dirpath[prefixlen:], dirs, files)
 
     @contextlib.contextmanager
     def backgroundclosing(self, ui, expectedcount=-1):
         """Allow files to be closed asynchronously.
 
         When this context manager is active, ``backgroundclose`` can be passed
         to ``__call__``/``open`` to result in the file possibly being closed
         asynchronously, on a background thread.
         """
         # This is an arbitrary restriction and could be changed if we ever
         # have a use case.
         vfs = getattr(self, 'vfs', self)
         if getattr(vfs, '_backgroundfilecloser', None):
             raise error.Abort(
                 _('can only have 1 active background file closer'))
 
         with backgroundfilecloser(ui, expectedcount=expectedcount) as bfc:
             try:
                 vfs._backgroundfilecloser = bfc
                 yield bfc
             finally:
                 vfs._backgroundfilecloser = None
 
 class vfs(abstractvfs):
     '''Operate files relative to a base directory
 
     This class is used to hide the details of COW semantics and
     remote file access from higher level code.
     '''
     def __init__(self, base, audit=True, expandpath=False, realpath=False):
         if expandpath:
             base = util.expandpath(base)
         if realpath:
             base = os.path.realpath(base)
         self.base = base
         self.mustaudit = audit
         self.createmode = None
         self._trustnlink = None
 
     @property
     def mustaudit(self):
         return self._audit
 
     @mustaudit.setter
     def mustaudit(self, onoff):
         self._audit = onoff
         if onoff:
             self.audit = pathutil.pathauditor(self.base)
         else:
             self.audit = util.always
 
     @util.propertycache
     def _cansymlink(self):
         return util.checklink(self.base)
 
     @util.propertycache
     def _chmod(self):
         return util.checkexec(self.base)
 
     def _fixfilemode(self, name):
         if self.createmode is None or not self._chmod:
             return
         os.chmod(name, self.createmode & 0o666)
 
     def __call__(self, path, mode="r", text=False, atomictemp=False,
                  notindexed=False, backgroundclose=False, checkambig=False):
         '''Open ``path`` file, which is relative to vfs root.
 
         Newly created directories are marked as "not to be indexed by
         the content indexing service", if ``notindexed`` is specified
         for "write" mode access.
 
         If ``backgroundclose`` is passed, the file may be closed asynchronously.
         It can only be used if the ``self.backgroundclosing()`` context manager
         is active. This should only be specified if the following criteria hold:
 
         1. There is a potential for writing thousands of files. Unless you
            are writing thousands of files, the performance benefits of
            asynchronously closing files is not realized.
         2. Files are opened exactly once for the ``backgroundclosing``
            active duration and are therefore free of race conditions between
            closing a file on a background thread and reopening it. (If the
            file were opened multiple times, there could be unflushed data
            because the original file handle hasn't been flushed/closed yet.)
 
         ``checkambig`` argument is passed to atomictemplfile (valid
         only for writing), and is useful only if target file is
         guarded by any lock (e.g. repo.lock or repo.wlock).
         '''
         if self._audit:
             r = util.checkosfilename(path)
             if r:
                 raise error.Abort("%s: %r" % (r, path))
         self.audit(path)
         f = self.join(path)
 
         if not text and "b" not in mode:
             mode += "b" # for that other OS
 
         nlink = -1
         if mode not in ('r', 'rb'):
             dirname, basename = util.split(f)
             # If basename is empty, then the path is malformed because it points
             # to a directory. Let the posixfile() call below raise IOError.
             if basename:
                 if atomictemp:
                     util.makedirs(dirname, self.createmode, notindexed)
                     return util.atomictempfile(f, mode, self.createmode,
                                                checkambig=checkambig)
                 try:
                     if 'w' in mode:
                         util.unlink(f)
                         nlink = 0
                     else:
                         # nlinks() may behave differently for files on Windows
                         # shares if the file is open.
                         with util.posixfile(f):
                             nlink = util.nlinks(f)
                             if nlink < 1:
                                 nlink = 2 # force mktempcopy (issue1922)
                 except (OSError, IOError) as e:
                     if e.errno != errno.ENOENT:
                         raise
                     nlink = 0
                     util.makedirs(dirname, self.createmode, notindexed)
                 if nlink > 0:
                     if self._trustnlink is None:
                         self._trustnlink = nlink > 1 or util.checknlink(f)
                     if nlink > 1 or not self._trustnlink:
                         util.rename(util.mktempcopy(f), f)
         fp = util.posixfile(f, mode)
         if nlink == 0:
             self._fixfilemode(f)
 
         if checkambig:
             if mode in ('r', 'rb'):
                 raise error.Abort(_('implementation error: mode %s is not'
                                     ' valid for checkambig=True') % mode)
             fp = checkambigatclosing(fp)
 
         if backgroundclose:
             if not self._backgroundfilecloser:
                 raise error.Abort(_('backgroundclose can only be used when a '
                                   'backgroundclosing context manager is active')
                                   )
 
             fp = delayclosedfile(fp, self._backgroundfilecloser)
 
         return fp
 
     def symlink(self, src, dst):
         self.audit(dst)
         linkname = self.join(dst)
         util.tryunlink(linkname)
 
         util.makedirs(os.path.dirname(linkname), self.createmode)
 
         if self._cansymlink:
             try:
                 os.symlink(src, linkname)
             except OSError as err:
                 raise OSError(err.errno, _('could not symlink to %r: %s') %
                               (src, err.strerror), linkname)
         else:
             self.write(dst, src)
 
     def join(self, path, *insidef):
         if path:
             return os.path.join(self.base, path, *insidef)
         else:
             return self.base
 
 opener = vfs
 
 class auditvfs(object):
     def __init__(self, vfs):
         self.vfs = vfs
 
     @property
     def mustaudit(self):
         return self.vfs.mustaudit
 
     @mustaudit.setter
     def mustaudit(self, onoff):
         self.vfs.mustaudit = onoff
 
     @property
     def options(self):
         return self.vfs.options
 
     @options.setter
     def options(self, value):
         self.vfs.options = value
 
 class filtervfs(abstractvfs, auditvfs):
     '''Wrapper vfs for filtering filenames with a function.'''
 
     def __init__(self, vfs, filter):
         auditvfs.__init__(self, vfs)
         self._filter = filter
 
     def __call__(self, path, *args, **kwargs):
         return self.vfs(self._filter(path), *args, **kwargs)
 
     def join(self, path, *insidef):
         if path:
             return self.vfs.join(self._filter(self.vfs.reljoin(path, *insidef)))
         else:
             return self.vfs.join(path)
 
 filteropener = filtervfs
 
 class readonlyvfs(abstractvfs, auditvfs):
     '''Wrapper vfs preventing any writing.'''
 
     def __init__(self, vfs):
         auditvfs.__init__(self, vfs)
 
     def __call__(self, path, mode='r', *args, **kw):
         if mode not in ('r', 'rb'):
             raise error.Abort(_('this vfs is read only'))
         return self.vfs(path, mode, *args, **kw)
 
     def join(self, path, *insidef):
         return self.vfs.join(path, *insidef)
 
 class closewrapbase(object):
     """Base class of wrapper, which hooks closing
 
     Do not instantiate outside of the vfs layer.
     """
     def __init__(self, fh):
         object.__setattr__(self, r'_origfh', fh)
 
     def __getattr__(self, attr):
         return getattr(self._origfh, attr)
 
     def __setattr__(self, attr, value):
         return setattr(self._origfh, attr, value)
 
     def __delattr__(self, attr):
         return delattr(self._origfh, attr)
 
     def __enter__(self):
         return self._origfh.__enter__()
 
     def __exit__(self, exc_type, exc_value, exc_tb):
         raise NotImplementedError('attempted instantiating ' + str(type(self)))
 
     def close(self):
         raise NotImplementedError('attempted instantiating ' + str(type(self)))
 
 class delayclosedfile(closewrapbase):
     """Proxy for a file object whose close is delayed.
 
     Do not instantiate outside of the vfs layer.
     """
     def __init__(self, fh, closer):
         super(delayclosedfile, self).__init__(fh)
         object.__setattr__(self, r'_closer', closer)
 
     def __exit__(self, exc_type, exc_value, exc_tb):
         self._closer.close(self._origfh)
 
     def close(self):
         self._closer.close(self._origfh)
 
 class backgroundfilecloser(object):
     """Coordinates background closing of file handles on multiple threads."""
     def __init__(self, ui, expectedcount=-1):
         self._running = False
         self._entered = False
         self._threads = []
         self._threadexception = None
 
         # Only Windows/NTFS has slow file closing. So only enable by default
         # on that platform. But allow to be enabled elsewhere for testing.
         defaultenabled = pycompat.osname == 'nt'
         enabled = ui.configbool('worker', 'backgroundclose', defaultenabled)
 
         if not enabled:
             return
 
         # There is overhead to starting and stopping the background threads.
         # Don't do background processing unless the file count is large enough
         # to justify it.
         minfilecount = ui.configint('worker', 'backgroundcloseminfilecount',
                                     2048)
         # FUTURE dynamically start background threads after minfilecount closes.
         # (We don't currently have any callers that don't know their file count)
         if expectedcount > 0 and expectedcount < minfilecount:
             return
 
         # Windows defaults to a limit of 512 open files. A buffer of 128
         # should give us enough headway.
         maxqueue = ui.configint('worker', 'backgroundclosemaxqueue', 384)
         threadcount = ui.configint('worker', 'backgroundclosethreadcount', 4)
 
         ui.debug('starting %d threads for background file closing\n' %
                  threadcount)
 
         self._queue = util.queue(maxsize=maxqueue)
         self._running = True
 
         for i in range(threadcount):
             t = threading.Thread(target=self._worker, name='backgroundcloser')
             self._threads.append(t)
             t.start()
 
     def __enter__(self):
         self._entered = True
         return self
 
     def __exit__(self, exc_type, exc_value, exc_tb):
         self._running = False
 
         # Wait for threads to finish closing so open files don't linger for
         # longer than lifetime of context manager.
         for t in self._threads:
             t.join()
 
     def _worker(self):
         """Main routine for worker thread."""
         while True:
             try:
                 fh = self._queue.get(block=True, timeout=0.100)
                 # Need to catch or the thread will terminate and
                 # we could orphan file descriptors.
                 try:
                     fh.close()
                 except Exception as e:
                     # Stash so can re-raise from main thread later.
                     self._threadexception = e
             except util.empty:
                 if not self._running:
                     break
 
     def close(self, fh):
         """Schedule a file for closing."""
         if not self._entered:
             raise error.Abort(_('can only call close() when context manager '
                               'active'))
 
         # If a background thread encountered an exception, raise now so we fail
         # fast. Otherwise we may potentially go on for minutes until the error
         # is acted on.
         if self._threadexception:
             e = self._threadexception
             self._threadexception = None
             raise e
 
         # If we're not actively running, close synchronously.
         if not self._running:
             fh.close()
             return
 
         self._queue.put(fh, block=True, timeout=None)
 
 class checkambigatclosing(closewrapbase):
     """Proxy for a file object, to avoid ambiguity of file stat
 
     See also util.filestat for detail about "ambiguity of file stat".
 
     This proxy is useful only if the target file is guarded by any
     lock (e.g. repo.lock or repo.wlock)
 
     Do not instantiate outside of the vfs layer.
     """
     def __init__(self, fh):
         super(checkambigatclosing, self).__init__(fh)
         object.__setattr__(self, r'_oldstat', util.filestat(fh.name))
 
     def _checkambig(self):
         oldstat = self._oldstat
         if oldstat.stat:
             newstat = util.filestat(self._origfh.name)
             if newstat.isambig(oldstat):
                 # stat of changed file is ambiguous to original one
                 newstat.avoidambig(self._origfh.name, oldstat)
 
     def __exit__(self, exc_type, exc_value, exc_tb):
         self._origfh.__exit__(exc_type, exc_value, exc_tb)
         self._checkambig()
 
     def close(self):
         self._origfh.close()
         self._checkambig()

mercurial/windows.py

@@ -1,472 +1,475
 # windows.py - Windows utility function implementations for Mercurial
 #
 #  Copyright 2005-2009 Matt Mackall <[e-mail]> and others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import errno
 import msvcrt
 import os
 import re
 import stat
 import sys
 
 from .i18n import _
 from . import (
     encoding,
     osutil,
     pycompat,
     win32,
 )
 
 try:
     import _winreg as winreg
     winreg.CloseKey
 except ImportError:
     import winreg
 
 executablepath = win32.executablepath
 getuser = win32.getuser
 hidewindow = win32.hidewindow
 makedir = win32.makedir
 nlinks = win32.nlinks
 oslink = win32.oslink
 samedevice = win32.samedevice
 samefile = win32.samefile
 setsignalhandler = win32.setsignalhandler
 spawndetached = win32.spawndetached
 split = os.path.split
 testpid = win32.testpid
 unlink = win32.unlink
 
 umask = 0o022
 
 class mixedfilemodewrapper(object):
     """Wraps a file handle when it is opened in read/write mode.
 
     fopen() and fdopen() on Windows have a specific-to-Windows requirement
     that files opened with mode r+, w+, or a+ make a call to a file positioning
     function when switching between reads and writes. Without this extra call,
     Python will raise a not very intuitive "IOError: [Errno 0] Error."
 
     This class wraps posixfile instances when the file is opened in read/write
     mode and automatically adds checks or inserts appropriate file positioning
     calls when necessary.
     """
     OPNONE = 0
     OPREAD = 1
     OPWRITE = 2
 
     def __init__(self, fp):
         object.__setattr__(self, r'_fp', fp)
         object.__setattr__(self, r'_lastop', 0)
 
     def __enter__(self):
         return self._fp.__enter__()
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         self._fp.__exit__(exc_type, exc_val, exc_tb)
 
     def __getattr__(self, name):
         return getattr(self._fp, name)
 
     def __setattr__(self, name, value):
         return self._fp.__setattr__(name, value)
 
     def _noopseek(self):
         self._fp.seek(0, os.SEEK_CUR)
 
     def seek(self, *args, **kwargs):
         object.__setattr__(self, r'_lastop', self.OPNONE)
         return self._fp.seek(*args, **kwargs)
 
     def write(self, d):
         if self._lastop == self.OPREAD:
             self._noopseek()
 
         object.__setattr__(self, r'_lastop', self.OPWRITE)
         return self._fp.write(d)
 
     def writelines(self, *args, **kwargs):
         if self._lastop == self.OPREAD:
             self._noopeseek()
 
         object.__setattr__(self, r'_lastop', self.OPWRITE)
         return self._fp.writelines(*args, **kwargs)
 
     def read(self, *args, **kwargs):
         if self._lastop == self.OPWRITE:
             self._noopseek()
 
         object.__setattr__(self, r'_lastop', self.OPREAD)
         return self._fp.read(*args, **kwargs)
 
     def readline(self, *args, **kwargs):
         if self._lastop == self.OPWRITE:
             self._noopseek()
 
         object.__setattr__(self, r'_lastop', self.OPREAD)
         return self._fp.readline(*args, **kwargs)
 
     def readlines(self, *args, **kwargs):
         if self._lastop == self.OPWRITE:
             self._noopseek()
 
         object.__setattr__(self, r'_lastop', self.OPREAD)
         return self._fp.readlines(*args, **kwargs)
 
 def posixfile(name, mode='r', buffering=-1):
     '''Open a file with even more POSIX-like semantics'''
     try:
         fp = osutil.posixfile(name, mode, buffering) # may raise WindowsError
 
         # The position when opening in append mode is implementation defined, so
         # make it consistent with other platforms, which position at EOF.
         if 'a' in mode:
             fp.seek(0, os.SEEK_END)
 
         if '+' in mode:
             return mixedfilemodewrapper(fp)
 
         return fp
     except WindowsError as err:
         # convert to a friendlier exception
         raise IOError(err.errno, '%s: %s' % (name, err.strerror))
 
+# may be wrapped by win32mbcs extension
+listdir = osutil.listdir
+
 class winstdout(object):
     '''stdout on windows misbehaves if sent through a pipe'''
 
     def __init__(self, fp):
         self.fp = fp
 
     def __getattr__(self, key):
         return getattr(self.fp, key)
 
     def close(self):
         try:
             self.fp.close()
         except IOError:
             pass
 
     def write(self, s):
         try:
             # This is workaround for "Not enough space" error on
             # writing large size of data to console.
             limit = 16000
             l = len(s)
             start = 0
             self.softspace = 0
             while start < l:
                 end = start + limit
                 self.fp.write(s[start:end])
                 start = end
         except IOError as inst:
             if inst.errno != 0:
                 raise
             self.close()
             raise IOError(errno.EPIPE, 'Broken pipe')
 
     def flush(self):
         try:
             return self.fp.flush()
         except IOError as inst:
             if inst.errno != errno.EINVAL:
                 raise
             self.close()
             raise IOError(errno.EPIPE, 'Broken pipe')
 
 def _is_win_9x():
     '''return true if run on windows 95, 98 or me.'''
     try:
         return sys.getwindowsversion()[3] == 1
     except AttributeError:
         return 'command' in encoding.environ.get('comspec', '')
 
 def openhardlinks():
     return not _is_win_9x()
 
 def parsepatchoutput(output_line):
     """parses the output produced by patch and returns the filename"""
     pf = output_line[14:]
     if pf[0] == '`':
         pf = pf[1:-1] # Remove the quotes
     return pf
 
 def sshargs(sshcmd, host, user, port):
     '''Build argument list for ssh or Plink'''
     pflag = 'plink' in sshcmd.lower() and '-P' or '-p'
     args = user and ("%s@%s" % (user, host)) or host
     return port and ("%s %s %s" % (args, pflag, port)) or args
 
 def setflags(f, l, x):
     pass
 
 def copymode(src, dst, mode=None):
     pass
 
 def checkexec(path):
     return False
 
 def checklink(path):
     return False
 
 def setbinary(fd):
     # When run without console, pipes may expose invalid
     # fileno(), usually set to -1.
     fno = getattr(fd, 'fileno', None)
     if fno is not None and fno() >= 0:
         msvcrt.setmode(fno(), os.O_BINARY)
 
 def pconvert(path):
     return path.replace(pycompat.ossep, '/')
 
 def localpath(path):
     return path.replace('/', '\\')
 
 def normpath(path):
     return pconvert(os.path.normpath(path))
 
 def normcase(path):
     return encoding.upper(path) # NTFS compares via upper()
 
 # see posix.py for definitions
 normcasespec = encoding.normcasespecs.upper
 normcasefallback = encoding.upperfallback
 
 def samestat(s1, s2):
     return False
 
 # A sequence of backslashes is special iff it precedes a double quote:
 # - if there's an even number of backslashes, the double quote is not
 #   quoted (i.e. it ends the quoted region)
 # - if there's an odd number of backslashes, the double quote is quoted
 # - in both cases, every pair of backslashes is unquoted into a single
 #   backslash
 # (See http://msdn2.microsoft.com/en-us/library/a1y7w461.aspx )
 # So, to quote a string, we must surround it in double quotes, double
 # the number of backslashes that precede double quotes and add another
 # backslash before every double quote (being careful with the double
 # quote we've appended to the end)
 _quotere = None
 _needsshellquote = None
 def shellquote(s):
     r"""
     >>> shellquote(r'C:\Users\xyz')
     '"C:\\Users\\xyz"'
     >>> shellquote(r'C:\Users\xyz/mixed')
     '"C:\\Users\\xyz/mixed"'
     >>> # Would be safe not to quote too, since it is all double backslashes
     >>> shellquote(r'C:\\Users\\xyz')
     '"C:\\\\Users\\\\xyz"'
     >>> # But this must be quoted
     >>> shellquote(r'C:\\Users\\xyz/abc')
     '"C:\\\\Users\\\\xyz/abc"'
     """
     global _quotere
     if _quotere is None:
         _quotere = re.compile(r'(\\*)("|\\$)')
     global _needsshellquote
     if _needsshellquote is None:
         # ":" is also treated as "safe character", because it is used as a part
         # of path name on Windows.  "\" is also part of a path name, but isn't
         # safe because shlex.split() (kind of) treats it as an escape char and
         # drops it.  It will leave the next character, even if it is another
         # "\".
         _needsshellquote = re.compile(r'[^a-zA-Z0-9._:/-]').search
     if s and not _needsshellquote(s) and not _quotere.search(s):
         # "s" shouldn't have to be quoted
         return s
     return '"%s"' % _quotere.sub(r'\1\1\\\2', s)
 
 def quotecommand(cmd):
     """Build a command string suitable for os.popen* calls."""
     if sys.version_info < (2, 7, 1):
         # Python versions since 2.7.1 do this extra quoting themselves
         return '"' + cmd + '"'
     return cmd
 
 def popen(command, mode='r'):
     # Work around "popen spawned process may not write to stdout
     # under windows"
     # http://bugs.python.org/issue1366
     command += " 2> %s" % os.devnull
     return os.popen(quotecommand(command), mode)
 
 def explainexit(code):
     return _("exited with status %d") % code, code
 
 # if you change this stub into a real check, please try to implement the
 # username and groupname functions above, too.
 def isowner(st):
     return True
 
 def findexe(command):
     '''Find executable for command searching like cmd.exe does.
     If command is a basename then PATH is searched for command.
     PATH isn't searched if command is an absolute or relative path.
     An extension from PATHEXT is found and added if not present.
     If command isn't found None is returned.'''
     pathext = encoding.environ.get('PATHEXT', '.COM;.EXE;.BAT;.CMD')
     pathexts = [ext for ext in pathext.lower().split(pycompat.ospathsep)]
     if os.path.splitext(command)[1].lower() in pathexts:
         pathexts = ['']
 
     def findexisting(pathcommand):
         'Will append extension (if needed) and return existing file'
         for ext in pathexts:
             executable = pathcommand + ext
             if os.path.exists(executable):
                 return executable
         return None
 
     if pycompat.ossep in command:
         return findexisting(command)
 
     for path in encoding.environ.get('PATH', '').split(pycompat.ospathsep):
         executable = findexisting(os.path.join(path, command))
         if executable is not None:
             return executable
     return findexisting(os.path.expanduser(os.path.expandvars(command)))
 
 _wantedkinds = set([stat.S_IFREG, stat.S_IFLNK])
 
 def statfiles(files):
     '''Stat each file in files. Yield each stat, or None if a file
     does not exist or has a type we don't care about.
 
     Cluster and cache stat per directory to minimize number of OS stat calls.'''
     dircache = {} # dirname -> filename -> status | None if file does not exist
     getkind = stat.S_IFMT
     for nf in files:
         nf  = normcase(nf)
         dir, base = os.path.split(nf)
         if not dir:
             dir = '.'
         cache = dircache.get(dir, None)
         if cache is None:
             try:
                 dmap = dict([(normcase(n), s)
-                             for n, k, s in osutil.listdir(dir, True)
+                             for n, k, s in listdir(dir, True)
                              if getkind(s.st_mode) in _wantedkinds])
             except OSError as err:
                 # Python >= 2.5 returns ENOENT and adds winerror field
                 # EINVAL is raised if dir is not a directory.
                 if err.errno not in (errno.ENOENT, errno.EINVAL,
                                      errno.ENOTDIR):
                     raise
                 dmap = {}
             cache = dircache.setdefault(dir, dmap)
         yield cache.get(base, None)
 
 def username(uid=None):
     """Return the name of the user with the given uid.
 
     If uid is None, return the name of the current user."""
     return None
 
 def groupname(gid=None):
     """Return the name of the group with the given gid.
 
     If gid is None, return the name of the current group."""
     return None
 
 def removedirs(name):
     """special version of os.removedirs that does not remove symlinked
     directories or junction points if they actually contain files"""
-    if osutil.listdir(name):
+    if listdir(name):
         return
     os.rmdir(name)
     head, tail = os.path.split(name)
     if not tail:
         head, tail = os.path.split(head)
     while head and tail:
         try:
-            if osutil.listdir(head):
+            if listdir(head):
                 return
             os.rmdir(head)
         except (ValueError, OSError):
             break
         head, tail = os.path.split(head)
 
 def rename(src, dst):
     '''atomically rename file src to dst, replacing dst if it exists'''
     try:
         os.rename(src, dst)
     except OSError as e:
         if e.errno != errno.EEXIST:
             raise
         unlink(dst)
         os.rename(src, dst)
 
 def gethgcmd():
     return [sys.executable] + sys.argv[:1]
 
 def groupmembers(name):
     # Don't support groups on Windows for now
     raise KeyError
 
 def isexec(f):
     return False
 
 class cachestat(object):
     def __init__(self, path):
         pass
 
     def cacheable(self):
         return False
 
 def lookupreg(key, valname=None, scope=None):
     ''' Look up a key/value name in the Windows registry.
 
     valname: value name. If unspecified, the default value for the key
     is used.
     scope: optionally specify scope for registry lookup, this can be
     a sequence of scopes to look up in order. Default (CURRENT_USER,
     LOCAL_MACHINE).
     '''
     if scope is None:
         scope = (winreg.HKEY_CURRENT_USER, winreg.HKEY_LOCAL_MACHINE)
     elif not isinstance(scope, (list, tuple)):
         scope = (scope,)
     for s in scope:
         try:
             val = winreg.QueryValueEx(winreg.OpenKey(s, key), valname)[0]
             # never let a Unicode string escape into the wild
             return encoding.unitolocal(val)
         except EnvironmentError:
             pass
 
 expandglobs = True
 
 def statislink(st):
     '''check whether a stat result is a symlink'''
     return False
 
 def statisexec(st):
     '''check whether a stat result is an executable file'''
     return False
 
 def poll(fds):
     # see posix.py for description
     raise NotImplementedError()
 
 def readpipe(pipe):
     """Read all available data from a pipe."""
     chunks = []
     while True:
         size = win32.peekpipe(pipe)
         if not size:
             break
 
         s = pipe.read(size)
         if not s:
             break
         chunks.append(s)
 
     return ''.join(chunks)
 
 def bindunixsocket(sock, path):
     raise NotImplementedError('unsupported platform')