pycompat.py
514 lines
| 15.1 KiB
| text/x-python
|
PythonLexer
/ mercurial / pycompat.py
|
r28818 | # pycompat.py - portability shim for python 3 | ||
# | ||||
# This software may be used and distributed according to the terms of the | ||||
# GNU General Public License version 2 or any later version. | ||||
"""Mercurial portability shim for python 3. | ||||
This contains aliases to hide python version-specific details from the core. | ||||
""" | ||||
from __future__ import absolute_import | ||||
|
r30578 | import getopt | ||
|
r36196 | import inspect | ||
|
r43697 | import json | ||
|
r30302 | import os | ||
|
r30678 | import shlex | ||
|
r29584 | import sys | ||
|
r38182 | import tempfile | ||
|
r29584 | |||
|
r43346 | ispy3 = sys.version_info[0] >= 3 | ||
|
r43906 | ispypy = '__pypy__' in sys.builtin_module_names | ||
|
r44212 | TYPE_CHECKING = False | ||
if not globals(): # hide this from non-pytype users | ||||
import typing | ||||
TYPE_CHECKING = typing.TYPE_CHECKING | ||||
|
r30030 | |||
if not ispy3: | ||||
|
r31934 | import cookielib | ||
|
r29324 | import cPickle as pickle | ||
|
r29455 | import httplib | ||
|
r37863 | import Queue as queue | ||
|
r29433 | import SocketServer as socketserver | ||
|
r29432 | import xmlrpclib | ||
|
r37646 | |||
from .thirdparty.concurrent import futures | ||||
|
r37687 | |||
def future_set_exception_info(f, exc_info): | ||||
f.set_exception_info(*exc_info) | ||||
|
r43346 | |||
|
r29584 | else: | ||
|
r37646 | import concurrent.futures as futures | ||
|
r31942 | import http.cookiejar as cookielib | ||
|
r29584 | import http.client as httplib | ||
import pickle | ||||
|
r37863 | import queue as queue | ||
|
r29584 | import socketserver | ||
|
r29432 | import xmlrpc.client as xmlrpclib | ||
|
r29431 | |||
|
r37687 | def future_set_exception_info(f, exc_info): | ||
f.set_exception(exc_info[0]) | ||||
|
r43346 | |||
|
r31774 | def identity(a): | ||
return a | ||||
|
r43346 | |||
|
r38594 | def _rapply(f, xs): | ||
if xs is None: | ||||
# assume None means non-value of optional data | ||||
return xs | ||||
if isinstance(xs, (list, set, tuple)): | ||||
return type(xs)(_rapply(f, x) for x in xs) | ||||
if isinstance(xs, dict): | ||||
return type(xs)((_rapply(f, k), _rapply(f, v)) for k, v in xs.items()) | ||||
return f(xs) | ||||
|
r43346 | |||
|
r38594 | def rapply(f, xs): | ||
"""Apply function recursively to every item preserving the data structure | ||||
>>> def f(x): | ||||
... return 'f(%s)' % x | ||||
>>> rapply(f, None) is None | ||||
True | ||||
>>> rapply(f, 'a') | ||||
'f(a)' | ||||
>>> rapply(f, {'a'}) == {'f(a)'} | ||||
True | ||||
>>> rapply(f, ['a', 'b', None, {'c': 'd'}, []]) | ||||
['f(a)', 'f(b)', None, {'f(c)': 'f(d)'}, []] | ||||
>>> xs = [object()] | ||||
>>> rapply(identity, xs) is xs | ||||
True | ||||
""" | ||||
if f is identity: | ||||
# fast path mainly for py2 | ||||
return xs | ||||
return _rapply(f, xs) | ||||
|
r43346 | |||
|
r30030 | if ispy3: | ||
|
r29797 | import builtins | ||
|
r43697 | import codecs | ||
|
r29799 | import functools | ||
|
r31372 | import io | ||
|
r31424 | import struct | ||
|
r31372 | |||
|
r43613 | if os.name == r'nt' and sys.version_info >= (3, 6): | ||
# MBCS (or ANSI) filesystem encoding must be used as before. | ||||
# Otherwise non-ASCII filenames in existing repositories would be | ||||
# corrupted. | ||||
# This must be set once prior to any fsencode/fsdecode calls. | ||||
|
r44207 | sys._enablelegacywindowsfsencoding() # pytype: disable=module-attr | ||
|
r43613 | |||
|
r30119 | fsencode = os.fsencode | ||
|
r30300 | fsdecode = os.fsdecode | ||
|
r36666 | oscurdir = os.curdir.encode('ascii') | ||
|
r31775 | oslinesep = os.linesep.encode('ascii') | ||
|
r30302 | osname = os.name.encode('ascii') | ||
|
r30303 | ospathsep = os.pathsep.encode('ascii') | ||
|
r36665 | ospardir = os.pardir.encode('ascii') | ||
|
r30303 | ossep = os.sep.encode('ascii') | ||
|
r30623 | osaltsep = os.altsep | ||
if osaltsep: | ||||
osaltsep = osaltsep.encode('ascii') | ||||
|
r44229 | osdevnull = os.devnull.encode('ascii') | ||
|
r39843 | |||
|
r30624 | sysplatform = sys.platform.encode('ascii') | ||
|
r30668 | sysexecutable = sys.executable | ||
if sysexecutable: | ||||
sysexecutable = os.fsencode(sysexecutable) | ||||
|
r36976 | bytesio = io.BytesIO | ||
# TODO deprecate stringio name, as it is a lie on Python 3. | ||||
stringio = bytesio | ||||
|
r36952 | |||
def maplist(*args): | ||||
return list(map(*args)) | ||||
|
r37082 | def rangelist(*args): | ||
return list(range(*args)) | ||||
|
r36952 | def ziplist(*args): | ||
return list(zip(*args)) | ||||
|
r33853 | rawinput = input | ||
|
r36196 | getargspec = inspect.getfullargspec | ||
|
r30334 | |||
|
r39490 | long = int | ||
|
r30472 | # TODO: .buffer might not exist if std streams were replaced; we'll need | ||
# a silly wrapper to make a bytes stream backed by a unicode one. | ||||
stdin = sys.stdin.buffer | ||||
stdout = sys.stdout.buffer | ||||
stderr = sys.stderr.buffer | ||||
|
r30334 | # Since Python 3 converts argv to wchar_t type by Py_DecodeLocale() on Unix, | ||
# we can use os.fsencode() to get back bytes argv. | ||||
# | ||||
# https://hg.python.org/cpython/file/v3.5.1/Programs/python.c#l55 | ||||
# | ||||
|
r43613 | # On Windows, the native argv is unicode and is converted to MBCS bytes | ||
# since we do enable the legacy filesystem encoding. | ||||
|
r31277 | if getattr(sys, 'argv', None) is not None: | ||
sysargv = list(map(os.fsencode, sys.argv)) | ||||
|
r29797 | |||
|
r43906 | bytechr = struct.Struct('>B').pack | ||
|
r36279 | byterepr = b'%r'.__mod__ | ||
|
r31253 | |||
|
r31439 | class bytestr(bytes): | ||
"""A bytes which mostly acts as a Python 2 str | ||||
>>> bytestr(), bytestr(bytearray(b'foo')), bytestr(u'ascii'), bytestr(1) | ||||
|
r35921 | ('', 'foo', 'ascii', '1') | ||
|
r31439 | >>> s = bytestr(b'foo') | ||
>>> assert s is bytestr(s) | ||||
|
r32450 | __bytes__() should be called if provided: | ||
>>> class bytesable(object): | ||||
... def __bytes__(self): | ||||
... return b'bytes' | ||||
>>> bytestr(bytesable()) | ||||
|
r35921 | 'bytes' | ||
|
r32450 | |||
|
r31439 | There's no implicit conversion from non-ascii str as its encoding is | ||
unknown: | ||||
>>> bytestr(chr(0x80)) # doctest: +ELLIPSIS | ||||
Traceback (most recent call last): | ||||
... | ||||
UnicodeEncodeError: ... | ||||
Comparison between bytestr and bytes should work: | ||||
>>> assert bytestr(b'foo') == b'foo' | ||||
>>> assert b'foo' == bytestr(b'foo') | ||||
>>> assert b'f' in bytestr(b'foo') | ||||
>>> assert bytestr(b'f') in b'foo' | ||||
Sliced elements should be bytes, not integer: | ||||
>>> s[1], s[:2] | ||||
(b'o', b'fo') | ||||
>>> list(s), list(reversed(s)) | ||||
([b'f', b'o', b'o'], [b'o', b'o', b'f']) | ||||
As bytestr type isn't propagated across operations, you need to cast | ||||
bytes to bytestr explicitly: | ||||
>>> s = bytestr(b'foo').upper() | ||||
>>> t = bytestr(s) | ||||
>>> s[0], t[0] | ||||
(70, b'F') | ||||
Be careful to not pass a bytestr object to a function which expects | ||||
bytearray-like behavior. | ||||
>>> t = bytes(t) # cast to bytes | ||||
>>> assert type(t) is bytes | ||||
""" | ||||
def __new__(cls, s=b''): | ||||
if isinstance(s, bytestr): | ||||
return s | ||||
|
r43346 | if not isinstance( | ||
s, (bytes, bytearray) | ||||
) and not hasattr( # hasattr-py3-only | ||||
s, u'__bytes__' | ||||
): | ||||
|
r43361 | s = str(s).encode('ascii') | ||
|
r31439 | return bytes.__new__(cls, s) | ||
def __getitem__(self, key): | ||||
s = bytes.__getitem__(self, key) | ||||
if not isinstance(s, bytes): | ||||
s = bytechr(s) | ||||
return s | ||||
def __iter__(self): | ||||
return iterbytestr(bytes.__iter__(self)) | ||||
|
r35921 | def __repr__(self): | ||
return bytes.__repr__(self)[1:] # drop b'' | ||||
|
r31382 | def iterbytestr(s): | ||
"""Iterate bytes as if it were a str object of Python 2""" | ||||
|
r31425 | return map(bytechr, s) | ||
|
r31382 | |||
|
r35922 | def maybebytestr(s): | ||
"""Promote bytes to bytestr""" | ||||
if isinstance(s, bytes): | ||||
return bytestr(s) | ||||
return s | ||||
|
r31820 | def sysbytes(s): | ||
"""Convert an internal str (e.g. keyword, __doc__) back to bytes | ||||
This never raises UnicodeEncodeError, but only ASCII characters | ||||
can be round-trip by sysstr(sysbytes(s)). | ||||
""" | ||||
|
r43361 | return s.encode('utf-8') | ||
|
r31820 | |||
|
r30032 | def sysstr(s): | ||
"""Return a keyword str to be passed to Python functions such as | ||||
getattr() and str.encode() | ||||
This never raises UnicodeDecodeError. Non-ascii characters are | ||||
considered invalid and mapped to arbitrary but unique code points | ||||
such that 'sysstr(a) != sysstr(b)' for all 'a != b'. | ||||
""" | ||||
if isinstance(s, builtins.str): | ||||
return s | ||||
|
r43361 | return s.decode('latin-1') | ||
|
r30032 | |||
|
r32859 | def strurl(url): | ||
"""Converts a bytes url back to str""" | ||||
|
r36662 | if isinstance(url, bytes): | ||
|
r43361 | return url.decode('ascii') | ||
|
r36662 | return url | ||
|
r32859 | |||
|
r32860 | def bytesurl(url): | ||
"""Converts a str url to bytes by encoding in ascii""" | ||||
|
r36662 | if isinstance(url, str): | ||
|
r43361 | return url.encode('ascii') | ||
|
r36662 | return url | ||
|
r32860 | |||
|
r32186 | def raisewithtb(exc, tb): | ||
"""Raise exception with the given traceback""" | ||||
raise exc.with_traceback(tb) | ||||
|
r32615 | def getdoc(obj): | ||
"""Get docstring as bytes; may be None so gettext() won't confuse it | ||||
with _('')""" | ||||
|
r43373 | doc = getattr(obj, '__doc__', None) | ||
|
r32615 | if doc is None: | ||
return doc | ||||
return sysbytes(doc) | ||||
|
r29799 | def _wrapattrfunc(f): | ||
@functools.wraps(f) | ||||
def w(object, name, *args): | ||||
|
r30032 | return f(object, sysstr(name), *args) | ||
|
r43346 | |||
|
r29799 | return w | ||
|
r29800 | # these wrappers are automagically imported by hgloader | ||
|
r29799 | delattr = _wrapattrfunc(builtins.delattr) | ||
getattr = _wrapattrfunc(builtins.getattr) | ||||
hasattr = _wrapattrfunc(builtins.hasattr) | ||||
setattr = _wrapattrfunc(builtins.setattr) | ||||
|
r29800 | xrange = builtins.range | ||
|
r31843 | unicode = str | ||
|
r29799 | |||
|
r39678 | def open(name, mode=b'r', buffering=-1, encoding=None): | ||
|
r36574 | return builtins.open(name, sysstr(mode), buffering, encoding) | ||
|
r31149 | |||
|
r37117 | safehasattr = _wrapattrfunc(builtins.hasattr) | ||
|
r35226 | def _getoptbwrapper(orig, args, shortlist, namelist): | ||
|
r32864 | """ | ||
Takes bytes arguments, converts them to unicode, pass them to | ||||
getopt.getopt(), convert the returned values back to bytes and then | ||||
return them for Python 3 compatibility as getopt.getopt() don't accepts | ||||
bytes on Python 3. | ||||
""" | ||||
|
r30578 | args = [a.decode('latin-1') for a in args] | ||
shortlist = shortlist.decode('latin-1') | ||||
namelist = [a.decode('latin-1') for a in namelist] | ||||
|
r35226 | opts, args = orig(args, shortlist, namelist) | ||
|
r43346 | opts = [(a[0].encode('latin-1'), a[1].encode('latin-1')) for a in opts] | ||
|
r30578 | args = [a.encode('latin-1') for a in args] | ||
return opts, args | ||||
|
r30579 | def strkwargs(dic): | ||
|
r32864 | """ | ||
Converts the keys of a python dictonary to str i.e. unicodes so that | ||||
they can be passed as keyword arguments as dictonaries with bytes keys | ||||
can't be passed as keyword arguments to functions on Python 3. | ||||
""" | ||||
|
r43376 | dic = dict((k.decode('latin-1'), v) for k, v in dic.items()) | ||
|
r30579 | return dic | ||
def byteskwargs(dic): | ||||
|
r32864 | """ | ||
Converts keys of python dictonaries to bytes as they were converted to | ||||
str to pass that dictonary as a keyword argument on Python 3. | ||||
""" | ||||
|
r43376 | dic = dict((k.encode('latin-1'), v) for k, v in dic.items()) | ||
|
r30579 | return dic | ||
|
r30678 | # TODO: handle shlex.shlex(). | ||
|
r36352 | def shlexsplit(s, comments=False, posix=True): | ||
|
r32864 | """ | ||
Takes bytes argument, convert it to str i.e. unicodes, pass that into | ||||
shlex.split(), convert the returned value to bytes and return that for | ||||
Python 3 compatibility as shelx.split() don't accept bytes on Python 3. | ||||
""" | ||||
|
r36352 | ret = shlex.split(s.decode('latin-1'), comments, posix) | ||
|
r30678 | return [a.encode('latin-1') for a in ret] | ||
|
r43375 | iteritems = lambda x: x.items() | ||
|
r43374 | itervalues = lambda x: x.values() | ||
|
r43289 | |||
|
r43697 | # Python 3.5's json.load and json.loads require str. We polyfill its | ||
# code for detecting encoding from bytes. | ||||
if sys.version_info[0:2] < (3, 6): | ||||
def _detect_encoding(b): | ||||
bstartswith = b.startswith | ||||
if bstartswith((codecs.BOM_UTF32_BE, codecs.BOM_UTF32_LE)): | ||||
return 'utf-32' | ||||
if bstartswith((codecs.BOM_UTF16_BE, codecs.BOM_UTF16_LE)): | ||||
return 'utf-16' | ||||
if bstartswith(codecs.BOM_UTF8): | ||||
return 'utf-8-sig' | ||||
if len(b) >= 4: | ||||
if not b[0]: | ||||
# 00 00 -- -- - utf-32-be | ||||
# 00 XX -- -- - utf-16-be | ||||
return 'utf-16-be' if b[1] else 'utf-32-be' | ||||
if not b[1]: | ||||
# XX 00 00 00 - utf-32-le | ||||
# XX 00 00 XX - utf-16-le | ||||
# XX 00 XX -- - utf-16-le | ||||
return 'utf-16-le' if b[2] or b[3] else 'utf-32-le' | ||||
elif len(b) == 2: | ||||
if not b[0]: | ||||
# 00 XX - utf-16-be | ||||
return 'utf-16-be' | ||||
if not b[1]: | ||||
# XX 00 - utf-16-le | ||||
return 'utf-16-le' | ||||
# default | ||||
return 'utf-8' | ||||
def json_loads(s, *args, **kwargs): | ||||
if isinstance(s, (bytes, bytearray)): | ||||
s = s.decode(_detect_encoding(s), 'surrogatepass') | ||||
return json.loads(s, *args, **kwargs) | ||||
else: | ||||
json_loads = json.loads | ||||
|
r30032 | else: | ||
|
r31372 | import cStringIO | ||
|
r38805 | xrange = xrange | ||
|
r38332 | unicode = unicode | ||
|
r31253 | bytechr = chr | ||
|
r36279 | byterepr = repr | ||
|
r31439 | bytestr = str | ||
|
r31382 | iterbytestr = iter | ||
|
r35922 | maybebytestr = identity | ||
|
r31820 | sysbytes = identity | ||
|
r31774 | sysstr = identity | ||
|
r32859 | strurl = identity | ||
|
r32860 | bytesurl = identity | ||
|
r43355 | open = open | ||
|
r43360 | delattr = delattr | ||
|
r43359 | getattr = getattr | ||
|
r43358 | hasattr = hasattr | ||
|
r43357 | setattr = setattr | ||
|
r30032 | |||
|
r32186 | # this can't be parsed on Python 3 | ||
|
r43387 | exec(b'def raisewithtb(exc, tb):\n raise exc, None, tb\n') | ||
|
r32186 | |||
|
r30133 | def fsencode(filename): | ||
|
r32864 | """ | ||
Partial backport from os.py in Python 3, which only accepts bytes. | ||||
In Python 2, our paths should only ever be bytes, a unicode path | ||||
indicates a bug. | ||||
""" | ||||
|
r30133 | if isinstance(filename, str): | ||
return filename | ||||
|
r30119 | else: | ||
|
r43809 | raise TypeError("expect str, not %s" % type(filename).__name__) | ||
|
r30119 | |||
|
r30300 | # In Python 2, fsdecode() has a very chance to receive bytes. So it's | ||
# better not to touch Python 2 part as it's already working fine. | ||||
|
r31774 | fsdecode = identity | ||
|
r30300 | |||
|
r32615 | def getdoc(obj): | ||
return getattr(obj, '__doc__', None) | ||||
|
r37117 | _notset = object() | ||
def safehasattr(thing, attr): | ||||
return getattr(thing, attr, _notset) is not _notset | ||||
|
r35226 | def _getoptbwrapper(orig, args, shortlist, namelist): | ||
return orig(args, shortlist, namelist) | ||||
|
r30578 | |||
|
r31774 | strkwargs = identity | ||
byteskwargs = identity | ||||
|
r30579 | |||
|
r36666 | oscurdir = os.curdir | ||
|
r31775 | oslinesep = os.linesep | ||
|
r30302 | osname = os.name | ||
|
r30303 | ospathsep = os.pathsep | ||
|
r36665 | ospardir = os.pardir | ||
|
r30303 | ossep = os.sep | ||
|
r30623 | osaltsep = os.altsep | ||
|
r44229 | osdevnull = os.devnull | ||
|
r39490 | long = long | ||
|
r30472 | stdin = sys.stdin | ||
stdout = sys.stdout | ||||
stderr = sys.stderr | ||||
|
r31277 | if getattr(sys, 'argv', None) is not None: | ||
sysargv = sys.argv | ||||
|
r30624 | sysplatform = sys.platform | ||
|
r30668 | sysexecutable = sys.executable | ||
|
r30678 | shlexsplit = shlex.split | ||
|
r36976 | bytesio = cStringIO.StringIO | ||
stringio = bytesio | ||||
|
r31501 | maplist = map | ||
|
r37082 | rangelist = range | ||
|
r35406 | ziplist = zip | ||
|
r33853 | rawinput = raw_input | ||
|
r36196 | getargspec = inspect.getargspec | ||
|
r43375 | iteritems = lambda x: x.iteritems() | ||
|
r43374 | itervalues = lambda x: x.itervalues() | ||
|
r43697 | json_loads = json.loads | ||
|
r34640 | |||
|
r39678 | isjython = sysplatform.startswith(b'java') | ||
|
r34645 | |||
|
r40563 | isdarwin = sysplatform.startswith(b'darwin') | ||
islinux = sysplatform.startswith(b'linux') | ||||
|
r39678 | isposix = osname == b'posix' | ||
iswindows = osname == b'nt' | ||||
|
r35226 | |||
|
r43346 | |||
|
r35226 | def getoptb(args, shortlist, namelist): | ||
return _getoptbwrapper(getopt.getopt, args, shortlist, namelist) | ||||
|
r43346 | |||
|
r35226 | def gnugetoptb(args, shortlist, namelist): | ||
return _getoptbwrapper(getopt.gnu_getopt, args, shortlist, namelist) | ||||
|
r38182 | |||
|
r43346 | |||
|
r38183 | def mkdtemp(suffix=b'', prefix=b'tmp', dir=None): | ||
return tempfile.mkdtemp(suffix, prefix, dir) | ||||
|
r43346 | |||
|
r38182 | # text=True is not supported; use util.from/tonativeeol() instead | ||
def mkstemp(suffix=b'', prefix=b'tmp', dir=None): | ||||
return tempfile.mkstemp(suffix, prefix, dir) | ||||
|
r38184 | |||
|
r43346 | |||
|
r38184 | # mode must include 'b'ytes as encoding= is not supported | ||
|
r43346 | def namedtempfile( | ||
mode=b'w+b', bufsize=-1, suffix=b'', prefix=b'tmp', dir=None, delete=True | ||||
): | ||||
|
r38184 | mode = sysstr(mode) | ||
|
r43906 | assert 'b' in mode | ||
|
r43346 | return tempfile.NamedTemporaryFile( | ||
mode, bufsize, suffix=suffix, prefix=prefix, dir=dir, delete=delete | ||||
) | ||||