encoding.py
742 lines
| 22.9 KiB
| text/x-python
|
PythonLexer
/ mercurial / encoding.py
|
r8226 | # encoding.py - character transcoding support for Mercurial | ||
# | ||||
|
r47575 | # Copyright 2005-2009 Olivia Mackall <olivia@selenic.com> and others | ||
|
r8226 | # | ||
# This software may be used and distributed according to the terms of the | ||||
|
r10263 | # GNU General Public License version 2 or any later version. | ||
|
r7948 | |||
|
r34139 | from __future__ import absolute_import, print_function | ||
|
r27355 | |||
import locale | ||||
import os | ||||
r48421 | import re | |||
|
r27355 | import unicodedata | ||
|
r43359 | from .pycompat import getattr | ||
|
r27355 | from . import ( | ||
error, | ||||
|
r32372 | policy, | ||
|
r30030 | pycompat, | ||
|
r27355 | ) | ||
|
r7948 | |||
|
r43346 | from .pure import charencode as charencodepure | ||
|
r33925 | |||
|
r44212 | if pycompat.TYPE_CHECKING: | ||
|
r43802 | from typing import ( | ||
Any, | ||||
Callable, | ||||
List, | ||||
Text, | ||||
Type, | ||||
TypeVar, | ||||
Union, | ||||
) | ||||
# keep pyflakes happy | ||||
for t in (Any, Callable, List, Text, Type, Union): | ||||
assert t | ||||
|
r44075 | _Tlocalstr = TypeVar('_Tlocalstr', bound='localstr') | ||
|
r43802 | |||
|
r43906 | charencode = policy.importmod('charencode') | ||
|
r33756 | |||
|
r33927 | isasciistr = charencode.isasciistr | ||
|
r33756 | asciilower = charencode.asciilower | ||
asciiupper = charencode.asciiupper | ||||
|
r33926 | _jsonescapeu8fast = charencode.jsonescapeu8fast | ||
|
r33756 | |||
|
r30033 | _sysstr = pycompat.sysstr | ||
|
r30030 | if pycompat.ispy3: | ||
|
r28507 | unichr = chr | ||
|
r23596 | # These unicode characters are ignored by HFS+ (Apple Technote 1150, | ||
# "Unicode Subtleties"), so we need to ignore them in some places for | ||||
# sanity. | ||||
|
r43346 | _ignore = [ | ||
unichr(int(x, 16)).encode("utf-8") | ||||
|
r43347 | for x in b"200c 200d 200e 200f 202a 202b 202c 202d 202e " | ||
b"206a 206b 206c 206d 206e 206f feff".split() | ||||
|
r43346 | ] | ||
|
r23596 | # verify the next function will work | ||
|
r43347 | assert all(i.startswith((b"\xe2", b"\xef")) for i in _ignore) | ||
|
r23596 | |||
|
r43346 | |||
|
r23596 | def hfsignoreclean(s): | ||
|
r43802 | # type: (bytes) -> bytes | ||
|
r23596 | """Remove codepoints ignored by HFS+ from s. | ||
>>> hfsignoreclean(u'.h\u200cg'.encode('utf-8')) | ||||
'.hg' | ||||
>>> hfsignoreclean(u'.h\ufeffg'.encode('utf-8')) | ||||
'.hg' | ||||
""" | ||||
|
r43347 | if b"\xe2" in s or b"\xef" in s: | ||
|
r23596 | for c in _ignore: | ||
|
r43347 | s = s.replace(c, b'') | ||
|
r23596 | return s | ||
|
r43346 | |||
|
r30034 | # encoding.environ is provided read-only, which may not be used to modify | ||
# the process environment | ||||
|
r43346 | _nativeenviron = not pycompat.ispy3 or os.supports_bytes_environ | ||
|
r30034 | if not pycompat.ispy3: | ||
|
r32185 | environ = os.environ # re-exports | ||
|
r30034 | elif _nativeenviron: | ||
|
r32185 | environ = os.environb # re-exports | ||
|
r30034 | else: | ||
# preferred encoding isn't known yet; use utf-8 to avoid unicode error | ||||
# and recreate it once encoding is settled | ||||
|
r44937 | environ = { | ||
k.encode('utf-8'): v.encode('utf-8') | ||||
|
r43346 | for k, v in os.environ.items() # re-exports | ||
|
r44937 | } | ||
|
r30034 | |||
|
r39871 | _encodingrewrites = { | ||
|
r43347 | b'646': b'ascii', | ||
b'ANSI_X3.4-1968': b'ascii', | ||||
|
r11892 | } | ||
|
r38633 | # cp65001 is a Windows variant of utf-8, which isn't supported on Python 2. | ||
# No idea if it should be rewritten to the canonical name 'utf-8' on Python 3. | ||||
# https://bugs.python.org/issue13216 | ||||
if pycompat.iswindows and not pycompat.ispy3: | ||||
|
r43347 | _encodingrewrites[b'cp65001'] = b'utf-8' | ||
|
r7948 | |||
try: | ||||
|
r43347 | encoding = environ.get(b"HGENCODING") | ||
|
r7948 | if not encoding: | ||
|
r43347 | encoding = locale.getpreferredencoding().encode('ascii') or b'ascii' | ||
|
r39871 | encoding = _encodingrewrites.get(encoding, encoding) | ||
|
r7948 | except locale.Error: | ||
|
r43347 | encoding = b'ascii' | ||
encodingmode = environ.get(b"HGENCODINGMODE", b"strict") | ||||
fallbackencoding = b'ISO-8859-1' | ||||
|
r7948 | |||
|
r43346 | |||
|
r33810 | class localstr(bytes): | ||
|
r46554 | """This class allows strings that are unmodified to be | ||
round-tripped to the local encoding and back""" | ||||
|
r43346 | |||
|
r13046 | def __new__(cls, u, l): | ||
|
r33810 | s = bytes.__new__(cls, l) | ||
|
r13046 | s._utf8 = u | ||
return s | ||||
|
r43346 | |||
|
r44212 | if pycompat.TYPE_CHECKING: | ||
|
r44080 | # pseudo implementation to help pytype see localstr() constructor | ||
def __init__(self, u, l): | ||||
# type: (bytes, bytes) -> None | ||||
super(localstr, self).__init__(l) | ||||
self._utf8 = u | ||||
|
r13046 | def __hash__(self): | ||
|
r43346 | return hash(self._utf8) # avoid collisions in local string space | ||
|
r13046 | |||
|
r37966 | class safelocalstr(bytes): | ||
"""Tagged string denoting it was previously an internal UTF-8 string, | ||||
and can be converted back to UTF-8 losslessly | ||||
>>> assert safelocalstr(b'\\xc3') == b'\\xc3' | ||||
>>> assert b'\\xc3' == safelocalstr(b'\\xc3') | ||||
>>> assert b'\\xc3' in {safelocalstr(b'\\xc3'): 0} | ||||
>>> assert safelocalstr(b'\\xc3') in {b'\\xc3': 0} | ||||
""" | ||||
|
r43346 | |||
|
r7948 | def tolocal(s): | ||
|
r44076 | # type: (bytes) -> bytes | ||
|
r7948 | """ | ||
Convert a string from internal UTF-8 to local encoding | ||||
All internal strings should be UTF-8 but some repos before the | ||||
implementation of locale support may contain latin1 or possibly | ||||
other character sets. We attempt to decode everything strictly | ||||
using UTF-8, then Latin-1, and failing that, we use UTF-8 and | ||||
replace unknown characters. | ||||
|
r13046 | |||
The localstr class is used to cache the known UTF-8 encoding of | ||||
strings next to their local representation to allow lossless | ||||
round-trip conversion back to UTF-8. | ||||
|
r34133 | >>> u = b'foo: \\xc3\\xa4' # utf-8 | ||
|
r13046 | >>> l = tolocal(u) | ||
>>> l | ||||
'foo: ?' | ||||
>>> fromlocal(l) | ||||
'foo: \\xc3\\xa4' | ||||
|
r34133 | >>> u2 = b'foo: \\xc3\\xa1' | ||
|
r13046 | >>> d = { l: 1, tolocal(u2): 2 } | ||
|
r18378 | >>> len(d) # no collision | ||
2 | ||||
|
r34133 | >>> b'foo: ?' in d | ||
|
r13046 | False | ||
|
r34133 | >>> l1 = b'foo: \\xe4' # historical latin1 fallback | ||
|
r13046 | >>> l = tolocal(l1) | ||
>>> l | ||||
'foo: ?' | ||||
>>> fromlocal(l) # magically in utf-8 | ||||
'foo: \\xc3\\xa4' | ||||
|
r7948 | """ | ||
|
r13046 | |||
|
r33928 | if isasciistr(s): | ||
return s | ||||
|
r16274 | try: | ||
|
r7948 | try: | ||
|
r16274 | # make sure string is actually stored in UTF-8 | ||
u = s.decode('UTF-8') | ||||
|
r43347 | if encoding == b'UTF-8': | ||
|
r16274 | # fast path | ||
return s | ||||
|
r43809 | r = u.encode(_sysstr(encoding), "replace") | ||
|
r30033 | if u == r.decode(_sysstr(encoding)): | ||
|
r13940 | # r is a safe, non-lossy encoding of s | ||
|
r37966 | return safelocalstr(r) | ||
|
r16274 | return localstr(s, r) | ||
except UnicodeDecodeError: | ||||
# we should only get here if we're looking at an ancient changeset | ||||
try: | ||||
|
r30033 | u = s.decode(_sysstr(fallbackencoding)) | ||
|
r43809 | r = u.encode(_sysstr(encoding), "replace") | ||
|
r30033 | if u == r.decode(_sysstr(encoding)): | ||
|
r16274 | # r is a safe, non-lossy encoding of s | ||
|
r37966 | return safelocalstr(r) | ||
|
r13940 | return localstr(u.encode('UTF-8'), r) | ||
|
r16274 | except UnicodeDecodeError: | ||
|
r43346 | u = s.decode("utf-8", "replace") # last ditch | ||
|
r30033 | # can't round-trip | ||
|
r43809 | return u.encode(_sysstr(encoding), "replace") | ||
|
r25660 | except LookupError as k: | ||
|
r46273 | raise error.Abort( | ||
pycompat.bytestr(k), hint=b"please check your locale settings" | ||||
) | ||||
|
r7948 | |||
|
r43346 | |||
|
r7948 | def fromlocal(s): | ||
|
r44036 | # type: (bytes) -> bytes | ||
|
r7948 | """ | ||
Convert a string from the local character encoding to UTF-8 | ||||
We attempt to decode strings using the encoding mode set by | ||||
HGENCODINGMODE, which defaults to 'strict'. In this mode, unknown | ||||
characters will cause an error message. Other modes include | ||||
'replace', which replaces unknown characters with a special | ||||
Unicode character, and 'ignore', which drops the character. | ||||
""" | ||||
|
r13046 | |||
# can we do a lossless round-trip? | ||||
if isinstance(s, localstr): | ||||
return s._utf8 | ||||
|
r33928 | if isasciistr(s): | ||
return s | ||||
|
r13046 | |||
|
r7948 | try: | ||
|
r30033 | u = s.decode(_sysstr(encoding), _sysstr(encodingmode)) | ||
return u.encode("utf-8") | ||||
|
r25660 | except UnicodeDecodeError as inst: | ||
|
r43346 | sub = s[max(0, inst.start - 10) : inst.start + 10] | ||
raise error.Abort( | ||||
|
r43347 | b"decoding near '%s': %s!" % (sub, pycompat.bytestr(inst)) | ||
|
r43346 | ) | ||
|
r25660 | except LookupError as k: | ||
|
r48820 | raise error.Abort( | ||
pycompat.bytestr(k), hint=b"please check your locale settings" | ||||
) | ||||
|
r7948 | |||
|
r43346 | |||
|
r31447 | def unitolocal(u): | ||
|
r43802 | # type: (Text) -> bytes | ||
|
r31447 | """Convert a unicode string to a byte string of local encoding""" | ||
return tolocal(u.encode('utf-8')) | ||||
|
r43346 | |||
|
r31447 | def unifromlocal(s): | ||
|
r43802 | # type: (bytes) -> Text | ||
|
r31447 | """Convert a byte string of local encoding to a unicode string""" | ||
return fromlocal(s).decode('utf-8') | ||||
|
r43346 | |||
|
r33022 | def unimethod(bytesfunc): | ||
|
r43802 | # type: (Callable[[Any], bytes]) -> Callable[[Any], Text] | ||
|
r33022 | """Create a proxy method that forwards __unicode__() and __str__() of | ||
Python 3 to __bytes__()""" | ||||
|
r43346 | |||
|
r33022 | def unifunc(obj): | ||
return unifromlocal(bytesfunc(obj)) | ||||
|
r43346 | |||
|
r33022 | return unifunc | ||
|
r43346 | |||
|
r31448 | # converter functions between native str and byte string. use these if the | ||
# character encoding is not aware (e.g. exception message) or is known to | ||||
# be locale dependent (e.g. date formatting.) | ||||
if pycompat.ispy3: | ||||
strtolocal = unitolocal | ||||
strfromlocal = unifromlocal | ||||
|
r33022 | strmethod = unimethod | ||
|
r31448 | else: | ||
|
r43770 | |||
def strtolocal(s): | ||||
# type: (str) -> bytes | ||||
|
r44078 | return s # pytype: disable=bad-return-type | ||
|
r43770 | |||
def strfromlocal(s): | ||||
# type: (bytes) -> str | ||||
|
r44078 | return s # pytype: disable=bad-return-type | ||
|
r43770 | |||
|
r33022 | strmethod = pycompat.identity | ||
|
r31448 | |||
|
r48359 | |||
def lower(s): | ||||
# type: (bytes) -> bytes | ||||
"""best-effort encoding-aware case-folding of local string s""" | ||||
try: | ||||
return asciilower(s) | ||||
except UnicodeDecodeError: | ||||
pass | ||||
try: | ||||
if isinstance(s, localstr): | ||||
u = s._utf8.decode("utf-8") | ||||
else: | ||||
u = s.decode(_sysstr(encoding), _sysstr(encodingmode)) | ||||
lu = u.lower() | ||||
if u == lu: | ||||
return s # preserve localstring | ||||
return lu.encode(_sysstr(encoding)) | ||||
except UnicodeError: | ||||
return s.lower() # we don't know how to fold this except in ASCII | ||||
except LookupError as k: | ||||
|
r48820 | raise error.Abort( | ||
pycompat.bytestr(k), hint=b"please check your locale settings" | ||||
) | ||||
|
r48359 | |||
def upper(s): | ||||
# type: (bytes) -> bytes | ||||
"""best-effort encoding-aware case-folding of local string s""" | ||||
try: | ||||
return asciiupper(s) | ||||
except UnicodeDecodeError: | ||||
return upperfallback(s) | ||||
def upperfallback(s): | ||||
# type: (Any) -> Any | ||||
try: | ||||
if isinstance(s, localstr): | ||||
u = s._utf8.decode("utf-8") | ||||
else: | ||||
u = s.decode(_sysstr(encoding), _sysstr(encodingmode)) | ||||
uu = u.upper() | ||||
if u == uu: | ||||
return s # preserve localstring | ||||
return uu.encode(_sysstr(encoding)) | ||||
except UnicodeError: | ||||
return s.upper() # we don't know how to fold this except in ASCII | ||||
except LookupError as k: | ||||
|
r48820 | raise error.Abort( | ||
pycompat.bytestr(k), hint=b"please check your locale settings" | ||||
) | ||||
|
r48359 | |||
|
r30034 | if not _nativeenviron: | ||
# now encoding and helper functions are available, recreate the environ | ||||
# dict to be exported to other modules | ||||
|
r48360 | if pycompat.iswindows and pycompat.ispy3: | ||
class WindowsEnviron(dict): | ||||
"""`os.environ` normalizes environment variables to uppercase on windows""" | ||||
def get(self, key, default=None): | ||||
return super().get(upper(key), default) | ||||
environ = WindowsEnviron() | ||||
for k, v in os.environ.items(): # re-exports | ||||
environ[tolocal(k.encode('utf-8'))] = tolocal(v.encode('utf-8')) | ||||
|
r30034 | |||
r48421 | DRIVE_RE = re.compile(b'^[a-z]:') | |||
|
r39843 | if pycompat.ispy3: | ||
# os.getcwd() on Python 3 returns string, but it has os.getcwdb() which | ||||
# returns bytes. | ||||
|
r39844 | if pycompat.iswindows: | ||
# Python 3 on Windows issues a DeprecationWarning about using the bytes | ||||
# API when os.getcwdb() is called. | ||||
|
r47037 | # | ||
# Additionally, py3.8+ uppercases the drive letter when calling | ||||
# os.path.realpath(), which is used on ``repo.root``. Since those | ||||
# strings are compared in various places as simple strings, also call | ||||
# realpath here. See https://bugs.python.org/issue40368 | ||||
r48421 | # | |||
# However this is not reliable, so lets explicitly make this drive | ||||
# letter upper case. | ||||
# | ||||
# note: we should consider dropping realpath here since it seems to | ||||
# change the semantic of `getcwd`. | ||||
def getcwd(): | ||||
cwd = os.getcwd() # re-exports | ||||
cwd = os.path.realpath(cwd) | ||||
cwd = strtolocal(cwd) | ||||
if DRIVE_RE.match(cwd): | ||||
cwd = cwd[0:1].upper() + cwd[1:] | ||||
return cwd | ||||
|
r39844 | else: | ||
getcwd = os.getcwdb # re-exports | ||||
|
r39843 | else: | ||
getcwd = os.getcwd # re-exports | ||||
|
r12866 | # How to treat ambiguous-width characters. Set to 'wide' to treat as wide. | ||
|
r43346 | _wide = _sysstr( | ||
|
r43347 | environ.get(b"HGENCODINGAMBIGUOUS", b"narrow") == b"wide" | ||
and b"WFA" | ||||
or b"WF" | ||||
|
r43346 | ) | ||
|
r12866 | |||
|
r7948 | def colwidth(s): | ||
|
r43802 | # type: (bytes) -> int | ||
|
r44226 | """Find the column width of a string for display in the local encoding""" | ||
|
r43906 | return ucolwidth(s.decode(_sysstr(encoding), 'replace')) | ||
|
r15066 | |||
|
r43346 | |||
|
r15066 | def ucolwidth(d): | ||
|
r43802 | # type: (Text) -> int | ||
|
r44226 | """Find the column width of a Unicode string for display""" | ||
|
r14951 | eaw = getattr(unicodedata, 'east_asian_width', None) | ||
if eaw is not None: | ||||
|
r32537 | return sum([eaw(c) in _wide and 2 or 1 for c in d]) | ||
|
r7948 | return len(d) | ||
|
r43346 | |||
|
r15143 | def getcols(s, start, c): | ||
|
r43802 | # type: (bytes, int, int) -> bytes | ||
|
r46554 | """Use colwidth to find a c-column substring of s starting at byte | ||
index start""" | ||||
|
r38806 | for x in pycompat.xrange(start + c, len(s)): | ||
|
r15143 | t = s[start:x] | ||
if colwidth(t) == c: | ||||
return t | ||||
|
r44074 | raise ValueError('substring not found') | ||
|
r15143 | |||
|
r43346 | |||
|
r43347 | def trim(s, width, ellipsis=b'', leftside=False): | ||
|
r43802 | # type: (bytes, int, bytes, bool) -> bytes | ||
|
r21856 | """Trim string 's' to at most 'width' columns (including 'ellipsis'). | ||
|
r21861 | If 'leftside' is True, left side of string 's' is trimmed. | ||
'ellipsis' is always placed at trimmed side. | ||||
|
r34138 | >>> from .node import bin | ||
|
r34139 | >>> def bprint(s): | ||
... print(pycompat.sysstr(s)) | ||||
|
r34133 | >>> ellipsis = b'+++' | ||
|
r27355 | >>> from . import encoding | ||
|
r34133 | >>> encoding.encoding = b'utf-8' | ||
>>> t = b'1234567890' | ||||
|
r34139 | >>> bprint(trim(t, 12, ellipsis=ellipsis)) | ||
|
r21856 | 1234567890 | ||
|
r34139 | >>> bprint(trim(t, 10, ellipsis=ellipsis)) | ||
|
r21856 | 1234567890 | ||
|
r34139 | >>> bprint(trim(t, 8, ellipsis=ellipsis)) | ||
|
r21856 | 12345+++ | ||
|
r34139 | >>> bprint(trim(t, 8, ellipsis=ellipsis, leftside=True)) | ||
|
r21861 | +++67890 | ||
|
r34139 | >>> bprint(trim(t, 8)) | ||
|
r21856 | 12345678 | ||
|
r34139 | >>> bprint(trim(t, 8, leftside=True)) | ||
|
r21861 | 34567890 | ||
|
r34139 | >>> bprint(trim(t, 3, ellipsis=ellipsis)) | ||
|
r21856 | +++ | ||
|
r34139 | >>> bprint(trim(t, 1, ellipsis=ellipsis)) | ||
|
r21856 | + | ||
>>> u = u'\u3042\u3044\u3046\u3048\u304a' # 2 x 5 = 10 columns | ||||
|
r34137 | >>> t = u.encode(pycompat.sysstr(encoding.encoding)) | ||
|
r34139 | >>> bprint(trim(t, 12, ellipsis=ellipsis)) | ||
|
r21856 | \xe3\x81\x82\xe3\x81\x84\xe3\x81\x86\xe3\x81\x88\xe3\x81\x8a | ||
|
r34139 | >>> bprint(trim(t, 10, ellipsis=ellipsis)) | ||
|
r21856 | \xe3\x81\x82\xe3\x81\x84\xe3\x81\x86\xe3\x81\x88\xe3\x81\x8a | ||
|
r34139 | >>> bprint(trim(t, 8, ellipsis=ellipsis)) | ||
|
r21856 | \xe3\x81\x82\xe3\x81\x84+++ | ||
|
r34139 | >>> bprint(trim(t, 8, ellipsis=ellipsis, leftside=True)) | ||
|
r21861 | +++\xe3\x81\x88\xe3\x81\x8a | ||
|
r34139 | >>> bprint(trim(t, 5)) | ||
|
r21856 | \xe3\x81\x82\xe3\x81\x84 | ||
|
r34139 | >>> bprint(trim(t, 5, leftside=True)) | ||
|
r21861 | \xe3\x81\x88\xe3\x81\x8a | ||
|
r34139 | >>> bprint(trim(t, 4, ellipsis=ellipsis)) | ||
|
r21856 | +++ | ||
|
r34139 | >>> bprint(trim(t, 4, ellipsis=ellipsis, leftside=True)) | ||
|
r21861 | +++ | ||
|
r34138 | >>> t = bin(b'112233445566778899aa') # invalid byte sequence | ||
|
r34139 | >>> bprint(trim(t, 12, ellipsis=ellipsis)) | ||
|
r21856 | \x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa | ||
|
r34139 | >>> bprint(trim(t, 10, ellipsis=ellipsis)) | ||
|
r21856 | \x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa | ||
|
r34139 | >>> bprint(trim(t, 8, ellipsis=ellipsis)) | ||
|
r21856 | \x11\x22\x33\x44\x55+++ | ||
|
r34139 | >>> bprint(trim(t, 8, ellipsis=ellipsis, leftside=True)) | ||
|
r21861 | +++\x66\x77\x88\x99\xaa | ||
|
r34139 | >>> bprint(trim(t, 8)) | ||
|
r21856 | \x11\x22\x33\x44\x55\x66\x77\x88 | ||
|
r34139 | >>> bprint(trim(t, 8, leftside=True)) | ||
|
r21861 | \x33\x44\x55\x66\x77\x88\x99\xaa | ||
|
r34139 | >>> bprint(trim(t, 3, ellipsis=ellipsis)) | ||
|
r21856 | +++ | ||
|
r34139 | >>> bprint(trim(t, 1, ellipsis=ellipsis)) | ||
|
r21856 | + | ||
""" | ||||
try: | ||||
|
r30033 | u = s.decode(_sysstr(encoding)) | ||
|
r21856 | except UnicodeDecodeError: | ||
|
r43346 | if len(s) <= width: # trimming is not needed | ||
|
r21856 | return s | ||
width -= len(ellipsis) | ||||
|
r43346 | if width <= 0: # no enough room even for ellipsis | ||
return ellipsis[: width + len(ellipsis)] | ||||
|
r21861 | if leftside: | ||
return ellipsis + s[-width:] | ||||
|
r21856 | return s[:width] + ellipsis | ||
|
r43346 | if ucolwidth(u) <= width: # trimming is not needed | ||
|
r21856 | return s | ||
width -= len(ellipsis) | ||||
|
r43346 | if width <= 0: # no enough room even for ellipsis | ||
return ellipsis[: width + len(ellipsis)] | ||||
|
r21856 | |||
|
r21861 | if leftside: | ||
uslice = lambda i: u[i:] | ||||
concat = lambda s: ellipsis + s | ||||
else: | ||||
uslice = lambda i: u[:-i] | ||||
concat = lambda s: s + ellipsis | ||||
|
r38806 | for i in pycompat.xrange(1, len(u)): | ||
|
r21856 | usub = uslice(i) | ||
if ucolwidth(usub) <= width: | ||||
|
r30033 | return concat(usub.encode(_sysstr(encoding))) | ||
|
r43346 | return ellipsis # no enough room for multi-column characters | ||
|
r21856 | |||
|
r24593 | class normcasespecs(object): | ||
|
r46554 | """what a platform's normcase does to ASCII strings | ||
|
r24593 | |||
This is specified per platform, and should be consistent with what normcase | ||||
on that platform actually does. | ||||
lower: normcase lowercases ASCII strings | ||||
upper: normcase uppercases ASCII strings | ||||
|
r24608 | other: the fallback function should always be called | ||
|
r46554 | This should be kept in sync with normcase_spec in util.h.""" | ||
|
r43346 | |||
|
r24593 | lower = -1 | ||
upper = 1 | ||||
other = 0 | ||||
|
r43346 | |||
|
r28068 | def jsonescape(s, paranoid=False): | ||
|
r43802 | # type: (Any, Any) -> Any | ||
|
r46554 | """returns a string suitable for JSON | ||
|
r22426 | |||
JSON is problematic for us because it doesn't support non-Unicode | ||||
bytes. To deal with this, we take the following approach: | ||||
|
r37966 | - localstr/safelocalstr objects are converted back to UTF-8 | ||
|
r22426 | - valid UTF-8/ASCII strings are passed as-is | ||
- other strings are converted to UTF-8b surrogate encoding | ||||
- apply JSON-specified string escaping | ||||
(escapes are doubled in these tests) | ||||
|
r34133 | >>> jsonescape(b'this is a test') | ||
|
r22426 | 'this is a test' | ||
|
r34133 | >>> jsonescape(b'escape characters: \\0 \\x0b \\x7f') | ||
|
r27881 | 'escape characters: \\\\u0000 \\\\u000b \\\\u007f' | ||
|
r34133 | >>> jsonescape(b'escape characters: \\b \\t \\n \\f \\r \\" \\\\') | ||
|
r33926 | 'escape characters: \\\\b \\\\t \\\\n \\\\f \\\\r \\\\" \\\\\\\\' | ||
|
r34133 | >>> jsonescape(b'a weird byte: \\xdd') | ||
|
r22426 | 'a weird byte: \\xed\\xb3\\x9d' | ||
|
r34133 | >>> jsonescape(b'utf-8: caf\\xc3\\xa9') | ||
|
r22426 | 'utf-8: caf\\xc3\\xa9' | ||
|
r34133 | >>> jsonescape(b'') | ||
|
r22426 | '' | ||
|
r28068 | |||
|
r28069 | If paranoid, non-ascii and common troublesome characters are also escaped. | ||
This is suitable for web output. | ||||
|
r28068 | |||
|
r34133 | >>> s = b'escape characters: \\0 \\x0b \\x7f' | ||
|
r33926 | >>> assert jsonescape(s) == jsonescape(s, paranoid=True) | ||
|
r34133 | >>> s = b'escape characters: \\b \\t \\n \\f \\r \\" \\\\' | ||
|
r33926 | >>> assert jsonescape(s) == jsonescape(s, paranoid=True) | ||
|
r34133 | >>> jsonescape(b'escape boundary: \\x7e \\x7f \\xc2\\x80', paranoid=True) | ||
|
r28068 | 'escape boundary: ~ \\\\u007f \\\\u0080' | ||
|
r34133 | >>> jsonescape(b'a weird byte: \\xdd', paranoid=True) | ||
|
r28068 | 'a weird byte: \\\\udcdd' | ||
|
r34133 | >>> jsonescape(b'utf-8: caf\\xc3\\xa9', paranoid=True) | ||
|
r28068 | 'utf-8: caf\\\\u00e9' | ||
|
r34133 | >>> jsonescape(b'non-BMP: \\xf0\\x9d\\x84\\x9e', paranoid=True) | ||
|
r28068 | 'non-BMP: \\\\ud834\\\\udd1e' | ||
|
r34133 | >>> jsonescape(b'<foo@example.org>', paranoid=True) | ||
|
r28069 | '\\\\u003cfoo@example.org\\\\u003e' | ||
|
r46554 | """ | ||
|
r22426 | |||
|
r28068 | u8chars = toutf8b(s) | ||
try: | ||||
|
r33925 | return _jsonescapeu8fast(u8chars, paranoid) | ||
except ValueError: | ||||
|
r28068 | pass | ||
|
r33925 | return charencodepure.jsonescapeu8fallback(u8chars, paranoid) | ||
|
r22426 | |||
|
r43346 | |||
|
r34215 | # We need to decode/encode U+DCxx codes transparently since invalid UTF-8 | ||
# bytes are mapped to that range. | ||||
if pycompat.ispy3: | ||||
_utf8strict = r'surrogatepass' | ||||
else: | ||||
_utf8strict = r'strict' | ||||
|
r26875 | _utf8len = [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 4] | ||
|
r43346 | |||
|
r26875 | def getutf8char(s, pos): | ||
|
r44077 | # type: (bytes, int) -> bytes | ||
|
r46554 | """get the next full utf-8 character in the given string, starting at pos | ||
|
r26875 | |||
Raises a UnicodeError if the given location does not start a valid | ||||
utf-8 character. | ||||
|
r46554 | """ | ||
|
r26875 | |||
# find how many bytes to attempt decoding from first nibble | ||||
|
r43346 | l = _utf8len[ord(s[pos : pos + 1]) >> 4] | ||
if not l: # ascii | ||||
return s[pos : pos + 1] | ||||
|
r26875 | |||
|
r43346 | c = s[pos : pos + l] | ||
|
r26875 | # validate with attempted decode | ||
|
r34215 | c.decode("utf-8", _utf8strict) | ||
|
r26875 | return c | ||
|
r43346 | |||
|
r16133 | def toutf8b(s): | ||
|
r44077 | # type: (bytes) -> bytes | ||
|
r46554 | """convert a local, possibly-binary string into UTF-8b | ||
|
r16133 | |||
This is intended as a generic method to preserve data when working | ||||
with schemes like JSON and XML that have no provision for | ||||
arbitrary byte strings. As Mercurial often doesn't know | ||||
what encoding data is in, we use so-called UTF-8b. | ||||
If a string is already valid UTF-8 (or ASCII), it passes unmodified. | ||||
Otherwise, unsupported bytes are mapped to UTF-16 surrogate range, | ||||
uDC00-uDCFF. | ||||
Principles of operation: | ||||
|
r17424 | - ASCII and UTF-8 data successfully round-trips and is understood | ||
|
r16133 | by Unicode-oriented clients | ||
- filenames and file contents in arbitrary other encodings can have | ||||
be round-tripped or recovered by clueful clients | ||||
- local strings that have a cached known UTF-8 encoding (aka | ||||
localstr) get sent as UTF-8 so Unicode-oriented clients get the | ||||
Unicode data they want | ||||
|
r37966 | - non-lossy local strings (aka safelocalstr) get sent as UTF-8 as well | ||
|
r16133 | - because we must preserve UTF-8 bytestring in places such as | ||
filenames, metadata can't be roundtripped without help | ||||
(Note: "UTF-8b" often refers to decoding a mix of valid UTF-8 and | ||||
arbitrary bytes into an internal Unicode format that can be | ||||
re-encoded back into the original. Here we are exposing the | ||||
internal surrogate encoding as a UTF-8 string.) | ||||
|
r46554 | """ | ||
|
r16133 | |||
|
r37965 | if isinstance(s, localstr): | ||
# assume that the original UTF-8 sequence would never contain | ||||
# invalid characters in U+DCxx range | ||||
return s._utf8 | ||||
|
r37966 | elif isinstance(s, safelocalstr): | ||
# already verified that s is non-lossy in legacy encoding, which | ||||
# shouldn't contain characters in U+DCxx range | ||||
return fromlocal(s) | ||||
|
r37965 | elif isasciistr(s): | ||
|
r33929 | return s | ||
|
r43347 | if b"\xed" not in s: | ||
|
r26879 | try: | ||
|
r34215 | s.decode('utf-8', _utf8strict) | ||
|
r26879 | return s | ||
except UnicodeDecodeError: | ||||
pass | ||||
|
r26878 | |||
|
r34213 | s = pycompat.bytestr(s) | ||
|
r43347 | r = b"" | ||
|
r26878 | pos = 0 | ||
l = len(s) | ||||
while pos < l: | ||||
try: | ||||
c = getutf8char(s, pos) | ||||
|
r43347 | if b"\xed\xb0\x80" <= c <= b"\xed\xb3\xbf": | ||
|
r26879 | # have to re-escape existing U+DCxx characters | ||
|
r43346 | c = unichr(0xDC00 + ord(s[pos])).encode('utf-8', _utf8strict) | ||
|
r26879 | pos += 1 | ||
else: | ||||
pos += len(c) | ||||
|
r26878 | except UnicodeDecodeError: | ||
|
r43346 | c = unichr(0xDC00 + ord(s[pos])).encode('utf-8', _utf8strict) | ||
|
r26878 | pos += 1 | ||
r += c | ||||
return r | ||||
|
r16133 | |||
|
r43346 | |||
|
r16133 | def fromutf8b(s): | ||
|
r44076 | # type: (bytes) -> bytes | ||
|
r46554 | """Given a UTF-8b string, return a local, possibly-binary string. | ||
|
r16133 | |||
return the original binary string. This | ||||
is a round-trip process for strings like filenames, but metadata | ||||
that's was passed through tolocal will remain in UTF-8. | ||||
|
r26963 | >>> roundtrip = lambda x: fromutf8b(toutf8b(x)) == x | ||
|
r34133 | >>> m = b"\\xc3\\xa9\\x99abcd" | ||
|
r26963 | >>> toutf8b(m) | ||
|
r16133 | '\\xc3\\xa9\\xed\\xb2\\x99abcd' | ||
|
r26963 | >>> roundtrip(m) | ||
True | ||||
|
r34133 | >>> roundtrip(b"\\xc2\\xc2\\x80") | ||
|
r26963 | True | ||
|
r34133 | >>> roundtrip(b"\\xef\\xbf\\xbd") | ||
|
r26963 | True | ||
|
r34133 | >>> roundtrip(b"\\xef\\xef\\xbf\\xbd") | ||
|
r16133 | True | ||
|
r34133 | >>> roundtrip(b"\\xf1\\x80\\x80\\x80\\x80") | ||
|
r27699 | True | ||
|
r46554 | """ | ||
|
r16133 | |||
|
r33929 | if isasciistr(s): | ||
return s | ||||
|
r16133 | # fast path - look for uDxxx prefixes in s | ||
|
r43347 | if b"\xed" not in s: | ||
|
r16133 | return s | ||
|
r27699 | # We could do this with the unicode type but some Python builds | ||
# use UTF-16 internally (issue5031) which causes non-BMP code | ||||
# points to be escaped. Instead, we use our handy getutf8char | ||||
# helper again to walk the string without "decoding" it. | ||||
|
r34213 | s = pycompat.bytestr(s) | ||
|
r43347 | r = b"" | ||
|
r27699 | pos = 0 | ||
l = len(s) | ||||
while pos < l: | ||||
c = getutf8char(s, pos) | ||||
pos += len(c) | ||||
# unescape U+DCxx characters | ||||
|
r43347 | if b"\xed\xb0\x80" <= c <= b"\xed\xb3\xbf": | ||
|
r43346 | c = pycompat.bytechr(ord(c.decode("utf-8", _utf8strict)) & 0xFF) | ||
|
r27699 | r += c | ||
|
r16133 | return r | ||