Show More
@@ -1,281 +1,281 b'' | |||||
1 | # encoding.py - character transcoding support for Mercurial |
|
1 | # encoding.py - character transcoding support for Mercurial | |
2 | # |
|
2 | # | |
3 | # Copyright 2005-2009 Matt Mackall <mpm@selenic.com> and others |
|
3 | # Copyright 2005-2009 Matt Mackall <mpm@selenic.com> and others | |
4 | # |
|
4 | # | |
5 | # This software may be used and distributed according to the terms of the |
|
5 | # This software may be used and distributed according to the terms of the | |
6 | # GNU General Public License version 2 or any later version. |
|
6 | # GNU General Public License version 2 or any later version. | |
7 |
|
7 | |||
8 | import error |
|
8 | import error | |
9 | import unicodedata, locale, os |
|
9 | import unicodedata, locale, os | |
10 |
|
10 | |||
11 | def _getpreferredencoding(): |
|
11 | def _getpreferredencoding(): | |
12 | ''' |
|
12 | ''' | |
13 | On darwin, getpreferredencoding ignores the locale environment and |
|
13 | On darwin, getpreferredencoding ignores the locale environment and | |
14 | always returns mac-roman. http://bugs.python.org/issue6202 fixes this |
|
14 | always returns mac-roman. http://bugs.python.org/issue6202 fixes this | |
15 | for Python 2.7 and up. This is the same corrected code for earlier |
|
15 | for Python 2.7 and up. This is the same corrected code for earlier | |
16 | Python versions. |
|
16 | Python versions. | |
17 |
|
17 | |||
18 | However, we can't use a version check for this method, as some distributions |
|
18 | However, we can't use a version check for this method, as some distributions | |
19 | patch Python to fix this. Instead, we use it as a 'fixer' for the mac-roman |
|
19 | patch Python to fix this. Instead, we use it as a 'fixer' for the mac-roman | |
20 | encoding, as it is unlikely that this encoding is the actually expected. |
|
20 | encoding, as it is unlikely that this encoding is the actually expected. | |
21 | ''' |
|
21 | ''' | |
22 | try: |
|
22 | try: | |
23 | locale.CODESET |
|
23 | locale.CODESET | |
24 | except AttributeError: |
|
24 | except AttributeError: | |
25 | # Fall back to parsing environment variables :-( |
|
25 | # Fall back to parsing environment variables :-( | |
26 | return locale.getdefaultlocale()[1] |
|
26 | return locale.getdefaultlocale()[1] | |
27 |
|
27 | |||
28 | oldloc = locale.setlocale(locale.LC_CTYPE) |
|
28 | oldloc = locale.setlocale(locale.LC_CTYPE) | |
29 | locale.setlocale(locale.LC_CTYPE, "") |
|
29 | locale.setlocale(locale.LC_CTYPE, "") | |
30 | result = locale.nl_langinfo(locale.CODESET) |
|
30 | result = locale.nl_langinfo(locale.CODESET) | |
31 | locale.setlocale(locale.LC_CTYPE, oldloc) |
|
31 | locale.setlocale(locale.LC_CTYPE, oldloc) | |
32 |
|
32 | |||
33 | return result |
|
33 | return result | |
34 |
|
34 | |||
35 | _encodingfixers = { |
|
35 | _encodingfixers = { | |
36 | '646': lambda: 'ascii', |
|
36 | '646': lambda: 'ascii', | |
37 | 'ANSI_X3.4-1968': lambda: 'ascii', |
|
37 | 'ANSI_X3.4-1968': lambda: 'ascii', | |
38 | 'mac-roman': _getpreferredencoding |
|
38 | 'mac-roman': _getpreferredencoding | |
39 | } |
|
39 | } | |
40 |
|
40 | |||
41 | try: |
|
41 | try: | |
42 | encoding = os.environ.get("HGENCODING") |
|
42 | encoding = os.environ.get("HGENCODING") | |
43 | if not encoding: |
|
43 | if not encoding: | |
44 | encoding = locale.getpreferredencoding() or 'ascii' |
|
44 | encoding = locale.getpreferredencoding() or 'ascii' | |
45 | encoding = _encodingfixers.get(encoding, lambda: encoding)() |
|
45 | encoding = _encodingfixers.get(encoding, lambda: encoding)() | |
46 | except locale.Error: |
|
46 | except locale.Error: | |
47 | encoding = 'ascii' |
|
47 | encoding = 'ascii' | |
48 | encodingmode = os.environ.get("HGENCODINGMODE", "strict") |
|
48 | encodingmode = os.environ.get("HGENCODINGMODE", "strict") | |
49 | fallbackencoding = 'ISO-8859-1' |
|
49 | fallbackencoding = 'ISO-8859-1' | |
50 |
|
50 | |||
51 | class localstr(str): |
|
51 | class localstr(str): | |
52 | '''This class allows strings that are unmodified to be |
|
52 | '''This class allows strings that are unmodified to be | |
53 | round-tripped to the local encoding and back''' |
|
53 | round-tripped to the local encoding and back''' | |
54 | def __new__(cls, u, l): |
|
54 | def __new__(cls, u, l): | |
55 | s = str.__new__(cls, l) |
|
55 | s = str.__new__(cls, l) | |
56 | s._utf8 = u |
|
56 | s._utf8 = u | |
57 | return s |
|
57 | return s | |
58 | def __hash__(self): |
|
58 | def __hash__(self): | |
59 | return hash(self._utf8) # avoid collisions in local string space |
|
59 | return hash(self._utf8) # avoid collisions in local string space | |
60 |
|
60 | |||
61 | def tolocal(s): |
|
61 | def tolocal(s): | |
62 | """ |
|
62 | """ | |
63 | Convert a string from internal UTF-8 to local encoding |
|
63 | Convert a string from internal UTF-8 to local encoding | |
64 |
|
64 | |||
65 | All internal strings should be UTF-8 but some repos before the |
|
65 | All internal strings should be UTF-8 but some repos before the | |
66 | implementation of locale support may contain latin1 or possibly |
|
66 | implementation of locale support may contain latin1 or possibly | |
67 | other character sets. We attempt to decode everything strictly |
|
67 | other character sets. We attempt to decode everything strictly | |
68 | using UTF-8, then Latin-1, and failing that, we use UTF-8 and |
|
68 | using UTF-8, then Latin-1, and failing that, we use UTF-8 and | |
69 | replace unknown characters. |
|
69 | replace unknown characters. | |
70 |
|
70 | |||
71 | The localstr class is used to cache the known UTF-8 encoding of |
|
71 | The localstr class is used to cache the known UTF-8 encoding of | |
72 | strings next to their local representation to allow lossless |
|
72 | strings next to their local representation to allow lossless | |
73 | round-trip conversion back to UTF-8. |
|
73 | round-trip conversion back to UTF-8. | |
74 |
|
74 | |||
75 | >>> u = 'foo: \\xc3\\xa4' # utf-8 |
|
75 | >>> u = 'foo: \\xc3\\xa4' # utf-8 | |
76 | >>> l = tolocal(u) |
|
76 | >>> l = tolocal(u) | |
77 | >>> l |
|
77 | >>> l | |
78 | 'foo: ?' |
|
78 | 'foo: ?' | |
79 | >>> fromlocal(l) |
|
79 | >>> fromlocal(l) | |
80 | 'foo: \\xc3\\xa4' |
|
80 | 'foo: \\xc3\\xa4' | |
81 | >>> u2 = 'foo: \\xc3\\xa1' |
|
81 | >>> u2 = 'foo: \\xc3\\xa1' | |
82 | >>> d = { l: 1, tolocal(u2): 2 } |
|
82 | >>> d = { l: 1, tolocal(u2): 2 } | |
83 | >>> d # no collision |
|
83 | >>> d # no collision | |
84 | {'foo: ?': 1, 'foo: ?': 2} |
|
84 | {'foo: ?': 1, 'foo: ?': 2} | |
85 | >>> 'foo: ?' in d |
|
85 | >>> 'foo: ?' in d | |
86 | False |
|
86 | False | |
87 | >>> l1 = 'foo: \\xe4' # historical latin1 fallback |
|
87 | >>> l1 = 'foo: \\xe4' # historical latin1 fallback | |
88 | >>> l = tolocal(l1) |
|
88 | >>> l = tolocal(l1) | |
89 | >>> l |
|
89 | >>> l | |
90 | 'foo: ?' |
|
90 | 'foo: ?' | |
91 | >>> fromlocal(l) # magically in utf-8 |
|
91 | >>> fromlocal(l) # magically in utf-8 | |
92 | 'foo: \\xc3\\xa4' |
|
92 | 'foo: \\xc3\\xa4' | |
93 | """ |
|
93 | """ | |
94 |
|
94 | |||
95 | try: |
|
95 | try: | |
96 | try: |
|
96 | try: | |
97 | # make sure string is actually stored in UTF-8 |
|
97 | # make sure string is actually stored in UTF-8 | |
98 | u = s.decode('UTF-8') |
|
98 | u = s.decode('UTF-8') | |
99 | if encoding == 'UTF-8': |
|
99 | if encoding == 'UTF-8': | |
100 | # fast path |
|
100 | # fast path | |
101 | return s |
|
101 | return s | |
102 | r = u.encode(encoding, "replace") |
|
102 | r = u.encode(encoding, "replace") | |
103 | if u == r.decode(encoding): |
|
103 | if u == r.decode(encoding): | |
104 | # r is a safe, non-lossy encoding of s |
|
104 | # r is a safe, non-lossy encoding of s | |
105 | return r |
|
105 | return r | |
106 | return localstr(s, r) |
|
106 | return localstr(s, r) | |
107 | except UnicodeDecodeError: |
|
107 | except UnicodeDecodeError: | |
108 | # we should only get here if we're looking at an ancient changeset |
|
108 | # we should only get here if we're looking at an ancient changeset | |
109 | try: |
|
109 | try: | |
110 | u = s.decode(fallbackencoding) |
|
110 | u = s.decode(fallbackencoding) | |
111 | r = u.encode(encoding, "replace") |
|
111 | r = u.encode(encoding, "replace") | |
112 | if u == r.decode(encoding): |
|
112 | if u == r.decode(encoding): | |
113 | # r is a safe, non-lossy encoding of s |
|
113 | # r is a safe, non-lossy encoding of s | |
114 | return r |
|
114 | return r | |
115 | return localstr(u.encode('UTF-8'), r) |
|
115 | return localstr(u.encode('UTF-8'), r) | |
116 | except UnicodeDecodeError: |
|
116 | except UnicodeDecodeError: | |
117 | u = s.decode("utf-8", "replace") # last ditch |
|
117 | u = s.decode("utf-8", "replace") # last ditch | |
118 | return u.encode(encoding, "replace") # can't round-trip |
|
118 | return u.encode(encoding, "replace") # can't round-trip | |
119 | except LookupError, k: |
|
119 | except LookupError, k: | |
120 | raise error.Abort(k, hint="please check your locale settings") |
|
120 | raise error.Abort(k, hint="please check your locale settings") | |
121 |
|
121 | |||
122 | def fromlocal(s): |
|
122 | def fromlocal(s): | |
123 | """ |
|
123 | """ | |
124 | Convert a string from the local character encoding to UTF-8 |
|
124 | Convert a string from the local character encoding to UTF-8 | |
125 |
|
125 | |||
126 | We attempt to decode strings using the encoding mode set by |
|
126 | We attempt to decode strings using the encoding mode set by | |
127 | HGENCODINGMODE, which defaults to 'strict'. In this mode, unknown |
|
127 | HGENCODINGMODE, which defaults to 'strict'. In this mode, unknown | |
128 | characters will cause an error message. Other modes include |
|
128 | characters will cause an error message. Other modes include | |
129 | 'replace', which replaces unknown characters with a special |
|
129 | 'replace', which replaces unknown characters with a special | |
130 | Unicode character, and 'ignore', which drops the character. |
|
130 | Unicode character, and 'ignore', which drops the character. | |
131 | """ |
|
131 | """ | |
132 |
|
132 | |||
133 | # can we do a lossless round-trip? |
|
133 | # can we do a lossless round-trip? | |
134 | if isinstance(s, localstr): |
|
134 | if isinstance(s, localstr): | |
135 | return s._utf8 |
|
135 | return s._utf8 | |
136 |
|
136 | |||
137 | try: |
|
137 | try: | |
138 | return s.decode(encoding, encodingmode).encode("utf-8") |
|
138 | return s.decode(encoding, encodingmode).encode("utf-8") | |
139 | except UnicodeDecodeError, inst: |
|
139 | except UnicodeDecodeError, inst: | |
140 | sub = s[max(0, inst.start - 10):inst.start + 10] |
|
140 | sub = s[max(0, inst.start - 10):inst.start + 10] | |
141 | raise error.Abort("decoding near '%s': %s!" % (sub, inst)) |
|
141 | raise error.Abort("decoding near '%s': %s!" % (sub, inst)) | |
142 | except LookupError, k: |
|
142 | except LookupError, k: | |
143 | raise error.Abort(k, hint="please check your locale settings") |
|
143 | raise error.Abort(k, hint="please check your locale settings") | |
144 |
|
144 | |||
145 | # How to treat ambiguous-width characters. Set to 'wide' to treat as wide. |
|
145 | # How to treat ambiguous-width characters. Set to 'wide' to treat as wide. | |
146 | wide = (os.environ.get("HGENCODINGAMBIGUOUS", "narrow") == "wide" |
|
146 | wide = (os.environ.get("HGENCODINGAMBIGUOUS", "narrow") == "wide" | |
147 | and "WFA" or "WF") |
|
147 | and "WFA" or "WF") | |
148 |
|
148 | |||
149 | def colwidth(s): |
|
149 | def colwidth(s): | |
150 | "Find the column width of a string for display in the local encoding" |
|
150 | "Find the column width of a string for display in the local encoding" | |
151 | return ucolwidth(s.decode(encoding, 'replace')) |
|
151 | return ucolwidth(s.decode(encoding, 'replace')) | |
152 |
|
152 | |||
153 | def ucolwidth(d): |
|
153 | def ucolwidth(d): | |
154 | "Find the column width of a Unicode string for display" |
|
154 | "Find the column width of a Unicode string for display" | |
155 | eaw = getattr(unicodedata, 'east_asian_width', None) |
|
155 | eaw = getattr(unicodedata, 'east_asian_width', None) | |
156 | if eaw is not None: |
|
156 | if eaw is not None: | |
157 | return sum([eaw(c) in wide and 2 or 1 for c in d]) |
|
157 | return sum([eaw(c) in wide and 2 or 1 for c in d]) | |
158 | return len(d) |
|
158 | return len(d) | |
159 |
|
159 | |||
160 | def getcols(s, start, c): |
|
160 | def getcols(s, start, c): | |
161 | '''Use colwidth to find a c-column substring of s starting at byte |
|
161 | '''Use colwidth to find a c-column substring of s starting at byte | |
162 | index start''' |
|
162 | index start''' | |
163 | for x in xrange(start + c, len(s)): |
|
163 | for x in xrange(start + c, len(s)): | |
164 | t = s[start:x] |
|
164 | t = s[start:x] | |
165 | if colwidth(t) == c: |
|
165 | if colwidth(t) == c: | |
166 | return t |
|
166 | return t | |
167 |
|
167 | |||
168 | def lower(s): |
|
168 | def lower(s): | |
169 | "best-effort encoding-aware case-folding of local string s" |
|
169 | "best-effort encoding-aware case-folding of local string s" | |
170 | try: |
|
170 | try: | |
171 | return s.encode('ascii').lower() |
|
171 | return s.encode('ascii').lower() | |
172 |
except Unicode |
|
172 | except UnicodeError: | |
173 | pass |
|
173 | pass | |
174 | try: |
|
174 | try: | |
175 | if isinstance(s, localstr): |
|
175 | if isinstance(s, localstr): | |
176 | u = s._utf8.decode("utf-8") |
|
176 | u = s._utf8.decode("utf-8") | |
177 | else: |
|
177 | else: | |
178 | u = s.decode(encoding, encodingmode) |
|
178 | u = s.decode(encoding, encodingmode) | |
179 |
|
179 | |||
180 | lu = u.lower() |
|
180 | lu = u.lower() | |
181 | if u == lu: |
|
181 | if u == lu: | |
182 | return s # preserve localstring |
|
182 | return s # preserve localstring | |
183 | return lu.encode(encoding) |
|
183 | return lu.encode(encoding) | |
184 | except UnicodeError: |
|
184 | except UnicodeError: | |
185 | return s.lower() # we don't know how to fold this except in ASCII |
|
185 | return s.lower() # we don't know how to fold this except in ASCII | |
186 | except LookupError, k: |
|
186 | except LookupError, k: | |
187 | raise error.Abort(k, hint="please check your locale settings") |
|
187 | raise error.Abort(k, hint="please check your locale settings") | |
188 |
|
188 | |||
189 | def upper(s): |
|
189 | def upper(s): | |
190 | "best-effort encoding-aware case-folding of local string s" |
|
190 | "best-effort encoding-aware case-folding of local string s" | |
191 | try: |
|
191 | try: | |
192 | if isinstance(s, localstr): |
|
192 | if isinstance(s, localstr): | |
193 | u = s._utf8.decode("utf-8") |
|
193 | u = s._utf8.decode("utf-8") | |
194 | else: |
|
194 | else: | |
195 | u = s.decode(encoding, encodingmode) |
|
195 | u = s.decode(encoding, encodingmode) | |
196 |
|
196 | |||
197 | uu = u.upper() |
|
197 | uu = u.upper() | |
198 | if u == uu: |
|
198 | if u == uu: | |
199 | return s # preserve localstring |
|
199 | return s # preserve localstring | |
200 | return uu.encode(encoding) |
|
200 | return uu.encode(encoding) | |
201 | except UnicodeError: |
|
201 | except UnicodeError: | |
202 | return s.upper() # we don't know how to fold this except in ASCII |
|
202 | return s.upper() # we don't know how to fold this except in ASCII | |
203 | except LookupError, k: |
|
203 | except LookupError, k: | |
204 | raise error.Abort(k, hint="please check your locale settings") |
|
204 | raise error.Abort(k, hint="please check your locale settings") | |
205 |
|
205 | |||
206 | def toutf8b(s): |
|
206 | def toutf8b(s): | |
207 | '''convert a local, possibly-binary string into UTF-8b |
|
207 | '''convert a local, possibly-binary string into UTF-8b | |
208 |
|
208 | |||
209 | This is intended as a generic method to preserve data when working |
|
209 | This is intended as a generic method to preserve data when working | |
210 | with schemes like JSON and XML that have no provision for |
|
210 | with schemes like JSON and XML that have no provision for | |
211 | arbitrary byte strings. As Mercurial often doesn't know |
|
211 | arbitrary byte strings. As Mercurial often doesn't know | |
212 | what encoding data is in, we use so-called UTF-8b. |
|
212 | what encoding data is in, we use so-called UTF-8b. | |
213 |
|
213 | |||
214 | If a string is already valid UTF-8 (or ASCII), it passes unmodified. |
|
214 | If a string is already valid UTF-8 (or ASCII), it passes unmodified. | |
215 | Otherwise, unsupported bytes are mapped to UTF-16 surrogate range, |
|
215 | Otherwise, unsupported bytes are mapped to UTF-16 surrogate range, | |
216 | uDC00-uDCFF. |
|
216 | uDC00-uDCFF. | |
217 |
|
217 | |||
218 | Principles of operation: |
|
218 | Principles of operation: | |
219 |
|
219 | |||
220 | - ASCII and UTF-8 data sucessfully round-trips and is understood |
|
220 | - ASCII and UTF-8 data sucessfully round-trips and is understood | |
221 | by Unicode-oriented clients |
|
221 | by Unicode-oriented clients | |
222 | - filenames and file contents in arbitrary other encodings can have |
|
222 | - filenames and file contents in arbitrary other encodings can have | |
223 | be round-tripped or recovered by clueful clients |
|
223 | be round-tripped or recovered by clueful clients | |
224 | - local strings that have a cached known UTF-8 encoding (aka |
|
224 | - local strings that have a cached known UTF-8 encoding (aka | |
225 | localstr) get sent as UTF-8 so Unicode-oriented clients get the |
|
225 | localstr) get sent as UTF-8 so Unicode-oriented clients get the | |
226 | Unicode data they want |
|
226 | Unicode data they want | |
227 | - because we must preserve UTF-8 bytestring in places such as |
|
227 | - because we must preserve UTF-8 bytestring in places such as | |
228 | filenames, metadata can't be roundtripped without help |
|
228 | filenames, metadata can't be roundtripped without help | |
229 |
|
229 | |||
230 | (Note: "UTF-8b" often refers to decoding a mix of valid UTF-8 and |
|
230 | (Note: "UTF-8b" often refers to decoding a mix of valid UTF-8 and | |
231 | arbitrary bytes into an internal Unicode format that can be |
|
231 | arbitrary bytes into an internal Unicode format that can be | |
232 | re-encoded back into the original. Here we are exposing the |
|
232 | re-encoded back into the original. Here we are exposing the | |
233 | internal surrogate encoding as a UTF-8 string.) |
|
233 | internal surrogate encoding as a UTF-8 string.) | |
234 | ''' |
|
234 | ''' | |
235 |
|
235 | |||
236 | if isinstance(s, localstr): |
|
236 | if isinstance(s, localstr): | |
237 | return s._utf8 |
|
237 | return s._utf8 | |
238 |
|
238 | |||
239 | try: |
|
239 | try: | |
240 | if s.decode('utf-8'): |
|
240 | if s.decode('utf-8'): | |
241 | return s |
|
241 | return s | |
242 | except UnicodeDecodeError: |
|
242 | except UnicodeDecodeError: | |
243 | # surrogate-encode any characters that don't round-trip |
|
243 | # surrogate-encode any characters that don't round-trip | |
244 | s2 = s.decode('utf-8', 'ignore').encode('utf-8') |
|
244 | s2 = s.decode('utf-8', 'ignore').encode('utf-8') | |
245 | r = "" |
|
245 | r = "" | |
246 | pos = 0 |
|
246 | pos = 0 | |
247 | for c in s: |
|
247 | for c in s: | |
248 | if s2[pos:pos + 1] == c: |
|
248 | if s2[pos:pos + 1] == c: | |
249 | r += c |
|
249 | r += c | |
250 | pos += 1 |
|
250 | pos += 1 | |
251 | else: |
|
251 | else: | |
252 | r += unichr(0xdc00 + ord(c)).encode('utf-8') |
|
252 | r += unichr(0xdc00 + ord(c)).encode('utf-8') | |
253 | return r |
|
253 | return r | |
254 |
|
254 | |||
255 | def fromutf8b(s): |
|
255 | def fromutf8b(s): | |
256 | '''Given a UTF-8b string, return a local, possibly-binary string. |
|
256 | '''Given a UTF-8b string, return a local, possibly-binary string. | |
257 |
|
257 | |||
258 | return the original binary string. This |
|
258 | return the original binary string. This | |
259 | is a round-trip process for strings like filenames, but metadata |
|
259 | is a round-trip process for strings like filenames, but metadata | |
260 | that's was passed through tolocal will remain in UTF-8. |
|
260 | that's was passed through tolocal will remain in UTF-8. | |
261 |
|
261 | |||
262 | >>> m = "\\xc3\\xa9\\x99abcd" |
|
262 | >>> m = "\\xc3\\xa9\\x99abcd" | |
263 | >>> n = toutf8b(m) |
|
263 | >>> n = toutf8b(m) | |
264 | >>> n |
|
264 | >>> n | |
265 | '\\xc3\\xa9\\xed\\xb2\\x99abcd' |
|
265 | '\\xc3\\xa9\\xed\\xb2\\x99abcd' | |
266 | >>> fromutf8b(n) == m |
|
266 | >>> fromutf8b(n) == m | |
267 | True |
|
267 | True | |
268 | ''' |
|
268 | ''' | |
269 |
|
269 | |||
270 | # fast path - look for uDxxx prefixes in s |
|
270 | # fast path - look for uDxxx prefixes in s | |
271 | if "\xed" not in s: |
|
271 | if "\xed" not in s: | |
272 | return s |
|
272 | return s | |
273 |
|
273 | |||
274 | u = s.decode("utf-8") |
|
274 | u = s.decode("utf-8") | |
275 | r = "" |
|
275 | r = "" | |
276 | for c in u: |
|
276 | for c in u: | |
277 | if ord(c) & 0xff00 == 0xdc00: |
|
277 | if ord(c) & 0xff00 == 0xdc00: | |
278 | r += chr(ord(c) & 0xff) |
|
278 | r += chr(ord(c) & 0xff) | |
279 | else: |
|
279 | else: | |
280 | r += c.encode("utf-8") |
|
280 | r += c.encode("utf-8") | |
281 | return r |
|
281 | return r |
General Comments 0
You need to be logged in to leave comments.
Login now