encoding: avoid localstr when a string can be encoded losslessly (issue2763)

localstr's hash method exists to prevent bogus matching on lossy local
encodings. For instance, we don't want 'caf?' to match 'café' in an
ASCII locale.

But when café can be losslessly encoded in the local charset, we can
simply use a normal string and avoid the hashing trick.


This avoids using localstr's hash method, which would prevent a match between

mercurial/encoding.py

@@ -1,146 +1,150 b''
 # encoding.py - character transcoding support for Mercurial
 #
 #  Copyright 2005-2009 Matt Mackall <[e-mail]> and others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 import error
 import unicodedata, locale, os
 
 def _getpreferredencoding():
     '''
     On darwin, getpreferredencoding ignores the locale environment and
     always returns mac-roman. http://bugs.python.org/issue6202 fixes this
     for Python 2.7 and up. This is the same corrected code for earlier
     Python versions.
 
     However, we can't use a version check for this method, as some distributions
     patch Python to fix this. Instead, we use it as a 'fixer' for the mac-roman
     encoding, as it is unlikely that this encoding is the actually expected.
     '''
     try:
         locale.CODESET
     except AttributeError:
         # Fall back to parsing environment variables :-(
         return locale.getdefaultlocale()[1]
 
     oldloc = locale.setlocale(locale.LC_CTYPE)
     locale.setlocale(locale.LC_CTYPE, "")
     result = locale.nl_langinfo(locale.CODESET)
     locale.setlocale(locale.LC_CTYPE, oldloc)
 
     return result
 
 _encodingfixers = {
     '646': lambda: 'ascii',
     'ANSI_X3.4-1968': lambda: 'ascii',
     'mac-roman': _getpreferredencoding
 }
 
 try:
     encoding = os.environ.get("HGENCODING")
     if not encoding:
         encoding = locale.getpreferredencoding() or 'ascii'
         encoding = _encodingfixers.get(encoding, lambda: encoding)()
 except locale.Error:
     encoding = 'ascii'
 encodingmode = os.environ.get("HGENCODINGMODE", "strict")
 fallbackencoding = 'ISO-8859-1'
 
 class localstr(str):
     '''This class allows strings that are unmodified to be
     round-tripped to the local encoding and back'''
     def __new__(cls, u, l):
         s = str.__new__(cls, l)
         s._utf8 = u
         return s
     def __hash__(self):
         return hash(self._utf8) # avoid collisions in local string space
 
 def tolocal(s):
     """
     Convert a string from internal UTF-8 to local encoding
 
     All internal strings should be UTF-8 but some repos before the
     implementation of locale support may contain latin1 or possibly
     other character sets. We attempt to decode everything strictly
     using UTF-8, then Latin-1, and failing that, we use UTF-8 and
     replace unknown characters.
 
     The localstr class is used to cache the known UTF-8 encoding of
     strings next to their local representation to allow lossless
     round-trip conversion back to UTF-8.
 
     >>> u = 'foo: \\xc3\\xa4' # utf-8
     >>> l = tolocal(u)
     >>> l
     'foo: ?'
     >>> fromlocal(l)
     'foo: \\xc3\\xa4'
     >>> u2 = 'foo: \\xc3\\xa1'
     >>> d = { l: 1, tolocal(u2): 2 }
     >>> d # no collision
     {'foo: ?': 1, 'foo: ?': 2}
     >>> 'foo: ?' in d
     False
     >>> l1 = 'foo: \\xe4' # historical latin1 fallback
     >>> l = tolocal(l1)
     >>> l
     'foo: ?'
     >>> fromlocal(l) # magically in utf-8
     'foo: \\xc3\\xa4'
     """
 
     for e in ('UTF-8', fallbackencoding):
         try:
             u = s.decode(e) # attempt strict decoding
-            if e == 'UTF-8':
-                return localstr(s, u.encode(encoding, "replace"))
+            r = u.encode(encoding, "replace")
+            if u == r.decode(encoding):
+                # r is a safe, non-lossy encoding of s
+                return r
+            elif e == 'UTF-8':
+                return localstr(s, r)
             else:
-                return localstr(u.encode('UTF-8'),
-                                u.encode(encoding, "replace"))
+                return localstr(u.encode('UTF-8'), r)
+
         except LookupError, k:
             raise error.Abort("%s, please check your locale settings" % k)
         except UnicodeDecodeError:
             pass
     u = s.decode("utf-8", "replace") # last ditch
     return u.encode(encoding, "replace") # can't round-trip
 
 def fromlocal(s):
     """
     Convert a string from the local character encoding to UTF-8
 
     We attempt to decode strings using the encoding mode set by
     HGENCODINGMODE, which defaults to 'strict'. In this mode, unknown
     characters will cause an error message. Other modes include
     'replace', which replaces unknown characters with a special
     Unicode character, and 'ignore', which drops the character.
     """
 
     # can we do a lossless round-trip?
     if isinstance(s, localstr):
         return s._utf8
 
     try:
         return s.decode(encoding, encodingmode).encode("utf-8")
     except UnicodeDecodeError, inst:
         sub = s[max(0, inst.start - 10):inst.start + 10]
         raise error.Abort("decoding near '%s': %s!" % (sub, inst))
     except LookupError, k:
         raise error.Abort("%s, please check your locale settings" % k)
 
 # How to treat ambiguous-width characters. Set to 'wide' to treat as wide.
 ambiguous = os.environ.get("HGENCODINGAMBIGUOUS", "narrow")
 
 def colwidth(s):
     "Find the column width of a UTF-8 string for display"
     d = s.decode(encoding, 'replace')
     if hasattr(unicodedata, 'east_asian_width'):
         wide = "WF"
         if ambiguous == "wide":
             wide = "WFA"
         w = unicodedata.east_asian_width
         return sum([w(c) in wide and 2 or 1 for c in d])
     return len(d)
 

tests/test-encoding.t

@@ -1,243 +1,249 b''
 Test character encoding
 
   $ hg init t
   $ cd t
 
 we need a repo with some legacy latin-1 changesets
 
   $ hg unbundle $TESTDIR/legacy-encoding.hg
   adding changesets
   adding manifests
   adding file changes
   added 2 changesets with 2 changes to 1 files
   (run 'hg update' to get a working copy)
   $ hg co
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ python << EOF
   > f = file('latin-1', 'w'); f.write("latin-1 e' encoded: \xe9"); f.close()
   > f = file('utf-8', 'w'); f.write("utf-8 e' encoded: \xc3\xa9"); f.close()
   > f = file('latin-1-tag', 'w'); f.write("\xe9"); f.close()
   > EOF
 
 should fail with encoding error
 
   $ echo "plain old ascii" > a
   $ hg st
   M a
   ? latin-1
   ? latin-1-tag
   ? utf-8
   $ HGENCODING=ascii hg ci -l latin-1
   transaction abort!
   rollback completed
   abort: decoding near ' encoded: \xe9': 'ascii' codec can't decode byte 0xe9 in position 20: ordinal not in range(128)! (esc)
   [255]
 
 these should work
 
   $ echo "latin-1" > a
   $ HGENCODING=latin-1 hg ci -l latin-1
   $ echo "utf-8" > a
   $ HGENCODING=utf-8 hg ci -l utf-8
   $ HGENCODING=latin-1 hg tag `cat latin-1-tag`
   $ HGENCODING=latin-1 hg branch `cat latin-1-tag`
   marked working directory as branch \xe9 (esc)
   $ HGENCODING=latin-1 hg ci -m 'latin1 branch'
   $ rm .hg/branch
 
 hg log (ascii)
 
   $ hg --encoding ascii log
   changeset:   5:093c6077d1c8
   branch:      ?
   tag:         tip
   user:        test
   date:        Thu Jan 01 00:00:00 1970 +0000
   summary:     latin1 branch
   
   changeset:   4:94db611b4196
   user:        test
   date:        Thu Jan 01 00:00:00 1970 +0000
   summary:     Added tag ? for changeset ca661e7520de
   
   changeset:   3:ca661e7520de
   tag:         ?
   user:        test
   date:        Thu Jan 01 00:00:00 1970 +0000
   summary:     utf-8 e' encoded: ?
   
   changeset:   2:650c6f3d55dd
   user:        test
   date:        Thu Jan 01 00:00:00 1970 +0000
   summary:     latin-1 e' encoded: ?
   
   changeset:   1:0e5b7e3f9c4a
   user:        test
   date:        Mon Jan 12 13:46:40 1970 +0000
   summary:     koi8-r: ????? = u'\u0440\u0442\u0443\u0442\u044c'
   
   changeset:   0:1e78a93102a3
   user:        test
   date:        Mon Jan 12 13:46:40 1970 +0000
   summary:     latin-1 e': ? = u'\xe9'
   
 
 hg log (latin-1)
 
   $ hg --encoding latin-1 log
   changeset:   5:093c6077d1c8
   branch:      \xe9 (esc)
   tag:         tip
   user:        test
   date:        Thu Jan 01 00:00:00 1970 +0000
   summary:     latin1 branch
   
   changeset:   4:94db611b4196
   user:        test
   date:        Thu Jan 01 00:00:00 1970 +0000
   summary:     Added tag \xe9 for changeset ca661e7520de (esc)
   
   changeset:   3:ca661e7520de
   tag:         \xe9 (esc)
   user:        test
   date:        Thu Jan 01 00:00:00 1970 +0000
   summary:     utf-8 e' encoded: \xe9 (esc)
   
   changeset:   2:650c6f3d55dd
   user:        test
   date:        Thu Jan 01 00:00:00 1970 +0000
   summary:     latin-1 e' encoded: \xe9 (esc)
   
   changeset:   1:0e5b7e3f9c4a
   user:        test
   date:        Mon Jan 12 13:46:40 1970 +0000
   summary:     koi8-r: \xd2\xd4\xd5\xd4\xd8 = u'\\u0440\\u0442\\u0443\\u0442\\u044c' (esc)
   
   changeset:   0:1e78a93102a3
   user:        test
   date:        Mon Jan 12 13:46:40 1970 +0000
   summary:     latin-1 e': \xe9 = u'\\xe9' (esc)
   
 
 hg log (utf-8)
 
   $ hg --encoding utf-8 log
   changeset:   5:093c6077d1c8
   branch:      \xc3\xa9 (esc)
   tag:         tip
   user:        test
   date:        Thu Jan 01 00:00:00 1970 +0000
   summary:     latin1 branch
   
   changeset:   4:94db611b4196
   user:        test
   date:        Thu Jan 01 00:00:00 1970 +0000
   summary:     Added tag \xc3\xa9 for changeset ca661e7520de (esc)
   
   changeset:   3:ca661e7520de
   tag:         \xc3\xa9 (esc)
   user:        test
   date:        Thu Jan 01 00:00:00 1970 +0000
   summary:     utf-8 e' encoded: \xc3\xa9 (esc)
   
   changeset:   2:650c6f3d55dd
   user:        test
   date:        Thu Jan 01 00:00:00 1970 +0000
   summary:     latin-1 e' encoded: \xc3\xa9 (esc)
   
   changeset:   1:0e5b7e3f9c4a
   user:        test
   date:        Mon Jan 12 13:46:40 1970 +0000
   summary:     koi8-r: \xc3\x92\xc3\x94\xc3\x95\xc3\x94\xc3\x98 = u'\\u0440\\u0442\\u0443\\u0442\\u044c' (esc)
   
   changeset:   0:1e78a93102a3
   user:        test
   date:        Mon Jan 12 13:46:40 1970 +0000
   summary:     latin-1 e': \xc3\xa9 = u'\\xe9' (esc)
   
 
 hg tags (ascii)
 
   $ HGENCODING=ascii hg tags
   tip                                5:093c6077d1c8
   ?                                  3:ca661e7520de
 
 hg tags (latin-1)
 
   $ HGENCODING=latin-1 hg tags
   tip                                5:093c6077d1c8
   \xe9                                  3:ca661e7520de (esc)
 
 hg tags (utf-8)
 
   $ HGENCODING=utf-8 hg tags
   tip                                5:093c6077d1c8
   \xc3\xa9                                  3:ca661e7520de (esc)
 
 hg branches (ascii)
 
   $ HGENCODING=ascii hg branches
   ?                              5:093c6077d1c8
   default                        4:94db611b4196 (inactive)
 
 hg branches (latin-1)
 
   $ HGENCODING=latin-1 hg branches
   \xe9                              5:093c6077d1c8 (esc)
   default                        4:94db611b4196 (inactive)
 
 hg branches (utf-8)
 
   $ HGENCODING=utf-8 hg branches
   \xc3\xa9                              5:093c6077d1c8 (esc)
   default                        4:94db611b4196 (inactive)
   $ echo '[ui]' >> .hg/hgrc
   $ echo 'fallbackencoding = koi8-r' >> .hg/hgrc
 
 hg log (utf-8)
 
   $ HGENCODING=utf-8 hg log
   changeset:   5:093c6077d1c8
   branch:      \xc3\xa9 (esc)
   tag:         tip
   user:        test
   date:        Thu Jan 01 00:00:00 1970 +0000
   summary:     latin1 branch
   
   changeset:   4:94db611b4196
   user:        test
   date:        Thu Jan 01 00:00:00 1970 +0000
   summary:     Added tag \xc3\xa9 for changeset ca661e7520de (esc)
   
   changeset:   3:ca661e7520de
   tag:         \xc3\xa9 (esc)
   user:        test
   date:        Thu Jan 01 00:00:00 1970 +0000
   summary:     utf-8 e' encoded: \xc3\xa9 (esc)
   
   changeset:   2:650c6f3d55dd
   user:        test
   date:        Thu Jan 01 00:00:00 1970 +0000
   summary:     latin-1 e' encoded: \xc3\xa9 (esc)
   
   changeset:   1:0e5b7e3f9c4a
   user:        test
   date:        Mon Jan 12 13:46:40 1970 +0000
   summary:     koi8-r: \xd1\x80\xd1\x82\xd1\x83\xd1\x82\xd1\x8c = u'\\u0440\\u0442\\u0443\\u0442\\u044c' (esc)
   
   changeset:   0:1e78a93102a3
   user:        test
   date:        Mon Jan 12 13:46:40 1970 +0000
   summary:     latin-1 e': \xd0\x98 = u'\\xe9' (esc)
   
 
 hg log (dolphin)
 
   $ HGENCODING=dolphin hg log
   abort: unknown encoding: dolphin, please check your locale settings
   [255]
   $ HGENCODING=ascii hg branch `cat latin-1-tag`
   abort: decoding near '\xe9': 'ascii' codec can't decode byte 0xe9 in position 0: ordinal not in range(128)! (esc)
   [255]
   $ cp latin-1-tag .hg/branch
   $ HGENCODING=latin-1 hg ci -m 'auto-promote legacy name'
+
+Test roundtrip encoding of lookup tables when not using UTF-8 (issue2763)
+
+  $ HGENCODING=latin-1 hg up `cat latin-1-tag`
+  0 files updated, 0 files merged, 1 files removed, 0 files unresolved
+