i18n: cache translated messages per encoding

This is a simpler workaround alternative to D958, "i18n: clean msgcache when
encoding changes." The cache won't be bloated unless you run tons of commands
with different --encoding options on command server, or serve many repositories
of different web.encoding options on hgweb.

The test was originally written by Jun Wu.

Differential Revision: https://phab.mercurial-scm.org/D1053

mercurial/i18n.py

@@ -1,109 +1,110
 # i18n.py - internationalization support for mercurial
 #
 # Copyright 2005, 2006 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import gettext as gettextmod
 import locale
 import os
 import sys
 
 from . import (
     encoding,
     pycompat,
 )
 
 # modelled after templater.templatepath:
 if getattr(sys, 'frozen', None) is not None:
     module = pycompat.sysexecutable
 else:
     module = pycompat.fsencode(__file__)
 
 try:
     unicode
 except NameError:
     unicode = str
 
 _languages = None
 if (pycompat.iswindows
     and 'LANGUAGE' not in encoding.environ
     and 'LC_ALL' not in encoding.environ
     and 'LC_MESSAGES' not in encoding.environ
     and 'LANG' not in encoding.environ):
     # Try to detect UI language by "User Interface Language Management" API
     # if no locale variables are set. Note that locale.getdefaultlocale()
     # uses GetLocaleInfo(), which may be different from UI language.
     # (See http://msdn.microsoft.com/en-us/library/dd374098(v=VS.85).aspx )
     try:
         import ctypes
         langid = ctypes.windll.kernel32.GetUserDefaultUILanguage()
         _languages = [locale.windows_locale[langid]]
     except (ImportError, AttributeError, KeyError):
         # ctypes not found or unknown langid
         pass
 
 _ugettext = None
 
 def setdatapath(datapath):
     datapath = pycompat.fsdecode(datapath)
     localedir = os.path.join(datapath, pycompat.sysstr('locale'))
     t = gettextmod.translation('hg', localedir, _languages, fallback=True)
     global _ugettext
     try:
         _ugettext = t.ugettext
     except AttributeError:
         _ugettext = t.gettext
 
-_msgcache = {}
+_msgcache = {}  # encoding: {message: translation}
 
 def gettext(message):
     """Translate message.
 
     The message is looked up in the catalog to get a Unicode string,
     which is encoded in the local encoding before being returned.
 
     Important: message is restricted to characters in the encoding
     given by sys.getdefaultencoding() which is most likely 'ascii'.
     """
     # If message is None, t.ugettext will return u'None' as the
     # translation whereas our callers expect us to return None.
     if message is None or not _ugettext:
         return message
 
-    if message not in _msgcache:
+    cache = _msgcache.setdefault(encoding.encoding, {})
+    if message not in cache:
         if type(message) is unicode:
             # goofy unicode docstrings in test
             paragraphs = message.split(u'\n\n')
         else:
             paragraphs = [p.decode("ascii") for p in message.split('\n\n')]
         # Be careful not to translate the empty string -- it holds the
         # meta data of the .po file.
         u = u'\n\n'.join([p and _ugettext(p) or u'' for p in paragraphs])
         try:
             # encoding.tolocal cannot be used since it will first try to
             # decode the Unicode string. Calling u.decode(enc) really
             # means u.encode(sys.getdefaultencoding()).decode(enc). Since
             # the Python encoding defaults to 'ascii', this fails if the
             # translated string use non-ASCII characters.
             encodingstr = pycompat.sysstr(encoding.encoding)
-            _msgcache[message] = u.encode(encodingstr, "replace")
+            cache[message] = u.encode(encodingstr, "replace")
         except LookupError:
             # An unknown encoding results in a LookupError.
-            _msgcache[message] = message
-    return _msgcache[message]
+            cache[message] = message
+    return cache[message]
 
 def _plain():
     if ('HGPLAIN' not in encoding.environ
         and 'HGPLAINEXCEPT' not in encoding.environ):
         return False
     exceptions = encoding.environ.get('HGPLAINEXCEPT', '').strip().split(',')
     return 'i18n' not in exceptions
 
 if _plain():
     _ = lambda message: message
 else:
     _ = gettext

tests/test-i18n.t

@@ -1,50 +1,70
 (Translations are optional)
 
 #if gettext no-outer-repo
 
 Test that translations are compiled and installed correctly.
 
 Default encoding in tests is "ascii" and the translation is encoded
 using the "replace" error handler:
 
   $ LANGUAGE=pt_BR hg tip
   abortado: n?o foi encontrado um reposit?rio em '$TESTTMP' (.hg n?o encontrado)!
   [255]
 
 Using a more accommodating encoding:
 
   $ HGENCODING=UTF-8 LANGUAGE=pt_BR hg tip
   abortado: n\xc3\xa3o foi encontrado um reposit\xc3\xb3rio em '$TESTTMP' (.hg n\xc3\xa3o encontrado)! (esc)
   [255]
 
 Different encoding:
 
   $ HGENCODING=Latin-1 LANGUAGE=pt_BR hg tip
   abortado: n\xe3o foi encontrado um reposit\xf3rio em '$TESTTMP' (.hg n\xe3o encontrado)! (esc)
   [255]
 
 #endif
 
 #if gettext
 
 Test keyword search in translated help text:
 
   $ HGENCODING=UTF-8 LANGUAGE=de hg help -k Aktualisiert
   Themen:
   
    subrepos Unterarchive
   
   Befehle:
   
    pull   Ruft \xc3\x84nderungen von der angegebenen Quelle ab (esc)
    update Aktualisiert das Arbeitsverzeichnis (oder wechselt die Version)
 
 #endif
 
 Check Mercurial specific translation problems in each *.po files, and
 tool itself by doctest
 
   $ cd "$TESTDIR"/../i18n
   $ $PYTHON check-translation.py *.po
   $ $PYTHON check-translation.py --doctest
   $ cd $TESTTMP
+
+Check i18n cache isn't reused after encoding change:
+
+  $ cat > $TESTTMP/encodingchange.py << EOF
+  > from mercurial import encoding, registrar
+  > from mercurial.i18n import _
+  > cmdtable = {}
+  > command = registrar.command(cmdtable)
+  > @command(b'encodingchange', norepo=True)
+  > def encodingchange(ui):
+  >     for encode in (b'ascii', b'UTF-8', b'ascii', b'UTF-8'):
+  >         encoding.encoding = encode
+  >         ui.write(b'%s\n' % _(b'(EXPERIMENTAL)'))
+  > EOF
+
+  $ LANGUAGE=ja hg --config extensions.encodingchange=$TESTTMP/encodingchange.py encodingchange
+  (?????)
+  (\xe5\xae\x9f\xe9\xa8\x93\xe7\x9a\x84\xe5\xae\x9f\xe8\xa3\x85) (esc)
+  (?????)
+  (\xe5\xae\x9f\xe9\xa8\x93\xe7\x9a\x84\xe5\xae\x9f\xe8\xa3\x85) (esc)