py3: provide encoding.environ which is a dict of bytes

This can't be moved to pycompat.py since we need encoding.tolocal() to
build bytes dict from unicode os.environ.

mercurial/encoding.py

@@ -47,6 +47,19 b' def hfsignoreclean(s):'
             s = s.replace(c, '')
     return s
 
+# encoding.environ is provided read-only, which may not be used to modify
+# the process environment
+_nativeenviron = (not pycompat.ispy3 or os.supports_bytes_environ)
+if not pycompat.ispy3:
+    environ = os.environ
+elif _nativeenviron:
+    environ = os.environb
+else:
+    # preferred encoding isn't known yet; use utf-8 to avoid unicode error
+    # and recreate it once encoding is settled
+    environ = dict((k.encode(u'utf-8'), v.encode(u'utf-8'))
+                   for k, v in os.environ.items())
+
 def _getpreferredencoding():
     '''
     On darwin, getpreferredencoding ignores the locale environment and
@@ -78,13 +91,13 b' def _getpreferredencoding():'
 }
 
 try:
-    encoding = os.environ.get("HGENCODING")
+    encoding = environ.get("HGENCODING")
     if not encoding:
         encoding = locale.getpreferredencoding() or 'ascii'
         encoding = _encodingfixers.get(encoding, lambda: encoding)()
 except locale.Error:
     encoding = 'ascii'
-encodingmode = os.environ.get("HGENCODINGMODE", "strict")
+encodingmode = environ.get("HGENCODINGMODE", "strict")
 fallbackencoding = 'ISO-8859-1'
 
 class localstr(str):
@@ -183,8 +196,14 b' def fromlocal(s):'
     except LookupError as k:
         raise error.Abort(k, hint="please check your locale settings")
 
+if not _nativeenviron:
+    # now encoding and helper functions are available, recreate the environ
+    # dict to be exported to other modules
+    environ = dict((tolocal(k.encode(u'utf-8')), tolocal(v.encode(u'utf-8')))
+                   for k, v in os.environ.items())
+
 # How to treat ambiguous-width characters. Set to 'wide' to treat as wide.
-wide = (os.environ.get("HGENCODINGAMBIGUOUS", "narrow") == "wide"
+wide = (environ.get("HGENCODINGAMBIGUOUS", "narrow") == "wide"
         and "WFA" or "WF")
 
 def colwidth(s):

tests/test-check-py3-compat.t

@@ -17,89 +17,90 b''
   > | xargs $PYTHON3 contrib/check-py3-compat.py \
   > | sed 's/[0-9][0-9]*)$/*)/'
   doc/hgmanpage.py: invalid syntax: invalid syntax (<unknown>, line *)
-  hgext/acl.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/automv.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/blackbox.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/bugzilla.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/censor.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/chgserver.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/children.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/churn.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/clonebundles.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/color.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/convert/bzr.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/convert/common.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/convert/convcmd.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/convert/cvs.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/convert/cvsps.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/convert/darcs.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/convert/filemap.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/convert/git.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/convert/gnuarch.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/convert/hg.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/convert/monotone.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/convert/p4.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/convert/subversion.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
+  hgext/acl.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/automv.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/blackbox.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/bugzilla.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/censor.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/chgserver.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/children.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/churn.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/clonebundles.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/color.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/convert/bzr.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/convert/common.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/convert/convcmd.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/convert/cvs.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/convert/cvsps.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/convert/darcs.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/convert/filemap.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/convert/git.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/convert/gnuarch.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/convert/hg.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/convert/monotone.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/convert/p4.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/convert/subversion.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
   hgext/convert/transport.py: error importing module: <ImportError> No module named 'svn.client' (line *)
-  hgext/eol.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/extdiff.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/factotum.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/fetch.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/fsmonitor/state.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/fsmonitor/watchmanclient.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/gpg.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/graphlog.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/hgk.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/histedit.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/journal.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/keyword.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/largefiles/basestore.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/largefiles/lfcommands.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/largefiles/lfutil.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/largefiles/localstore.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/largefiles/overrides.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/largefiles/proto.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/largefiles/remotestore.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/largefiles/reposetup.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/largefiles/storefactory.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/largefiles/uisetup.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
+  hgext/eol.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/extdiff.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/factotum.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/fetch.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/fsmonitor/state.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/fsmonitor/watchmanclient.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/gpg.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/graphlog.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/hgk.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/histedit.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/journal.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/keyword.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/largefiles/basestore.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/largefiles/lfcommands.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/largefiles/lfutil.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/largefiles/localstore.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/largefiles/overrides.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/largefiles/proto.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/largefiles/remotestore.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/largefiles/reposetup.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/largefiles/storefactory.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/largefiles/uisetup.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
   hgext/largefiles/wirestore.py: error importing module: <SystemError> Parent module 'hgext.largefiles' not loaded, cannot perform relative import (line *)
-  hgext/mq.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/notify.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/pager.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/patchbomb.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/purge.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/rebase.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/record.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/relink.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/schemes.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/share.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/shelve.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/strip.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/transplant.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/win32mbcs.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  hgext/win32text.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  mercurial/archival.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  mercurial/bookmarks.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  mercurial/branchmap.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  mercurial/bundle2.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  mercurial/bundlerepo.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  mercurial/byterange.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  mercurial/changegroup.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  mercurial/changelog.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  mercurial/cmdutil.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  mercurial/commands.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  mercurial/commandserver.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  mercurial/config.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  mercurial/context.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  mercurial/copies.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  mercurial/crecord.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  mercurial/dagparser.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  mercurial/dagutil.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  mercurial/destutil.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  mercurial/dirstate.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  mercurial/discovery.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
-  mercurial/dispatch.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
+  hgext/mq.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/notify.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/pager.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/patchbomb.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/purge.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/rebase.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/record.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/relink.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/schemes.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/share.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/shelve.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/strip.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/transplant.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/win32mbcs.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  hgext/win32text.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  mercurial/archival.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  mercurial/bookmarks.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  mercurial/branchmap.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  mercurial/bundle2.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  mercurial/bundlerepo.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  mercurial/byterange.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  mercurial/changegroup.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  mercurial/changelog.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  mercurial/cmdutil.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  mercurial/commands.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  mercurial/commandserver.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  mercurial/config.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  mercurial/context.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  mercurial/copies.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  mercurial/crecord.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  mercurial/dagparser.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  mercurial/dagutil.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  mercurial/destutil.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  mercurial/dirstate.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  mercurial/discovery.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  mercurial/dispatch.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
+  mercurial/encoding.py: error importing module: <TypeError> bytes expected, not str (line *)
   mercurial/exchange.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
   mercurial/extensions.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
   mercurial/fancyopts.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
@@ -165,10 +166,11 b''
   mercurial/ui.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*)
   mercurial/unionrepo.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*)
   mercurial/url.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*)
-  mercurial/verify.py: error importing module: <TypeError> unorderable types: str() >= tuple() (line *)
+  mercurial/util.py: error importing module: <TypeError> encode() argument 1 must be str, not bytes (line *)
+  mercurial/verify.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*)
   mercurial/win32.py: error importing module: <ImportError> No module named 'msvcrt' (line *)
   mercurial/windows.py: error importing module: <ImportError> No module named 'msvcrt' (line *)
-  mercurial/wireproto.py: error importing module: <TypeError> unorderable types: str() >= tuple() (line *)
+  mercurial/wireproto.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:*)
 
 #endif
 
@@ -176,5 +178,5 b''
   $ hg files 'set:(**.py) and grep(pygments)' | sed 's|\\|/|g' \
   > | xargs $PYTHON3 contrib/check-py3-compat.py \
   > | sed 's/[0-9][0-9]*)$/*)/'
-  hgext/highlight/highlight.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*)
+  hgext/highlight/highlight.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*)
 #endif