py3: import builtin wrappers automagically by code transformer

This should be less invasive than mucking builtins.

Since tokenize.untokenize() looks start/end positions of tokens, we calculates
them from the NEWLINE token of the future import.

mercurial/__init__.py

@@ -170,7 +170,7 b' if sys.version_info[0] >= 3:'
             spec.loader = hgloader(spec.name, spec.origin)
             return spec
 
-    def replacetokens(tokens):
+    def replacetokens(tokens, fullname):
         """Transform a stream of tokens from raw to Python 3.
 
         It is called by the custom module loading machinery to rewrite
@@ -184,6 +184,7 b' if sys.version_info[0] >= 3:'
         REMEMBER TO CHANGE ``BYTECODEHEADER`` WHEN CHANGING THIS FUNCTION
         OR CACHED FILES WON'T GET INVALIDATED PROPERLY.
         """
+        futureimpline = False
         for i, t in enumerate(tokens):
             # Convert most string literals to byte literals. String literals
             # in Python 2 are bytes. String literals in Python 3 are unicode.
@@ -217,6 +218,29 b' if sys.version_info[0] >= 3:'
                                           t.line)
                 continue
 
+            # Insert compatibility imports at "from __future__ import" line.
+            # No '\n' should be added to preserve line numbers.
+            if (t.type == token.NAME and t.string == 'import' and
+                all(u.type == token.NAME for u in tokens[i - 2:i]) and
+                [u.string for u in tokens[i - 2:i]] == ['from', '__future__']):
+                futureimpline = True
+            if t.type == token.NEWLINE and futureimpline:
+                futureimpline = False
+                if fullname == 'mercurial.pycompat':
+                    yield t
+                    continue
+                r, c = t.start
+                l = (b'; from mercurial.pycompat import '
+                     b'delattr, getattr, hasattr, setattr, xrange\n')
+                for u in tokenize.tokenize(io.BytesIO(l).readline):
+                    if u.type in (tokenize.ENCODING, token.ENDMARKER):
+                        continue
+                    yield tokenize.TokenInfo(u.type, u.string,
+                                             (r, c + u.start[1]),
+                                             (r, c + u.end[1]),
+                                             '')
+                continue
+
             try:
                 nexttoken = tokens[i + 1]
             except IndexError:
@@ -279,7 +303,7 b' if sys.version_info[0] >= 3:'
     # ``replacetoken`` or any mechanism that changes semantics of module
     # loading is changed. Otherwise cached bytecode may get loaded without
     # the new transformation mechanisms applied.
-    BYTECODEHEADER = b'HG\x00\x01'
+    BYTECODEHEADER = b'HG\x00\x02'
 
     class hgloader(importlib.machinery.SourceFileLoader):
         """Custom module loader that transforms source code.
@@ -338,7 +362,7 b' if sys.version_info[0] >= 3:'
             """Perform token transformation before compilation."""
             buf = io.BytesIO(data)
             tokens = tokenize.tokenize(buf.readline)
-            data = tokenize.untokenize(replacetokens(list(tokens)))
+            data = tokenize.untokenize(replacetokens(list(tokens), self.name))
             # Python's built-in importer strips frames from exceptions raised
             # for this code. Unfortunately, that mechanism isn't extensible
             # and our frame will be blamed for the import failure. There

mercurial/pycompat.py

@@ -32,7 +32,6 b' else:'
 if sys.version_info[0] >= 3:
     import builtins
     import functools
-    builtins.xrange = range
 
     def _wrapattrfunc(f):
         @functools.wraps(f)
@@ -42,10 +41,12 b' if sys.version_info[0] >= 3:'
             return f(object, name, *args)
         return w
 
+    # these wrappers are automagically imported by hgloader
     delattr = _wrapattrfunc(builtins.delattr)
     getattr = _wrapattrfunc(builtins.getattr)
     hasattr = _wrapattrfunc(builtins.hasattr)
     setattr = _wrapattrfunc(builtins.setattr)
+    xrange = builtins.range
 
 stringio = io.StringIO
 empty = _queue.Empty

tests/test-check-py3-compat.t

@@ -48,7 +48,7 b''
   hgext/gpg.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
   hgext/graphlog.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
   hgext/hgk.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
-  hgext/highlight/highlight.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
+  hgext/highlight/highlight.py: error importing module: <ImportError> No module named 'pygments' (line 13)
   hgext/histedit.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
   hgext/journal.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
   hgext/keyword.py: error importing: <TypeError> str expected, not bytes (error at encoding.py:*) (glob)
@@ -122,52 +122,51 b''
   mercurial/hook.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
   mercurial/httpconnection.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
   mercurial/httppeer.py: error importing: <TypeError> str expected, not bytes (error at i18n.py:*) (glob)
-  mercurial/keepalive.py: error importing: <TypeError> getattr(): attribute name must be string (error at pycompat.py:*) (glob)
-  mercurial/localrepo.py: error importing: <TypeError> getattr(): attribute name must be string (error at pycompat.py:*) (glob)
-  mercurial/lock.py: error importing: <TypeError> getattr(): attribute name must be string (error at pycompat.py:*) (glob)
-  mercurial/mail.py: error importing: <TypeError> getattr(): attribute name must be string (error at pycompat.py:*) (glob)
-  mercurial/manifest.py: error importing: <TypeError> getattr(): attribute name must be string (error at pycompat.py:*) (glob)
-  mercurial/match.py: error importing: <TypeError> getattr(): attribute name must be string (error at pycompat.py:*) (glob)
-  mercurial/mdiff.py: error importing: <TypeError> getattr(): attribute name must be string (error at pycompat.py:*) (glob)
-  mercurial/merge.py: error importing: <TypeError> getattr(): attribute name must be string (error at pycompat.py:*) (glob)
-  mercurial/minirst.py: error importing: <TypeError> getattr(): attribute name must be string (error at pycompat.py:*) (glob)
-  mercurial/namespaces.py: error importing: <TypeError> getattr(): attribute name must be string (error at pycompat.py:*) (glob)
-  mercurial/obsolete.py: error importing: <TypeError> getattr(): attribute name must be string (error at pycompat.py:*) (glob)
-  mercurial/patch.py: error importing: <TypeError> getattr(): attribute name must be string (error at pycompat.py:*) (glob)
-  mercurial/pathutil.py: error importing: <TypeError> getattr(): attribute name must be string (error at pycompat.py:*) (glob)
-  mercurial/peer.py: error importing: <TypeError> getattr(): attribute name must be string (error at pycompat.py:*) (glob)
-  mercurial/pure/mpatch.py: error importing: <TypeError> getattr(): attribute name must be string (error at pycompat.py:*) (glob)
-  mercurial/pure/parsers.py: error importing: <TypeError> getattr(): attribute name must be string (error at pycompat.py:*) (glob)
-  mercurial/pushkey.py: error importing: <TypeError> getattr(): attribute name must be string (error at pycompat.py:*) (glob)
-  mercurial/pvec.py: error importing: <TypeError> getattr(): attribute name must be string (error at pycompat.py:*) (glob)
-  mercurial/registrar.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/repair.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/repoview.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/revlog.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/revset.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/scmutil.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/scmwindows.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/similar.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/simplemerge.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/sshpeer.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/sshserver.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/sslutil.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/statichttprepo.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/store.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/streamclone.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/subrepo.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/tagmerge.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/tags.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/templatefilters.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/templatekw.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/templater.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/transaction.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/ui.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/unionrepo.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/url.py: error importing: <TypeError> getattr(): attribute name must be string (error at util.py:*) (glob)
-  mercurial/verify.py: error importing: <TypeError> attribute name must be string, not 'bytes' (error at mdiff.py:*) (glob)
+  mercurial/keepalive.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/localrepo.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/lock.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/mail.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/manifest.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/match.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/mdiff.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/merge.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/minirst.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/namespaces.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/obsolete.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/patch.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/pathutil.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/peer.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/profiling.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/pushkey.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/pvec.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/registrar.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/repair.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/repoview.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/revlog.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/revset.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/scmutil.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/scmwindows.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/similar.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/simplemerge.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/sshpeer.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/sshserver.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/sslutil.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/statichttprepo.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/store.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/streamclone.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/subrepo.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/tagmerge.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/tags.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/templatefilters.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/templatekw.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/templater.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/transaction.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/ui.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/unionrepo.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/url.py: error importing: <TypeError> __slots__ items must be strings, not 'bytes' (error at util.py:558)
+  mercurial/verify.py: error importing module: <TypeError> unorderable types: str() >= tuple() (line 884)
   mercurial/win32.py: error importing module: <ImportError> No module named 'msvcrt' (line *) (glob)
   mercurial/windows.py: error importing module: <ImportError> No module named 'msvcrt' (line *) (glob)
-  mercurial/wireproto.py: error importing module: <TypeError> a bytes-like object is required, not 'str' (line *) (glob)
+  mercurial/wireproto.py: error importing module: <TypeError> unorderable types: str() >= tuple() (line 884)
 
 #endif