tests: try to import modules with Python 3

All of mercurial.* is now using absolute_import. Most of
mercurial.* is able to ast parse with Python 3. The next big
hurdle is being able to import modules using Python 3.

This patch adds testing of hgext.* and mercurial.* module imports
in Python 3. As the new test output shows, most modules can't
import under Python 3. However, many of the failures are due
to a common problem in a highly imported module (e.g. the bytes vs
str issue in node.py).

contrib/check-py3-compat.py

@@ -10,7 +10,10 b''
 from __future__ import absolute_import, print_function
 
 import ast
+import imp
+import os
 import sys
+import traceback
 
 def check_compat_py2(f):
     """Check Python 3 compatibility for a file with Python 2"""
@@ -47,6 +50,27 b' def check_compat_py3(f):'
         print('%s: invalid syntax: %s' % (f, e))
         return
 
+    # Try to import the module.
+    # For now we only support mercurial.* and hgext.* modules because figuring
+    # out module paths for things not in a package can be confusing.
+    if f.startswith(('hgext/', 'mercurial/')) and not f.endswith('__init__.py'):
+        assert f.endswith('.py')
+        name = f.replace('/', '.')[:-3]
+        with open(f, 'r') as fh:
+            try:
+                imp.load_module(name, fh, '', ('py', 'r', imp.PY_SOURCE))
+            except Exception as e:
+                exc_type, exc_value, tb = sys.exc_info()
+                frame = traceback.extract_tb(tb)[-1]
+
+                if frame.filename:
+                    filename = os.path.basename(frame.filename)
+                    print('%s: error importing: <%s> %s (error at %s:%d)' % (
+                          f, type(e).__name__, e, filename, frame.lineno))
+                else:
+                    print('%s: error importing module: <%s> %s (line %d)' % (
+                          f, type(e).__name__, e, frame.lineno))
+
 if __name__ == '__main__':
     if sys.version_info[0] == 2:
         fn = check_compat_py2

tests/test-check-py3-compat.t

@@ -125,10 +125,163 b''
   contrib/import-checker.py: invalid syntax: Missing parentheses in call to 'print' (<unknown>, line 569)
   contrib/revsetbenchmarks.py: invalid syntax: invalid syntax (<unknown>, line 186)
   doc/hgmanpage.py: invalid syntax: invalid syntax (<unknown>, line 286)
+  hgext/acl.py: error importing: <ImportError> No module named 'cStringIO' (error at parsers.py:10)
+  hgext/automv.py: error importing module: <SyntaxError> invalid syntax (commands.py, line 3324) (line 29)
+  hgext/blackbox.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  hgext/bugzilla.py: error importing module: <ImportError> No module named 'urlparse' (line 284)
+  hgext/censor.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  hgext/chgserver.py: error importing module: <ImportError> No module named 'SocketServer' (line 43)
+  hgext/children.py: error importing: <ImportError> No module named 'cStringIO' (error at cmdutil.py:10)
+  hgext/churn.py: error importing: <ImportError> No module named 'cStringIO' (error at cmdutil.py:10)
+  hgext/clonebundles.py: error importing: <ImportError> No module named 'cStringIO' (error at cmdutil.py:10)
   hgext/color.py: invalid syntax: invalid syntax (<unknown>, line 551)
+  hgext/convert/bzr.py: error importing module: <SystemError> Parent module 'hgext.convert' not loaded, cannot perform relative import (line 18)
+  hgext/convert/common.py: error importing module: <ImportError> No module named 'cPickle' (line 10)
+  hgext/convert/convcmd.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  hgext/convert/cvs.py: error importing module: <ImportError> No module named 'cStringIO' (line 9)
+  hgext/convert/cvsps.py: error importing module: <ImportError> No module named 'cPickle' (line 9)
+  hgext/convert/darcs.py: error importing: <ImportError> No module named 'cStringIO' (error at parsers.py:10)
+  hgext/convert/filemap.py: error importing module: <SystemError> Parent module 'hgext.convert' not loaded, cannot perform relative import (line 14)
+  hgext/convert/git.py: error importing: <ImportError> No module named 'cStringIO' (error at parsers.py:10)
+  hgext/convert/gnuarch.py: error importing: <ImportError> No module named 'cStringIO' (error at parsers.py:10)
+  hgext/convert/hg.py: error importing module: <ImportError> No module named 'cStringIO' (line 21)
+  hgext/convert/monotone.py: error importing: <ImportError> No module named 'cStringIO' (error at parsers.py:10)
+  hgext/convert/p4.py: error importing: <ImportError> No module named 'cStringIO' (error at parsers.py:10)
+  hgext/convert/subversion.py: error importing module: <ImportError> No module named 'cPickle' (line 6)
+  hgext/convert/transport.py: error importing module: <ImportError> No module named 'svn.client' (line 21)
+  hgext/eol.py: error importing: <ImportError> No module named 'cStringIO' (error at parsers.py:10)
+  hgext/extdiff.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  hgext/factotum.py: error importing: <ImportError> No module named 'cStringIO' (error at url.py:13)
+  hgext/fetch.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  hgext/fsmonitor/state.py: error importing: <ImportError> No module named 'cStringIO' (error at parsers.py:10)
+  hgext/fsmonitor/watchmanclient.py: error importing: <ImportError> No module named 'cStringIO' (error at parsers.py:10)
+  hgext/gpg.py: error importing: <ImportError> No module named 'cStringIO' (error at parsers.py:10)
+  hgext/graphlog.py: error importing: <ImportError> No module named 'cStringIO' (error at cmdutil.py:10)
+  hgext/hgcia.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  hgext/hgk.py: error importing: <ImportError> No module named 'cStringIO' (error at cmdutil.py:10)
+  hgext/highlight/highlight.py: error importing: <ImportError> No module named 'cStringIO' (error at parsers.py:10)
+  hgext/histedit.py: error importing module: <SyntaxError> invalid syntax (bundle2.py, line 977) (line 177)
+  hgext/keyword.py: error importing: <ImportError> No module named 'cStringIO' (error at parsers.py:10)
+  hgext/largefiles/basestore.py: error importing: <ImportError> No module named 'cStringIO' (error at parsers.py:10)
+  hgext/largefiles/lfcommands.py: error importing: <ImportError> No module named 'cStringIO' (error at parsers.py:10)
+  hgext/largefiles/lfutil.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  hgext/largefiles/localstore.py: error importing module: <ImportError> No module named 'lfutil' (line 13)
+  hgext/largefiles/overrides.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  hgext/largefiles/proto.py: error importing module: <ImportError> No module named 'urllib2' (line 7)
+  hgext/largefiles/remotestore.py: error importing module: <ImportError> No module named 'urllib2' (line 9)
+  hgext/largefiles/reposetup.py: error importing: <ImportError> No module named 'cStringIO' (error at parsers.py:10)
+  hgext/largefiles/uisetup.py: error importing module: <SyntaxError> invalid syntax (archival.py, line 234) (line 11)
+  hgext/largefiles/wirestore.py: error importing module: <ImportError> No module named 'lfutil' (line 8)
+  hgext/mq.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  hgext/notify.py: error importing: <ImportError> No module named 'cStringIO' (error at cmdutil.py:10)
+  hgext/pager.py: error importing: <ImportError> No module named 'cStringIO' (error at cmdutil.py:10)
+  hgext/patchbomb.py: error importing module: <ImportError> No module named 'cStringIO' (line 68)
+  hgext/purge.py: error importing: <ImportError> No module named 'cStringIO' (error at cmdutil.py:10)
+  hgext/rebase.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  hgext/record.py: error importing: <ImportError> No module named 'cStringIO' (error at cmdutil.py:10)
+  hgext/relink.py: error importing: <ImportError> No module named 'cStringIO' (error at cmdutil.py:10)
+  hgext/schemes.py: error importing: <ImportError> No module named 'cStringIO' (error at cmdutil.py:10)
+  hgext/share.py: error importing: <ImportError> No module named 'cStringIO' (error at cmdutil.py:10)
+  hgext/shelve.py: error importing module: <SyntaxError> invalid syntax (bundle2.py, line 977) (line 28)
+  hgext/strip.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  hgext/transplant.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  hgext/win32text.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  mercurial/ancestor.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
   mercurial/archival.py: invalid syntax: invalid syntax (<unknown>, line 234)
+  mercurial/bookmarks.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  mercurial/branchmap.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
   mercurial/bundle2.py: invalid syntax: invalid syntax (<unknown>, line 977)
+  mercurial/bundlerepo.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  mercurial/byterange.py: error importing module: <ImportError> No module named 'urllib2' (line 30)
+  mercurial/changegroup.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  mercurial/changelog.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  mercurial/cmdutil.py: error importing module: <ImportError> No module named 'cStringIO' (line 10)
   mercurial/commands.py: invalid syntax: invalid syntax (<unknown>, line 3324)
+  mercurial/commandserver.py: error importing module: <ImportError> No module named 'SocketServer' (line 10)
+  mercurial/config.py: error importing: <ImportError> No module named 'cStringIO' (error at parsers.py:10)
+  mercurial/context.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  mercurial/copies.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  mercurial/crecord.py: error importing module: <ImportError> No module named 'cStringIO' (line 13)
+  mercurial/dagutil.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  mercurial/destutil.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  mercurial/dirstate.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  mercurial/discovery.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  mercurial/dispatch.py: error importing: <ImportError> No module named 'cStringIO' (error at cmdutil.py:10)
+  mercurial/exchange.py: error importing module: <ImportError> No module named 'urllib2' (line 12)
+  mercurial/extensions.py: error importing: <ImportError> No module named 'cStringIO' (error at cmdutil.py:10)
+  mercurial/filelog.py: error importing: <ImportError> No module named 'cStringIO' (error at mpatch.py:10)
+  mercurial/filemerge.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  mercurial/fileset.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  mercurial/formatter.py: error importing module: <ImportError> No module named 'cPickle' (line 10)
+  mercurial/graphmod.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  mercurial/hbisect.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  mercurial/help.py: error importing: <ImportError> No module named 'cStringIO' (error at cmdutil.py:10)
+  mercurial/hg.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  mercurial/hgweb/common.py: error importing module: <ImportError> No module named 'BaseHTTPServer' (line 11)
+  mercurial/hgweb/hgweb_mod.py: error importing module: <SystemError> Parent module 'mercurial.hgweb' not loaded, cannot perform relative import (line 14)
+  mercurial/hgweb/hgwebdir_mod.py: error importing module: <SystemError> Parent module 'mercurial.hgweb' not loaded, cannot perform relative import (line 15)
+  mercurial/hgweb/protocol.py: error importing module: <ImportError> No module named 'cStringIO' (line 10)
+  mercurial/hgweb/request.py: error importing module: <SystemError> Parent module 'mercurial.hgweb' not loaded, cannot perform relative import (line 15)
+  mercurial/hgweb/server.py: error importing module: <ImportError> No module named 'BaseHTTPServer' (line 11)
+  mercurial/hgweb/webcommands.py: error importing module: <SystemError> Parent module 'mercurial.hgweb' not loaded, cannot perform relative import (line 16)
+  mercurial/hgweb/webutil.py: error importing module: <SystemError> Parent module 'mercurial.hgweb' not loaded, cannot perform relative import (line 16)
+  mercurial/hgweb/wsgicgi.py: error importing module: <SystemError> Parent module 'mercurial.hgweb' not loaded, cannot perform relative import (line 16)
+  mercurial/hook.py: error importing: <ImportError> No module named 'cStringIO' (error at cmdutil.py:10)
+  mercurial/httpclient/_readers.py: error importing module: <ImportError> No module named 'httplib' (line 36)
+  mercurial/httpconnection.py: error importing module: <ImportError> No module named 'urllib2' (line 17)
+  mercurial/httppeer.py: error importing module: <ImportError> No module named 'httplib' (line 12)
+  mercurial/keepalive.py: error importing module: <ImportError> No module named 'httplib' (line 113)
+  mercurial/localrepo.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  mercurial/lock.py: error importing: <ImportError> No module named 'cStringIO' (error at parsers.py:10)
+  mercurial/mail.py: error importing: <ImportError> No module named 'cStringIO' (error at parsers.py:10)
+  mercurial/manifest.py: error importing: <ImportError> No module named 'cStringIO' (error at mpatch.py:10)
+  mercurial/match.py: error importing: <ImportError> No module named 'cStringIO' (error at parsers.py:10)
+  mercurial/mdiff.py: error importing: <ImportError> No module named 'cStringIO' (error at mpatch.py:10)
+  mercurial/merge.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  mercurial/minirst.py: error importing: <ImportError> No module named 'cStringIO' (error at parsers.py:10)
+  mercurial/namespaces.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  mercurial/node.py: error importing module: <TypeError> a bytes-like object is required, not 'str' (line 18)
+  mercurial/obsolete.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  mercurial/patch.py: error importing module: <ImportError> No module named 'cStringIO' (line 11)
+  mercurial/pathutil.py: error importing: <ImportError> No module named 'cStringIO' (error at parsers.py:10)
+  mercurial/peer.py: error importing: <ImportError> No module named 'cStringIO' (error at parsers.py:10)
+  mercurial/phases.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  mercurial/pure/mpatch.py: error importing module: <ImportError> No module named 'cStringIO' (line 10)
+  mercurial/pure/parsers.py: error importing module: <ImportError> No module named 'cStringIO' (line 10)
+  mercurial/pushkey.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  mercurial/pvec.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  mercurial/registrar.py: error importing: <ImportError> No module named 'cStringIO' (error at parsers.py:10)
+  mercurial/repair.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  mercurial/repoview.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  mercurial/revlog.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  mercurial/revset.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  mercurial/scmutil.py: error importing module: <ImportError> No module named 'Queue' (line 10)
+  mercurial/scmwindows.py: error importing module: <ImportError> No module named '_winreg' (line 3)
+  mercurial/setdiscovery.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  mercurial/similar.py: error importing: <ImportError> No module named 'cStringIO' (error at mpatch.py:10)
+  mercurial/simplemerge.py: error importing: <ImportError> No module named 'cStringIO' (error at mpatch.py:10)
+  mercurial/sshpeer.py: error importing: <ImportError> No module named 'cStringIO' (error at parsers.py:10)
+  mercurial/sshserver.py: error importing: <ImportError> No module named 'cStringIO' (error at cmdutil.py:10)
+  mercurial/sslutil.py: error importing: <ImportError> No module named 'cStringIO' (error at parsers.py:10)
+  mercurial/statichttprepo.py: error importing module: <ImportError> No module named 'urllib2' (line 15)
+  mercurial/store.py: error importing: <ImportError> No module named 'cStringIO' (error at parsers.py:10)
+  mercurial/streamclone.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  mercurial/subrepo.py: error importing: <ImportError> No module named 'cStringIO' (error at cmdutil.py:10)
+  mercurial/tagmerge.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  mercurial/tags.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  mercurial/templatefilters.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  mercurial/templatekw.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  mercurial/templater.py: error importing: <ImportError> No module named 'cStringIO' (error at parsers.py:10)
+  mercurial/transaction.py: error importing: <ImportError> No module named 'cStringIO' (error at parsers.py:10)
+  mercurial/treediscovery.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  mercurial/ui.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  mercurial/unionrepo.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  mercurial/url.py: error importing module: <ImportError> No module named 'cStringIO' (line 13)
+  mercurial/util.py: error importing: <ImportError> No module named 'cStringIO' (error at parsers.py:10)
+  mercurial/verify.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
+  mercurial/win32.py: error importing module: <ImportError> No module named 'msvcrt' (line 12)
+  mercurial/windows.py: error importing module: <ImportError> No module named '_winreg' (line 10)
+  mercurial/wireproto.py: error importing: <TypeError> a bytes-like object is required, not 'str' (error at node.py:18)
   tests/filterpyflakes.py: invalid syntax: Missing parentheses in call to 'print' (<unknown>, line 61)
   tests/generate-working-copy-states.py: invalid syntax: Missing parentheses in call to 'print' (<unknown>, line 69)
   tests/get-with-headers.py: invalid syntax: Missing parentheses in call to 'print' (<unknown>, line 44)