policy: eliminate ".pure." from module name only if marked as dual

So we can switch cext/pure modules to new layout one by one.

contrib/check-py3-compat.py

@@ -1,95 +1,107 b''
 #!/usr/bin/env python
 #
 # check-py3-compat - check Python 3 compatibility of Mercurial files
 #
 # Copyright 2015 Gregory Szorc <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import, print_function
 
 import ast
 import os
 import sys
 import traceback
 
+# Modules that have both Python and C implementations.
+_dualmodules = (
+    'base85.py',
+    'bdiff.py',
+    'diffhelpers.py',
+    'mpatch.py',
+    'osutil.py',
+    'parsers.py',
+)
+
 def check_compat_py2(f):
     """Check Python 3 compatibility for a file with Python 2"""
     with open(f, 'rb') as fh:
         content = fh.read()
     root = ast.parse(content)
 
     # Ignore empty files.
     if not root.body:
         return
 
     futures = set()
     haveprint = False
     for node in ast.walk(root):
         if isinstance(node, ast.ImportFrom):
             if node.module == '__future__':
                 futures |= set(n.name for n in node.names)
         elif isinstance(node, ast.Print):
             haveprint = True
 
     if 'absolute_import' not in futures:
         print('%s not using absolute_import' % f)
     if haveprint and 'print_function' not in futures:
         print('%s requires print_function' % f)
 
 def check_compat_py3(f):
     """Check Python 3 compatibility of a file with Python 3."""
     import importlib  # not available on Python 2.6
     with open(f, 'rb') as fh:
         content = fh.read()
 
     try:
         ast.parse(content)
     except SyntaxError as e:
         print('%s: invalid syntax: %s' % (f, e))
         return
 
     # Try to import the module.
     # For now we only support mercurial.* and hgext.* modules because figuring
     # out module paths for things not in a package can be confusing.
     if f.startswith(('hgext/', 'mercurial/')) and not f.endswith('__init__.py'):
         assert f.endswith('.py')
-        name = f.replace('/', '.')[:-3].replace('.pure.', '.')
+        name = f.replace('/', '.')[:-3]
+        if f.endswith(_dualmodules):
+            name = name.replace('.pure.', '.')
         try:
             importlib.import_module(name)
         except Exception as e:
             exc_type, exc_value, tb = sys.exc_info()
             # We walk the stack and ignore frames from our custom importer,
             # import mechanisms, and stdlib modules. This kinda/sorta
             # emulates CPython behavior in import.c while also attempting
             # to pin blame on a Mercurial file.
             for frame in reversed(traceback.extract_tb(tb)):
                 if frame.name == '_call_with_frames_removed':
                     continue
                 if 'importlib' in frame.filename:
                     continue
                 if 'mercurial/__init__.py' in frame.filename:
                     continue
                 if frame.filename.startswith(sys.prefix):
                     continue
                 break
 
             if frame.filename:
                 filename = os.path.basename(frame.filename)
                 print('%s: error importing: <%s> %s (error at %s:%d)' % (
                       f, type(e).__name__, e, filename, frame.lineno))
             else:
                 print('%s: error importing module: <%s> %s (line %d)' % (
                       f, type(e).__name__, e, frame.lineno))
 
 if __name__ == '__main__':
     if sys.version_info[0] == 2:
         fn = check_compat_py2
     else:
         fn = check_compat_py3
 
     for f in sys.argv[1:]:
         fn(f)
 
     sys.exit(0)

contrib/import-checker.py

@@ -1,725 +1,736 b''
 #!/usr/bin/env python
 
 from __future__ import absolute_import, print_function
 
 import ast
 import collections
 import os
 import re
 import sys
 
 # Import a minimal set of stdlib modules needed for list_stdlib_modules()
 # to work when run from a virtualenv.  The modules were chosen empirically
 # so that the return value matches the return value without virtualenv.
 if True: # disable lexical sorting checks
     import BaseHTTPServer
     import zlib
 
 # Whitelist of modules that symbols can be directly imported from.
 allowsymbolimports = (
     '__future__',
     'mercurial.hgweb.common',
     'mercurial.hgweb.request',
     'mercurial.i18n',
     'mercurial.node',
 )
 
+# Modules that have both Python and C implementations.
+_dualmodules = (
+    'base85.py',
+    'bdiff.py',
+    'diffhelpers.py',
+    'mpatch.py',
+    'osutil.py',
+    'parsers.py',
+)
+
 # Modules that must be aliased because they are commonly confused with
 # common variables and can create aliasing and readability issues.
 requirealias = {
     'ui': 'uimod',
 }
 
 def usingabsolute(root):
     """Whether absolute imports are being used."""
     if sys.version_info[0] >= 3:
         return True
 
     for node in ast.walk(root):
         if isinstance(node, ast.ImportFrom):
             if node.module == '__future__':
                 for n in node.names:
                     if n.name == 'absolute_import':
                         return True
 
     return False
 
 def walklocal(root):
     """Recursively yield all descendant nodes but not in a different scope"""
     todo = collections.deque(ast.iter_child_nodes(root))
     yield root, False
     while todo:
         node = todo.popleft()
         newscope = isinstance(node, ast.FunctionDef)
         if not newscope:
             todo.extend(ast.iter_child_nodes(node))
         yield node, newscope
 
 def dotted_name_of_path(path, trimpure=False):
     """Given a relative path to a source file, return its dotted module name.
 
     >>> dotted_name_of_path('mercurial/error.py')
     'mercurial.error'
     >>> dotted_name_of_path('mercurial/pure/parsers.py', trimpure=True)
     'mercurial.parsers'
     >>> dotted_name_of_path('zlibmodule.so')
     'zlib'
     """
     parts = path.replace(os.sep, '/').split('/')
     parts[-1] = parts[-1].split('.', 1)[0] # remove .py and .so and .ARCH.so
     if parts[-1].endswith('module'):
         parts[-1] = parts[-1][:-6]
     if trimpure:
         return '.'.join(p for p in parts if p != 'pure')
     return '.'.join(parts)
 
 def fromlocalfunc(modulename, localmods):
     """Get a function to examine which locally defined module the
     target source imports via a specified name.
 
     `modulename` is an `dotted_name_of_path()`-ed source file path,
     which may have `.__init__` at the end of it, of the target source.
 
     `localmods` is a dict (or set), of which key is an absolute
     `dotted_name_of_path()`-ed source file path of locally defined (=
     Mercurial specific) modules.
 
     This function assumes that module names not existing in
     `localmods` are from the Python standard library.
 
     This function returns the function, which takes `name` argument,
     and returns `(absname, dottedpath, hassubmod)` tuple if `name`
     matches against locally defined module. Otherwise, it returns
     False.
 
     It is assumed that `name` doesn't have `.__init__`.
 
     `absname` is an absolute module name of specified `name`
     (e.g. "hgext.convert"). This can be used to compose prefix for sub
     modules or so.
 
     `dottedpath` is a `dotted_name_of_path()`-ed source file path
     (e.g. "hgext.convert.__init__") of `name`. This is used to look
     module up in `localmods` again.
 
     `hassubmod` is whether it may have sub modules under it (for
     convenient, even though this is also equivalent to "absname !=
     dottednpath")
 
     >>> localmods = {'foo.__init__': True, 'foo.foo1': True,
     ...              'foo.bar.__init__': True, 'foo.bar.bar1': True,
     ...              'baz.__init__': True, 'baz.baz1': True }
     >>> fromlocal = fromlocalfunc('foo.xxx', localmods)
     >>> # relative
     >>> fromlocal('foo1')
     ('foo.foo1', 'foo.foo1', False)
     >>> fromlocal('bar')
     ('foo.bar', 'foo.bar.__init__', True)
     >>> fromlocal('bar.bar1')
     ('foo.bar.bar1', 'foo.bar.bar1', False)
     >>> # absolute
     >>> fromlocal('baz')
     ('baz', 'baz.__init__', True)
     >>> fromlocal('baz.baz1')
     ('baz.baz1', 'baz.baz1', False)
     >>> # unknown = maybe standard library
     >>> fromlocal('os')
     False
     >>> fromlocal(None, 1)
     ('foo', 'foo.__init__', True)
     >>> fromlocal('foo1', 1)
     ('foo.foo1', 'foo.foo1', False)
     >>> fromlocal2 = fromlocalfunc('foo.xxx.yyy', localmods)
     >>> fromlocal2(None, 2)
     ('foo', 'foo.__init__', True)
     >>> fromlocal2('bar2', 1)
     False
     >>> fromlocal2('bar', 2)
     ('foo.bar', 'foo.bar.__init__', True)
     """
     prefix = '.'.join(modulename.split('.')[:-1])
     if prefix:
         prefix += '.'
     def fromlocal(name, level=0):
         # name is false value when relative imports are used.
         if not name:
             # If relative imports are used, level must not be absolute.
             assert level > 0
             candidates = ['.'.join(modulename.split('.')[:-level])]
         else:
             if not level:
                 # Check relative name first.
                 candidates = [prefix + name, name]
             else:
                 candidates = ['.'.join(modulename.split('.')[:-level]) +
                               '.' + name]
 
         for n in candidates:
             if n in localmods:
                 return (n, n, False)
             dottedpath = n + '.__init__'
             if dottedpath in localmods:
                 return (n, dottedpath, True)
         return False
     return fromlocal
 
 def list_stdlib_modules():
     """List the modules present in the stdlib.
 
     >>> mods = set(list_stdlib_modules())
     >>> 'BaseHTTPServer' in mods
     True
 
     os.path isn't really a module, so it's missing:
 
     >>> 'os.path' in mods
     False
 
     sys requires special treatment, because it's baked into the
     interpreter, but it should still appear:
 
     >>> 'sys' in mods
     True
 
     >>> 'collections' in mods
     True
 
     >>> 'cStringIO' in mods
     True
 
     >>> 'cffi' in mods
     True
     """
     for m in sys.builtin_module_names:
         yield m
     # These modules only exist on windows, but we should always
     # consider them stdlib.
     for m in ['msvcrt', '_winreg']:
         yield m
     yield 'builtins' # python3 only
     for m in 'fcntl', 'grp', 'pwd', 'termios':  # Unix only
         yield m
     for m in 'cPickle', 'datetime': # in Python (not C) on PyPy
         yield m
     for m in ['cffi']:
         yield m
     stdlib_prefixes = set([sys.prefix, sys.exec_prefix])
     # We need to supplement the list of prefixes for the search to work
     # when run from within a virtualenv.
     for mod in (BaseHTTPServer, zlib):
         try:
             # Not all module objects have a __file__ attribute.
             filename = mod.__file__
         except AttributeError:
             continue
         dirname = os.path.dirname(filename)
         for prefix in stdlib_prefixes:
             if dirname.startswith(prefix):
                 # Then this directory is redundant.
                 break
         else:
             stdlib_prefixes.add(dirname)
     for libpath in sys.path:
         # We want to walk everything in sys.path that starts with
         # something in stdlib_prefixes.
         if not any(libpath.startswith(p) for p in stdlib_prefixes):
             continue
         for top, dirs, files in os.walk(libpath):
             for i, d in reversed(list(enumerate(dirs))):
                 if (not os.path.exists(os.path.join(top, d, '__init__.py'))
                     or top == libpath and d in ('hgext', 'mercurial')):
                     del dirs[i]
             for name in files:
                 if not name.endswith(('.py', '.so', '.pyc', '.pyo', '.pyd')):
                     continue
                 if name.startswith('__init__.py'):
                     full_path = top
                 else:
                     full_path = os.path.join(top, name)
                 rel_path = full_path[len(libpath) + 1:]
                 mod = dotted_name_of_path(rel_path)
                 yield mod
 
 stdlib_modules = set(list_stdlib_modules())
 
 def imported_modules(source, modulename, f, localmods, ignore_nested=False):
     """Given the source of a file as a string, yield the names
     imported by that file.
 
     Args:
       source: The python source to examine as a string.
       modulename: of specified python source (may have `__init__`)
       localmods: dict of locally defined module names (may have `__init__`)
       ignore_nested: If true, import statements that do not start in
                      column zero will be ignored.
 
     Returns:
       A list of absolute module names imported by the given source.
 
     >>> f = 'foo/xxx.py'
     >>> modulename = 'foo.xxx'
     >>> localmods = {'foo.__init__': True,
     ...              'foo.foo1': True, 'foo.foo2': True,
     ...              'foo.bar.__init__': True, 'foo.bar.bar1': True,
     ...              'baz.__init__': True, 'baz.baz1': True }
     >>> # standard library (= not locally defined ones)
     >>> sorted(imported_modules(
     ...        'from stdlib1 import foo, bar; import stdlib2',
     ...        modulename, f, localmods))
     []
     >>> # relative importing
     >>> sorted(imported_modules(
     ...        'import foo1; from bar import bar1',
     ...        modulename, f, localmods))
     ['foo.bar.bar1', 'foo.foo1']
     >>> sorted(imported_modules(
     ...        'from bar.bar1 import name1, name2, name3',
     ...        modulename, f, localmods))
     ['foo.bar.bar1']
     >>> # absolute importing
     >>> sorted(imported_modules(
     ...        'from baz import baz1, name1',
     ...        modulename, f, localmods))
     ['baz.__init__', 'baz.baz1']
     >>> # mixed importing, even though it shouldn't be recommended
     >>> sorted(imported_modules(
     ...        'import stdlib, foo1, baz',
     ...        modulename, f, localmods))
     ['baz.__init__', 'foo.foo1']
     >>> # ignore_nested
     >>> sorted(imported_modules(
     ... '''import foo
     ... def wat():
     ...     import bar
     ... ''', modulename, f, localmods))
     ['foo.__init__', 'foo.bar.__init__']
     >>> sorted(imported_modules(
     ... '''import foo
     ... def wat():
     ...     import bar
     ... ''', modulename, f, localmods, ignore_nested=True))
     ['foo.__init__']
     """
     fromlocal = fromlocalfunc(modulename, localmods)
     for node in ast.walk(ast.parse(source, f)):
         if ignore_nested and getattr(node, 'col_offset', 0) > 0:
             continue
         if isinstance(node, ast.Import):
             for n in node.names:
                 found = fromlocal(n.name)
                 if not found:
                     # this should import standard library
                     continue
                 yield found[1]
         elif isinstance(node, ast.ImportFrom):
             found = fromlocal(node.module, node.level)
             if not found:
                 # this should import standard library
                 continue
 
             absname, dottedpath, hassubmod = found
             if not hassubmod:
                 # "dottedpath" is not a package; must be imported
                 yield dottedpath
                 # examination of "node.names" should be redundant
                 # e.g.: from mercurial.node import nullid, nullrev
                 continue
 
             modnotfound = False
             prefix = absname + '.'
             for n in node.names:
                 found = fromlocal(prefix + n.name)
                 if not found:
                     # this should be a function or a property of "node.module"
                     modnotfound = True
                     continue
                 yield found[1]
             if modnotfound:
                 # "dottedpath" is a package, but imported because of non-module
                 # lookup
                 yield dottedpath
 
 def verify_import_convention(module, source, localmods):
     """Verify imports match our established coding convention.
 
     We have 2 conventions: legacy and modern. The modern convention is in
     effect when using absolute imports.
 
     The legacy convention only looks for mixed imports. The modern convention
     is much more thorough.
     """
     root = ast.parse(source)
     absolute = usingabsolute(root)
 
     if absolute:
         return verify_modern_convention(module, root, localmods)
     else:
         return verify_stdlib_on_own_line(root)
 
 def verify_modern_convention(module, root, localmods, root_col_offset=0):
     """Verify a file conforms to the modern import convention rules.
 
     The rules of the modern convention are:
 
     * Ordering is stdlib followed by local imports. Each group is lexically
       sorted.
     * Importing multiple modules via "import X, Y" is not allowed: use
       separate import statements.
     * Importing multiple modules via "from X import ..." is allowed if using
       parenthesis and one entry per line.
     * Only 1 relative import statement per import level ("from .", "from ..")
       is allowed.
     * Relative imports from higher levels must occur before lower levels. e.g.
       "from .." must be before "from .".
     * Imports from peer packages should use relative import (e.g. do not
       "import mercurial.foo" from a "mercurial.*" module).
     * Symbols can only be imported from specific modules (see
       `allowsymbolimports`). For other modules, first import the module then
       assign the symbol to a module-level variable. In addition, these imports
       must be performed before other local imports. This rule only
       applies to import statements outside of any blocks.
     * Relative imports from the standard library are not allowed.
     * Certain modules must be aliased to alternate names to avoid aliasing
       and readability problems. See `requirealias`.
     """
     topmodule = module.split('.')[0]
     fromlocal = fromlocalfunc(module, localmods)
 
     # Whether a local/non-stdlib import has been performed.
     seenlocal = None
     # Whether a local/non-stdlib, non-symbol import has been seen.
     seennonsymbollocal = False
     # The last name to be imported (for sorting).
     lastname = None
     laststdlib = None
     # Relative import levels encountered so far.
     seenlevels = set()
 
     for node, newscope in walklocal(root):
         def msg(fmt, *args):
             return (fmt % args, node.lineno)
         if newscope:
             # Check for local imports in function
             for r in verify_modern_convention(module, node, localmods,
                                               node.col_offset + 4):
                 yield r
         elif isinstance(node, ast.Import):
             # Disallow "import foo, bar" and require separate imports
             # for each module.
             if len(node.names) > 1:
                 yield msg('multiple imported names: %s',
                           ', '.join(n.name for n in node.names))
 
             name = node.names[0].name
             asname = node.names[0].asname
 
             stdlib = name in stdlib_modules
 
             # Ignore sorting rules on imports inside blocks.
             if node.col_offset == root_col_offset:
                 if lastname and name < lastname and laststdlib == stdlib:
                     yield msg('imports not lexically sorted: %s < %s',
                               name, lastname)
 
             lastname = name
             laststdlib = stdlib
 
             # stdlib imports should be before local imports.
             if stdlib and seenlocal and node.col_offset == root_col_offset:
                 yield msg('stdlib import "%s" follows local import: %s',
                           name, seenlocal)
 
             if not stdlib:
                 seenlocal = name
 
             # Import of sibling modules should use relative imports.
             topname = name.split('.')[0]
             if topname == topmodule:
                 yield msg('import should be relative: %s', name)
 
             if name in requirealias and asname != requirealias[name]:
                 yield msg('%s module must be "as" aliased to %s',
                           name, requirealias[name])
 
         elif isinstance(node, ast.ImportFrom):
             # Resolve the full imported module name.
             if node.level > 0:
                 fullname = '.'.join(module.split('.')[:-node.level])
                 if node.module:
                     fullname += '.%s' % node.module
             else:
                 assert node.module
                 fullname = node.module
 
                 topname = fullname.split('.')[0]
                 if topname == topmodule:
                     yield msg('import should be relative: %s', fullname)
 
             # __future__ is special since it needs to come first and use
             # symbol import.
             if fullname != '__future__':
                 if not fullname or fullname in stdlib_modules:
                     yield msg('relative import of stdlib module')
                 else:
                     seenlocal = fullname
 
             # Direct symbol import is only allowed from certain modules and
             # must occur before non-symbol imports.
             found = fromlocal(node.module, node.level)
             if found and found[2]:  # node.module is a package
                 prefix = found[0] + '.'
                 symbols = [n.name for n in node.names
                            if not fromlocal(prefix + n.name)]
             else:
                 symbols = [n.name for n in node.names]
             if node.module and node.col_offset == root_col_offset:
                 if symbols and fullname not in allowsymbolimports:
                     yield msg('direct symbol import %s from %s',
                               ', '.join(symbols), fullname)
 
                 if symbols and seennonsymbollocal:
                     yield msg('symbol import follows non-symbol import: %s',
                               fullname)
             if not symbols and fullname not in stdlib_modules:
                 seennonsymbollocal = True
 
             if not node.module:
                 assert node.level
 
                 # Only allow 1 group per level.
                 if (node.level in seenlevels
                     and node.col_offset == root_col_offset):
                     yield msg('multiple "from %s import" statements',
                               '.' * node.level)
 
                 # Higher-level groups come before lower-level groups.
                 if any(node.level > l for l in seenlevels):
                     yield msg('higher-level import should come first: %s',
                               fullname)
 
                 seenlevels.add(node.level)
 
             # Entries in "from .X import ( ... )" lists must be lexically
             # sorted.
             lastentryname = None
 
             for n in node.names:
                 if lastentryname and n.name < lastentryname:
                     yield msg('imports from %s not lexically sorted: %s < %s',
                               fullname, n.name, lastentryname)
 
                 lastentryname = n.name
 
                 if n.name in requirealias and n.asname != requirealias[n.name]:
                     yield msg('%s from %s must be "as" aliased to %s',
                               n.name, fullname, requirealias[n.name])
 
 def verify_stdlib_on_own_line(root):
     """Given some python source, verify that stdlib imports are done
     in separate statements from relative local module imports.
 
     >>> list(verify_stdlib_on_own_line(ast.parse('import sys, foo')))
     [('mixed imports\\n   stdlib:    sys\\n   relative:  foo', 1)]
     >>> list(verify_stdlib_on_own_line(ast.parse('import sys, os')))
     []
     >>> list(verify_stdlib_on_own_line(ast.parse('import foo, bar')))
     []
     """
     for node in ast.walk(root):
         if isinstance(node, ast.Import):
             from_stdlib = {False: [], True: []}
             for n in node.names:
                 from_stdlib[n.name in stdlib_modules].append(n.name)
             if from_stdlib[True] and from_stdlib[False]:
                 yield ('mixed imports\n   stdlib:    %s\n   relative:  %s' %
                        (', '.join(sorted(from_stdlib[True])),
                         ', '.join(sorted(from_stdlib[False]))), node.lineno)
 
 class CircularImport(Exception):
     pass
 
 def checkmod(mod, imports):
     shortest = {}
     visit = [[mod]]
     while visit:
         path = visit.pop(0)
         for i in sorted(imports.get(path[-1], [])):
             if len(path) < shortest.get(i, 1000):
                 shortest[i] = len(path)
                 if i in path:
                     if i == path[0]:
                         raise CircularImport(path)
                     continue
                 visit.append(path + [i])
 
 def rotatecycle(cycle):
     """arrange a cycle so that the lexicographically first module listed first
 
     >>> rotatecycle(['foo', 'bar'])
     ['bar', 'foo', 'bar']
     """
     lowest = min(cycle)
     idx = cycle.index(lowest)
     return cycle[idx:] + cycle[:idx] + [lowest]
 
 def find_cycles(imports):
     """Find cycles in an already-loaded import graph.
 
     All module names recorded in `imports` should be absolute one.
 
     >>> from __future__ import print_function
     >>> imports = {'top.foo': ['top.bar', 'os.path', 'top.qux'],
     ...            'top.bar': ['top.baz', 'sys'],
     ...            'top.baz': ['top.foo'],
     ...            'top.qux': ['top.foo']}
     >>> print('\\n'.join(sorted(find_cycles(imports))))
     top.bar -> top.baz -> top.foo -> top.bar
     top.foo -> top.qux -> top.foo
     """
     cycles = set()
     for mod in sorted(imports.keys()):
         try:
             checkmod(mod, imports)
         except CircularImport as e:
             cycle = e.args[0]
             cycles.add(" -> ".join(rotatecycle(cycle)))
     return cycles
 
 def _cycle_sortkey(c):
     return len(c), c
 
 def embedded(f, modname, src):
     """Extract embedded python code
 
     >>> def test(fn, lines):
     ...     for s, m, f, l in embedded(fn, "example", lines):
     ...         print("%s %s %s" % (m, f, l))
     ...         print(repr(s))
     >>> lines = [
     ...   'comment',
     ...   '  >>> from __future__ import print_function',
     ...   "  >>> ' multiline",
     ...   "  ... string'",
     ...   '  ',
     ...   'comment',
     ...   '  $ cat > foo.py <<EOF',
     ...   '  > from __future__ import print_function',
     ...   '  > EOF',
     ... ]
     >>> test("example.t", lines)
     example[2] doctest.py 2
     "from __future__ import print_function\\n' multiline\\nstring'\\n"
     example[7] foo.py 7
     'from __future__ import print_function\\n'
     """
     inlinepython = 0
     shpython = 0
     script = []
     prefix = 6
     t = ''
     n = 0
     for l in src:
         n += 1
         if not l.endswith(b'\n'):
             l += b'\n'
         if l.startswith(b'  >>> '): # python inlines
             if shpython:
                 print("%s:%d: Parse Error" % (f, n))
             if not inlinepython:
                 # We've just entered a Python block.
                 inlinepython = n
                 t = 'doctest.py'
             script.append(l[prefix:])
             continue
         if l.startswith(b'  ... '): # python inlines
             script.append(l[prefix:])
             continue
         cat = re.search(r"\$ \s*cat\s*>\s*(\S+\.py)\s*<<\s*EOF", l)
         if cat:
             if inlinepython:
                 yield ''.join(script), ("%s[%d]" %
                        (modname, inlinepython)), t, inlinepython
                 script = []
                 inlinepython = 0
             shpython = n
             t = cat.group(1)
             continue
         if shpython and l.startswith(b'  > '): # sh continuation
             if l == b'  > EOF\n':
                 yield ''.join(script), ("%s[%d]" %
                        (modname, shpython)), t, shpython
                 script = []
                 shpython = 0
             else:
                 script.append(l[4:])
             continue
         if inlinepython and l == b'  \n':
             yield ''.join(script), ("%s[%d]" %
                    (modname, inlinepython)), t, inlinepython
             script = []
             inlinepython = 0
             continue
 
 def sources(f, modname):
     """Yields possibly multiple sources from a filepath
 
     input: filepath, modulename
     yields:  script(string), modulename, filepath, linenumber
 
     For embedded scripts, the modulename and filepath will be different
     from the function arguments. linenumber is an offset relative to
     the input file.
     """
     py = False
     if not f.endswith('.t'):
         with open(f) as src:
             yield src.read(), modname, f, 0
             py = True
     if py or f.endswith('.t'):
         with open(f) as src:
             for script, modname, t, line in embedded(f, modname, src):
                 yield script, modname, t, line
 
 def main(argv):
     if len(argv) < 2 or (argv[1] == '-' and len(argv) > 2):
         print('Usage: %s {-|file [file] [file] ...}')
         return 1
     if argv[1] == '-':
         argv = argv[:1]
         argv.extend(l.rstrip() for l in sys.stdin.readlines())
     localmods = {}
     used_imports = {}
     any_errors = False
     for source_path in argv[1:]:
-        modname = dotted_name_of_path(source_path, trimpure=True)
+        trimpure = source_path.endswith(_dualmodules)
+        modname = dotted_name_of_path(source_path, trimpure=trimpure)
         localmods[modname] = source_path
     for localmodname, source_path in sorted(localmods.items()):
         for src, modname, name, line in sources(source_path, localmodname):
             try:
                 used_imports[modname] = sorted(
                     imported_modules(src, modname, name, localmods,
                                      ignore_nested=True))
                 for error, lineno in verify_import_convention(modname, src,
                                                               localmods):
                     any_errors = True
                     print('%s:%d: %s' % (source_path, lineno + line, error))
             except SyntaxError as e:
                 print('%s:%d: SyntaxError: %s' %
                       (source_path, e.lineno + line, e))
     cycles = find_cycles(used_imports)
     if cycles:
         firstmods = set()
         for c in sorted(cycles, key=_cycle_sortkey):
             first = c.split()[0]
             # As a rough cut, ignore any cycle that starts with the
             # same module as some other cycle. Otherwise we see lots
             # of cycles that are effectively duplicates.
             if first in firstmods:
                 continue
             print('Import cycle:', c)
             firstmods.add(first)
         any_errors = True
     return any_errors != 0
 
 if __name__ == '__main__':
     sys.exit(int(main(sys.argv)))

mercurial/__init__.py

@@ -1,403 +1,404 b''
 # __init__.py - Startup and module loading logic for Mercurial.
 #
 # Copyright 2015 Gregory Szorc <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import imp
 import os
 import sys
 import zipimport
 
 from . import (
     policy
 )
 
 __all__ = []
 
 modulepolicy = policy.policy
 
 # Modules that have both Python and C implementations. See also the
 # set of .py files under mercurial/pure/.
 _dualmodules = set([
     'mercurial.base85',
     'mercurial.bdiff',
     'mercurial.diffhelpers',
     'mercurial.mpatch',
     'mercurial.osutil',
     'mercurial.parsers',
 ])
 
 class hgimporter(object):
     """Object that conforms to import hook interface defined in PEP-302."""
     def find_module(self, name, path=None):
         # We only care about modules that have both C and pure implementations.
         if name in _dualmodules:
             return self
         return None
 
     def load_module(self, name):
         mod = sys.modules.get(name, None)
         if mod:
             return mod
 
         mercurial = sys.modules['mercurial']
 
         # The zip importer behaves sufficiently differently from the default
         # importer to warrant its own code path.
         loader = getattr(mercurial, '__loader__', None)
         if isinstance(loader, zipimport.zipimporter):
             def ziploader(*paths):
                 """Obtain a zipimporter for a directory under the main zip."""
                 path = os.path.join(loader.archive, *paths)
                 zl = sys.path_importer_cache.get(path)
                 if not zl:
                     zl = zipimport.zipimporter(path)
                 return zl
 
             try:
                 if modulepolicy in policy.policynoc:
                     raise ImportError()
 
                 zl = ziploader('mercurial')
                 mod = zl.load_module(name)
                 # Unlike imp, ziploader doesn't expose module metadata that
                 # indicates the type of module. So just assume what we found
                 # is OK (even though it could be a pure Python module).
             except ImportError:
                 if modulepolicy == b'c':
                     raise
                 zl = ziploader('mercurial', 'pure')
                 mod = zl.load_module(name)
 
             sys.modules[name] = mod
             return mod
 
         # Unlike the default importer which searches special locations and
         # sys.path, we only look in the directory where "mercurial" was
         # imported from.
 
         # imp.find_module doesn't support submodules (modules with ".").
         # Instead you have to pass the parent package's __path__ attribute
         # as the path argument.
         stem = name.split('.')[-1]
 
         try:
             if modulepolicy in policy.policynoc:
                 raise ImportError()
 
             modinfo = imp.find_module(stem, mercurial.__path__)
 
             # The Mercurial installer used to copy files from
             # mercurial/pure/*.py to mercurial/*.py. Therefore, it's possible
             # for some installations to have .py files under mercurial/*.
             # Loading Python modules when we expected C versions could result
             # in a) poor performance b) loading a version from a previous
             # Mercurial version, potentially leading to incompatibility. Either
             # scenario is bad. So we verify that modules loaded from
             # mercurial/* are C extensions. If the current policy allows the
             # loading of .py modules, the module will be re-imported from
             # mercurial/pure/* below.
             if modinfo[2][2] != imp.C_EXTENSION:
                 raise ImportError('.py version of %s found where C '
                                   'version should exist' % name)
 
         except ImportError:
             if modulepolicy == b'c':
                 raise
 
             # Could not load the C extension and pure Python is allowed. So
             # try to load them.
             from . import pure
             modinfo = imp.find_module(stem, pure.__path__)
             if not modinfo:
                 raise ImportError('could not find mercurial module %s' %
                                   name)
 
         mod = imp.load_module(name, *modinfo)
         sys.modules[name] = mod
         return mod
 
 # Python 3 uses a custom module loader that transforms source code between
 # source file reading and compilation. This is done by registering a custom
 # finder that changes the spec for Mercurial modules to use a custom loader.
 if sys.version_info[0] >= 3:
     from . import pure
     import importlib
     import io
     import token
     import tokenize
 
     class hgpathentryfinder(importlib.abc.MetaPathFinder):
         """A sys.meta_path finder that uses a custom module loader."""
         def find_spec(self, fullname, path, target=None):
             # Only handle Mercurial-related modules.
             if not fullname.startswith(('mercurial.', 'hgext.', 'hgext3rd.')):
                 return None
             # zstd is already dual-version clean, don't try and mangle it
             if fullname.startswith('mercurial.zstd'):
                 return None
 
             # This assumes Python 3 doesn't support loading C modules.
             if fullname in _dualmodules:
                 stem = fullname.split('.')[-1]
                 fullname = 'mercurial.pure.%s' % stem
                 target = pure
                 assert len(path) == 1
                 path = [os.path.join(path[0], 'pure')]
 
             # Try to find the module using other registered finders.
             spec = None
             for finder in sys.meta_path:
                 if finder == self:
                     continue
 
                 spec = finder.find_spec(fullname, path, target=target)
                 if spec:
                     break
 
             # This is a Mercurial-related module but we couldn't find it
             # using the previously-registered finders. This likely means
             # the module doesn't exist.
             if not spec:
                 return None
 
-            if fullname.startswith('mercurial.pure.'):
+            if (fullname.startswith('mercurial.pure.')
+                and fullname.replace('.pure.', '.') in _dualmodules):
                 spec.name = spec.name.replace('.pure.', '.')
 
             # TODO need to support loaders from alternate specs, like zip
             # loaders.
             spec.loader = hgloader(spec.name, spec.origin)
             return spec
 
     def replacetokens(tokens, fullname):
         """Transform a stream of tokens from raw to Python 3.
 
         It is called by the custom module loading machinery to rewrite
         source/tokens between source decoding and compilation.
 
         Returns a generator of possibly rewritten tokens.
 
         The input token list may be mutated as part of processing. However,
         its changes do not necessarily match the output token stream.
 
         REMEMBER TO CHANGE ``BYTECODEHEADER`` WHEN CHANGING THIS FUNCTION
         OR CACHED FILES WON'T GET INVALIDATED PROPERLY.
         """
         futureimpline = False
 
         # The following utility functions access the tokens list and i index of
         # the for i, t enumerate(tokens) loop below
         def _isop(j, *o):
             """Assert that tokens[j] is an OP with one of the given values"""
             try:
                 return tokens[j].type == token.OP and tokens[j].string in o
             except IndexError:
                 return False
 
         def _findargnofcall(n):
             """Find arg n of a call expression (start at 0)
 
             Returns index of the first token of that argument, or None if
             there is not that many arguments.
 
             Assumes that token[i + 1] is '('.
 
             """
             nested = 0
             for j in range(i + 2, len(tokens)):
                 if _isop(j, ')', ']', '}'):
                     # end of call, tuple, subscription or dict / set
                     nested -= 1
                     if nested < 0:
                         return None
                 elif n == 0:
                     # this is the starting position of arg
                     return j
                 elif _isop(j, '(', '[', '{'):
                     nested += 1
                 elif _isop(j, ',') and nested == 0:
                     n -= 1
 
             return None
 
         def _ensureunicode(j):
             """Make sure the token at j is a unicode string
 
             This rewrites a string token to include the unicode literal prefix
             so the string transformer won't add the byte prefix.
 
             Ignores tokens that are not strings. Assumes bounds checking has
             already been done.
 
             """
             st = tokens[j]
             if st.type == token.STRING and st.string.startswith(("'", '"')):
                 tokens[j] = st._replace(string='u%s' % st.string)
 
         for i, t in enumerate(tokens):
             # Convert most string literals to byte literals. String literals
             # in Python 2 are bytes. String literals in Python 3 are unicode.
             # Most strings in Mercurial are bytes and unicode strings are rare.
             # Rather than rewrite all string literals to use ``b''`` to indicate
             # byte strings, we apply this token transformer to insert the ``b``
             # prefix nearly everywhere.
             if t.type == token.STRING:
                 s = t.string
 
                 # Preserve docstrings as string literals. This is inconsistent
                 # with regular unprefixed strings. However, the
                 # "from __future__" parsing (which allows a module docstring to
                 # exist before it) doesn't properly handle the docstring if it
                 # is b''' prefixed, leading to a SyntaxError. We leave all
                 # docstrings as unprefixed to avoid this. This means Mercurial
                 # components touching docstrings need to handle unicode,
                 # unfortunately.
                 if s[0:3] in ("'''", '"""'):
                     yield t
                     continue
 
                 # If the first character isn't a quote, it is likely a string
                 # prefixing character (such as 'b', 'u', or 'r'. Ignore.
                 if s[0] not in ("'", '"'):
                     yield t
                     continue
 
                 # String literal. Prefix to make a b'' string.
                 yield t._replace(string='b%s' % t.string)
                 continue
 
             # Insert compatibility imports at "from __future__ import" line.
             # No '\n' should be added to preserve line numbers.
             if (t.type == token.NAME and t.string == 'import' and
                 all(u.type == token.NAME for u in tokens[i - 2:i]) and
                 [u.string for u in tokens[i - 2:i]] == ['from', '__future__']):
                 futureimpline = True
             if t.type == token.NEWLINE and futureimpline:
                 futureimpline = False
                 if fullname == 'mercurial.pycompat':
                     yield t
                     continue
                 r, c = t.start
                 l = (b'; from mercurial.pycompat import '
                      b'delattr, getattr, hasattr, setattr, xrange, '
                      b'open, unicode\n')
                 for u in tokenize.tokenize(io.BytesIO(l).readline):
                     if u.type in (tokenize.ENCODING, token.ENDMARKER):
                         continue
                     yield u._replace(
                         start=(r, c + u.start[1]), end=(r, c + u.end[1]))
                 continue
 
             # This looks like a function call.
             if t.type == token.NAME and _isop(i + 1, '('):
                 fn = t.string
 
                 # *attr() builtins don't accept byte strings to 2nd argument.
                 if (fn in ('getattr', 'setattr', 'hasattr', 'safehasattr') and
                         not _isop(i - 1, '.')):
                     arg1idx = _findargnofcall(1)
                     if arg1idx is not None:
                         _ensureunicode(arg1idx)
 
                 # .encode() and .decode() on str/bytes/unicode don't accept
                 # byte strings on Python 3.
                 elif fn in ('encode', 'decode') and _isop(i - 1, '.'):
                     for argn in range(2):
                         argidx = _findargnofcall(argn)
                         if argidx is not None:
                             _ensureunicode(argidx)
 
                 # It changes iteritems/values to items/values as they are not
                 # present in Python 3 world.
                 elif fn in ('iteritems', 'itervalues'):
                     yield t._replace(string=fn[4:])
                     continue
 
             # Emit unmodified token.
             yield t
 
     # Header to add to bytecode files. This MUST be changed when
     # ``replacetoken`` or any mechanism that changes semantics of module
     # loading is changed. Otherwise cached bytecode may get loaded without
     # the new transformation mechanisms applied.
     BYTECODEHEADER = b'HG\x00\x0a'
 
     class hgloader(importlib.machinery.SourceFileLoader):
         """Custom module loader that transforms source code.
 
         When the source code is converted to a code object, we transform
         certain patterns to be Python 3 compatible. This allows us to write code
         that is natively Python 2 and compatible with Python 3 without
         making the code excessively ugly.
 
         We do this by transforming the token stream between parse and compile.
 
         Implementing transformations invalidates caching assumptions made
         by the built-in importer. The built-in importer stores a header on
         saved bytecode files indicating the Python/bytecode version. If the
         version changes, the cached bytecode is ignored. The Mercurial
         transformations could change at any time. This means we need to check
         that cached bytecode was generated with the current transformation
         code or there could be a mismatch between cached bytecode and what
         would be generated from this class.
 
         We supplement the bytecode caching layer by wrapping ``get_data``
         and ``set_data``. These functions are called when the
         ``SourceFileLoader`` retrieves and saves bytecode cache files,
         respectively. We simply add an additional header on the file. As
         long as the version in this file is changed when semantics change,
         cached bytecode should be invalidated when transformations change.
 
         The added header has the form ``HG<VERSION>``. That is a literal
         ``HG`` with 2 binary bytes indicating the transformation version.
         """
         def get_data(self, path):
             data = super(hgloader, self).get_data(path)
 
             if not path.endswith(tuple(importlib.machinery.BYTECODE_SUFFIXES)):
                 return data
 
             # There should be a header indicating the Mercurial transformation
             # version. If it doesn't exist or doesn't match the current version,
             # we raise an OSError because that is what
             # ``SourceFileLoader.get_code()`` expects when loading bytecode
             # paths to indicate the cached file is "bad."
             if data[0:2] != b'HG':
                 raise OSError('no hg header')
             if data[0:4] != BYTECODEHEADER:
                 raise OSError('hg header version mismatch')
 
             return data[4:]
 
         def set_data(self, path, data, *args, **kwargs):
             if path.endswith(tuple(importlib.machinery.BYTECODE_SUFFIXES)):
                 data = BYTECODEHEADER + data
 
             return super(hgloader, self).set_data(path, data, *args, **kwargs)
 
         def source_to_code(self, data, path):
             """Perform token transformation before compilation."""
             buf = io.BytesIO(data)
             tokens = tokenize.tokenize(buf.readline)
             data = tokenize.untokenize(replacetokens(list(tokens), self.name))
             # Python's built-in importer strips frames from exceptions raised
             # for this code. Unfortunately, that mechanism isn't extensible
             # and our frame will be blamed for the import failure. There
             # are extremely hacky ways to do frame stripping. We haven't
             # implemented them because they are very ugly.
             return super(hgloader, self).source_to_code(data, path)
 
 # We automagically register our custom importer as a side-effect of loading.
 # This is necessary to ensure that any entry points are able to import
 # mercurial.* modules without having to perform this registration themselves.
 if sys.version_info[0] >= 3:
     _importercls = hgpathentryfinder
 else:
     _importercls = hgimporter
 if not any(isinstance(x, _importercls) for x in sys.meta_path):
     # meta_path is used before any implicit finders and before sys.path.
     sys.meta_path.insert(0, _importercls())