py3: convert __doc__ back to bytes in help.py

pycompat.getdoc() is pretty simple, but we wouldn't want to write handling
of None inline.

mercurial/help.py

@@ -1,667 +1,668 b''
 # help.py - help data for mercurial
 #
 # Copyright 2006 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import itertools
 import os
 import textwrap
 
 from .i18n import (
     _,
     gettext,
 )
 from . import (
     cmdutil,
     encoding,
     error,
     extensions,
     filemerge,
     fileset,
     minirst,
     pycompat,
     revset,
     templatefilters,
     templatekw,
     templater,
     util,
 )
 from .hgweb import (
     webcommands,
 )
 
 _exclkeywords = {
     "(ADVANCED)",
     "(DEPRECATED)",
     "(EXPERIMENTAL)",
     # i18n: "(ADVANCED)" is a keyword, must be translated consistently
     _("(ADVANCED)"),
     # i18n: "(DEPRECATED)" is a keyword, must be translated consistently
     _("(DEPRECATED)"),
     # i18n: "(EXPERIMENTAL)" is a keyword, must be translated consistently
     _("(EXPERIMENTAL)"),
 }
 
 def listexts(header, exts, indent=1, showdeprecated=False):
     '''return a text listing of the given extensions'''
     rst = []
     if exts:
         for name, desc in sorted(exts.iteritems()):
             if not showdeprecated and any(w in desc for w in _exclkeywords):
                 continue
             rst.append('%s:%s: %s\n' % (' ' * indent, name, desc))
     if rst:
         rst.insert(0, '\n%s\n\n' % header)
     return rst
 
 def extshelp(ui):
     rst = loaddoc('extensions')(ui).splitlines(True)
     rst.extend(listexts(
         _('enabled extensions:'), extensions.enabled(), showdeprecated=True))
     rst.extend(listexts(_('disabled extensions:'), extensions.disabled()))
     doc = ''.join(rst)
     return doc
 
 def optrst(header, options, verbose):
     data = []
     multioccur = False
     for option in options:
         if len(option) == 5:
             shortopt, longopt, default, desc, optlabel = option
         else:
             shortopt, longopt, default, desc = option
             optlabel = _("VALUE") # default label
 
         if not verbose and any(w in desc for w in _exclkeywords):
             continue
 
         so = ''
         if shortopt:
             so = '-' + shortopt
         lo = '--' + longopt
         if default:
             desc += _(" (default: %s)") % default
 
         if isinstance(default, list):
             lo += " %s [+]" % optlabel
             multioccur = True
         elif (default is not None) and not isinstance(default, bool):
             lo += " %s" % optlabel
 
         data.append((so, lo, desc))
 
     if multioccur:
         header += (_(" ([+] can be repeated)"))
 
     rst = ['\n%s:\n\n' % header]
     rst.extend(minirst.maketable(data, 1))
 
     return ''.join(rst)
 
 def indicateomitted(rst, omitted, notomitted=None):
     rst.append('\n\n.. container:: omitted\n\n    %s\n\n' % omitted)
     if notomitted:
         rst.append('\n\n.. container:: notomitted\n\n    %s\n\n' % notomitted)
 
 def filtercmd(ui, cmd, kw, doc):
     if not ui.debugflag and cmd.startswith("debug") and kw != "debug":
         return True
     if not ui.verbose and doc and any(w in doc for w in _exclkeywords):
         return True
     return False
 
 def topicmatch(ui, commands, kw):
     """Return help topics matching kw.
 
     Returns {'section': [(name, summary), ...], ...} where section is
     one of topics, commands, extensions, or extensioncommands.
     """
     kw = encoding.lower(kw)
     def lowercontains(container):
         return kw in encoding.lower(container)  # translated in helptable
     results = {'topics': [],
                'commands': [],
                'extensions': [],
                'extensioncommands': [],
                }
     for names, header, doc in helptable:
         # Old extensions may use a str as doc.
         if (sum(map(lowercontains, names))
             or lowercontains(header)
             or (callable(doc) and lowercontains(doc(ui)))):
             results['topics'].append((names[0], header))
     for cmd, entry in commands.table.iteritems():
         if len(entry) == 3:
             summary = entry[2]
         else:
             summary = ''
         # translate docs *before* searching there
-        docs = _(getattr(entry[0], '__doc__', None)) or ''
+        docs = _(pycompat.getdoc(entry[0])) or ''
         if kw in cmd or lowercontains(summary) or lowercontains(docs):
             doclines = docs.splitlines()
             if doclines:
                 summary = doclines[0]
             cmdname = cmd.partition('|')[0].lstrip('^')
             if filtercmd(ui, cmdname, kw, docs):
                 continue
             results['commands'].append((cmdname, summary))
     for name, docs in itertools.chain(
         extensions.enabled(False).iteritems(),
         extensions.disabled().iteritems()):
         if not docs:
             continue
         mod = extensions.load(ui, name, '')
         name = name.rpartition('.')[-1]
         if lowercontains(name) or lowercontains(docs):
             # extension docs are already translated
             results['extensions'].append((name, docs.splitlines()[0]))
         for cmd, entry in getattr(mod, 'cmdtable', {}).iteritems():
             if kw in cmd or (len(entry) > 2 and lowercontains(entry[2])):
                 cmdname = cmd.partition('|')[0].lstrip('^')
-                if entry[0].__doc__:
-                    cmddoc = gettext(entry[0].__doc__).splitlines()[0]
+                cmddoc = pycompat.getdoc(entry[0])
+                if cmddoc:
+                    cmddoc = gettext(cmddoc).splitlines()[0]
                 else:
                     cmddoc = _('(no help text available)')
                 if filtercmd(ui, cmdname, kw, cmddoc):
                     continue
                 results['extensioncommands'].append((cmdname, cmddoc))
     return results
 
 def loaddoc(topic, subdir=None):
     """Return a delayed loader for help/topic.txt."""
 
     def loader(ui):
         docdir = os.path.join(util.datapath, 'help')
         if subdir:
             docdir = os.path.join(docdir, subdir)
         path = os.path.join(docdir, topic + ".txt")
         doc = gettext(util.readfile(path))
         for rewriter in helphooks.get(topic, []):
             doc = rewriter(ui, topic, doc)
         return doc
 
     return loader
 
 internalstable = sorted([
     (['bundles'], _('Bundles'),
      loaddoc('bundles', subdir='internals')),
     (['censor'], _('Censor'),
      loaddoc('censor', subdir='internals')),
     (['changegroups'], _('Changegroups'),
      loaddoc('changegroups', subdir='internals')),
     (['requirements'], _('Repository Requirements'),
      loaddoc('requirements', subdir='internals')),
     (['revlogs'], _('Revision Logs'),
      loaddoc('revlogs', subdir='internals')),
     (['wireprotocol'], _('Wire Protocol'),
      loaddoc('wireprotocol', subdir='internals')),
 ])
 
 def internalshelp(ui):
     """Generate the index for the "internals" topic."""
     lines = ['To access a subtopic, use "hg help internals.{subtopic-name}"\n',
              '\n']
     for names, header, doc in internalstable:
         lines.append(' :%s: %s\n' % (names[0], header))
 
     return ''.join(lines)
 
 helptable = sorted([
     (['bundlespec'], _("Bundle File Formats"), loaddoc('bundlespec')),
     (['color'], _("Colorizing Outputs"), loaddoc('color')),
     (["config", "hgrc"], _("Configuration Files"), loaddoc('config')),
     (["dates"], _("Date Formats"), loaddoc('dates')),
     (["patterns"], _("File Name Patterns"), loaddoc('patterns')),
     (['environment', 'env'], _('Environment Variables'),
      loaddoc('environment')),
     (['revisions', 'revs', 'revsets', 'revset', 'multirevs', 'mrevs'],
       _('Specifying Revisions'), loaddoc('revisions')),
     (['filesets', 'fileset'], _("Specifying File Sets"), loaddoc('filesets')),
     (['diffs'], _('Diff Formats'), loaddoc('diffs')),
     (['merge-tools', 'mergetools', 'mergetool'], _('Merge Tools'),
      loaddoc('merge-tools')),
     (['templating', 'templates', 'template', 'style'], _('Template Usage'),
      loaddoc('templates')),
     (['urls'], _('URL Paths'), loaddoc('urls')),
     (["extensions"], _("Using Additional Features"), extshelp),
     (["subrepos", "subrepo"], _("Subrepositories"), loaddoc('subrepos')),
     (["hgweb"], _("Configuring hgweb"), loaddoc('hgweb')),
     (["glossary"], _("Glossary"), loaddoc('glossary')),
     (["hgignore", "ignore"], _("Syntax for Mercurial Ignore Files"),
      loaddoc('hgignore')),
     (["phases"], _("Working with Phases"), loaddoc('phases')),
     (['scripting'], _('Using Mercurial from scripts and automation'),
      loaddoc('scripting')),
     (['internals'], _("Technical implementation topics"),
      internalshelp),
     (['pager'], _("Pager Support"), loaddoc('pager')),
 ])
 
 # Maps topics with sub-topics to a list of their sub-topics.
 subtopics = {
     'internals': internalstable,
 }
 
 # Map topics to lists of callable taking the current topic help and
 # returning the updated version
 helphooks = {}
 
 def addtopichook(topic, rewriter):
     helphooks.setdefault(topic, []).append(rewriter)
 
 def makeitemsdoc(ui, topic, doc, marker, items, dedent=False):
     """Extract docstring from the items key to function mapping, build a
     single documentation block and use it to overwrite the marker in doc.
     """
     entries = []
     for name in sorted(items):
-        text = (items[name].__doc__ or '').rstrip()
+        text = (pycompat.getdoc(items[name]) or '').rstrip()
         if (not text
             or not ui.verbose and any(w in text for w in _exclkeywords)):
             continue
         text = gettext(text)
         if dedent:
             # Abuse latin1 to use textwrap.dedent() on bytes.
             text = textwrap.dedent(text.decode('latin1')).encode('latin1')
         lines = text.splitlines()
         doclines = [(lines[0])]
         for l in lines[1:]:
             # Stop once we find some Python doctest
             if l.strip().startswith('>>>'):
                 break
             if dedent:
                 doclines.append(l.rstrip())
             else:
                 doclines.append('  ' + l.strip())
         entries.append('\n'.join(doclines))
     entries = '\n\n'.join(entries)
     return doc.replace(marker, entries)
 
 def addtopicsymbols(topic, marker, symbols, dedent=False):
     def add(ui, topic, doc):
         return makeitemsdoc(ui, topic, doc, marker, symbols, dedent=dedent)
     addtopichook(topic, add)
 
 addtopicsymbols('bundlespec', '.. bundlecompressionmarker',
                 util.bundlecompressiontopics())
 addtopicsymbols('filesets', '.. predicatesmarker', fileset.symbols)
 addtopicsymbols('merge-tools', '.. internaltoolsmarker',
                 filemerge.internalsdoc)
 addtopicsymbols('revisions', '.. predicatesmarker', revset.symbols)
 addtopicsymbols('templates', '.. keywordsmarker', templatekw.keywords)
 addtopicsymbols('templates', '.. filtersmarker', templatefilters.filters)
 addtopicsymbols('templates', '.. functionsmarker', templater.funcs)
 addtopicsymbols('hgweb', '.. webcommandsmarker', webcommands.commands,
                 dedent=True)
 
 def help_(ui, commands, name, unknowncmd=False, full=True, subtopic=None,
           **opts):
     '''
     Generate the help for 'name' as unformatted restructured text. If
     'name' is None, describe the commands available.
     '''
 
     opts = pycompat.byteskwargs(opts)
 
     def helpcmd(name, subtopic=None):
         try:
             aliases, entry = cmdutil.findcmd(name, commands.table,
                                              strict=unknowncmd)
         except error.AmbiguousCommand as inst:
             # py3k fix: except vars can't be used outside the scope of the
             # except block, nor can be used inside a lambda. python issue4617
             prefix = inst.args[0]
             select = lambda c: c.lstrip('^').startswith(prefix)
             rst = helplist(select)
             return rst
 
         rst = []
 
         # check if it's an invalid alias and display its error if it is
         if getattr(entry[0], 'badalias', None):
             rst.append(entry[0].badalias + '\n')
             if entry[0].unknowncmd:
                 try:
                     rst.extend(helpextcmd(entry[0].cmdname))
                 except error.UnknownCommand:
                     pass
             return rst
 
         # synopsis
         if len(entry) > 2:
             if entry[2].startswith('hg'):
                 rst.append("%s\n" % entry[2])
             else:
                 rst.append('hg %s %s\n' % (aliases[0], entry[2]))
         else:
             rst.append('hg %s\n' % aliases[0])
         # aliases
         if full and not ui.quiet and len(aliases) > 1:
             rst.append(_("\naliases: %s\n") % ', '.join(aliases[1:]))
         rst.append('\n')
 
         # description
-        doc = gettext(entry[0].__doc__)
+        doc = gettext(pycompat.getdoc(entry[0]))
         if not doc:
             doc = _("(no help text available)")
         if util.safehasattr(entry[0], 'definition'):  # aliased command
             source = entry[0].source
             if entry[0].definition.startswith('!'):  # shell alias
                 doc = (_('shell alias for::\n\n    %s\n\ndefined by: %s\n') %
                        (entry[0].definition[1:], source))
             else:
                 doc = (_('alias for: hg %s\n\n%s\n\ndefined by: %s\n') %
                        (entry[0].definition, doc, source))
         doc = doc.splitlines(True)
         if ui.quiet or not full:
             rst.append(doc[0])
         else:
             rst.extend(doc)
         rst.append('\n')
 
         # check if this command shadows a non-trivial (multi-line)
         # extension help text
         try:
             mod = extensions.find(name)
-            doc = gettext(mod.__doc__) or ''
+            doc = gettext(pycompat.getdoc(mod)) or ''
             if '\n' in doc.strip():
                 msg = _("(use 'hg help -e %s' to show help for "
                         "the %s extension)") % (name, name)
                 rst.append('\n%s\n' % msg)
         except KeyError:
             pass
 
         # options
         if not ui.quiet and entry[1]:
             rst.append(optrst(_("options"), entry[1], ui.verbose))
 
         if ui.verbose:
             rst.append(optrst(_("global options"),
                               commands.globalopts, ui.verbose))
 
         if not ui.verbose:
             if not full:
                 rst.append(_("\n(use 'hg %s -h' to show more help)\n")
                            % name)
             elif not ui.quiet:
                 rst.append(_('\n(some details hidden, use --verbose '
                                'to show complete help)'))
 
         return rst
 
 
     def helplist(select=None, **opts):
         # list of commands
         if name == "shortlist":
             header = _('basic commands:\n\n')
         elif name == "debug":
             header = _('debug commands (internal and unsupported):\n\n')
         else:
             header = _('list of commands:\n\n')
 
         h = {}
         cmds = {}
         for c, e in commands.table.iteritems():
             f = c.partition("|")[0]
             if select and not select(f):
                 continue
             if (not select and name != 'shortlist' and
                 e[0].__module__ != commands.__name__):
                 continue
             if name == "shortlist" and not f.startswith("^"):
                 continue
             f = f.lstrip("^")
-            doc = e[0].__doc__
+            doc = pycompat.getdoc(e[0])
             if filtercmd(ui, f, name, doc):
                 continue
             doc = gettext(doc)
             if not doc:
                 doc = _("(no help text available)")
             h[f] = doc.splitlines()[0].rstrip()
             cmds[f] = c.lstrip("^")
 
         rst = []
         if not h:
             if not ui.quiet:
                 rst.append(_('no commands defined\n'))
             return rst
 
         if not ui.quiet:
             rst.append(header)
         fns = sorted(h)
         for f in fns:
             if ui.verbose:
                 commacmds = cmds[f].replace("|",", ")
                 rst.append(" :%s: %s\n" % (commacmds, h[f]))
             else:
                 rst.append(' :%s: %s\n' % (f, h[f]))
 
         ex = opts.get
         anyopts = (ex('keyword') or not (ex('command') or ex('extension')))
         if not name and anyopts:
             exts = listexts(_('enabled extensions:'), extensions.enabled())
             if exts:
                 rst.append('\n')
                 rst.extend(exts)
 
             rst.append(_("\nadditional help topics:\n\n"))
             topics = []
             for names, header, doc in helptable:
                 topics.append((names[0], header))
             for t, desc in topics:
                 rst.append(" :%s: %s\n" % (t, desc))
 
         if ui.quiet:
             pass
         elif ui.verbose:
             rst.append('\n%s\n' % optrst(_("global options"),
                                          commands.globalopts, ui.verbose))
             if name == 'shortlist':
                 rst.append(_("\n(use 'hg help' for the full list "
                              "of commands)\n"))
         else:
             if name == 'shortlist':
                 rst.append(_("\n(use 'hg help' for the full list of commands "
                              "or 'hg -v' for details)\n"))
             elif name and not full:
                 rst.append(_("\n(use 'hg help %s' to show the full help "
                              "text)\n") % name)
             elif name and cmds and name in cmds.keys():
                 rst.append(_("\n(use 'hg help -v -e %s' to show built-in "
                              "aliases and global options)\n") % name)
             else:
                 rst.append(_("\n(use 'hg help -v%s' to show built-in aliases "
                              "and global options)\n")
                            % (name and " " + name or ""))
         return rst
 
     def helptopic(name, subtopic=None):
         # Look for sub-topic entry first.
         header, doc = None, None
         if subtopic and name in subtopics:
             for names, header, doc in subtopics[name]:
                 if subtopic in names:
                     break
 
         if not header:
             for names, header, doc in helptable:
                 if name in names:
                     break
             else:
                 raise error.UnknownCommand(name)
 
         rst = [minirst.section(header)]
 
         # description
         if not doc:
             rst.append("    %s\n" % _("(no help text available)"))
         if callable(doc):
             rst += ["    %s\n" % l for l in doc(ui).splitlines()]
 
         if not ui.verbose:
             omitted = _('(some details hidden, use --verbose'
                          ' to show complete help)')
             indicateomitted(rst, omitted)
 
         try:
             cmdutil.findcmd(name, commands.table)
             rst.append(_("\nuse 'hg help -c %s' to see help for "
                        "the %s command\n") % (name, name))
         except error.UnknownCommand:
             pass
         return rst
 
     def helpext(name, subtopic=None):
         try:
             mod = extensions.find(name)
-            doc = gettext(mod.__doc__) or _('no help text available')
+            doc = gettext(pycompat.getdoc(mod)) or _('no help text available')
         except KeyError:
             mod = None
             doc = extensions.disabledext(name)
             if not doc:
                 raise error.UnknownCommand(name)
 
         if '\n' not in doc:
             head, tail = doc, ""
         else:
             head, tail = doc.split('\n', 1)
         rst = [_('%s extension - %s\n\n') % (name.rpartition('.')[-1], head)]
         if tail:
             rst.extend(tail.splitlines(True))
             rst.append('\n')
 
         if not ui.verbose:
             omitted = _('(some details hidden, use --verbose'
                          ' to show complete help)')
             indicateomitted(rst, omitted)
 
         if mod:
             try:
                 ct = mod.cmdtable
             except AttributeError:
                 ct = {}
             modcmds = set([c.partition('|')[0] for c in ct])
             rst.extend(helplist(modcmds.__contains__))
         else:
             rst.append(_("(use 'hg help extensions' for information on enabling"
                        " extensions)\n"))
         return rst
 
     def helpextcmd(name, subtopic=None):
         cmd, ext, mod = extensions.disabledcmd(ui, name,
                                                ui.configbool('ui', 'strict'))
-        doc = gettext(mod.__doc__).splitlines()[0]
+        doc = gettext(pycompat.getdoc(mod)).splitlines()[0]
 
         rst = listexts(_("'%s' is provided by the following "
                               "extension:") % cmd, {ext: doc}, indent=4,
                        showdeprecated=True)
         rst.append('\n')
         rst.append(_("(use 'hg help extensions' for information on enabling "
                    "extensions)\n"))
         return rst
 
 
     rst = []
     kw = opts.get('keyword')
     if kw or name is None and any(opts[o] for o in opts):
         matches = topicmatch(ui, commands, name or '')
         helpareas = []
         if opts.get('extension'):
             helpareas += [('extensions', _('Extensions'))]
         if opts.get('command'):
             helpareas += [('commands', _('Commands'))]
         if not helpareas:
             helpareas = [('topics', _('Topics')),
                          ('commands', _('Commands')),
                          ('extensions', _('Extensions')),
                          ('extensioncommands', _('Extension Commands'))]
         for t, title in helpareas:
             if matches[t]:
                 rst.append('%s:\n\n' % title)
                 rst.extend(minirst.maketable(sorted(matches[t]), 1))
                 rst.append('\n')
         if not rst:
             msg = _('no matches')
             hint = _("try 'hg help' for a list of topics")
             raise error.Abort(msg, hint=hint)
     elif name and name != 'shortlist':
         queries = []
         if unknowncmd:
             queries += [helpextcmd]
         if opts.get('extension'):
             queries += [helpext]
         if opts.get('command'):
             queries += [helpcmd]
         if not queries:
             queries = (helptopic, helpcmd, helpext, helpextcmd)
         for f in queries:
             try:
                 rst = f(name, subtopic)
                 break
             except error.UnknownCommand:
                 pass
         else:
             if unknowncmd:
                 raise error.UnknownCommand(name)
             else:
                 msg = _('no such help topic: %s') % name
                 hint = _("try 'hg help --keyword %s'") % name
                 raise error.Abort(msg, hint=hint)
     else:
         # program name
         if not ui.quiet:
             rst = [_("Mercurial Distributed SCM\n"), '\n']
         rst.extend(helplist(None, **pycompat.strkwargs(opts)))
 
     return ''.join(rst)
 
 def formattedhelp(ui, commands, name, keep=None, unknowncmd=False, full=True,
                   **opts):
     """get help for a given topic (as a dotted name) as rendered rst
 
     Either returns the rendered help text or raises an exception.
     """
     if keep is None:
         keep = []
     else:
         keep = list(keep) # make a copy so we can mutate this later
     fullname = name
     section = None
     subtopic = None
     if name and '.' in name:
         name, remaining = name.split('.', 1)
         remaining = encoding.lower(remaining)
         if '.' in remaining:
             subtopic, section = remaining.split('.', 1)
         else:
             if name in subtopics:
                 subtopic = remaining
             else:
                 section = remaining
     textwidth = ui.configint('ui', 'textwidth', 78)
     termwidth = ui.termwidth() - 2
     if textwidth <= 0 or termwidth < textwidth:
         textwidth = termwidth
     text = help_(ui, commands, name,
                  subtopic=subtopic, unknowncmd=unknowncmd, full=full, **opts)
 
     formatted, pruned = minirst.format(text, textwidth, keep=keep,
                                        section=section)
 
     # We could have been given a weird ".foo" section without a name
     # to look for, or we could have simply failed to found "foo.bar"
     # because bar isn't a section of foo
     if section and not (formatted and name):
         raise error.Abort(_("help section not found: %s") % fullname)
 
     if 'verbose' in pruned:
         keep.append('omitted')
     else:
         keep.append('notomitted')
     formatted, pruned = minirst.format(text, textwidth, keep=keep,
                                        section=section)
     return formatted

mercurial/pycompat.py

@@ -1,429 +1,440 b''
 # pycompat.py - portability shim for python 3
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """Mercurial portability shim for python 3.
 
 This contains aliases to hide python version-specific details from the core.
 """
 
 from __future__ import absolute_import
 
 import getopt
 import os
 import shlex
 import sys
 
 ispy3 = (sys.version_info[0] >= 3)
 
 if not ispy3:
     import cookielib
     import cPickle as pickle
     import httplib
     import Queue as _queue
     import SocketServer as socketserver
     import xmlrpclib
 else:
     import http.cookiejar as cookielib
     import http.client as httplib
     import pickle
     import queue as _queue
     import socketserver
     import xmlrpc.client as xmlrpclib
 
 def identity(a):
     return a
 
 if ispy3:
     import builtins
     import functools
     import io
     import struct
 
     fsencode = os.fsencode
     fsdecode = os.fsdecode
     # A bytes version of os.name.
     oslinesep = os.linesep.encode('ascii')
     osname = os.name.encode('ascii')
     ospathsep = os.pathsep.encode('ascii')
     ossep = os.sep.encode('ascii')
     osaltsep = os.altsep
     if osaltsep:
         osaltsep = osaltsep.encode('ascii')
     # os.getcwd() on Python 3 returns string, but it has os.getcwdb() which
     # returns bytes.
     getcwd = os.getcwdb
     sysplatform = sys.platform.encode('ascii')
     sysexecutable = sys.executable
     if sysexecutable:
         sysexecutable = os.fsencode(sysexecutable)
     stringio = io.BytesIO
     maplist = lambda *args: list(map(*args))
 
     # TODO: .buffer might not exist if std streams were replaced; we'll need
     # a silly wrapper to make a bytes stream backed by a unicode one.
     stdin = sys.stdin.buffer
     stdout = sys.stdout.buffer
     stderr = sys.stderr.buffer
 
     # Since Python 3 converts argv to wchar_t type by Py_DecodeLocale() on Unix,
     # we can use os.fsencode() to get back bytes argv.
     #
     # https://hg.python.org/cpython/file/v3.5.1/Programs/python.c#l55
     #
     # TODO: On Windows, the native argv is wchar_t, so we'll need a different
     # workaround to simulate the Python 2 (i.e. ANSI Win32 API) behavior.
     if getattr(sys, 'argv', None) is not None:
         sysargv = list(map(os.fsencode, sys.argv))
 
     bytechr = struct.Struct('>B').pack
 
     class bytestr(bytes):
         """A bytes which mostly acts as a Python 2 str
 
         >>> bytestr(), bytestr(bytearray(b'foo')), bytestr(u'ascii'), bytestr(1)
         (b'', b'foo', b'ascii', b'1')
         >>> s = bytestr(b'foo')
         >>> assert s is bytestr(s)
 
         __bytes__() should be called if provided:
 
         >>> class bytesable(object):
         ...     def __bytes__(self):
         ...         return b'bytes'
         >>> bytestr(bytesable())
         b'bytes'
 
         There's no implicit conversion from non-ascii str as its encoding is
         unknown:
 
         >>> bytestr(chr(0x80)) # doctest: +ELLIPSIS
         Traceback (most recent call last):
           ...
         UnicodeEncodeError: ...
 
         Comparison between bytestr and bytes should work:
 
         >>> assert bytestr(b'foo') == b'foo'
         >>> assert b'foo' == bytestr(b'foo')
         >>> assert b'f' in bytestr(b'foo')
         >>> assert bytestr(b'f') in b'foo'
 
         Sliced elements should be bytes, not integer:
 
         >>> s[1], s[:2]
         (b'o', b'fo')
         >>> list(s), list(reversed(s))
         ([b'f', b'o', b'o'], [b'o', b'o', b'f'])
 
         As bytestr type isn't propagated across operations, you need to cast
         bytes to bytestr explicitly:
 
         >>> s = bytestr(b'foo').upper()
         >>> t = bytestr(s)
         >>> s[0], t[0]
         (70, b'F')
 
         Be careful to not pass a bytestr object to a function which expects
         bytearray-like behavior.
 
         >>> t = bytes(t)  # cast to bytes
         >>> assert type(t) is bytes
         """
 
         def __new__(cls, s=b''):
             if isinstance(s, bytestr):
                 return s
             if (not isinstance(s, (bytes, bytearray))
                 and not hasattr(s, u'__bytes__')):  # hasattr-py3-only
                 s = str(s).encode(u'ascii')
             return bytes.__new__(cls, s)
 
         def __getitem__(self, key):
             s = bytes.__getitem__(self, key)
             if not isinstance(s, bytes):
                 s = bytechr(s)
             return s
 
         def __iter__(self):
             return iterbytestr(bytes.__iter__(self))
 
     def iterbytestr(s):
         """Iterate bytes as if it were a str object of Python 2"""
         return map(bytechr, s)
 
     def sysbytes(s):
         """Convert an internal str (e.g. keyword, __doc__) back to bytes
 
         This never raises UnicodeEncodeError, but only ASCII characters
         can be round-trip by sysstr(sysbytes(s)).
         """
         return s.encode(u'utf-8')
 
     def sysstr(s):
         """Return a keyword str to be passed to Python functions such as
         getattr() and str.encode()
 
         This never raises UnicodeDecodeError. Non-ascii characters are
         considered invalid and mapped to arbitrary but unique code points
         such that 'sysstr(a) != sysstr(b)' for all 'a != b'.
         """
         if isinstance(s, builtins.str):
             return s
         return s.decode(u'latin-1')
 
     def raisewithtb(exc, tb):
         """Raise exception with the given traceback"""
         raise exc.with_traceback(tb)
 
+    def getdoc(obj):
+        """Get docstring as bytes; may be None so gettext() won't confuse it
+        with _('')"""
+        doc = getattr(obj, u'__doc__', None)
+        if doc is None:
+            return doc
+        return sysbytes(doc)
+
     def _wrapattrfunc(f):
         @functools.wraps(f)
         def w(object, name, *args):
             return f(object, sysstr(name), *args)
         return w
 
     # these wrappers are automagically imported by hgloader
     delattr = _wrapattrfunc(builtins.delattr)
     getattr = _wrapattrfunc(builtins.getattr)
     hasattr = _wrapattrfunc(builtins.hasattr)
     setattr = _wrapattrfunc(builtins.setattr)
     xrange = builtins.range
     unicode = str
 
     def open(name, mode='r', buffering=-1):
         return builtins.open(name, sysstr(mode), buffering)
 
     # getopt.getopt() on Python 3 deals with unicodes internally so we cannot
     # pass bytes there. Passing unicodes will result in unicodes as return
     # values which we need to convert again to bytes.
     def getoptb(args, shortlist, namelist):
         args = [a.decode('latin-1') for a in args]
         shortlist = shortlist.decode('latin-1')
         namelist = [a.decode('latin-1') for a in namelist]
         opts, args = getopt.getopt(args, shortlist, namelist)
         opts = [(a[0].encode('latin-1'), a[1].encode('latin-1'))
                 for a in opts]
         args = [a.encode('latin-1') for a in args]
         return opts, args
 
     # keys of keyword arguments in Python need to be strings which are unicodes
     # Python 3. This function takes keyword arguments, convert the keys to str.
     def strkwargs(dic):
         dic = dict((k.decode('latin-1'), v) for k, v in dic.iteritems())
         return dic
 
     # keys of keyword arguments need to be unicode while passing into
     # a function. This function helps us to convert those keys back to bytes
     # again as we need to deal with bytes.
     def byteskwargs(dic):
         dic = dict((k.encode('latin-1'), v) for k, v in dic.iteritems())
         return dic
 
     # shlex.split() accepts unicodes on Python 3. This function takes bytes
     # argument, convert it into unicodes, pass into shlex.split(), convert the
     # returned value to bytes and return that.
     # TODO: handle shlex.shlex().
     def shlexsplit(s):
         ret = shlex.split(s.decode('latin-1'))
         return [a.encode('latin-1') for a in ret]
 
 else:
     import cStringIO
 
     bytechr = chr
     bytestr = str
     iterbytestr = iter
     sysbytes = identity
     sysstr = identity
 
     # this can't be parsed on Python 3
     exec('def raisewithtb(exc, tb):\n'
          '    raise exc, None, tb\n')
 
     # Partial backport from os.py in Python 3, which only accepts bytes.
     # In Python 2, our paths should only ever be bytes, a unicode path
     # indicates a bug.
     def fsencode(filename):
         if isinstance(filename, str):
             return filename
         else:
             raise TypeError(
                 "expect str, not %s" % type(filename).__name__)
 
     # In Python 2, fsdecode() has a very chance to receive bytes. So it's
     # better not to touch Python 2 part as it's already working fine.
     fsdecode = identity
 
+    def getdoc(obj):
+        return getattr(obj, '__doc__', None)
+
     def getoptb(args, shortlist, namelist):
         return getopt.getopt(args, shortlist, namelist)
 
     strkwargs = identity
     byteskwargs = identity
 
     oslinesep = os.linesep
     osname = os.name
     ospathsep = os.pathsep
     ossep = os.sep
     osaltsep = os.altsep
     stdin = sys.stdin
     stdout = sys.stdout
     stderr = sys.stderr
     if getattr(sys, 'argv', None) is not None:
         sysargv = sys.argv
     sysplatform = sys.platform
     getcwd = os.getcwd
     sysexecutable = sys.executable
     shlexsplit = shlex.split
     stringio = cStringIO.StringIO
     maplist = map
 
 empty = _queue.Empty
 queue = _queue.Queue
 
 class _pycompatstub(object):
     def __init__(self):
         self._aliases = {}
 
     def _registeraliases(self, origin, items):
         """Add items that will be populated at the first access"""
         items = map(sysstr, items)
         self._aliases.update(
             (item.replace(sysstr('_'), sysstr('')).lower(), (origin, item))
             for item in items)
 
     def _registeralias(self, origin, attr, name):
         """Alias ``origin``.``attr`` as ``name``"""
         self._aliases[sysstr(name)] = (origin, sysstr(attr))
 
     def __getattr__(self, name):
         try:
             origin, item = self._aliases[name]
         except KeyError:
             raise AttributeError(name)
         self.__dict__[name] = obj = getattr(origin, item)
         return obj
 
 httpserver = _pycompatstub()
 urlreq = _pycompatstub()
 urlerr = _pycompatstub()
 if not ispy3:
     import BaseHTTPServer
     import CGIHTTPServer
     import SimpleHTTPServer
     import urllib2
     import urllib
     import urlparse
     urlreq._registeraliases(urllib, (
         "addclosehook",
         "addinfourl",
         "ftpwrapper",
         "pathname2url",
         "quote",
         "splitattr",
         "splitpasswd",
         "splitport",
         "splituser",
         "unquote",
         "url2pathname",
         "urlencode",
     ))
     urlreq._registeraliases(urllib2, (
         "AbstractHTTPHandler",
         "BaseHandler",
         "build_opener",
         "FileHandler",
         "FTPHandler",
         "HTTPBasicAuthHandler",
         "HTTPDigestAuthHandler",
         "HTTPHandler",
         "HTTPPasswordMgrWithDefaultRealm",
         "HTTPSHandler",
         "install_opener",
         "ProxyHandler",
         "Request",
         "urlopen",
     ))
     urlreq._registeraliases(urlparse, (
         "urlparse",
         "urlunparse",
     ))
     urlerr._registeraliases(urllib2, (
         "HTTPError",
         "URLError",
     ))
     httpserver._registeraliases(BaseHTTPServer, (
         "HTTPServer",
         "BaseHTTPRequestHandler",
     ))
     httpserver._registeraliases(SimpleHTTPServer, (
         "SimpleHTTPRequestHandler",
     ))
     httpserver._registeraliases(CGIHTTPServer, (
         "CGIHTTPRequestHandler",
     ))
 
 else:
     import urllib.parse
     urlreq._registeraliases(urllib.parse, (
         "splitattr",
         "splitpasswd",
         "splitport",
         "splituser",
         "urlparse",
         "urlunparse",
     ))
     urlreq._registeralias(urllib.parse, "unquote_to_bytes", "unquote")
     import urllib.request
     urlreq._registeraliases(urllib.request, (
         "AbstractHTTPHandler",
         "BaseHandler",
         "build_opener",
         "FileHandler",
         "FTPHandler",
         "ftpwrapper",
         "HTTPHandler",
         "HTTPSHandler",
         "install_opener",
         "pathname2url",
         "HTTPBasicAuthHandler",
         "HTTPDigestAuthHandler",
         "HTTPPasswordMgrWithDefaultRealm",
         "ProxyHandler",
         "Request",
         "url2pathname",
         "urlopen",
     ))
     import urllib.response
     urlreq._registeraliases(urllib.response, (
         "addclosehook",
         "addinfourl",
     ))
     import urllib.error
     urlerr._registeraliases(urllib.error, (
         "HTTPError",
         "URLError",
     ))
     import http.server
     httpserver._registeraliases(http.server, (
         "HTTPServer",
         "BaseHTTPRequestHandler",
         "SimpleHTTPRequestHandler",
         "CGIHTTPRequestHandler",
     ))
 
     # urllib.parse.quote() accepts both str and bytes, decodes bytes
     # (if necessary), and returns str. This is wonky. We provide a custom
     # implementation that only accepts bytes and emits bytes.
     def quote(s, safe=r'/'):
         s = urllib.parse.quote_from_bytes(s, safe=safe)
         return s.encode('ascii', 'strict')
 
     # urllib.parse.urlencode() returns str. We use this function to make
     # sure we return bytes.
     def urlencode(query, doseq=False):
             s = urllib.parse.urlencode(query, doseq=doseq)
             return s.encode('ascii')
 
     urlreq.quote = quote
     urlreq.urlencode = urlencode

tests/test-py3-commands.t

@@ -1,198 +1,212 b''
 #require py3exe
 
 This test helps in keeping a track on which commands we can run on
 Python 3 and see what kind of errors are coming up.
 The full traceback is hidden to have a stable output.
   $ HGBIN=`which hg`
 
   $ for cmd in version debuginstall ; do
   >   echo $cmd
   >   $PYTHON3 $HGBIN $cmd 2>&1 2>&1 | tail -1
   > done
   version
   warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
   debuginstall
   no problems detected
 
 #if test-repo
 Make a clone so that any features in the developer's .hg/hgrc that
 might confuse Python 3 don't break this test. When we can do commit in
 Python 3, we'll stop doing this. We use e76ed1e480ef for the clone
 because it has different files than 273ce12ad8f1, so we can test both
 `files` from dirstate and `files` loaded from a specific revision.
 
   $ hg clone -r e76ed1e480ef "`dirname "$TESTDIR"`" testrepo 2>&1 | tail -1
   15 files updated, 0 files merged, 0 files removed, 0 files unresolved
 
 Test using -R, which exercises some URL code:
   $ $PYTHON3 $HGBIN -R testrepo files -r 273ce12ad8f1 | tail -1
   testrepo/tkmerge
 
 Now prove `hg files` is reading the whole manifest. We have to grep
 out some potential warnings that come from hgrc as yet.
   $ cd testrepo
   $ $PYTHON3 $HGBIN files -r 273ce12ad8f1
   .hgignore
   PKG-INFO
   README
   hg
   mercurial/__init__.py
   mercurial/byterange.py
   mercurial/fancyopts.py
   mercurial/hg.py
   mercurial/mdiff.py
   mercurial/revlog.py
   mercurial/transaction.py
   notes.txt
   setup.py
   tkmerge
 
   $ $PYTHON3 $HGBIN files -r 273ce12ad8f1 | wc -l
   \s*14 (re)
   $ $PYTHON3 $HGBIN files | wc -l
   \s*15 (re)
 
 Test if log-like commands work:
 
   $ $PYTHON3 $HGBIN tip
   changeset:   10:e76ed1e480ef
   tag:         tip
   user:        [e-mail]
   date:        Tue May 03 23:37:43 2005 -0800
   summary:     Fix linking of changeset revs when merging
   
 
   $ $PYTHON3 $HGBIN log -r0
   changeset:   0:9117c6561b0b
   user:        [e-mail]
   date:        Tue May 03 13:16:10 2005 -0800
   summary:     Add back links from file revisions to changeset revisions
   
 
   $ cd ..
 #endif
 
 Test if `hg config` works:
 
   $ $PYTHON3 $HGBIN config
   devel.all-warnings=true
   devel.default-date=0 0
   largefiles.usercache=$TESTTMP/.cache/largefiles
   ui.slash=True
   ui.interactive=False
   ui.mergemarkers=detailed
   ui.promptecho=True
   web.address=localhost
   web.ipv6=False
 
   $ cat > included-hgrc <<EOF
   > [extensions]
   > babar = imaginary_elephant
   > EOF
   $ cat >> $HGRCPATH <<EOF
   > %include $TESTTMP/included-hgrc
   > EOF
   $ $PYTHON3 $HGBIN version | tail -1
   *** failed to import extension babar from imaginary_elephant: *: 'imaginary_elephant' (glob)
   warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 
   $ rm included-hgrc
   $ touch included-hgrc
 
 Test bytes-ness of policy.policy with HGMODULEPOLICY
 
   $ HGMODULEPOLICY=py
   $ export HGMODULEPOLICY
   $ $PYTHON3 `which hg` debuginstall 2>&1 2>&1 | tail -1
   no problems detected
 
 `hg init` can create empty repos
 `hg status works fine`
 `hg summary` also works!
 
   $ $PYTHON3 `which hg` init py3repo
   $ cd py3repo
   $ echo "This is the file 'iota'." > iota
   $ $PYTHON3 $HGBIN status
   ? iota
   $ $PYTHON3 $HGBIN add iota
   $ $PYTHON3 $HGBIN status
   A iota
   $ $PYTHON3 $HGBIN commit --message 'commit performed in Python 3'
   $ $PYTHON3 $HGBIN status
 
   $ mkdir A
   $ echo "This is the file 'mu'." > A/mu
   $ $PYTHON3 $HGBIN addremove
   adding A/mu
   $ $PYTHON3 $HGBIN status
   A A/mu
   $ HGEDITOR='echo message > ' $PYTHON3 $HGBIN commit
   $ $PYTHON3 $HGBIN status
   $ $PYHON3 $HGBIN summary
   parent: 1:e1e9167203d4 tip
    message
   branch: default
   commit: (clean)
   update: (current)
   phases: 2 draft
 
+Test weird unicode-vs-bytes stuff
+
+  $ $PYTHON3 $HGBIN help | egrep -v '^ |^$'
+  Mercurial Distributed SCM
+  list of commands:
+  additional help topics:
+  (use 'hg help -v' to show built-in aliases and global options)
+
+  $ $PYTHON3 $HGBIN help help | egrep -v '^ |^$'
+  hg help [-ecks] [TOPIC]
+  show help for a given topic or a help overview
+  options ([+] can be repeated):
+  (some details hidden, use --verbose to show complete help)
+
 Prove the repo is valid using the Python 2 `hg`:
   $ hg verify
   checking changesets
   checking manifests
   crosschecking files in changesets and manifests
   checking files
   2 files, 2 changesets, 2 total revisions
   $ hg log
   changeset:   1:e1e9167203d4
   tag:         tip
   user:        test
   date:        Thu Jan 01 00:00:00 1970 +0000
   summary:     message
   
   changeset:   0:71c96e924262
   user:        test
   date:        Thu Jan 01 00:00:00 1970 +0000
   summary:     commit performed in Python 3
   
 
   $ hg log -G
   @  changeset:   1:e1e9167203d4
   |  tag:         tip
   |  user:        test
   |  date:        Thu Jan 01 00:00:00 1970 +0000
   |  summary:     message
   |
   o  changeset:   0:71c96e924262
      user:        test
      date:        Thu Jan 01 00:00:00 1970 +0000
      summary:     commit performed in Python 3
   
   $ hg log -Tjson
   [
    {
     "rev": 1,
     "node": "e1e9167203d450ca2f558af628955b5f5afd4489",
     "branch": "default",
     "phase": "draft",
     "user": "test",
     "date": [0, 0],
     "desc": "message",
     "bookmarks": [],
     "tags": ["tip"],
     "parents": ["71c96e924262969ff0d8d3d695b0f75412ccc3d8"]
    },
    {
     "rev": 0,
     "node": "71c96e924262969ff0d8d3d695b0f75412ccc3d8",
     "branch": "default",
     "phase": "draft",
     "user": "test",
     "date": [0, 0],
     "desc": "commit performed in Python 3",
     "bookmarks": [],
     "tags": [],
     "parents": ["0000000000000000000000000000000000000000"]
    }
   ]