py3: get bytes-repr of network errors portably

This resolves a lot of weird issues in Python 3 around error strings.

Differential Revision: https://phab.mercurial-scm.org/D2295

mercurial/hgweb/hgweb_mod.py

@@ -1,483 +1,483 b''
 # hgweb/hgweb_mod.py - Web interface for a repository.
 #
 # Copyright 21 May 2005 - (c) 2005 Jake Edge <[e-mail]>
 # Copyright 2005-2007 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import contextlib
 import os
 
 from .common import (
     ErrorResponse,
     HTTP_BAD_REQUEST,
     HTTP_NOT_FOUND,
     HTTP_NOT_MODIFIED,
     HTTP_OK,
     HTTP_SERVER_ERROR,
     caching,
     cspvalues,
     permhooks,
 )
 from .request import wsgirequest
 
 from .. import (
     encoding,
     error,
     hg,
     hook,
     profiling,
     pycompat,
     repoview,
     templatefilters,
     templater,
     ui as uimod,
     util,
     wireprotoserver,
 )
 
 from . import (
     webcommands,
     webutil,
     wsgicgi,
 )
 
 perms = {
     'changegroup': 'pull',
     'changegroupsubset': 'pull',
     'getbundle': 'pull',
     'stream_out': 'pull',
     'listkeys': 'pull',
     'unbundle': 'push',
     'pushkey': 'push',
 }
 
 archivespecs = util.sortdict((
     ('zip', ('application/zip', 'zip', '.zip', None)),
     ('gz', ('application/x-gzip', 'tgz', '.tar.gz', None)),
     ('bz2', ('application/x-bzip2', 'tbz2', '.tar.bz2', None)),
 ))
 
 def getstyle(req, configfn, templatepath):
     fromreq = req.form.get('style', [None])[0]
     if fromreq is not None:
         fromreq = pycompat.sysbytes(fromreq)
     styles = (
         fromreq,
         configfn('web', 'style'),
         'paper',
     )
     return styles, templater.stylemap(styles, templatepath)
 
 def makebreadcrumb(url, prefix=''):
     '''Return a 'URL breadcrumb' list
 
     A 'URL breadcrumb' is a list of URL-name pairs,
     corresponding to each of the path items on a URL.
     This can be used to create path navigation entries.
     '''
     if url.endswith('/'):
         url = url[:-1]
     if prefix:
         url = '/' + prefix + url
     relpath = url
     if relpath.startswith('/'):
         relpath = relpath[1:]
 
     breadcrumb = []
     urlel = url
     pathitems = [''] + relpath.split('/')
     for pathel in reversed(pathitems):
         if not pathel or not urlel:
             break
         breadcrumb.append({'url': urlel, 'name': pathel})
         urlel = os.path.dirname(urlel)
     return reversed(breadcrumb)
 
 class requestcontext(object):
     """Holds state/context for an individual request.
 
     Servers can be multi-threaded. Holding state on the WSGI application
     is prone to race conditions. Instances of this class exist to hold
     mutable and race-free state for requests.
     """
     def __init__(self, app, repo):
         self.repo = repo
         self.reponame = app.reponame
 
         self.archivespecs = archivespecs
 
         self.maxchanges = self.configint('web', 'maxchanges')
         self.stripecount = self.configint('web', 'stripes')
         self.maxshortchanges = self.configint('web', 'maxshortchanges')
         self.maxfiles = self.configint('web', 'maxfiles')
         self.allowpull = self.configbool('web', 'allow-pull')
 
         # we use untrusted=False to prevent a repo owner from using
         # web.templates in .hg/hgrc to get access to any file readable
         # by the user running the CGI script
         self.templatepath = self.config('web', 'templates', untrusted=False)
 
         # This object is more expensive to build than simple config values.
         # It is shared across requests. The app will replace the object
         # if it is updated. Since this is a reference and nothing should
         # modify the underlying object, it should be constant for the lifetime
         # of the request.
         self.websubtable = app.websubtable
 
         self.csp, self.nonce = cspvalues(self.repo.ui)
 
     # Trust the settings from the .hg/hgrc files by default.
     def config(self, section, name, default=uimod._unset, untrusted=True):
         return self.repo.ui.config(section, name, default,
                                    untrusted=untrusted)
 
     def configbool(self, section, name, default=uimod._unset, untrusted=True):
         return self.repo.ui.configbool(section, name, default,
                                        untrusted=untrusted)
 
     def configint(self, section, name, default=uimod._unset, untrusted=True):
         return self.repo.ui.configint(section, name, default,
                                       untrusted=untrusted)
 
     def configlist(self, section, name, default=uimod._unset, untrusted=True):
         return self.repo.ui.configlist(section, name, default,
                                        untrusted=untrusted)
 
     def archivelist(self, nodeid):
         allowed = self.configlist('web', 'allow_archive')
         for typ, spec in self.archivespecs.iteritems():
             if typ in allowed or self.configbool('web', 'allow%s' % typ):
                 yield {'type': typ, 'extension': spec[2], 'node': nodeid}
 
     def templater(self, req):
         # determine scheme, port and server name
         # this is needed to create absolute urls
 
         proto = req.env.get('wsgi.url_scheme')
         if proto == 'https':
             proto = 'https'
             default_port = '443'
         else:
             proto = 'http'
             default_port = '80'
 
         port = req.env[r'SERVER_PORT']
         port = port != default_port and (r':' + port) or r''
         urlbase = r'%s://%s%s' % (proto, req.env[r'SERVER_NAME'], port)
         logourl = self.config('web', 'logourl')
         logoimg = self.config('web', 'logoimg')
         staticurl = self.config('web', 'staticurl') or req.url + 'static/'
         if not staticurl.endswith('/'):
             staticurl += '/'
 
         # some functions for the templater
 
         def motd(**map):
             yield self.config('web', 'motd')
 
         # figure out which style to use
 
         vars = {}
         styles, (style, mapfile) = getstyle(req, self.config,
                                             self.templatepath)
         if style == styles[0]:
             vars['style'] = style
 
         start = '&' if req.url[-1] == r'?' else '?'
         sessionvars = webutil.sessionvars(vars, start)
 
         if not self.reponame:
             self.reponame = (self.config('web', 'name', '')
                              or req.env.get('REPO_NAME')
                              or req.url.strip('/') or self.repo.root)
 
         def websubfilter(text):
             return templatefilters.websub(text, self.websubtable)
 
         # create the templater
 
         defaults = {
             'url': req.url,
             'logourl': logourl,
             'logoimg': logoimg,
             'staticurl': staticurl,
             'urlbase': urlbase,
             'repo': self.reponame,
             'encoding': encoding.encoding,
             'motd': motd,
             'sessionvars': sessionvars,
             'pathdef': makebreadcrumb(req.url),
             'style': style,
             'nonce': self.nonce,
         }
         tmpl = templater.templater.frommapfile(mapfile,
                                                filters={'websub': websubfilter},
                                                defaults=defaults)
         return tmpl
 
 
 class hgweb(object):
     """HTTP server for individual repositories.
 
     Instances of this class serve HTTP responses for a particular
     repository.
 
     Instances are typically used as WSGI applications.
 
     Some servers are multi-threaded. On these servers, there may
     be multiple active threads inside __call__.
     """
     def __init__(self, repo, name=None, baseui=None):
         if isinstance(repo, str):
             if baseui:
                 u = baseui.copy()
             else:
                 u = uimod.ui.load()
             r = hg.repository(u, repo)
         else:
             # we trust caller to give us a private copy
             r = repo
 
         r.ui.setconfig('ui', 'report_untrusted', 'off', 'hgweb')
         r.baseui.setconfig('ui', 'report_untrusted', 'off', 'hgweb')
         r.ui.setconfig('ui', 'nontty', 'true', 'hgweb')
         r.baseui.setconfig('ui', 'nontty', 'true', 'hgweb')
         # resolve file patterns relative to repo root
         r.ui.setconfig('ui', 'forcecwd', r.root, 'hgweb')
         r.baseui.setconfig('ui', 'forcecwd', r.root, 'hgweb')
         # displaying bundling progress bar while serving feel wrong and may
         # break some wsgi implementation.
         r.ui.setconfig('progress', 'disable', 'true', 'hgweb')
         r.baseui.setconfig('progress', 'disable', 'true', 'hgweb')
         self._repos = [hg.cachedlocalrepo(self._webifyrepo(r))]
         self._lastrepo = self._repos[0]
         hook.redirect(True)
         self.reponame = name
 
     def _webifyrepo(self, repo):
         repo = getwebview(repo)
         self.websubtable = webutil.getwebsubs(repo)
         return repo
 
     @contextlib.contextmanager
     def _obtainrepo(self):
         """Obtain a repo unique to the caller.
 
         Internally we maintain a stack of cachedlocalrepo instances
         to be handed out. If one is available, we pop it and return it,
         ensuring it is up to date in the process. If one is not available,
         we clone the most recently used repo instance and return it.
 
         It is currently possible for the stack to grow without bounds
         if the server allows infinite threads. However, servers should
         have a thread limit, thus establishing our limit.
         """
         if self._repos:
             cached = self._repos.pop()
             r, created = cached.fetch()
         else:
             cached = self._lastrepo.copy()
             r, created = cached.fetch()
         if created:
             r = self._webifyrepo(r)
 
         self._lastrepo = cached
         self.mtime = cached.mtime
         try:
             yield r
         finally:
             self._repos.append(cached)
 
     def run(self):
         """Start a server from CGI environment.
 
         Modern servers should be using WSGI and should avoid this
         method, if possible.
         """
         if not encoding.environ.get('GATEWAY_INTERFACE',
                                     '').startswith("CGI/1."):
             raise RuntimeError("This function is only intended to be "
                                "called while running as a CGI script.")
         wsgicgi.launch(self)
 
     def __call__(self, env, respond):
         """Run the WSGI application.
 
         This may be called by multiple threads.
         """
         req = wsgirequest(env, respond)
         return self.run_wsgi(req)
 
     def run_wsgi(self, req):
         """Internal method to run the WSGI application.
 
         This is typically only called by Mercurial. External consumers
         should be using instances of this class as the WSGI application.
         """
         with self._obtainrepo() as repo:
             profile = repo.ui.configbool('profiling', 'enabled')
             with profiling.profile(repo.ui, enabled=profile):
                 for r in self._runwsgi(req, repo):
                     yield r
 
     def _runwsgi(self, req, repo):
         rctx = requestcontext(self, repo)
 
         # This state is global across all threads.
         encoding.encoding = rctx.config('web', 'encoding')
         rctx.repo.ui.environ = req.env
 
         if rctx.csp:
             # hgwebdir may have added CSP header. Since we generate our own,
             # replace it.
             req.headers = [h for h in req.headers
                            if h[0] != 'Content-Security-Policy']
             req.headers.append(('Content-Security-Policy', rctx.csp))
 
         # work with CGI variables to create coherent structure
         # use SCRIPT_NAME, PATH_INFO and QUERY_STRING as well as our REPO_NAME
 
         req.url = req.env[r'SCRIPT_NAME']
         if not req.url.endswith('/'):
             req.url += '/'
         if req.env.get('REPO_NAME'):
             req.url += req.env[r'REPO_NAME'] + r'/'
 
         if r'PATH_INFO' in req.env:
             parts = req.env[r'PATH_INFO'].strip('/').split('/')
             repo_parts = req.env.get(r'REPO_NAME', r'').split(r'/')
             if parts[:len(repo_parts)] == repo_parts:
                 parts = parts[len(repo_parts):]
             query = '/'.join(parts)
         else:
             query = req.env[r'QUERY_STRING'].partition(r'&')[0]
             query = query.partition(r';')[0]
 
         # Route it to a wire protocol handler if it looks like a wire protocol
         # request.
         protohandler = wireprotoserver.parsehttprequest(rctx.repo, req, query)
 
         if protohandler:
             cmd = protohandler['cmd']
             try:
                 if query:
                     raise ErrorResponse(HTTP_NOT_FOUND)
                 if cmd in perms:
                     self.check_perm(rctx, req, perms[cmd])
             except ErrorResponse as inst:
                 return protohandler['handleerror'](inst)
 
             return protohandler['dispatch']()
 
         # translate user-visible url structure to internal structure
 
         args = query.split('/', 2)
         if r'cmd' not in req.form and args and args[0]:
             cmd = args.pop(0)
             style = cmd.rfind('-')
             if style != -1:
                 req.form['style'] = [cmd[:style]]
                 cmd = cmd[style + 1:]
 
             # avoid accepting e.g. style parameter as command
             if util.safehasattr(webcommands, cmd):
                 req.form[r'cmd'] = [cmd]
 
             if cmd == 'static':
                 req.form['file'] = ['/'.join(args)]
             else:
                 if args and args[0]:
                     node = args.pop(0).replace('%2F', '/')
                     req.form['node'] = [node]
                 if args:
                     req.form['file'] = args
 
             ua = req.env.get('HTTP_USER_AGENT', '')
             if cmd == 'rev' and 'mercurial' in ua:
                 req.form['style'] = ['raw']
 
             if cmd == 'archive':
                 fn = req.form['node'][0]
                 for type_, spec in rctx.archivespecs.iteritems():
                     ext = spec[2]
                     if fn.endswith(ext):
                         req.form['node'] = [fn[:-len(ext)]]
                         req.form['type'] = [type_]
         else:
             cmd = pycompat.sysbytes(req.form.get(r'cmd', [r''])[0])
 
         # process the web interface request
 
         try:
             tmpl = rctx.templater(req)
             ctype = tmpl('mimetype', encoding=encoding.encoding)
             ctype = templater.stringify(ctype)
 
             # check read permissions non-static content
             if cmd != 'static':
                 self.check_perm(rctx, req, None)
 
             if cmd == '':
                 req.form[r'cmd'] = [tmpl.cache['default']]
                 cmd = req.form[r'cmd'][0]
 
             # Don't enable caching if using a CSP nonce because then it wouldn't
             # be a nonce.
             if rctx.configbool('web', 'cache') and not rctx.nonce:
                 caching(self, req) # sets ETag header or raises NOT_MODIFIED
             if cmd not in webcommands.__all__:
                 msg = 'no such method: %s' % cmd
                 raise ErrorResponse(HTTP_BAD_REQUEST, msg)
             elif cmd == 'file' and r'raw' in req.form.get(r'style', []):
                 rctx.ctype = ctype
                 content = webcommands.rawfile(rctx, req, tmpl)
             else:
                 content = getattr(webcommands, cmd)(rctx, req, tmpl)
                 req.respond(HTTP_OK, ctype)
 
             return content
 
         except (error.LookupError, error.RepoLookupError) as err:
             req.respond(HTTP_NOT_FOUND, ctype)
-            msg = str(err)
+            msg = pycompat.bytestr(err)
             if (util.safehasattr(err, 'name') and
                 not isinstance(err,  error.ManifestLookupError)):
                 msg = 'revision not found: %s' % err.name
             return tmpl('error', error=msg)
         except (error.RepoError, error.RevlogError) as inst:
             req.respond(HTTP_SERVER_ERROR, ctype)
-            return tmpl('error', error=str(inst))
+            return tmpl('error', error=pycompat.bytestr(inst))
         except ErrorResponse as inst:
             req.respond(inst, ctype)
             if inst.code == HTTP_NOT_MODIFIED:
                 # Not allowed to return a body on a 304
                 return ['']
-            return tmpl('error', error=str(inst))
+            return tmpl('error', error=pycompat.bytestr(inst))
 
     def check_perm(self, rctx, req, op):
         for permhook in permhooks:
             permhook(rctx, req, op)
 
 def getwebview(repo):
     """The 'web.view' config controls changeset filter to hgweb. Possible
     values are ``served``, ``visible`` and ``all``. Default is ``served``.
     The ``served`` filter only shows changesets that can be pulled from the
     hgweb instance.  The``visible`` filter includes secret changesets but
     still excludes "hidden" one.
 
     See the repoview module for details.
 
     The option has been around undocumented since Mercurial 2.5, but no
     user ever asked about it. So we better keep it undocumented for now."""
     # experimental config: web.view
     viewconfig = repo.ui.config('web', 'view', untrusted=True)
     if viewconfig == 'all':
         return repo.unfiltered()
     elif viewconfig in repoview.filtertable:
         return repo.filtered(viewconfig)
     else:
         return repo.filtered('served')

mercurial/hgweb/request.py

@@ -1,155 +1,156 b''
 # hgweb/request.py - An http request from either CGI or the standalone server.
 #
 # Copyright 21 May 2005 - (c) 2005 Jake Edge <[e-mail]>
 # Copyright 2005, 2006 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import cgi
 import errno
 import socket
 
 from .common import (
     ErrorResponse,
     HTTP_NOT_MODIFIED,
     statusmessage,
 )
 
 from .. import (
     pycompat,
     util,
 )
 
 shortcuts = {
     'cl': [('cmd', ['changelog']), ('rev', None)],
     'sl': [('cmd', ['shortlog']), ('rev', None)],
     'cs': [('cmd', ['changeset']), ('node', None)],
     'f': [('cmd', ['file']), ('filenode', None)],
     'fl': [('cmd', ['filelog']), ('filenode', None)],
     'fd': [('cmd', ['filediff']), ('node', None)],
     'fa': [('cmd', ['annotate']), ('filenode', None)],
     'mf': [('cmd', ['manifest']), ('manifest', None)],
     'ca': [('cmd', ['archive']), ('node', None)],
     'tags': [('cmd', ['tags'])],
     'tip': [('cmd', ['changeset']), ('node', ['tip'])],
     'static': [('cmd', ['static']), ('file', None)]
 }
 
 def normalize(form):
     # first expand the shortcuts
     for k in shortcuts:
         if k in form:
             for name, value in shortcuts[k]:
                 if value is None:
                     value = form[k]
                 form[name] = value
             del form[k]
     # And strip the values
     for k, v in form.iteritems():
         form[k] = [i.strip() for i in v]
     return form
 
 class wsgirequest(object):
     """Higher-level API for a WSGI request.
 
     WSGI applications are invoked with 2 arguments. They are used to
     instantiate instances of this class, which provides higher-level APIs
     for obtaining request parameters, writing HTTP output, etc.
     """
     def __init__(self, wsgienv, start_response):
         version = wsgienv[r'wsgi.version']
         if (version < (1, 0)) or (version >= (2, 0)):
             raise RuntimeError("Unknown and unsupported WSGI version %d.%d"
                                % version)
         self.inp = wsgienv[r'wsgi.input']
         self.err = wsgienv[r'wsgi.errors']
         self.threaded = wsgienv[r'wsgi.multithread']
         self.multiprocess = wsgienv[r'wsgi.multiprocess']
         self.run_once = wsgienv[r'wsgi.run_once']
         self.env = wsgienv
         self.form = normalize(cgi.parse(self.inp,
                                         self.env,
                                         keep_blank_values=1))
         self._start_response = start_response
         self.server_write = None
         self.headers = []
 
     def __iter__(self):
         return iter([])
 
     def read(self, count=-1):
         return self.inp.read(count)
 
     def drain(self):
         '''need to read all data from request, httplib is half-duplex'''
         length = int(self.env.get('CONTENT_LENGTH') or 0)
         for s in util.filechunkiter(self.inp, limit=length):
             pass
 
     def respond(self, status, type, filename=None, body=None):
         if not isinstance(type, str):
             type = pycompat.sysstr(type)
         if self._start_response is not None:
             self.headers.append((r'Content-Type', type))
             if filename:
                 filename = (filename.rpartition('/')[-1]
                             .replace('\\', '\\\\').replace('"', '\\"'))
                 self.headers.append(('Content-Disposition',
                                      'inline; filename="%s"' % filename))
             if body is not None:
                 self.headers.append((r'Content-Length', str(len(body))))
 
             for k, v in self.headers:
                 if not isinstance(v, str):
                     raise TypeError('header value must be string: %r' % (v,))
 
             if isinstance(status, ErrorResponse):
                 self.headers.extend(status.headers)
                 if status.code == HTTP_NOT_MODIFIED:
                     # RFC 2616 Section 10.3.5: 304 Not Modified has cases where
                     # it MUST NOT include any headers other than these and no
                     # body
                     self.headers = [(k, v) for (k, v) in self.headers if
                                     k in ('Date', 'ETag', 'Expires',
                                           'Cache-Control', 'Vary')]
                 status = statusmessage(status.code, pycompat.bytestr(status))
             elif status == 200:
                 status = '200 Script output follows'
             elif isinstance(status, int):
                 status = statusmessage(status)
 
-            self.server_write = self._start_response(status, self.headers)
+            self.server_write = self._start_response(
+                pycompat.sysstr(status), self.headers)
             self._start_response = None
             self.headers = []
         if body is not None:
             self.write(body)
             self.server_write = None
 
     def write(self, thing):
         if thing:
             try:
                 self.server_write(thing)
             except socket.error as inst:
                 if inst[0] != errno.ECONNRESET:
                     raise
 
     def writelines(self, lines):
         for line in lines:
             self.write(line)
 
     def flush(self):
         return None
 
     def close(self):
         return None
 
 def wsgiapplication(app_maker):
     '''For compatibility with old CGI scripts. A plain hgweb() or hgwebdir()
     can and should now be used as a WSGI application.'''
     application = app_maker()
     def run_wsgi(env, respond):
         return application(env, respond)
     return run_wsgi

mercurial/hgweb/webutil.py

@@ -1,654 +1,654 b''
 # hgweb/webutil.py - utility library for the web interface.
 #
 # Copyright 21 May 2005 - (c) 2005 Jake Edge <[e-mail]>
 # Copyright 2005-2007 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import copy
 import difflib
 import os
 import re
 
 from ..i18n import _
 from ..node import hex, nullid, short
 
 from .common import (
     ErrorResponse,
     HTTP_BAD_REQUEST,
     HTTP_NOT_FOUND,
     paritygen,
 )
 
 from .. import (
     context,
     error,
     match,
     mdiff,
     patch,
     pathutil,
     pycompat,
     templatefilters,
     templatekw,
     ui as uimod,
     util,
 )
 
 def up(p):
     if p[0] != "/":
         p = "/" + p
     if p[-1] == "/":
         p = p[:-1]
     up = os.path.dirname(p)
     if up == "/":
         return "/"
     return up + "/"
 
 def _navseq(step, firststep=None):
     if firststep:
         yield firststep
         if firststep >= 20 and firststep <= 40:
             firststep = 50
             yield firststep
         assert step > 0
         assert firststep > 0
         while step <= firststep:
             step *= 10
     while True:
         yield 1 * step
         yield 3 * step
         step *= 10
 
 class revnav(object):
 
     def __init__(self, repo):
         """Navigation generation object
 
         :repo: repo object we generate nav for
         """
         # used for hex generation
         self._revlog = repo.changelog
 
     def __nonzero__(self):
         """return True if any revision to navigate over"""
         return self._first() is not None
 
     __bool__ = __nonzero__
 
     def _first(self):
         """return the minimum non-filtered changeset or None"""
         try:
             return next(iter(self._revlog))
         except StopIteration:
             return None
 
     def hex(self, rev):
         return hex(self._revlog.node(rev))
 
     def gen(self, pos, pagelen, limit):
         """computes label and revision id for navigation link
 
         :pos: is the revision relative to which we generate navigation.
         :pagelen: the size of each navigation page
         :limit: how far shall we link
 
         The return is:
             - a single element tuple
             - containing a dictionary with a `before` and `after` key
             - values are generator functions taking arbitrary number of kwargs
             - yield items are dictionaries with `label` and `node` keys
         """
         if not self:
             # empty repo
             return ({'before': (), 'after': ()},)
 
         targets = []
         for f in _navseq(1, pagelen):
             if f > limit:
                 break
             targets.append(pos + f)
             targets.append(pos - f)
         targets.sort()
 
         first = self._first()
         navbefore = [("(%i)" % first, self.hex(first))]
         navafter = []
         for rev in targets:
             if rev not in self._revlog:
                 continue
             if pos < rev < limit:
                 navafter.append(("+%d" % abs(rev - pos), self.hex(rev)))
             if 0 < rev < pos:
                 navbefore.append(("-%d" % abs(rev - pos), self.hex(rev)))
 
 
         navafter.append(("tip", "tip"))
 
         data = lambda i: {"label": i[0], "node": i[1]}
         return ({'before': lambda **map: (data(i) for i in navbefore),
                  'after':  lambda **map: (data(i) for i in navafter)},)
 
 class filerevnav(revnav):
 
     def __init__(self, repo, path):
         """Navigation generation object
 
         :repo: repo object we generate nav for
         :path: path of the file we generate nav for
         """
         # used for iteration
         self._changelog = repo.unfiltered().changelog
         # used for hex generation
         self._revlog = repo.file(path)
 
     def hex(self, rev):
         return hex(self._changelog.node(self._revlog.linkrev(rev)))
 
 class _siblings(object):
     def __init__(self, siblings=None, hiderev=None):
         if siblings is None:
             siblings = []
         self.siblings = [s for s in siblings if s.node() != nullid]
         if len(self.siblings) == 1 and self.siblings[0].rev() == hiderev:
             self.siblings = []
 
     def __iter__(self):
         for s in self.siblings:
             d = {
                 'node': s.hex(),
                 'rev': s.rev(),
                 'user': s.user(),
                 'date': s.date(),
                 'description': s.description(),
                 'branch': s.branch(),
             }
             if util.safehasattr(s, 'path'):
                 d['file'] = s.path()
             yield d
 
     def __len__(self):
         return len(self.siblings)
 
 def difffeatureopts(req, ui, section):
     diffopts = patch.difffeatureopts(ui, untrusted=True,
                                      section=section, whitespace=True)
 
     for k in ('ignorews', 'ignorewsamount', 'ignorewseol', 'ignoreblanklines'):
         v = req.form.get(k, [None])[0]
         if v is not None:
             v = util.parsebool(v)
             setattr(diffopts, k, v if v is not None else True)
 
     return diffopts
 
 def annotate(req, fctx, ui):
     diffopts = difffeatureopts(req, ui, 'annotate')
     return fctx.annotate(follow=True, linenumber=True, diffopts=diffopts)
 
 def parents(ctx, hide=None):
     if isinstance(ctx, context.basefilectx):
         introrev = ctx.introrev()
         if ctx.changectx().rev() != introrev:
             return _siblings([ctx.repo()[introrev]], hide)
     return _siblings(ctx.parents(), hide)
 
 def children(ctx, hide=None):
     return _siblings(ctx.children(), hide)
 
 def renamelink(fctx):
     r = fctx.renamed()
     if r:
         return [{'file': r[0], 'node': hex(r[1])}]
     return []
 
 def nodetagsdict(repo, node):
     return [{"name": i} for i in repo.nodetags(node)]
 
 def nodebookmarksdict(repo, node):
     return [{"name": i} for i in repo.nodebookmarks(node)]
 
 def nodebranchdict(repo, ctx):
     branches = []
     branch = ctx.branch()
     # If this is an empty repo, ctx.node() == nullid,
     # ctx.branch() == 'default'.
     try:
         branchnode = repo.branchtip(branch)
     except error.RepoLookupError:
         branchnode = None
     if branchnode == ctx.node():
         branches.append({"name": branch})
     return branches
 
 def nodeinbranch(repo, ctx):
     branches = []
     branch = ctx.branch()
     try:
         branchnode = repo.branchtip(branch)
     except error.RepoLookupError:
         branchnode = None
     if branch != 'default' and branchnode != ctx.node():
         branches.append({"name": branch})
     return branches
 
 def nodebranchnodefault(ctx):
     branches = []
     branch = ctx.branch()
     if branch != 'default':
         branches.append({"name": branch})
     return branches
 
 def showtag(repo, tmpl, t1, node=nullid, **args):
     for t in repo.nodetags(node):
         yield tmpl(t1, tag=t, **args)
 
 def showbookmark(repo, tmpl, t1, node=nullid, **args):
     for t in repo.nodebookmarks(node):
         yield tmpl(t1, bookmark=t, **args)
 
 def branchentries(repo, stripecount, limit=0):
     tips = []
     heads = repo.heads()
     parity = paritygen(stripecount)
     sortkey = lambda item: (not item[1], item[0].rev())
 
     def entries(**map):
         count = 0
         if not tips:
             for tag, hs, tip, closed in repo.branchmap().iterbranches():
                 tips.append((repo[tip], closed))
         for ctx, closed in sorted(tips, key=sortkey, reverse=True):
             if limit > 0 and count >= limit:
                 return
             count += 1
             if closed:
                 status = 'closed'
             elif ctx.node() not in heads:
                 status = 'inactive'
             else:
                 status = 'open'
             yield {
                 'parity': next(parity),
                 'branch': ctx.branch(),
                 'status': status,
                 'node': ctx.hex(),
                 'date': ctx.date()
             }
 
     return entries
 
 def cleanpath(repo, path):
     path = path.lstrip('/')
     return pathutil.canonpath(repo.root, '', path)
 
 def changeidctx(repo, changeid):
     try:
         ctx = repo[changeid]
     except error.RepoError:
         man = repo.manifestlog._revlog
         ctx = repo[man.linkrev(man.rev(man.lookup(changeid)))]
 
     return ctx
 
 def changectx(repo, req):
     changeid = "tip"
     if 'node' in req.form:
         changeid = req.form['node'][0]
         ipos = changeid.find(':')
         if ipos != -1:
             changeid = changeid[(ipos + 1):]
     elif 'manifest' in req.form:
         changeid = req.form['manifest'][0]
 
     return changeidctx(repo, changeid)
 
 def basechangectx(repo, req):
     if 'node' in req.form:
         changeid = req.form['node'][0]
         ipos = changeid.find(':')
         if ipos != -1:
             changeid = changeid[:ipos]
             return changeidctx(repo, changeid)
 
     return None
 
 def filectx(repo, req):
     if 'file' not in req.form:
         raise ErrorResponse(HTTP_NOT_FOUND, 'file not given')
     path = cleanpath(repo, req.form['file'][0])
     if 'node' in req.form:
         changeid = req.form['node'][0]
     elif 'filenode' in req.form:
         changeid = req.form['filenode'][0]
     else:
         raise ErrorResponse(HTTP_NOT_FOUND, 'node or filenode not given')
     try:
         fctx = repo[changeid][path]
     except error.RepoError:
         fctx = repo.filectx(path, fileid=changeid)
 
     return fctx
 
 def linerange(req):
     linerange = req.form.get('linerange')
     if linerange is None:
         return None
     if len(linerange) > 1:
         raise ErrorResponse(HTTP_BAD_REQUEST,
                             'redundant linerange parameter')
     try:
         fromline, toline = map(int, linerange[0].split(':', 1))
     except ValueError:
         raise ErrorResponse(HTTP_BAD_REQUEST,
                             'invalid linerange parameter')
     try:
         return util.processlinerange(fromline, toline)
     except error.ParseError as exc:
-        raise ErrorResponse(HTTP_BAD_REQUEST, str(exc))
+        raise ErrorResponse(HTTP_BAD_REQUEST, pycompat.bytestr(exc))
 
 def formatlinerange(fromline, toline):
     return '%d:%d' % (fromline + 1, toline)
 
 def succsandmarkers(repo, ctx):
     for item in templatekw.showsuccsandmarkers(repo, ctx):
         item['successors'] = _siblings(repo[successor]
                                        for successor in item['successors'])
         yield item
 
 def commonentry(repo, ctx):
     node = ctx.node()
     return {
         'rev': ctx.rev(),
         'node': hex(node),
         'author': ctx.user(),
         'desc': ctx.description(),
         'date': ctx.date(),
         'extra': ctx.extra(),
         'phase': ctx.phasestr(),
         'obsolete': ctx.obsolete(),
         'succsandmarkers': lambda **x: succsandmarkers(repo, ctx),
         'instabilities': [{"instability": i} for i in ctx.instabilities()],
         'branch': nodebranchnodefault(ctx),
         'inbranch': nodeinbranch(repo, ctx),
         'branches': nodebranchdict(repo, ctx),
         'tags': nodetagsdict(repo, node),
         'bookmarks': nodebookmarksdict(repo, node),
         'parent': lambda **x: parents(ctx),
         'child': lambda **x: children(ctx),
     }
 
 def changelistentry(web, ctx, tmpl):
     '''Obtain a dictionary to be used for entries in a changelist.
 
     This function is called when producing items for the "entries" list passed
     to the "shortlog" and "changelog" templates.
     '''
     repo = web.repo
     rev = ctx.rev()
     n = ctx.node()
     showtags = showtag(repo, tmpl, 'changelogtag', n)
     files = listfilediffs(tmpl, ctx.files(), n, web.maxfiles)
 
     entry = commonentry(repo, ctx)
     entry.update(
         allparents=lambda **x: parents(ctx),
         parent=lambda **x: parents(ctx, rev - 1),
         child=lambda **x: children(ctx, rev + 1),
         changelogtag=showtags,
         files=files,
     )
     return entry
 
 def symrevorshortnode(req, ctx):
     if 'node' in req.form:
         return templatefilters.revescape(req.form['node'][0])
     else:
         return short(ctx.node())
 
 def changesetentry(web, req, tmpl, ctx):
     '''Obtain a dictionary to be used to render the "changeset" template.'''
 
     showtags = showtag(web.repo, tmpl, 'changesettag', ctx.node())
     showbookmarks = showbookmark(web.repo, tmpl, 'changesetbookmark',
                                  ctx.node())
     showbranch = nodebranchnodefault(ctx)
 
     files = []
     parity = paritygen(web.stripecount)
     for blockno, f in enumerate(ctx.files()):
         template = 'filenodelink' if f in ctx else 'filenolink'
         files.append(tmpl(template,
                           node=ctx.hex(), file=f, blockno=blockno + 1,
                           parity=next(parity)))
 
     basectx = basechangectx(web.repo, req)
     if basectx is None:
         basectx = ctx.p1()
 
     style = web.config('web', 'style')
     if 'style' in req.form:
         style = req.form['style'][0]
 
     diff = diffs(web, tmpl, ctx, basectx, None, style)
 
     parity = paritygen(web.stripecount)
     diffstatsgen = diffstatgen(ctx, basectx)
     diffstats = diffstat(tmpl, ctx, diffstatsgen, parity)
 
     return dict(
         diff=diff,
         symrev=symrevorshortnode(req, ctx),
         basenode=basectx.hex(),
         changesettag=showtags,
         changesetbookmark=showbookmarks,
         changesetbranch=showbranch,
         files=files,
         diffsummary=lambda **x: diffsummary(diffstatsgen),
         diffstat=diffstats,
         archives=web.archivelist(ctx.hex()),
         **commonentry(web.repo, ctx))
 
 def listfilediffs(tmpl, files, node, max):
     for f in files[:max]:
         yield tmpl('filedifflink', node=hex(node), file=f)
     if len(files) > max:
         yield tmpl('fileellipses')
 
 def diffs(web, tmpl, ctx, basectx, files, style, linerange=None,
           lineidprefix=''):
 
     def prettyprintlines(lines, blockno):
         for lineno, l in enumerate(lines, 1):
             difflineno = "%d.%d" % (blockno, lineno)
             if l.startswith('+'):
                 ltype = "difflineplus"
             elif l.startswith('-'):
                 ltype = "difflineminus"
             elif l.startswith('@'):
                 ltype = "difflineat"
             else:
                 ltype = "diffline"
             yield tmpl(ltype,
                        line=l,
                        lineno=lineno,
                        lineid=lineidprefix + "l%s" % difflineno,
                        linenumber="% 8s" % difflineno)
 
     repo = web.repo
     if files:
         m = match.exact(repo.root, repo.getcwd(), files)
     else:
         m = match.always(repo.root, repo.getcwd())
 
     diffopts = patch.diffopts(repo.ui, untrusted=True)
     node1 = basectx.node()
     node2 = ctx.node()
     parity = paritygen(web.stripecount)
 
     diffhunks = patch.diffhunks(repo, node1, node2, m, opts=diffopts)
     for blockno, (fctx1, fctx2, header, hunks) in enumerate(diffhunks, 1):
         if style != 'raw':
             header = header[1:]
         lines = [h + '\n' for h in header]
         for hunkrange, hunklines in hunks:
             if linerange is not None and hunkrange is not None:
                 s1, l1, s2, l2 = hunkrange
                 if not mdiff.hunkinrange((s2, l2), linerange):
                     continue
             lines.extend(hunklines)
         if lines:
             yield tmpl('diffblock', parity=next(parity), blockno=blockno,
                        lines=prettyprintlines(lines, blockno))
 
 def compare(tmpl, context, leftlines, rightlines):
     '''Generator function that provides side-by-side comparison data.'''
 
     def compline(type, leftlineno, leftline, rightlineno, rightline):
         lineid = leftlineno and ("l%s" % leftlineno) or ''
         lineid += rightlineno and ("r%s" % rightlineno) or ''
         return tmpl('comparisonline',
                     type=type,
                     lineid=lineid,
                     leftlineno=leftlineno,
                     leftlinenumber="% 6s" % (leftlineno or ''),
                     leftline=leftline or '',
                     rightlineno=rightlineno,
                     rightlinenumber="% 6s" % (rightlineno or ''),
                     rightline=rightline or '')
 
     def getblock(opcodes):
         for type, llo, lhi, rlo, rhi in opcodes:
             len1 = lhi - llo
             len2 = rhi - rlo
             count = min(len1, len2)
             for i in xrange(count):
                 yield compline(type=type,
                                leftlineno=llo + i + 1,
                                leftline=leftlines[llo + i],
                                rightlineno=rlo + i + 1,
                                rightline=rightlines[rlo + i])
             if len1 > len2:
                 for i in xrange(llo + count, lhi):
                     yield compline(type=type,
                                    leftlineno=i + 1,
                                    leftline=leftlines[i],
                                    rightlineno=None,
                                    rightline=None)
             elif len2 > len1:
                 for i in xrange(rlo + count, rhi):
                     yield compline(type=type,
                                    leftlineno=None,
                                    leftline=None,
                                    rightlineno=i + 1,
                                    rightline=rightlines[i])
 
     s = difflib.SequenceMatcher(None, leftlines, rightlines)
     if context < 0:
         yield tmpl('comparisonblock', lines=getblock(s.get_opcodes()))
     else:
         for oc in s.get_grouped_opcodes(n=context):
             yield tmpl('comparisonblock', lines=getblock(oc))
 
 def diffstatgen(ctx, basectx):
     '''Generator function that provides the diffstat data.'''
 
     stats = patch.diffstatdata(
         util.iterlines(ctx.diff(basectx, noprefix=False)))
     maxname, maxtotal, addtotal, removetotal, binary = patch.diffstatsum(stats)
     while True:
         yield stats, maxname, maxtotal, addtotal, removetotal, binary
 
 def diffsummary(statgen):
     '''Return a short summary of the diff.'''
 
     stats, maxname, maxtotal, addtotal, removetotal, binary = next(statgen)
     return _(' %d files changed, %d insertions(+), %d deletions(-)\n') % (
              len(stats), addtotal, removetotal)
 
 def diffstat(tmpl, ctx, statgen, parity):
     '''Return a diffstat template for each file in the diff.'''
 
     stats, maxname, maxtotal, addtotal, removetotal, binary = next(statgen)
     files = ctx.files()
 
     def pct(i):
         if maxtotal == 0:
             return 0
         return (float(i) / maxtotal) * 100
 
     fileno = 0
     for filename, adds, removes, isbinary in stats:
         template = 'diffstatlink' if filename in files else 'diffstatnolink'
         total = adds + removes
         fileno += 1
         yield tmpl(template, node=ctx.hex(), file=filename, fileno=fileno,
                    total=total, addpct=pct(adds), removepct=pct(removes),
                    parity=next(parity))
 
 class sessionvars(object):
     def __init__(self, vars, start='?'):
         self.start = start
         self.vars = vars
     def __getitem__(self, key):
         return self.vars[key]
     def __setitem__(self, key, value):
         self.vars[key] = value
     def __copy__(self):
         return sessionvars(copy.copy(self.vars), self.start)
     def __iter__(self):
         separator = self.start
         for key, value in sorted(self.vars.iteritems()):
             yield {'name': key,
                    'value': pycompat.bytestr(value),
                    'separator': separator,
             }
             separator = '&'
 
 class wsgiui(uimod.ui):
     # default termwidth breaks under mod_wsgi
     def termwidth(self):
         return 80
 
 def getwebsubs(repo):
     websubtable = []
     websubdefs = repo.ui.configitems('websub')
     # we must maintain interhg backwards compatibility
     websubdefs += repo.ui.configitems('interhg')
     for key, pattern in websubdefs:
         # grab the delimiter from the character after the "s"
         unesc = pattern[1:2]
         delim = re.escape(unesc)
 
         # identify portions of the pattern, taking care to avoid escaped
         # delimiters. the replace format and flags are optional, but
         # delimiters are required.
         match = re.match(
             br'^s%s(.+)(?:(?<=\\\\)|(?<!\\))%s(.*)%s([ilmsux])*$'
             % (delim, delim, delim), pattern)
         if not match:
             repo.ui.warn(_("websub: invalid pattern for %s: %s\n")
                               % (key, pattern))
             continue
 
         # we need to unescape the delimiter for regexp and format
         delim_re = re.compile(br'(?<!\\)\\%s' % delim)
         regexp = delim_re.sub(unesc, match.group(1))
         format = delim_re.sub(unesc, match.group(2))
 
         # the pattern allows for 6 regexp flags, so set them if necessary
         flagin = match.group(3)
         flags = 0
         if flagin:
             for flag in flagin.upper():
                 flags |= re.__dict__[flag]
 
         try:
             regexp = re.compile(regexp, flags)
             websubtable.append((regexp, format))
         except re.error:
             repo.ui.warn(_("websub: invalid regexp for %s: %s\n")
                          % (key, regexp))
     return websubtable

mercurial/wireproto.py

@@ -1,1066 +1,1066 b''
 # wireproto.py - generic wire protocol support functions
 #
 # Copyright 2005-2010 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import hashlib
 import os
 import tempfile
 
 from .i18n import _
 from .node import (
     bin,
     hex,
     nullid,
 )
 
 from . import (
     bundle2,
     changegroup as changegroupmod,
     discovery,
     encoding,
     error,
     exchange,
     peer,
     pushkey as pushkeymod,
     pycompat,
     repository,
     streamclone,
     util,
     wireprototypes,
 )
 
 urlerr = util.urlerr
 urlreq = util.urlreq
 
 bytesresponse = wireprototypes.bytesresponse
 ooberror = wireprototypes.ooberror
 pushres = wireprototypes.pushres
 pusherr = wireprototypes.pusherr
 streamres = wireprototypes.streamres
 streamres_legacy = wireprototypes.streamreslegacy
 
 bundle2requiredmain = _('incompatible Mercurial client; bundle2 required')
 bundle2requiredhint = _('see https://www.mercurial-scm.org/wiki/'
                         'IncompatibleClient')
 bundle2required = '%s\n(%s)\n' % (bundle2requiredmain, bundle2requiredhint)
 
 class remoteiterbatcher(peer.iterbatcher):
     def __init__(self, remote):
         super(remoteiterbatcher, self).__init__()
         self._remote = remote
 
     def __getattr__(self, name):
         # Validate this method is batchable, since submit() only supports
         # batchable methods.
         fn = getattr(self._remote, name)
         if not getattr(fn, 'batchable', None):
             raise error.ProgrammingError('Attempted to batch a non-batchable '
                                          'call to %r' % name)
 
         return super(remoteiterbatcher, self).__getattr__(name)
 
     def submit(self):
         """Break the batch request into many patch calls and pipeline them.
 
         This is mostly valuable over http where request sizes can be
         limited, but can be used in other places as well.
         """
         # 2-tuple of (command, arguments) that represents what will be
         # sent over the wire.
         requests = []
 
         # 4-tuple of (command, final future, @batchable generator, remote
         # future).
         results = []
 
         for command, args, opts, finalfuture in self.calls:
             mtd = getattr(self._remote, command)
             batchable = mtd.batchable(mtd.__self__, *args, **opts)
 
             commandargs, fremote = next(batchable)
             assert fremote
             requests.append((command, commandargs))
             results.append((command, finalfuture, batchable, fremote))
 
         if requests:
             self._resultiter = self._remote._submitbatch(requests)
 
         self._results = results
 
     def results(self):
         for command, finalfuture, batchable, remotefuture in self._results:
             # Get the raw result, set it in the remote future, feed it
             # back into the @batchable generator so it can be decoded, and
             # set the result on the final future to this value.
             remoteresult = next(self._resultiter)
             remotefuture.set(remoteresult)
             finalfuture.set(next(batchable))
 
             # Verify our @batchable generators only emit 2 values.
             try:
                 next(batchable)
             except StopIteration:
                 pass
             else:
                 raise error.ProgrammingError('%s @batchable generator emitted '
                                              'unexpected value count' % command)
 
             yield finalfuture.value
 
 # Forward a couple of names from peer to make wireproto interactions
 # slightly more sensible.
 batchable = peer.batchable
 future = peer.future
 
 # list of nodes encoding / decoding
 
 def decodelist(l, sep=' '):
     if l:
         return [bin(v) for v in  l.split(sep)]
     return []
 
 def encodelist(l, sep=' '):
     try:
         return sep.join(map(hex, l))
     except TypeError:
         raise
 
 # batched call argument encoding
 
 def escapearg(plain):
     return (plain
             .replace(':', ':c')
             .replace(',', ':o')
             .replace(';', ':s')
             .replace('=', ':e'))
 
 def unescapearg(escaped):
     return (escaped
             .replace(':e', '=')
             .replace(':s', ';')
             .replace(':o', ',')
             .replace(':c', ':'))
 
 def encodebatchcmds(req):
     """Return a ``cmds`` argument value for the ``batch`` command."""
     cmds = []
     for op, argsdict in req:
         # Old servers didn't properly unescape argument names. So prevent
         # the sending of argument names that may not be decoded properly by
         # servers.
         assert all(escapearg(k) == k for k in argsdict)
 
         args = ','.join('%s=%s' % (escapearg(k), escapearg(v))
                         for k, v in argsdict.iteritems())
         cmds.append('%s %s' % (op, args))
 
     return ';'.join(cmds)
 
 # mapping of options accepted by getbundle and their types
 #
 # Meant to be extended by extensions. It is extensions responsibility to ensure
 # such options are properly processed in exchange.getbundle.
 #
 # supported types are:
 #
 # :nodes: list of binary nodes
 # :csv:   list of comma-separated values
 # :scsv:  list of comma-separated values return as set
 # :plain: string with no transformation needed.
 gboptsmap = {'heads':  'nodes',
              'bookmarks': 'boolean',
              'common': 'nodes',
              'obsmarkers': 'boolean',
              'phases': 'boolean',
              'bundlecaps': 'scsv',
              'listkeys': 'csv',
              'cg': 'boolean',
              'cbattempted': 'boolean',
              'stream': 'boolean',
 }
 
 # client side
 
 class wirepeer(repository.legacypeer):
     """Client-side interface for communicating with a peer repository.
 
     Methods commonly call wire protocol commands of the same name.
 
     See also httppeer.py and sshpeer.py for protocol-specific
     implementations of this interface.
     """
     # Begin of basewirepeer interface.
 
     def iterbatch(self):
         return remoteiterbatcher(self)
 
     @batchable
     def lookup(self, key):
         self.requirecap('lookup', _('look up remote revision'))
         f = future()
         yield {'key': encoding.fromlocal(key)}, f
         d = f.value
         success, data = d[:-1].split(" ", 1)
         if int(success):
             yield bin(data)
         else:
             self._abort(error.RepoError(data))
 
     @batchable
     def heads(self):
         f = future()
         yield {}, f
         d = f.value
         try:
             yield decodelist(d[:-1])
         except ValueError:
             self._abort(error.ResponseError(_("unexpected response:"), d))
 
     @batchable
     def known(self, nodes):
         f = future()
         yield {'nodes': encodelist(nodes)}, f
         d = f.value
         try:
             yield [bool(int(b)) for b in d]
         except ValueError:
             self._abort(error.ResponseError(_("unexpected response:"), d))
 
     @batchable
     def branchmap(self):
         f = future()
         yield {}, f
         d = f.value
         try:
             branchmap = {}
             for branchpart in d.splitlines():
                 branchname, branchheads = branchpart.split(' ', 1)
                 branchname = encoding.tolocal(urlreq.unquote(branchname))
                 branchheads = decodelist(branchheads)
                 branchmap[branchname] = branchheads
             yield branchmap
         except TypeError:
             self._abort(error.ResponseError(_("unexpected response:"), d))
 
     @batchable
     def listkeys(self, namespace):
         if not self.capable('pushkey'):
             yield {}, None
         f = future()
         self.ui.debug('preparing listkeys for "%s"\n' % namespace)
         yield {'namespace': encoding.fromlocal(namespace)}, f
         d = f.value
         self.ui.debug('received listkey for "%s": %i bytes\n'
                       % (namespace, len(d)))
         yield pushkeymod.decodekeys(d)
 
     @batchable
     def pushkey(self, namespace, key, old, new):
         if not self.capable('pushkey'):
             yield False, None
         f = future()
         self.ui.debug('preparing pushkey for "%s:%s"\n' % (namespace, key))
         yield {'namespace': encoding.fromlocal(namespace),
                'key': encoding.fromlocal(key),
                'old': encoding.fromlocal(old),
                'new': encoding.fromlocal(new)}, f
         d = f.value
         d, output = d.split('\n', 1)
         try:
             d = bool(int(d))
         except ValueError:
             raise error.ResponseError(
                 _('push failed (unexpected response):'), d)
         for l in output.splitlines(True):
             self.ui.status(_('remote: '), l)
         yield d
 
     def stream_out(self):
         return self._callstream('stream_out')
 
     def getbundle(self, source, **kwargs):
         kwargs = pycompat.byteskwargs(kwargs)
         self.requirecap('getbundle', _('look up remote changes'))
         opts = {}
         bundlecaps = kwargs.get('bundlecaps')
         if bundlecaps is not None:
             kwargs['bundlecaps'] = sorted(bundlecaps)
         else:
             bundlecaps = () # kwargs could have it to None
         for key, value in kwargs.iteritems():
             if value is None:
                 continue
             keytype = gboptsmap.get(key)
             if keytype is None:
                 raise error.ProgrammingError(
                     'Unexpectedly None keytype for key %s' % key)
             elif keytype == 'nodes':
                 value = encodelist(value)
             elif keytype in ('csv', 'scsv'):
                 value = ','.join(value)
             elif keytype == 'boolean':
                 value = '%i' % bool(value)
             elif keytype != 'plain':
                 raise KeyError('unknown getbundle option type %s'
                                % keytype)
             opts[key] = value
         f = self._callcompressable("getbundle", **pycompat.strkwargs(opts))
         if any((cap.startswith('HG2') for cap in bundlecaps)):
             return bundle2.getunbundler(self.ui, f)
         else:
             return changegroupmod.cg1unpacker(f, 'UN')
 
     def unbundle(self, cg, heads, url):
         '''Send cg (a readable file-like object representing the
         changegroup to push, typically a chunkbuffer object) to the
         remote server as a bundle.
 
         When pushing a bundle10 stream, return an integer indicating the
         result of the push (see changegroup.apply()).
 
         When pushing a bundle20 stream, return a bundle20 stream.
 
         `url` is the url the client thinks it's pushing to, which is
         visible to hooks.
         '''
 
         if heads != ['force'] and self.capable('unbundlehash'):
             heads = encodelist(['hashed',
                                 hashlib.sha1(''.join(sorted(heads))).digest()])
         else:
             heads = encodelist(heads)
 
         if util.safehasattr(cg, 'deltaheader'):
             # this a bundle10, do the old style call sequence
             ret, output = self._callpush("unbundle", cg, heads=heads)
             if ret == "":
                 raise error.ResponseError(
                     _('push failed:'), output)
             try:
                 ret = int(ret)
             except ValueError:
                 raise error.ResponseError(
                     _('push failed (unexpected response):'), ret)
 
             for l in output.splitlines(True):
                 self.ui.status(_('remote: '), l)
         else:
             # bundle2 push. Send a stream, fetch a stream.
             stream = self._calltwowaystream('unbundle', cg, heads=heads)
             ret = bundle2.getunbundler(self.ui, stream)
         return ret
 
     # End of basewirepeer interface.
 
     # Begin of baselegacywirepeer interface.
 
     def branches(self, nodes):
         n = encodelist(nodes)
         d = self._call("branches", nodes=n)
         try:
             br = [tuple(decodelist(b)) for b in d.splitlines()]
             return br
         except ValueError:
             self._abort(error.ResponseError(_("unexpected response:"), d))
 
     def between(self, pairs):
         batch = 8 # avoid giant requests
         r = []
         for i in xrange(0, len(pairs), batch):
             n = " ".join([encodelist(p, '-') for p in pairs[i:i + batch]])
             d = self._call("between", pairs=n)
             try:
                 r.extend(l and decodelist(l) or [] for l in d.splitlines())
             except ValueError:
                 self._abort(error.ResponseError(_("unexpected response:"), d))
         return r
 
     def changegroup(self, nodes, kind):
         n = encodelist(nodes)
         f = self._callcompressable("changegroup", roots=n)
         return changegroupmod.cg1unpacker(f, 'UN')
 
     def changegroupsubset(self, bases, heads, kind):
         self.requirecap('changegroupsubset', _('look up remote changes'))
         bases = encodelist(bases)
         heads = encodelist(heads)
         f = self._callcompressable("changegroupsubset",
                                    bases=bases, heads=heads)
         return changegroupmod.cg1unpacker(f, 'UN')
 
     # End of baselegacywirepeer interface.
 
     def _submitbatch(self, req):
         """run batch request <req> on the server
 
         Returns an iterator of the raw responses from the server.
         """
         rsp = self._callstream("batch", cmds=encodebatchcmds(req))
         chunk = rsp.read(1024)
         work = [chunk]
         while chunk:
             while ';' not in chunk and chunk:
                 chunk = rsp.read(1024)
                 work.append(chunk)
             merged = ''.join(work)
             while ';' in merged:
                 one, merged = merged.split(';', 1)
                 yield unescapearg(one)
             chunk = rsp.read(1024)
             work = [merged, chunk]
         yield unescapearg(''.join(work))
 
     def _submitone(self, op, args):
         return self._call(op, **pycompat.strkwargs(args))
 
     def debugwireargs(self, one, two, three=None, four=None, five=None):
         # don't pass optional arguments left at their default value
         opts = {}
         if three is not None:
             opts[r'three'] = three
         if four is not None:
             opts[r'four'] = four
         return self._call('debugwireargs', one=one, two=two, **opts)
 
     def _call(self, cmd, **args):
         """execute <cmd> on the server
 
         The command is expected to return a simple string.
 
         returns the server reply as a string."""
         raise NotImplementedError()
 
     def _callstream(self, cmd, **args):
         """execute <cmd> on the server
 
         The command is expected to return a stream. Note that if the
         command doesn't return a stream, _callstream behaves
         differently for ssh and http peers.
 
         returns the server reply as a file like object.
         """
         raise NotImplementedError()
 
     def _callcompressable(self, cmd, **args):
         """execute <cmd> on the server
 
         The command is expected to return a stream.
 
         The stream may have been compressed in some implementations. This
         function takes care of the decompression. This is the only difference
         with _callstream.
 
         returns the server reply as a file like object.
         """
         raise NotImplementedError()
 
     def _callpush(self, cmd, fp, **args):
         """execute a <cmd> on server
 
         The command is expected to be related to a push. Push has a special
         return method.
 
         returns the server reply as a (ret, output) tuple. ret is either
         empty (error) or a stringified int.
         """
         raise NotImplementedError()
 
     def _calltwowaystream(self, cmd, fp, **args):
         """execute <cmd> on server
 
         The command will send a stream to the server and get a stream in reply.
         """
         raise NotImplementedError()
 
     def _abort(self, exception):
         """clearly abort the wire protocol connection and raise the exception
         """
         raise NotImplementedError()
 
 # server side
 
 # wire protocol command can either return a string or one of these classes.
 
 def getdispatchrepo(repo, proto, command):
     """Obtain the repo used for processing wire protocol commands.
 
     The intent of this function is to serve as a monkeypatch point for
     extensions that need commands to operate on different repo views under
     specialized circumstances.
     """
     return repo.filtered('served')
 
 def dispatch(repo, proto, command):
     repo = getdispatchrepo(repo, proto, command)
     func, spec = commands[command]
     args = proto.getargs(spec)
     return func(repo, proto, *args)
 
 def options(cmd, keys, others):
     opts = {}
     for k in keys:
         if k in others:
             opts[k] = others[k]
             del others[k]
     if others:
         util.stderr.write("warning: %s ignored unexpected arguments %s\n"
                           % (cmd, ",".join(others)))
     return opts
 
 def bundle1allowed(repo, action):
     """Whether a bundle1 operation is allowed from the server.
 
     Priority is:
 
     1. server.bundle1gd.<action> (if generaldelta active)
     2. server.bundle1.<action>
     3. server.bundle1gd (if generaldelta active)
     4. server.bundle1
     """
     ui = repo.ui
     gd = 'generaldelta' in repo.requirements
 
     if gd:
         v = ui.configbool('server', 'bundle1gd.%s' % action)
         if v is not None:
             return v
 
     v = ui.configbool('server', 'bundle1.%s' % action)
     if v is not None:
         return v
 
     if gd:
         v = ui.configbool('server', 'bundle1gd')
         if v is not None:
             return v
 
     return ui.configbool('server', 'bundle1')
 
 def supportedcompengines(ui, role):
     """Obtain the list of supported compression engines for a request."""
     assert role in (util.CLIENTROLE, util.SERVERROLE)
 
     compengines = util.compengines.supportedwireengines(role)
 
     # Allow config to override default list and ordering.
     if role == util.SERVERROLE:
         configengines = ui.configlist('server', 'compressionengines')
         config = 'server.compressionengines'
     else:
         # This is currently implemented mainly to facilitate testing. In most
         # cases, the server should be in charge of choosing a compression engine
         # because a server has the most to lose from a sub-optimal choice. (e.g.
         # CPU DoS due to an expensive engine or a network DoS due to poor
         # compression ratio).
         configengines = ui.configlist('experimental',
                                       'clientcompressionengines')
         config = 'experimental.clientcompressionengines'
 
     # No explicit config. Filter out the ones that aren't supposed to be
     # advertised and return default ordering.
     if not configengines:
         attr = 'serverpriority' if role == util.SERVERROLE else 'clientpriority'
         return [e for e in compengines
                 if getattr(e.wireprotosupport(), attr) > 0]
 
     # If compression engines are listed in the config, assume there is a good
     # reason for it (like server operators wanting to achieve specific
     # performance characteristics). So fail fast if the config references
     # unusable compression engines.
     validnames = set(e.name() for e in compengines)
     invalidnames = set(e for e in configengines if e not in validnames)
     if invalidnames:
         raise error.Abort(_('invalid compression engine defined in %s: %s') %
                           (config, ', '.join(sorted(invalidnames))))
 
     compengines = [e for e in compengines if e.name() in configengines]
     compengines = sorted(compengines,
                          key=lambda e: configengines.index(e.name()))
 
     if not compengines:
         raise error.Abort(_('%s config option does not specify any known '
                             'compression engines') % config,
                           hint=_('usable compression engines: %s') %
                           ', '.sorted(validnames))
 
     return compengines
 
 class commandentry(object):
     """Represents a declared wire protocol command."""
     def __init__(self, func, args=''):
         self.func = func
         self.args = args
 
     def _merge(self, func, args):
         """Merge this instance with an incoming 2-tuple.
 
         This is called when a caller using the old 2-tuple API attempts
         to replace an instance. The incoming values are merged with
         data not captured by the 2-tuple and a new instance containing
         the union of the two objects is returned.
         """
         return commandentry(func, args)
 
     # Old code treats instances as 2-tuples. So expose that interface.
     def __iter__(self):
         yield self.func
         yield self.args
 
     def __getitem__(self, i):
         if i == 0:
             return self.func
         elif i == 1:
             return self.args
         else:
             raise IndexError('can only access elements 0 and 1')
 
 class commanddict(dict):
     """Container for registered wire protocol commands.
 
     It behaves like a dict. But __setitem__ is overwritten to allow silent
     coercion of values from 2-tuples for API compatibility.
     """
     def __setitem__(self, k, v):
         if isinstance(v, commandentry):
             pass
         # Cast 2-tuples to commandentry instances.
         elif isinstance(v, tuple):
             if len(v) != 2:
                 raise ValueError('command tuples must have exactly 2 elements')
 
             # It is common for extensions to wrap wire protocol commands via
             # e.g. ``wireproto.commands[x] = (newfn, args)``. Because callers
             # doing this aren't aware of the new API that uses objects to store
             # command entries, we automatically merge old state with new.
             if k in self:
                 v = self[k]._merge(v[0], v[1])
             else:
                 v = commandentry(v[0], v[1])
         else:
             raise ValueError('command entries must be commandentry instances '
                              'or 2-tuples')
 
         return super(commanddict, self).__setitem__(k, v)
 
     def commandavailable(self, command, proto):
         """Determine if a command is available for the requested protocol."""
         # For now, commands are available for all protocols. So do a simple
         # membership test.
         return command in self
 
 commands = commanddict()
 
 def wireprotocommand(name, args=''):
     """Decorator to declare a wire protocol command.
 
     ``name`` is the name of the wire protocol command being provided.
 
     ``args`` is a space-delimited list of named arguments that the command
     accepts. ``*`` is a special value that says to accept all arguments.
     """
     def register(func):
         commands[name] = commandentry(func, args)
         return func
     return register
 
 @wireprotocommand('batch', 'cmds *')
 def batch(repo, proto, cmds, others):
     repo = repo.filtered("served")
     res = []
     for pair in cmds.split(';'):
         op, args = pair.split(' ', 1)
         vals = {}
         for a in args.split(','):
             if a:
                 n, v = a.split('=')
                 vals[unescapearg(n)] = unescapearg(v)
         func, spec = commands[op]
         if spec:
             keys = spec.split()
             data = {}
             for k in keys:
                 if k == '*':
                     star = {}
                     for key in vals.keys():
                         if key not in keys:
                             star[key] = vals[key]
                     data['*'] = star
                 else:
                     data[k] = vals[k]
             result = func(repo, proto, *[data[k] for k in keys])
         else:
             result = func(repo, proto)
         if isinstance(result, ooberror):
             return result
 
         # For now, all batchable commands must return bytesresponse or
         # raw bytes (for backwards compatibility).
         assert isinstance(result, (bytesresponse, bytes))
         if isinstance(result, bytesresponse):
             result = result.data
         res.append(escapearg(result))
 
     return bytesresponse(';'.join(res))
 
 @wireprotocommand('between', 'pairs')
 def between(repo, proto, pairs):
     pairs = [decodelist(p, '-') for p in pairs.split(" ")]
     r = []
     for b in repo.between(pairs):
         r.append(encodelist(b) + "\n")
 
     return bytesresponse(''.join(r))
 
 @wireprotocommand('branchmap')
 def branchmap(repo, proto):
     branchmap = repo.branchmap()
     heads = []
     for branch, nodes in branchmap.iteritems():
         branchname = urlreq.quote(encoding.fromlocal(branch))
         branchnodes = encodelist(nodes)
         heads.append('%s %s' % (branchname, branchnodes))
 
     return bytesresponse('\n'.join(heads))
 
 @wireprotocommand('branches', 'nodes')
 def branches(repo, proto, nodes):
     nodes = decodelist(nodes)
     r = []
     for b in repo.branches(nodes):
         r.append(encodelist(b) + "\n")
 
     return bytesresponse(''.join(r))
 
 @wireprotocommand('clonebundles', '')
 def clonebundles(repo, proto):
     """Server command for returning info for available bundles to seed clones.
 
     Clients will parse this response and determine what bundle to fetch.
 
     Extensions may wrap this command to filter or dynamically emit data
     depending on the request. e.g. you could advertise URLs for the closest
     data center given the client's IP address.
     """
     return bytesresponse(repo.vfs.tryread('clonebundles.manifest'))
 
 wireprotocaps = ['lookup', 'changegroupsubset', 'branchmap', 'pushkey',
                  'known', 'getbundle', 'unbundlehash', 'batch']
 
 def _capabilities(repo, proto):
     """return a list of capabilities for a repo
 
     This function exists to allow extensions to easily wrap capabilities
     computation
 
     - returns a lists: easy to alter
     - change done here will be propagated to both `capabilities` and `hello`
       command without any other action needed.
     """
     # copy to prevent modification of the global list
     caps = list(wireprotocaps)
     if streamclone.allowservergeneration(repo):
         if repo.ui.configbool('server', 'preferuncompressed'):
             caps.append('stream-preferred')
         requiredformats = repo.requirements & repo.supportedformats
         # if our local revlogs are just revlogv1, add 'stream' cap
         if not requiredformats - {'revlogv1'}:
             caps.append('stream')
         # otherwise, add 'streamreqs' detailing our local revlog format
         else:
             caps.append('streamreqs=%s' % ','.join(sorted(requiredformats)))
     if repo.ui.configbool('experimental', 'bundle2-advertise'):
         capsblob = bundle2.encodecaps(bundle2.getrepocaps(repo, role='server'))
         caps.append('bundle2=' + urlreq.quote(capsblob))
     caps.append('unbundle=%s' % ','.join(bundle2.bundlepriority))
 
     if proto.name == 'http-v1':
         caps.append('httpheader=%d' %
                     repo.ui.configint('server', 'maxhttpheaderlen'))
         if repo.ui.configbool('experimental', 'httppostargs'):
             caps.append('httppostargs')
 
         # FUTURE advertise 0.2rx once support is implemented
         # FUTURE advertise minrx and mintx after consulting config option
         caps.append('httpmediatype=0.1rx,0.1tx,0.2tx')
 
         compengines = supportedcompengines(repo.ui, util.SERVERROLE)
         if compengines:
             comptypes = ','.join(urlreq.quote(e.wireprotosupport().name)
                                  for e in compengines)
             caps.append('compression=%s' % comptypes)
 
     return caps
 
 # If you are writing an extension and consider wrapping this function. Wrap
 # `_capabilities` instead.
 @wireprotocommand('capabilities')
 def capabilities(repo, proto):
     return bytesresponse(' '.join(_capabilities(repo, proto)))
 
 @wireprotocommand('changegroup', 'roots')
 def changegroup(repo, proto, roots):
     nodes = decodelist(roots)
     outgoing = discovery.outgoing(repo, missingroots=nodes,
                                   missingheads=repo.heads())
     cg = changegroupmod.makechangegroup(repo, outgoing, '01', 'serve')
     gen = iter(lambda: cg.read(32768), '')
     return streamres(gen=gen)
 
 @wireprotocommand('changegroupsubset', 'bases heads')
 def changegroupsubset(repo, proto, bases, heads):
     bases = decodelist(bases)
     heads = decodelist(heads)
     outgoing = discovery.outgoing(repo, missingroots=bases,
                                   missingheads=heads)
     cg = changegroupmod.makechangegroup(repo, outgoing, '01', 'serve')
     gen = iter(lambda: cg.read(32768), '')
     return streamres(gen=gen)
 
 @wireprotocommand('debugwireargs', 'one two *')
 def debugwireargs(repo, proto, one, two, others):
     # only accept optional args from the known set
     opts = options('debugwireargs', ['three', 'four'], others)
     return bytesresponse(repo.debugwireargs(one, two,
                                             **pycompat.strkwargs(opts)))
 
 @wireprotocommand('getbundle', '*')
 def getbundle(repo, proto, others):
     opts = options('getbundle', gboptsmap.keys(), others)
     for k, v in opts.iteritems():
         keytype = gboptsmap[k]
         if keytype == 'nodes':
             opts[k] = decodelist(v)
         elif keytype == 'csv':
             opts[k] = list(v.split(','))
         elif keytype == 'scsv':
             opts[k] = set(v.split(','))
         elif keytype == 'boolean':
             # Client should serialize False as '0', which is a non-empty string
             # so it evaluates as a True bool.
             if v == '0':
                 opts[k] = False
             else:
                 opts[k] = bool(v)
         elif keytype != 'plain':
             raise KeyError('unknown getbundle option type %s'
                            % keytype)
 
     if not bundle1allowed(repo, 'pull'):
         if not exchange.bundle2requested(opts.get('bundlecaps')):
             if proto.name == 'http-v1':
                 return ooberror(bundle2required)
             raise error.Abort(bundle2requiredmain,
                               hint=bundle2requiredhint)
 
     prefercompressed = True
 
     try:
         if repo.ui.configbool('server', 'disablefullbundle'):
             # Check to see if this is a full clone.
             clheads = set(repo.changelog.heads())
             changegroup = opts.get('cg', True)
             heads = set(opts.get('heads', set()))
             common = set(opts.get('common', set()))
             common.discard(nullid)
             if changegroup and not common and clheads == heads:
                 raise error.Abort(
                     _('server has pull-based clones disabled'),
                     hint=_('remove --pull if specified or upgrade Mercurial'))
 
         info, chunks = exchange.getbundlechunks(repo, 'serve',
                                                 **pycompat.strkwargs(opts))
         prefercompressed = info.get('prefercompressed', True)
     except error.Abort as exc:
         # cleanly forward Abort error to the client
         if not exchange.bundle2requested(opts.get('bundlecaps')):
             if proto.name == 'http-v1':
-                return ooberror(str(exc) + '\n')
+                return ooberror(pycompat.bytestr(exc) + '\n')
             raise # cannot do better for bundle1 + ssh
         # bundle2 request expect a bundle2 reply
         bundler = bundle2.bundle20(repo.ui)
-        manargs = [('message', str(exc))]
+        manargs = [('message', pycompat.bytestr(exc))]
         advargs = []
         if exc.hint is not None:
             advargs.append(('hint', exc.hint))
         bundler.addpart(bundle2.bundlepart('error:abort',
                                            manargs, advargs))
         chunks = bundler.getchunks()
         prefercompressed = False
 
     return streamres(gen=chunks, prefer_uncompressed=not prefercompressed)
 
 @wireprotocommand('heads')
 def heads(repo, proto):
     h = repo.heads()
     return bytesresponse(encodelist(h) + '\n')
 
 @wireprotocommand('hello')
 def hello(repo, proto):
     """Called as part of SSH handshake to obtain server info.
 
     Returns a list of lines describing interesting things about the
     server, in an RFC822-like format.
 
     Currently, the only one defined is ``capabilities``, which consists of a
     line of space separated tokens describing server abilities:
 
         capabilities: <token0> <token1> <token2>
     """
     caps = capabilities(repo, proto).data
     return bytesresponse('capabilities: %s\n' % caps)
 
 @wireprotocommand('listkeys', 'namespace')
 def listkeys(repo, proto, namespace):
     d = repo.listkeys(encoding.tolocal(namespace)).items()
     return bytesresponse(pushkeymod.encodekeys(d))
 
 @wireprotocommand('lookup', 'key')
 def lookup(repo, proto, key):
     try:
         k = encoding.tolocal(key)
         c = repo[k]
         r = c.hex()
         success = 1
     except Exception as inst:
         r = str(inst)
         success = 0
     return bytesresponse('%d %s\n' % (success, r))
 
 @wireprotocommand('known', 'nodes *')
 def known(repo, proto, nodes, others):
     v = ''.join(b and '1' or '0' for b in repo.known(decodelist(nodes)))
     return bytesresponse(v)
 
 @wireprotocommand('pushkey', 'namespace key old new')
 def pushkey(repo, proto, namespace, key, old, new):
     # compatibility with pre-1.8 clients which were accidentally
     # sending raw binary nodes rather than utf-8-encoded hex
     if len(new) == 20 and util.escapestr(new) != new:
         # looks like it could be a binary node
         try:
             new.decode('utf-8')
             new = encoding.tolocal(new) # but cleanly decodes as UTF-8
         except UnicodeDecodeError:
             pass # binary, leave unmodified
     else:
         new = encoding.tolocal(new) # normal path
 
     with proto.mayberedirectstdio() as output:
         r = repo.pushkey(encoding.tolocal(namespace), encoding.tolocal(key),
                          encoding.tolocal(old), new) or False
 
     output = output.getvalue() if output else ''
     return bytesresponse('%s\n%s' % (int(r), output))
 
 @wireprotocommand('stream_out')
 def stream(repo, proto):
     '''If the server supports streaming clone, it advertises the "stream"
     capability with a value representing the version and flags of the repo
     it is serving. Client checks to see if it understands the format.
     '''
     return streamres_legacy(streamclone.generatev1wireproto(repo))
 
 @wireprotocommand('unbundle', 'heads')
 def unbundle(repo, proto, heads):
     their_heads = decodelist(heads)
 
     with proto.mayberedirectstdio() as output:
         try:
             exchange.check_heads(repo, their_heads, 'preparing changes')
 
             # write bundle data to temporary file because it can be big
             fd, tempname = tempfile.mkstemp(prefix='hg-unbundle-')
             fp = os.fdopen(fd, pycompat.sysstr('wb+'))
             r = 0
             try:
                 proto.forwardpayload(fp)
                 fp.seek(0)
                 gen = exchange.readbundle(repo.ui, fp, None)
                 if (isinstance(gen, changegroupmod.cg1unpacker)
                     and not bundle1allowed(repo, 'push')):
                     if proto.name == 'http-v1':
                         # need to special case http because stderr do not get to
                         # the http client on failed push so we need to abuse
                         # some other error type to make sure the message get to
                         # the user.
                         return ooberror(bundle2required)
                     raise error.Abort(bundle2requiredmain,
                                       hint=bundle2requiredhint)
 
                 r = exchange.unbundle(repo, gen, their_heads, 'serve',
                                       proto.client())
                 if util.safehasattr(r, 'addpart'):
                     # The return looks streamable, we are in the bundle2 case
                     # and should return a stream.
                     return streamres_legacy(gen=r.getchunks())
                 return pushres(r, output.getvalue() if output else '')
 
             finally:
                 fp.close()
                 os.unlink(tempname)
 
         except (error.BundleValueError, error.Abort, error.PushRaced) as exc:
             # handle non-bundle2 case first
             if not getattr(exc, 'duringunbundle2', False):
                 try:
                     raise
                 except error.Abort:
                     # The old code we moved used util.stderr directly.
                     # We did not change it to minimise code change.
                     # This need to be moved to something proper.
                     # Feel free to do it.
                     util.stderr.write("abort: %s\n" % exc)
                     if exc.hint is not None:
                         util.stderr.write("(%s)\n" % exc.hint)
                     return pushres(0, output.getvalue() if output else '')
                 except error.PushRaced:
                     return pusherr(str(exc),
                                    output.getvalue() if output else '')
 
             bundler = bundle2.bundle20(repo.ui)
             for out in getattr(exc, '_bundle2salvagedoutput', ()):
                 bundler.addpart(out)
             try:
                 try:
                     raise
                 except error.PushkeyFailed as exc:
                     # check client caps
                     remotecaps = getattr(exc, '_replycaps', None)
                     if (remotecaps is not None
                             and 'pushkey' not in remotecaps.get('error', ())):
                         # no support remote side, fallback to Abort handler.
                         raise
                     part = bundler.newpart('error:pushkey')
                     part.addparam('in-reply-to', exc.partid)
                     if exc.namespace is not None:
                         part.addparam('namespace', exc.namespace,
                                       mandatory=False)
                     if exc.key is not None:
                         part.addparam('key', exc.key, mandatory=False)
                     if exc.new is not None:
                         part.addparam('new', exc.new, mandatory=False)
                     if exc.old is not None:
                         part.addparam('old', exc.old, mandatory=False)
                     if exc.ret is not None:
                         part.addparam('ret', exc.ret, mandatory=False)
             except error.BundleValueError as exc:
                 errpart = bundler.newpart('error:unsupportedcontent')
                 if exc.parttype is not None:
                     errpart.addparam('parttype', exc.parttype)
                 if exc.params:
                     errpart.addparam('params', '\0'.join(exc.params))
             except error.Abort as exc:
                 manargs = [('message', str(exc))]
                 advargs = []
                 if exc.hint is not None:
                     advargs.append(('hint', exc.hint))
                 bundler.addpart(bundle2.bundlepart('error:abort',
                                                    manargs, advargs))
             except error.PushRaced as exc:
                 bundler.newpart('error:pushraced', [('message', str(exc))])
             return streamres_legacy(gen=bundler.getchunks())

mercurial/wireprotoserver.py

@@ -1,648 +1,648 b''
 # Copyright 21 May 2005 - (c) 2005 Jake Edge <[e-mail]>
 # Copyright 2005-2007 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import abc
 import contextlib
 import struct
 import sys
 
 from .i18n import _
 from . import (
     encoding,
     error,
     hook,
     pycompat,
     util,
     wireproto,
     wireprototypes,
 )
 
 stringio = util.stringio
 
 urlerr = util.urlerr
 urlreq = util.urlreq
 
 HTTP_OK = 200
 
 HGTYPE = 'application/mercurial-0.1'
 HGTYPE2 = 'application/mercurial-0.2'
 HGERRTYPE = 'application/hg-error'
 
 # Names of the SSH protocol implementations.
 SSHV1 = 'ssh-v1'
 # This is advertised over the wire. Incremental the counter at the end
 # to reflect BC breakages.
 SSHV2 = 'exp-ssh-v2-0001'
 
 class baseprotocolhandler(object):
     """Abstract base class for wire protocol handlers.
 
     A wire protocol handler serves as an interface between protocol command
     handlers and the wire protocol transport layer. Protocol handlers provide
     methods to read command arguments, redirect stdio for the duration of
     the request, handle response types, etc.
     """
 
     __metaclass__ = abc.ABCMeta
 
     @abc.abstractproperty
     def name(self):
         """The name of the protocol implementation.
 
         Used for uniquely identifying the transport type.
         """
 
     @abc.abstractmethod
     def getargs(self, args):
         """return the value for arguments in <args>
 
         returns a list of values (same order as <args>)"""
 
     @abc.abstractmethod
     def forwardpayload(self, fp):
         """Read the raw payload and forward to a file.
 
         The payload is read in full before the function returns.
         """
 
     @abc.abstractmethod
     def mayberedirectstdio(self):
         """Context manager to possibly redirect stdio.
 
         The context manager yields a file-object like object that receives
         stdout and stderr output when the context manager is active. Or it
         yields ``None`` if no I/O redirection occurs.
 
         The intent of this context manager is to capture stdio output
         so it may be sent in the response. Some transports support streaming
         stdio to the client in real time. For these transports, stdio output
         won't be captured.
         """
 
     @abc.abstractmethod
     def client(self):
         """Returns a string representation of this client (as bytes)."""
 
 def decodevaluefromheaders(req, headerprefix):
     """Decode a long value from multiple HTTP request headers.
 
     Returns the value as a bytes, not a str.
     """
     chunks = []
     i = 1
     prefix = headerprefix.upper().replace(r'-', r'_')
     while True:
         v = req.env.get(r'HTTP_%s_%d' % (prefix, i))
         if v is None:
             break
         chunks.append(pycompat.bytesurl(v))
         i += 1
 
     return ''.join(chunks)
 
 class httpv1protocolhandler(baseprotocolhandler):
     def __init__(self, req, ui):
         self._req = req
         self._ui = ui
 
     @property
     def name(self):
         return 'http-v1'
 
     def getargs(self, args):
         knownargs = self._args()
         data = {}
         keys = args.split()
         for k in keys:
             if k == '*':
                 star = {}
                 for key in knownargs.keys():
                     if key != 'cmd' and key not in keys:
                         star[key] = knownargs[key][0]
                 data['*'] = star
             else:
                 data[k] = knownargs[k][0]
         return [data[k] for k in keys]
 
     def _args(self):
         args = util.rapply(pycompat.bytesurl, self._req.form.copy())
         postlen = int(self._req.env.get(r'HTTP_X_HGARGS_POST', 0))
         if postlen:
             args.update(urlreq.parseqs(
                 self._req.read(postlen), keep_blank_values=True))
             return args
 
         argvalue = decodevaluefromheaders(self._req, r'X-HgArg')
         args.update(urlreq.parseqs(argvalue, keep_blank_values=True))
         return args
 
     def forwardpayload(self, fp):
         length = int(self._req.env[r'CONTENT_LENGTH'])
         # If httppostargs is used, we need to read Content-Length
         # minus the amount that was consumed by args.
         length -= int(self._req.env.get(r'HTTP_X_HGARGS_POST', 0))
         for s in util.filechunkiter(self._req, limit=length):
             fp.write(s)
 
     @contextlib.contextmanager
     def mayberedirectstdio(self):
         oldout = self._ui.fout
         olderr = self._ui.ferr
 
         out = util.stringio()
 
         try:
             self._ui.fout = out
             self._ui.ferr = out
             yield out
         finally:
             self._ui.fout = oldout
             self._ui.ferr = olderr
 
     def client(self):
         return 'remote:%s:%s:%s' % (
             self._req.env.get('wsgi.url_scheme') or 'http',
             urlreq.quote(self._req.env.get('REMOTE_HOST', '')),
             urlreq.quote(self._req.env.get('REMOTE_USER', '')))
 
 # This method exists mostly so that extensions like remotefilelog can
 # disable a kludgey legacy method only over http. As of early 2018,
 # there are no other known users, so with any luck we can discard this
 # hook if remotefilelog becomes a first-party extension.
 def iscmd(cmd):
     return cmd in wireproto.commands
 
 def parsehttprequest(repo, req, query):
     """Parse the HTTP request for a wire protocol request.
 
     If the current request appears to be a wire protocol request, this
     function returns a dict with details about that request, including
     an ``abstractprotocolserver`` instance suitable for handling the
     request. Otherwise, ``None`` is returned.
 
     ``req`` is a ``wsgirequest`` instance.
     """
     # HTTP version 1 wire protocol requests are denoted by a "cmd" query
     # string parameter. If it isn't present, this isn't a wire protocol
     # request.
     if r'cmd' not in req.form:
         return None
 
     cmd = pycompat.sysbytes(req.form[r'cmd'][0])
 
     # The "cmd" request parameter is used by both the wire protocol and hgweb.
     # While not all wire protocol commands are available for all transports,
     # if we see a "cmd" value that resembles a known wire protocol command, we
     # route it to a protocol handler. This is better than routing possible
     # wire protocol requests to hgweb because it prevents hgweb from using
     # known wire protocol commands and it is less confusing for machine
     # clients.
     if not iscmd(cmd):
         return None
 
     proto = httpv1protocolhandler(req, repo.ui)
 
     return {
         'cmd': cmd,
         'proto': proto,
         'dispatch': lambda: _callhttp(repo, req, proto, cmd),
         'handleerror': lambda ex: _handlehttperror(ex, req, cmd),
     }
 
 def _httpresponsetype(ui, req, prefer_uncompressed):
     """Determine the appropriate response type and compression settings.
 
     Returns a tuple of (mediatype, compengine, engineopts).
     """
     # Determine the response media type and compression engine based
     # on the request parameters.
     protocaps = decodevaluefromheaders(req, r'X-HgProto').split(' ')
 
     if '0.2' in protocaps:
         # All clients are expected to support uncompressed data.
         if prefer_uncompressed:
             return HGTYPE2, util._noopengine(), {}
 
         # Default as defined by wire protocol spec.
         compformats = ['zlib', 'none']
         for cap in protocaps:
             if cap.startswith('comp='):
                 compformats = cap[5:].split(',')
                 break
 
         # Now find an agreed upon compression format.
         for engine in wireproto.supportedcompengines(ui, util.SERVERROLE):
             if engine.wireprotosupport().name in compformats:
                 opts = {}
                 level = ui.configint('server', '%slevel' % engine.name())
                 if level is not None:
                     opts['level'] = level
 
                 return HGTYPE2, engine, opts
 
         # No mutually supported compression format. Fall back to the
         # legacy protocol.
 
     # Don't allow untrusted settings because disabling compression or
     # setting a very high compression level could lead to flooding
     # the server's network or CPU.
     opts = {'level': ui.configint('server', 'zliblevel')}
     return HGTYPE, util.compengines['zlib'], opts
 
 def _callhttp(repo, req, proto, cmd):
     def genversion2(gen, engine, engineopts):
         # application/mercurial-0.2 always sends a payload header
         # identifying the compression engine.
         name = engine.wireprotosupport().name
         assert 0 < len(name) < 256
         yield struct.pack('B', len(name))
         yield name
 
         for chunk in gen:
             yield chunk
 
     rsp = wireproto.dispatch(repo, proto, cmd)
 
     if not wireproto.commands.commandavailable(cmd, proto):
         req.respond(HTTP_OK, HGERRTYPE,
                     body=_('requested wire protocol command is not available '
                            'over HTTP'))
         return []
 
     if isinstance(rsp, bytes):
         req.respond(HTTP_OK, HGTYPE, body=rsp)
         return []
     elif isinstance(rsp, wireprototypes.bytesresponse):
         req.respond(HTTP_OK, HGTYPE, body=rsp.data)
         return []
     elif isinstance(rsp, wireprototypes.streamreslegacy):
         gen = rsp.gen
         req.respond(HTTP_OK, HGTYPE)
         return gen
     elif isinstance(rsp, wireprototypes.streamres):
         gen = rsp.gen
 
         # This code for compression should not be streamres specific. It
         # is here because we only compress streamres at the moment.
         mediatype, engine, engineopts = _httpresponsetype(
             repo.ui, req, rsp.prefer_uncompressed)
         gen = engine.compressstream(gen, engineopts)
 
         if mediatype == HGTYPE2:
             gen = genversion2(gen, engine, engineopts)
 
         req.respond(HTTP_OK, mediatype)
         return gen
     elif isinstance(rsp, wireprototypes.pushres):
         rsp = '%d\n%s' % (rsp.res, rsp.output)
         req.respond(HTTP_OK, HGTYPE, body=rsp)
         return []
     elif isinstance(rsp, wireprototypes.pusherr):
         # This is the httplib workaround documented in _handlehttperror().
         req.drain()
 
         rsp = '0\n%s\n' % rsp.res
         req.respond(HTTP_OK, HGTYPE, body=rsp)
         return []
     elif isinstance(rsp, wireprototypes.ooberror):
         rsp = rsp.message
         req.respond(HTTP_OK, HGERRTYPE, body=rsp)
         return []
     raise error.ProgrammingError('hgweb.protocol internal failure', rsp)
 
 def _handlehttperror(e, req, cmd):
     """Called when an ErrorResponse is raised during HTTP request processing."""
 
     # Clients using Python's httplib are stateful: the HTTP client
     # won't process an HTTP response until all request data is
     # sent to the server. The intent of this code is to ensure
     # we always read HTTP request data from the client, thus
     # ensuring httplib transitions to a state that allows it to read
     # the HTTP response. In other words, it helps prevent deadlocks
     # on clients using httplib.
 
     if (req.env[r'REQUEST_METHOD'] == r'POST' and
         # But not if Expect: 100-continue is being used.
         (req.env.get('HTTP_EXPECT',
                      '').lower() != '100-continue') or
         # Or the non-httplib HTTP library is being advertised by
         # the client.
         req.env.get('X-HgHttp2', '')):
         req.drain()
     else:
         req.headers.append((r'Connection', r'Close'))
 
     # TODO This response body assumes the failed command was
     # "unbundle." That assumption is not always valid.
-    req.respond(e, HGTYPE, body='0\n%s\n' % e)
+    req.respond(e, HGTYPE, body='0\n%s\n' % pycompat.bytestr(e))
 
     return ''
 
 def _sshv1respondbytes(fout, value):
     """Send a bytes response for protocol version 1."""
     fout.write('%d\n' % len(value))
     fout.write(value)
     fout.flush()
 
 def _sshv1respondstream(fout, source):
     write = fout.write
     for chunk in source.gen:
         write(chunk)
     fout.flush()
 
 def _sshv1respondooberror(fout, ferr, rsp):
     ferr.write(b'%s\n-\n' % rsp)
     ferr.flush()
     fout.write(b'\n')
     fout.flush()
 
 class sshv1protocolhandler(baseprotocolhandler):
     """Handler for requests services via version 1 of SSH protocol."""
     def __init__(self, ui, fin, fout):
         self._ui = ui
         self._fin = fin
         self._fout = fout
 
     @property
     def name(self):
         return SSHV1
 
     def getargs(self, args):
         data = {}
         keys = args.split()
         for n in xrange(len(keys)):
             argline = self._fin.readline()[:-1]
             arg, l = argline.split()
             if arg not in keys:
                 raise error.Abort(_("unexpected parameter %r") % arg)
             if arg == '*':
                 star = {}
                 for k in xrange(int(l)):
                     argline = self._fin.readline()[:-1]
                     arg, l = argline.split()
                     val = self._fin.read(int(l))
                     star[arg] = val
                 data['*'] = star
             else:
                 val = self._fin.read(int(l))
                 data[arg] = val
         return [data[k] for k in keys]
 
     def forwardpayload(self, fpout):
         # The file is in the form:
         #
         # <chunk size>\n<chunk>
         # ...
         # 0\n
         _sshv1respondbytes(self._fout, b'')
         count = int(self._fin.readline())
         while count:
             fpout.write(self._fin.read(count))
             count = int(self._fin.readline())
 
     @contextlib.contextmanager
     def mayberedirectstdio(self):
         yield None
 
     def client(self):
         client = encoding.environ.get('SSH_CLIENT', '').split(' ', 1)[0]
         return 'remote:ssh:' + client
 
 class sshv2protocolhandler(sshv1protocolhandler):
     """Protocol handler for version 2 of the SSH protocol."""
 
 def _runsshserver(ui, repo, fin, fout):
     # This function operates like a state machine of sorts. The following
     # states are defined:
     #
     # protov1-serving
     #    Server is in protocol version 1 serving mode. Commands arrive on
     #    new lines. These commands are processed in this state, one command
     #    after the other.
     #
     # protov2-serving
     #    Server is in protocol version 2 serving mode.
     #
     # upgrade-initial
     #    The server is going to process an upgrade request.
     #
     # upgrade-v2-filter-legacy-handshake
     #    The protocol is being upgraded to version 2. The server is expecting
     #    the legacy handshake from version 1.
     #
     # upgrade-v2-finish
     #    The upgrade to version 2 of the protocol is imminent.
     #
     # shutdown
     #    The server is shutting down, possibly in reaction to a client event.
     #
     # And here are their transitions:
     #
     # protov1-serving -> shutdown
     #    When server receives an empty request or encounters another
     #    error.
     #
     # protov1-serving -> upgrade-initial
     #    An upgrade request line was seen.
     #
     # upgrade-initial -> upgrade-v2-filter-legacy-handshake
     #    Upgrade to version 2 in progress. Server is expecting to
     #    process a legacy handshake.
     #
     # upgrade-v2-filter-legacy-handshake -> shutdown
     #    Client did not fulfill upgrade handshake requirements.
     #
     # upgrade-v2-filter-legacy-handshake -> upgrade-v2-finish
     #    Client fulfilled version 2 upgrade requirements. Finishing that
     #    upgrade.
     #
     # upgrade-v2-finish -> protov2-serving
     #    Protocol upgrade to version 2 complete. Server can now speak protocol
     #    version 2.
     #
     # protov2-serving -> protov1-serving
     #    Ths happens by default since protocol version 2 is the same as
     #    version 1 except for the handshake.
 
     state = 'protov1-serving'
     proto = sshv1protocolhandler(ui, fin, fout)
     protoswitched = False
 
     while True:
         if state == 'protov1-serving':
             # Commands are issued on new lines.
             request = fin.readline()[:-1]
 
             # Empty lines signal to terminate the connection.
             if not request:
                 state = 'shutdown'
                 continue
 
             # It looks like a protocol upgrade request. Transition state to
             # handle it.
             if request.startswith(b'upgrade '):
                 if protoswitched:
                     _sshv1respondooberror(fout, ui.ferr,
                                           b'cannot upgrade protocols multiple '
                                           b'times')
                     state = 'shutdown'
                     continue
 
                 state = 'upgrade-initial'
                 continue
 
             available = wireproto.commands.commandavailable(request, proto)
 
             # This command isn't available. Send an empty response and go
             # back to waiting for a new command.
             if not available:
                 _sshv1respondbytes(fout, b'')
                 continue
 
             rsp = wireproto.dispatch(repo, proto, request)
 
             if isinstance(rsp, bytes):
                 _sshv1respondbytes(fout, rsp)
             elif isinstance(rsp, wireprototypes.bytesresponse):
                 _sshv1respondbytes(fout, rsp.data)
             elif isinstance(rsp, wireprototypes.streamres):
                 _sshv1respondstream(fout, rsp)
             elif isinstance(rsp, wireprototypes.streamreslegacy):
                 _sshv1respondstream(fout, rsp)
             elif isinstance(rsp, wireprototypes.pushres):
                 _sshv1respondbytes(fout, b'')
                 _sshv1respondbytes(fout, b'%d' % rsp.res)
             elif isinstance(rsp, wireprototypes.pusherr):
                 _sshv1respondbytes(fout, rsp.res)
             elif isinstance(rsp, wireprototypes.ooberror):
                 _sshv1respondooberror(fout, ui.ferr, rsp.message)
             else:
                 raise error.ProgrammingError('unhandled response type from '
                                              'wire protocol command: %s' % rsp)
 
         # For now, protocol version 2 serving just goes back to version 1.
         elif state == 'protov2-serving':
             state = 'protov1-serving'
             continue
 
         elif state == 'upgrade-initial':
             # We should never transition into this state if we've switched
             # protocols.
             assert not protoswitched
             assert proto.name == SSHV1
 
             # Expected: upgrade <token> <capabilities>
             # If we get something else, the request is malformed. It could be
             # from a future client that has altered the upgrade line content.
             # We treat this as an unknown command.
             try:
                 token, caps = request.split(b' ')[1:]
             except ValueError:
                 _sshv1respondbytes(fout, b'')
                 state = 'protov1-serving'
                 continue
 
             # Send empty response if we don't support upgrading protocols.
             if not ui.configbool('experimental', 'sshserver.support-v2'):
                 _sshv1respondbytes(fout, b'')
                 state = 'protov1-serving'
                 continue
 
             try:
                 caps = urlreq.parseqs(caps)
             except ValueError:
                 _sshv1respondbytes(fout, b'')
                 state = 'protov1-serving'
                 continue
 
             # We don't see an upgrade request to protocol version 2. Ignore
             # the upgrade request.
             wantedprotos = caps.get(b'proto', [b''])[0]
             if SSHV2 not in wantedprotos:
                 _sshv1respondbytes(fout, b'')
                 state = 'protov1-serving'
                 continue
 
             # It looks like we can honor this upgrade request to protocol 2.
             # Filter the rest of the handshake protocol request lines.
             state = 'upgrade-v2-filter-legacy-handshake'
             continue
 
         elif state == 'upgrade-v2-filter-legacy-handshake':
             # Client should have sent legacy handshake after an ``upgrade``
             # request. Expected lines:
             #
             #    hello
             #    between
             #    pairs 81
             #    0000...-0000...
 
             ok = True
             for line in (b'hello', b'between', b'pairs 81'):
                 request = fin.readline()[:-1]
 
                 if request != line:
                     _sshv1respondooberror(fout, ui.ferr,
                                           b'malformed handshake protocol: '
                                           b'missing %s' % line)
                     ok = False
                     state = 'shutdown'
                     break
 
             if not ok:
                 continue
 
             request = fin.read(81)
             if request != b'%s-%s' % (b'0' * 40, b'0' * 40):
                 _sshv1respondooberror(fout, ui.ferr,
                                       b'malformed handshake protocol: '
                                       b'missing between argument value')
                 state = 'shutdown'
                 continue
 
             state = 'upgrade-v2-finish'
             continue
 
         elif state == 'upgrade-v2-finish':
             # Send the upgrade response.
             fout.write(b'upgraded %s %s\n' % (token, SSHV2))
             servercaps = wireproto.capabilities(repo, proto)
             rsp = b'capabilities: %s' % servercaps.data
             fout.write(b'%d\n%s\n' % (len(rsp), rsp))
             fout.flush()
 
             proto = sshv2protocolhandler(ui, fin, fout)
             protoswitched = True
 
             state = 'protov2-serving'
             continue
 
         elif state == 'shutdown':
             break
 
         else:
             raise error.ProgrammingError('unhandled ssh server state: %s' %
                                          state)
 
 class sshserver(object):
     def __init__(self, ui, repo):
         self._ui = ui
         self._repo = repo
         self._fin = ui.fin
         self._fout = ui.fout
 
         hook.redirect(True)
         ui.fout = repo.ui.fout = ui.ferr
 
         # Prevent insertion/deletion of CRs
         util.setbinary(self._fin)
         util.setbinary(self._fout)
 
     def serve_forever(self):
         _runsshserver(self._ui, self._repo, self._fin, self._fout)
         sys.exit(0)