wireprotov2: pass ui into clientreactor and serverreactor

This will allow us to use config options to influence compression
settings.

Differential Revision: https://phab.mercurial-scm.org/D4919

mercurial/httppeer.py

@@ -1,986 +1,987
 # httppeer.py - HTTP repository proxy classes for mercurial
 #
 # Copyright 2005, 2006 Matt Mackall <[e-mail]>
 # Copyright 2006 Vadim Gelfer <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import errno
 import io
 import os
 import socket
 import struct
 import weakref
 
 from .i18n import _
 from . import (
     bundle2,
     error,
     httpconnection,
     pycompat,
     repository,
     statichttprepo,
     url as urlmod,
     util,
     wireprotoframing,
     wireprototypes,
     wireprotov1peer,
     wireprotov2peer,
     wireprotov2server,
 )
 from .utils import (
     cborutil,
     interfaceutil,
     stringutil,
 )
 
 httplib = util.httplib
 urlerr = util.urlerr
 urlreq = util.urlreq
 
 def encodevalueinheaders(value, header, limit):
     """Encode a string value into multiple HTTP headers.
 
     ``value`` will be encoded into 1 or more HTTP headers with the names
     ``header-<N>`` where ``<N>`` is an integer starting at 1. Each header
     name + value will be at most ``limit`` bytes long.
 
     Returns an iterable of 2-tuples consisting of header names and
     values as native strings.
     """
     # HTTP Headers are ASCII. Python 3 requires them to be unicodes,
     # not bytes. This function always takes bytes in as arguments.
     fmt = pycompat.strurl(header) + r'-%s'
     # Note: it is *NOT* a bug that the last bit here is a bytestring
     # and not a unicode: we're just getting the encoded length anyway,
     # and using an r-string to make it portable between Python 2 and 3
     # doesn't work because then the \r is a literal backslash-r
     # instead of a carriage return.
     valuelen = limit - len(fmt % r'000') - len(': \r\n')
     result = []
 
     n = 0
     for i in pycompat.xrange(0, len(value), valuelen):
         n += 1
         result.append((fmt % str(n), pycompat.strurl(value[i:i + valuelen])))
 
     return result
 
 class _multifile(object):
     def __init__(self, *fileobjs):
         for f in fileobjs:
             if not util.safehasattr(f, 'length'):
                 raise ValueError(
                     '_multifile only supports file objects that '
                     'have a length but this one does not:', type(f), f)
         self._fileobjs = fileobjs
         self._index = 0
 
     @property
     def length(self):
         return sum(f.length for f in self._fileobjs)
 
     def read(self, amt=None):
         if amt <= 0:
             return ''.join(f.read() for f in self._fileobjs)
         parts = []
         while amt and self._index < len(self._fileobjs):
             parts.append(self._fileobjs[self._index].read(amt))
             got = len(parts[-1])
             if got < amt:
                 self._index += 1
             amt -= got
         return ''.join(parts)
 
     def seek(self, offset, whence=os.SEEK_SET):
         if whence != os.SEEK_SET:
             raise NotImplementedError(
                 '_multifile does not support anything other'
                 ' than os.SEEK_SET for whence on seek()')
         if offset != 0:
             raise NotImplementedError(
                 '_multifile only supports seeking to start, but that '
                 'could be fixed if you need it')
         for f in self._fileobjs:
             f.seek(0)
         self._index = 0
 
 def makev1commandrequest(ui, requestbuilder, caps, capablefn,
                          repobaseurl, cmd, args):
     """Make an HTTP request to run a command for a version 1 client.
 
     ``caps`` is a set of known server capabilities. The value may be
     None if capabilities are not yet known.
 
     ``capablefn`` is a function to evaluate a capability.
 
     ``cmd``, ``args``, and ``data`` define the command, its arguments, and
     raw data to pass to it.
     """
     if cmd == 'pushkey':
         args['data'] = ''
     data = args.pop('data', None)
     headers = args.pop('headers', {})
 
     ui.debug("sending %s command\n" % cmd)
     q = [('cmd', cmd)]
     headersize = 0
     # Important: don't use self.capable() here or else you end up
     # with infinite recursion when trying to look up capabilities
     # for the first time.
     postargsok = caps is not None and 'httppostargs' in caps
 
     # Send arguments via POST.
     if postargsok and args:
         strargs = urlreq.urlencode(sorted(args.items()))
         if not data:
             data = strargs
         else:
             if isinstance(data, bytes):
                 i = io.BytesIO(data)
                 i.length = len(data)
                 data = i
             argsio = io.BytesIO(strargs)
             argsio.length = len(strargs)
             data = _multifile(argsio, data)
         headers[r'X-HgArgs-Post'] = len(strargs)
     elif args:
         # Calling self.capable() can infinite loop if we are calling
         # "capabilities". But that command should never accept wire
         # protocol arguments. So this should never happen.
         assert cmd != 'capabilities'
         httpheader = capablefn('httpheader')
         if httpheader:
             headersize = int(httpheader.split(',', 1)[0])
 
         # Send arguments via HTTP headers.
         if headersize > 0:
             # The headers can typically carry more data than the URL.
             encargs = urlreq.urlencode(sorted(args.items()))
             for header, value in encodevalueinheaders(encargs, 'X-HgArg',
                                                       headersize):
                 headers[header] = value
         # Send arguments via query string (Mercurial <1.9).
         else:
             q += sorted(args.items())
 
     qs = '?%s' % urlreq.urlencode(q)
     cu = "%s%s" % (repobaseurl, qs)
     size = 0
     if util.safehasattr(data, 'length'):
         size = data.length
     elif data is not None:
         size = len(data)
     if data is not None and r'Content-Type' not in headers:
         headers[r'Content-Type'] = r'application/mercurial-0.1'
 
     # Tell the server we accept application/mercurial-0.2 and multiple
     # compression formats if the server is capable of emitting those
     # payloads.
     # Note: Keep this set empty by default, as client advertisement of
     # protocol parameters should only occur after the handshake.
     protoparams = set()
 
     mediatypes = set()
     if caps is not None:
         mt = capablefn('httpmediatype')
         if mt:
             protoparams.add('0.1')
             mediatypes = set(mt.split(','))
 
         protoparams.add('partial-pull')
 
     if '0.2tx' in mediatypes:
         protoparams.add('0.2')
 
     if '0.2tx' in mediatypes and capablefn('compression'):
         # We /could/ compare supported compression formats and prune
         # non-mutually supported or error if nothing is mutually supported.
         # For now, send the full list to the server and have it error.
         comps = [e.wireprotosupport().name for e in
                  util.compengines.supportedwireengines(util.CLIENTROLE)]
         protoparams.add('comp=%s' % ','.join(comps))
 
     if protoparams:
         protoheaders = encodevalueinheaders(' '.join(sorted(protoparams)),
                                             'X-HgProto',
                                             headersize or 1024)
         for header, value in protoheaders:
             headers[header] = value
 
     varyheaders = []
     for header in headers:
         if header.lower().startswith(r'x-hg'):
             varyheaders.append(header)
 
     if varyheaders:
         headers[r'Vary'] = r','.join(sorted(varyheaders))
 
     req = requestbuilder(pycompat.strurl(cu), data, headers)
 
     if data is not None:
         ui.debug("sending %d bytes\n" % size)
         req.add_unredirected_header(r'Content-Length', r'%d' % size)
 
     return req, cu, qs
 
 def _reqdata(req):
     """Get request data, if any. If no data, returns None."""
     if pycompat.ispy3:
         return req.data
     if not req.has_data():
         return None
     return req.get_data()
 
 def sendrequest(ui, opener, req):
     """Send a prepared HTTP request.
 
     Returns the response object.
     """
     dbg = ui.debug
     if (ui.debugflag
         and ui.configbool('devel', 'debug.peer-request')):
         line = 'devel-peer-request: %s\n'
         dbg(line % '%s %s' % (pycompat.bytesurl(req.get_method()),
                               pycompat.bytesurl(req.get_full_url())))
         hgargssize = None
 
         for header, value in sorted(req.header_items()):
             header = pycompat.bytesurl(header)
             value = pycompat.bytesurl(value)
             if header.startswith('X-hgarg-'):
                 if hgargssize is None:
                     hgargssize = 0
                 hgargssize += len(value)
             else:
                 dbg(line % '  %s %s' % (header, value))
 
         if hgargssize is not None:
             dbg(line % '  %d bytes of commands arguments in headers'
                 % hgargssize)
         data = _reqdata(req)
         if data is not None:
             length = getattr(data, 'length', None)
             if length is None:
                 length = len(data)
             dbg(line % '  %d bytes of data' % length)
 
         start = util.timer()
 
     res = None
     try:
         res = opener.open(req)
     except urlerr.httperror as inst:
         if inst.code == 401:
             raise error.Abort(_('authorization failed'))
         raise
     except httplib.HTTPException as inst:
         ui.debug('http error requesting %s\n' %
                  util.hidepassword(req.get_full_url()))
         ui.traceback()
         raise IOError(None, inst)
     finally:
         if ui.debugflag and ui.configbool('devel', 'debug.peer-request'):
             code = res.code if res else -1
             dbg(line % '  finished in %.4f seconds (%d)'
                 % (util.timer() - start, code))
 
     # Insert error handlers for common I/O failures.
     urlmod.wrapresponse(res)
 
     return res
 
 class RedirectedRepoError(error.RepoError):
     def __init__(self, msg, respurl):
         super(RedirectedRepoError, self).__init__(msg)
         self.respurl = respurl
 
 def parsev1commandresponse(ui, baseurl, requrl, qs, resp, compressible,
                            allowcbor=False):
     # record the url we got redirected to
     redirected = False
     respurl = pycompat.bytesurl(resp.geturl())
     if respurl.endswith(qs):
         respurl = respurl[:-len(qs)]
         qsdropped = False
     else:
         qsdropped = True
 
     if baseurl.rstrip('/') != respurl.rstrip('/'):
         redirected = True
         if not ui.quiet:
             ui.warn(_('real URL is %s\n') % respurl)
 
     try:
         proto = pycompat.bytesurl(resp.getheader(r'content-type', r''))
     except AttributeError:
         proto = pycompat.bytesurl(resp.headers.get(r'content-type', r''))
 
     safeurl = util.hidepassword(baseurl)
     if proto.startswith('application/hg-error'):
         raise error.OutOfBandError(resp.read())
 
     # Pre 1.0 versions of Mercurial used text/plain and
     # application/hg-changegroup. We don't support such old servers.
     if not proto.startswith('application/mercurial-'):
         ui.debug("requested URL: '%s'\n" % util.hidepassword(requrl))
         msg = _("'%s' does not appear to be an hg repository:\n"
                 "---%%<--- (%s)\n%s\n---%%<---\n") % (
             safeurl, proto or 'no content-type', resp.read(1024))
 
         # Some servers may strip the query string from the redirect. We
         # raise a special error type so callers can react to this specially.
         if redirected and qsdropped:
             raise RedirectedRepoError(msg, respurl)
         else:
             raise error.RepoError(msg)
 
     try:
         subtype = proto.split('-', 1)[1]
 
         # Unless we end up supporting CBOR in the legacy wire protocol,
         # this should ONLY be encountered for the initial capabilities
         # request during handshake.
         if subtype == 'cbor':
             if allowcbor:
                 return respurl, proto, resp
             else:
                 raise error.RepoError(_('unexpected CBOR response from '
                                         'server'))
 
         version_info = tuple([int(n) for n in subtype.split('.')])
     except ValueError:
         raise error.RepoError(_("'%s' sent a broken Content-Type "
                                 "header (%s)") % (safeurl, proto))
 
     # TODO consider switching to a decompression reader that uses
     # generators.
     if version_info == (0, 1):
         if compressible:
             resp = util.compengines['zlib'].decompressorreader(resp)
 
     elif version_info == (0, 2):
         # application/mercurial-0.2 always identifies the compression
         # engine in the payload header.
         elen = struct.unpack('B', util.readexactly(resp, 1))[0]
         ename = util.readexactly(resp, elen)
         engine = util.compengines.forwiretype(ename)
 
         resp = engine.decompressorreader(resp)
     else:
         raise error.RepoError(_("'%s' uses newer protocol %s") %
                               (safeurl, subtype))
 
     return respurl, proto, resp
 
 class httppeer(wireprotov1peer.wirepeer):
     def __init__(self, ui, path, url, opener, requestbuilder, caps):
         self.ui = ui
         self._path = path
         self._url = url
         self._caps = caps
         self._urlopener = opener
         self._requestbuilder = requestbuilder
 
     def __del__(self):
         for h in self._urlopener.handlers:
             h.close()
             getattr(h, "close_all", lambda: None)()
 
     # Begin of ipeerconnection interface.
 
     def url(self):
         return self._path
 
     def local(self):
         return None
 
     def peer(self):
         return self
 
     def canpush(self):
         return True
 
     def close(self):
         self.ui.note(_('(sent %d HTTP requests and %d bytes; '
                        'received %d bytes in responses)\n') %
                      (self._urlopener.requestscount,
                       self._urlopener.sentbytescount,
                       self._urlopener.receivedbytescount))
 
     # End of ipeerconnection interface.
 
     # Begin of ipeercommands interface.
 
     def capabilities(self):
         return self._caps
 
     # End of ipeercommands interface.
 
     def _callstream(self, cmd, _compressible=False, **args):
         args = pycompat.byteskwargs(args)
 
         req, cu, qs = makev1commandrequest(self.ui, self._requestbuilder,
                                            self._caps, self.capable,
                                            self._url, cmd, args)
 
         resp = sendrequest(self.ui, self._urlopener, req)
 
         self._url, ct, resp = parsev1commandresponse(self.ui, self._url, cu, qs,
                                                      resp, _compressible)
 
         return resp
 
     def _call(self, cmd, **args):
         fp = self._callstream(cmd, **args)
         try:
             return fp.read()
         finally:
             # if using keepalive, allow connection to be reused
             fp.close()
 
     def _callpush(self, cmd, cg, **args):
         # have to stream bundle to a temp file because we do not have
         # http 1.1 chunked transfer.
 
         types = self.capable('unbundle')
         try:
             types = types.split(',')
         except AttributeError:
             # servers older than d1b16a746db6 will send 'unbundle' as a
             # boolean capability. They only support headerless/uncompressed
             # bundles.
             types = [""]
         for x in types:
             if x in bundle2.bundletypes:
                 type = x
                 break
 
         tempname = bundle2.writebundle(self.ui, cg, None, type)
         fp = httpconnection.httpsendfile(self.ui, tempname, "rb")
         headers = {r'Content-Type': r'application/mercurial-0.1'}
 
         try:
             r = self._call(cmd, data=fp, headers=headers, **args)
             vals = r.split('\n', 1)
             if len(vals) < 2:
                 raise error.ResponseError(_("unexpected response:"), r)
             return vals
         except urlerr.httperror:
             # Catch and re-raise these so we don't try and treat them
             # like generic socket errors. They lack any values in
             # .args on Python 3 which breaks our socket.error block.
             raise
         except socket.error as err:
             if err.args[0] in (errno.ECONNRESET, errno.EPIPE):
                 raise error.Abort(_('push failed: %s') % err.args[1])
             raise error.Abort(err.args[1])
         finally:
             fp.close()
             os.unlink(tempname)
 
     def _calltwowaystream(self, cmd, fp, **args):
         fh = None
         fp_ = None
         filename = None
         try:
             # dump bundle to disk
             fd, filename = pycompat.mkstemp(prefix="hg-bundle-", suffix=".hg")
             fh = os.fdopen(fd, r"wb")
             d = fp.read(4096)
             while d:
                 fh.write(d)
                 d = fp.read(4096)
             fh.close()
             # start http push
             fp_ = httpconnection.httpsendfile(self.ui, filename, "rb")
             headers = {r'Content-Type': r'application/mercurial-0.1'}
             return self._callstream(cmd, data=fp_, headers=headers, **args)
         finally:
             if fp_ is not None:
                 fp_.close()
             if fh is not None:
                 fh.close()
                 os.unlink(filename)
 
     def _callcompressable(self, cmd, **args):
         return self._callstream(cmd, _compressible=True, **args)
 
     def _abort(self, exception):
         raise exception
 
 def sendv2request(ui, opener, requestbuilder, apiurl, permission, requests,
                   redirect):
-    reactor = wireprotoframing.clientreactor(hasmultiplesend=False,
+    reactor = wireprotoframing.clientreactor(ui,
+                                             hasmultiplesend=False,
                                              buffersends=True)
 
     handler = wireprotov2peer.clienthandler(ui, reactor,
                                             opener=opener,
                                             requestbuilder=requestbuilder)
 
     url = '%s/%s' % (apiurl, permission)
 
     if len(requests) > 1:
         url += '/multirequest'
     else:
         url += '/%s' % requests[0][0]
 
     ui.debug('sending %d commands\n' % len(requests))
     for command, args, f in requests:
         ui.debug('sending command %s: %s\n' % (
             command, stringutil.pprint(args, indent=2)))
         assert not list(handler.callcommand(command, args, f,
                                             redirect=redirect))
 
     # TODO stream this.
     body = b''.join(map(bytes, handler.flushcommands()))
 
     # TODO modify user-agent to reflect v2
     headers = {
         r'Accept': wireprotov2server.FRAMINGTYPE,
         r'Content-Type': wireprotov2server.FRAMINGTYPE,
     }
 
     req = requestbuilder(pycompat.strurl(url), body, headers)
     req.add_unredirected_header(r'Content-Length', r'%d' % len(body))
 
     try:
         res = opener.open(req)
     except urlerr.httperror as e:
         if e.code == 401:
             raise error.Abort(_('authorization failed'))
 
         raise
     except httplib.HTTPException as e:
         ui.traceback()
         raise IOError(None, e)
 
     return handler, res
 
 class queuedcommandfuture(pycompat.futures.Future):
     """Wraps result() on command futures to trigger submission on call."""
 
     def result(self, timeout=None):
         if self.done():
             return pycompat.futures.Future.result(self, timeout)
 
         self._peerexecutor.sendcommands()
 
         # sendcommands() will restore the original __class__ and self.result
         # will resolve to Future.result.
         return self.result(timeout)
 
 @interfaceutil.implementer(repository.ipeercommandexecutor)
 class httpv2executor(object):
     def __init__(self, ui, opener, requestbuilder, apiurl, descriptor,
                  redirect):
         self._ui = ui
         self._opener = opener
         self._requestbuilder = requestbuilder
         self._apiurl = apiurl
         self._descriptor = descriptor
         self._redirect = redirect
         self._sent = False
         self._closed = False
         self._neededpermissions = set()
         self._calls = []
         self._futures = weakref.WeakSet()
         self._responseexecutor = None
         self._responsef = None
 
     def __enter__(self):
         return self
 
     def __exit__(self, exctype, excvalue, exctb):
         self.close()
 
     def callcommand(self, command, args):
         if self._sent:
             raise error.ProgrammingError('callcommand() cannot be used after '
                                          'commands are sent')
 
         if self._closed:
             raise error.ProgrammingError('callcommand() cannot be used after '
                                          'close()')
 
         # The service advertises which commands are available. So if we attempt
         # to call an unknown command or pass an unknown argument, we can screen
         # for this.
         if command not in self._descriptor['commands']:
             raise error.ProgrammingError(
                 'wire protocol command %s is not available' % command)
 
         cmdinfo = self._descriptor['commands'][command]
         unknownargs = set(args.keys()) - set(cmdinfo.get('args', {}))
 
         if unknownargs:
             raise error.ProgrammingError(
                 'wire protocol command %s does not accept argument: %s' % (
                     command, ', '.join(sorted(unknownargs))))
 
         self._neededpermissions |= set(cmdinfo['permissions'])
 
         # TODO we /could/ also validate types here, since the API descriptor
         # includes types...
 
         f = pycompat.futures.Future()
 
         # Monkeypatch it so result() triggers sendcommands(), otherwise result()
         # could deadlock.
         f.__class__ = queuedcommandfuture
         f._peerexecutor = self
 
         self._futures.add(f)
         self._calls.append((command, args, f))
 
         return f
 
     def sendcommands(self):
         if self._sent:
             return
 
         if not self._calls:
             return
 
         self._sent = True
 
         # Unhack any future types so caller sees a clean type and so we
         # break reference cycle.
         for f in self._futures:
             if isinstance(f, queuedcommandfuture):
                 f.__class__ = pycompat.futures.Future
                 f._peerexecutor = None
 
         # Mark the future as running and filter out cancelled futures.
         calls = [(command, args, f)
                  for command, args, f in self._calls
                  if f.set_running_or_notify_cancel()]
 
         # Clear out references, prevent improper object usage.
         self._calls = None
 
         if not calls:
             return
 
         permissions = set(self._neededpermissions)
 
         if 'push' in permissions and 'pull' in permissions:
             permissions.remove('pull')
 
         if len(permissions) > 1:
             raise error.RepoError(_('cannot make request requiring multiple '
                                     'permissions: %s') %
                                   _(', ').join(sorted(permissions)))
 
         permission = {
             'push': 'rw',
             'pull': 'ro',
         }[permissions.pop()]
 
         handler, resp = sendv2request(
             self._ui, self._opener, self._requestbuilder, self._apiurl,
             permission, calls, self._redirect)
 
         # TODO we probably want to validate the HTTP code, media type, etc.
 
         self._responseexecutor = pycompat.futures.ThreadPoolExecutor(1)
         self._responsef = self._responseexecutor.submit(self._handleresponse,
                                                         handler, resp)
 
     def close(self):
         if self._closed:
             return
 
         self.sendcommands()
 
         self._closed = True
 
         if not self._responsef:
             return
 
         # TODO ^C here may not result in immediate program termination.
 
         try:
             self._responsef.result()
         finally:
             self._responseexecutor.shutdown(wait=True)
             self._responsef = None
             self._responseexecutor = None
 
             # If any of our futures are still in progress, mark them as
             # errored, otherwise a result() could wait indefinitely.
             for f in self._futures:
                 if not f.done():
                     f.set_exception(error.ResponseError(
                         _('unfulfilled command response')))
 
             self._futures = None
 
     def _handleresponse(self, handler, resp):
         # Called in a thread to read the response.
 
         while handler.readdata(resp):
             pass
 
 # TODO implement interface for version 2 peers
 @interfaceutil.implementer(repository.ipeerconnection,
                            repository.ipeercapabilities,
                            repository.ipeerrequests)
 class httpv2peer(object):
     def __init__(self, ui, repourl, apipath, opener, requestbuilder,
                  apidescriptor):
         self.ui = ui
 
         if repourl.endswith('/'):
             repourl = repourl[:-1]
 
         self._url = repourl
         self._apipath = apipath
         self._apiurl = '%s/%s' % (repourl, apipath)
         self._opener = opener
         self._requestbuilder = requestbuilder
         self._descriptor = apidescriptor
 
         self._redirect = wireprotov2peer.supportedredirects(ui, apidescriptor)
 
     # Start of ipeerconnection.
 
     def url(self):
         return self._url
 
     def local(self):
         return None
 
     def peer(self):
         return self
 
     def canpush(self):
         # TODO change once implemented.
         return False
 
     def close(self):
         self.ui.note(_('(sent %d HTTP requests and %d bytes; '
                        'received %d bytes in responses)\n') %
                      (self._opener.requestscount,
                       self._opener.sentbytescount,
                       self._opener.receivedbytescount))
 
     # End of ipeerconnection.
 
     # Start of ipeercapabilities.
 
     def capable(self, name):
         # The capabilities used internally historically map to capabilities
         # advertised from the "capabilities" wire protocol command. However,
         # version 2 of that command works differently.
 
         # Maps to commands that are available.
         if name in ('branchmap', 'getbundle', 'known', 'lookup', 'pushkey'):
             return True
 
         # Other concepts.
         if name in ('bundle2'):
             return True
 
         # Alias command-* to presence of command of that name.
         if name.startswith('command-'):
             return name[len('command-'):] in self._descriptor['commands']
 
         return False
 
     def requirecap(self, name, purpose):
         if self.capable(name):
             return
 
         raise error.CapabilityError(
             _('cannot %s; client or remote repository does not support the %r '
               'capability') % (purpose, name))
 
     # End of ipeercapabilities.
 
     def _call(self, name, **args):
         with self.commandexecutor() as e:
             return e.callcommand(name, args).result()
 
     def commandexecutor(self):
         return httpv2executor(self.ui, self._opener, self._requestbuilder,
                               self._apiurl, self._descriptor, self._redirect)
 
 # Registry of API service names to metadata about peers that handle it.
 #
 # The following keys are meaningful:
 #
 # init
 #    Callable receiving (ui, repourl, servicepath, opener, requestbuilder,
 #                        apidescriptor) to create a peer.
 #
 # priority
 #    Integer priority for the service. If we could choose from multiple
 #    services, we choose the one with the highest priority.
 API_PEERS = {
     wireprototypes.HTTP_WIREPROTO_V2: {
         'init': httpv2peer,
         'priority': 50,
     },
 }
 
 def performhandshake(ui, url, opener, requestbuilder):
     # The handshake is a request to the capabilities command.
 
     caps = None
     def capable(x):
         raise error.ProgrammingError('should not be called')
 
     args = {}
 
     # The client advertises support for newer protocols by adding an
     # X-HgUpgrade-* header with a list of supported APIs and an
     # X-HgProto-* header advertising which serializing formats it supports.
     # We only support the HTTP version 2 transport and CBOR responses for
     # now.
     advertisev2 = ui.configbool('experimental', 'httppeer.advertise-v2')
 
     if advertisev2:
         args['headers'] = {
             r'X-HgProto-1': r'cbor',
         }
 
         args['headers'].update(
             encodevalueinheaders(' '.join(sorted(API_PEERS)),
                                  'X-HgUpgrade',
                                  # We don't know the header limit this early.
                                  # So make it small.
                                  1024))
 
     req, requrl, qs = makev1commandrequest(ui, requestbuilder, caps,
                                            capable, url, 'capabilities',
                                            args)
     resp = sendrequest(ui, opener, req)
 
     # The server may redirect us to the repo root, stripping the
     # ?cmd=capabilities query string from the URL. The server would likely
     # return HTML in this case and ``parsev1commandresponse()`` would raise.
     # We catch this special case and re-issue the capabilities request against
     # the new URL.
     #
     # We should ideally not do this, as a redirect that drops the query
     # string from the URL is arguably a server bug. (Garbage in, garbage out).
     # However,  Mercurial clients for several years appeared to handle this
     # issue without behavior degradation. And according to issue 5860, it may
     # be a longstanding bug in some server implementations. So we allow a
     # redirect that drops the query string to "just work."
     try:
         respurl, ct, resp = parsev1commandresponse(ui, url, requrl, qs, resp,
                                                    compressible=False,
                                                    allowcbor=advertisev2)
     except RedirectedRepoError as e:
         req, requrl, qs = makev1commandrequest(ui, requestbuilder, caps,
                                                capable, e.respurl,
                                                'capabilities', args)
         resp = sendrequest(ui, opener, req)
         respurl, ct, resp = parsev1commandresponse(ui, url, requrl, qs, resp,
                                                    compressible=False,
                                                    allowcbor=advertisev2)
 
     try:
         rawdata = resp.read()
     finally:
         resp.close()
 
     if not ct.startswith('application/mercurial-'):
         raise error.ProgrammingError('unexpected content-type: %s' % ct)
 
     if advertisev2:
         if ct == 'application/mercurial-cbor':
             try:
                 info = cborutil.decodeall(rawdata)[0]
             except cborutil.CBORDecodeError:
                 raise error.Abort(_('error decoding CBOR from remote server'),
                                   hint=_('try again and consider contacting '
                                          'the server operator'))
 
         # We got a legacy response. That's fine.
         elif ct in ('application/mercurial-0.1', 'application/mercurial-0.2'):
             info = {
                 'v1capabilities': set(rawdata.split())
             }
 
         else:
             raise error.RepoError(
                 _('unexpected response type from server: %s') % ct)
     else:
         info = {
             'v1capabilities': set(rawdata.split())
         }
 
     return respurl, info
 
 def makepeer(ui, path, opener=None, requestbuilder=urlreq.request):
     """Construct an appropriate HTTP peer instance.
 
     ``opener`` is an ``url.opener`` that should be used to establish
     connections, perform HTTP requests.
 
     ``requestbuilder`` is the type used for constructing HTTP requests.
     It exists as an argument so extensions can override the default.
     """
     u = util.url(path)
     if u.query or u.fragment:
         raise error.Abort(_('unsupported URL component: "%s"') %
                           (u.query or u.fragment))
 
     # urllib cannot handle URLs with embedded user or passwd.
     url, authinfo = u.authinfo()
     ui.debug('using %s\n' % url)
 
     opener = opener or urlmod.opener(ui, authinfo)
 
     respurl, info = performhandshake(ui, url, opener, requestbuilder)
 
     # Given the intersection of APIs that both we and the server support,
     # sort by their advertised priority and pick the first one.
     #
     # TODO consider making this request-based and interface driven. For
     # example, the caller could say "I want a peer that does X." It's quite
     # possible that not all peers would do that. Since we know the service
     # capabilities, we could filter out services not meeting the
     # requirements. Possibly by consulting the interfaces defined by the
     # peer type.
     apipeerchoices = set(info.get('apis', {}).keys()) & set(API_PEERS.keys())
 
     preferredchoices = sorted(apipeerchoices,
                               key=lambda x: API_PEERS[x]['priority'],
                               reverse=True)
 
     for service in preferredchoices:
         apipath = '%s/%s' % (info['apibase'].rstrip('/'), service)
 
         return API_PEERS[service]['init'](ui, respurl, apipath, opener,
                                           requestbuilder,
                                           info['apis'][service])
 
     # Failed to construct an API peer. Fall back to legacy.
     return httppeer(ui, path, respurl, opener, requestbuilder,
                     info['v1capabilities'])
 
 def instance(ui, path, create, intents=None, createopts=None):
     if create:
         raise error.Abort(_('cannot create new http repository'))
     try:
         if path.startswith('https:') and not urlmod.has_https:
             raise error.Abort(_('Python support for SSL and HTTPS '
                                 'is not installed'))
 
         inst = makepeer(ui, path)
 
         return inst
     except error.RepoError as httpexception:
         try:
             r = statichttprepo.instance(ui, "static-" + path, create)
             ui.note(_('(falling back to static-http)\n'))
             return r
         except error.RepoError:
             raise httpexception # use the original http RepoError instead

mercurial/wireprotoframing.py

@@ -1,1600 +1,1602
 # wireprotoframing.py - unified framing protocol for wire protocol
 #
 # Copyright 2018 Gregory Szorc <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 # This file contains functionality to support the unified frame-based wire
 # protocol. For details about the protocol, see
 # `hg help internals.wireprotocol`.
 
 from __future__ import absolute_import
 
 import collections
 import struct
 
 from .i18n import _
 from .thirdparty import (
     attr,
 )
 from . import (
     encoding,
     error,
     pycompat,
     util,
     wireprototypes,
 )
 from .utils import (
     cborutil,
     stringutil,
 )
 
 FRAME_HEADER_SIZE = 8
 DEFAULT_MAX_FRAME_SIZE = 32768
 
 STREAM_FLAG_BEGIN_STREAM = 0x01
 STREAM_FLAG_END_STREAM = 0x02
 STREAM_FLAG_ENCODING_APPLIED = 0x04
 
 STREAM_FLAGS = {
     b'stream-begin': STREAM_FLAG_BEGIN_STREAM,
     b'stream-end': STREAM_FLAG_END_STREAM,
     b'encoded': STREAM_FLAG_ENCODING_APPLIED,
 }
 
 FRAME_TYPE_COMMAND_REQUEST = 0x01
 FRAME_TYPE_COMMAND_DATA = 0x02
 FRAME_TYPE_COMMAND_RESPONSE = 0x03
 FRAME_TYPE_ERROR_RESPONSE = 0x05
 FRAME_TYPE_TEXT_OUTPUT = 0x06
 FRAME_TYPE_PROGRESS = 0x07
 FRAME_TYPE_SENDER_PROTOCOL_SETTINGS = 0x08
 FRAME_TYPE_STREAM_SETTINGS = 0x09
 
 FRAME_TYPES = {
     b'command-request': FRAME_TYPE_COMMAND_REQUEST,
     b'command-data': FRAME_TYPE_COMMAND_DATA,
     b'command-response': FRAME_TYPE_COMMAND_RESPONSE,
     b'error-response': FRAME_TYPE_ERROR_RESPONSE,
     b'text-output': FRAME_TYPE_TEXT_OUTPUT,
     b'progress': FRAME_TYPE_PROGRESS,
     b'sender-protocol-settings': FRAME_TYPE_SENDER_PROTOCOL_SETTINGS,
     b'stream-settings': FRAME_TYPE_STREAM_SETTINGS,
 }
 
 FLAG_COMMAND_REQUEST_NEW = 0x01
 FLAG_COMMAND_REQUEST_CONTINUATION = 0x02
 FLAG_COMMAND_REQUEST_MORE_FRAMES = 0x04
 FLAG_COMMAND_REQUEST_EXPECT_DATA = 0x08
 
 FLAGS_COMMAND_REQUEST = {
     b'new': FLAG_COMMAND_REQUEST_NEW,
     b'continuation': FLAG_COMMAND_REQUEST_CONTINUATION,
     b'more': FLAG_COMMAND_REQUEST_MORE_FRAMES,
     b'have-data': FLAG_COMMAND_REQUEST_EXPECT_DATA,
 }
 
 FLAG_COMMAND_DATA_CONTINUATION = 0x01
 FLAG_COMMAND_DATA_EOS = 0x02
 
 FLAGS_COMMAND_DATA = {
     b'continuation': FLAG_COMMAND_DATA_CONTINUATION,
     b'eos': FLAG_COMMAND_DATA_EOS,
 }
 
 FLAG_COMMAND_RESPONSE_CONTINUATION = 0x01
 FLAG_COMMAND_RESPONSE_EOS = 0x02
 
 FLAGS_COMMAND_RESPONSE = {
     b'continuation': FLAG_COMMAND_RESPONSE_CONTINUATION,
     b'eos': FLAG_COMMAND_RESPONSE_EOS,
 }
 
 FLAG_SENDER_PROTOCOL_SETTINGS_CONTINUATION = 0x01
 FLAG_SENDER_PROTOCOL_SETTINGS_EOS = 0x02
 
 FLAGS_SENDER_PROTOCOL_SETTINGS = {
     b'continuation': FLAG_SENDER_PROTOCOL_SETTINGS_CONTINUATION,
     b'eos': FLAG_SENDER_PROTOCOL_SETTINGS_EOS,
 }
 
 FLAG_STREAM_ENCODING_SETTINGS_CONTINUATION = 0x01
 FLAG_STREAM_ENCODING_SETTINGS_EOS = 0x02
 
 FLAGS_STREAM_ENCODING_SETTINGS = {
     b'continuation': FLAG_STREAM_ENCODING_SETTINGS_CONTINUATION,
     b'eos': FLAG_STREAM_ENCODING_SETTINGS_EOS,
 }
 
 # Maps frame types to their available flags.
 FRAME_TYPE_FLAGS = {
     FRAME_TYPE_COMMAND_REQUEST: FLAGS_COMMAND_REQUEST,
     FRAME_TYPE_COMMAND_DATA: FLAGS_COMMAND_DATA,
     FRAME_TYPE_COMMAND_RESPONSE: FLAGS_COMMAND_RESPONSE,
     FRAME_TYPE_ERROR_RESPONSE: {},
     FRAME_TYPE_TEXT_OUTPUT: {},
     FRAME_TYPE_PROGRESS: {},
     FRAME_TYPE_SENDER_PROTOCOL_SETTINGS: FLAGS_SENDER_PROTOCOL_SETTINGS,
     FRAME_TYPE_STREAM_SETTINGS: FLAGS_STREAM_ENCODING_SETTINGS,
 }
 
 ARGUMENT_RECORD_HEADER = struct.Struct(r'<HH')
 
 def humanflags(mapping, value):
     """Convert a numeric flags value to a human value, using a mapping table."""
     namemap = {v: k for k, v in mapping.iteritems()}
     flags = []
     val = 1
     while value >= val:
         if value & val:
             flags.append(namemap.get(val, '<unknown 0x%02x>' % val))
         val <<= 1
 
     return b'|'.join(flags)
 
 @attr.s(slots=True)
 class frameheader(object):
     """Represents the data in a frame header."""
 
     length = attr.ib()
     requestid = attr.ib()
     streamid = attr.ib()
     streamflags = attr.ib()
     typeid = attr.ib()
     flags = attr.ib()
 
 @attr.s(slots=True, repr=False)
 class frame(object):
     """Represents a parsed frame."""
 
     requestid = attr.ib()
     streamid = attr.ib()
     streamflags = attr.ib()
     typeid = attr.ib()
     flags = attr.ib()
     payload = attr.ib()
 
     @encoding.strmethod
     def __repr__(self):
         typename = '<unknown 0x%02x>' % self.typeid
         for name, value in FRAME_TYPES.iteritems():
             if value == self.typeid:
                 typename = name
                 break
 
         return ('frame(size=%d; request=%d; stream=%d; streamflags=%s; '
                 'type=%s; flags=%s)' % (
             len(self.payload), self.requestid, self.streamid,
             humanflags(STREAM_FLAGS, self.streamflags), typename,
             humanflags(FRAME_TYPE_FLAGS.get(self.typeid, {}), self.flags)))
 
 def makeframe(requestid, streamid, streamflags, typeid, flags, payload):
     """Assemble a frame into a byte array."""
     # TODO assert size of payload.
     frame = bytearray(FRAME_HEADER_SIZE + len(payload))
 
     # 24 bits length
     # 16 bits request id
     # 8 bits stream id
     # 8 bits stream flags
     # 4 bits type
     # 4 bits flags
 
     l = struct.pack(r'<I', len(payload))
     frame[0:3] = l[0:3]
     struct.pack_into(r'<HBB', frame, 3, requestid, streamid, streamflags)
     frame[7] = (typeid << 4) | flags
     frame[8:] = payload
 
     return frame
 
 def makeframefromhumanstring(s):
     """Create a frame from a human readable string
 
     Strings have the form:
 
         <request-id> <stream-id> <stream-flags> <type> <flags> <payload>
 
     This can be used by user-facing applications and tests for creating
     frames easily without having to type out a bunch of constants.
 
     Request ID and stream IDs are integers.
 
     Stream flags, frame type, and flags can be specified by integer or
     named constant.
 
     Flags can be delimited by `|` to bitwise OR them together.
 
     If the payload begins with ``cbor:``, the following string will be
     evaluated as Python literal and the resulting object will be fed into
     a CBOR encoder. Otherwise, the payload is interpreted as a Python
     byte string literal.
     """
     fields = s.split(b' ', 5)
     requestid, streamid, streamflags, frametype, frameflags, payload = fields
 
     requestid = int(requestid)
     streamid = int(streamid)
 
     finalstreamflags = 0
     for flag in streamflags.split(b'|'):
         if flag in STREAM_FLAGS:
             finalstreamflags |= STREAM_FLAGS[flag]
         else:
             finalstreamflags |= int(flag)
 
     if frametype in FRAME_TYPES:
         frametype = FRAME_TYPES[frametype]
     else:
         frametype = int(frametype)
 
     finalflags = 0
     validflags = FRAME_TYPE_FLAGS[frametype]
     for flag in frameflags.split(b'|'):
         if flag in validflags:
             finalflags |= validflags[flag]
         else:
             finalflags |= int(flag)
 
     if payload.startswith(b'cbor:'):
         payload = b''.join(cborutil.streamencode(
             stringutil.evalpythonliteral(payload[5:])))
 
     else:
         payload = stringutil.unescapestr(payload)
 
     return makeframe(requestid=requestid, streamid=streamid,
                      streamflags=finalstreamflags, typeid=frametype,
                      flags=finalflags, payload=payload)
 
 def parseheader(data):
     """Parse a unified framing protocol frame header from a buffer.
 
     The header is expected to be in the buffer at offset 0 and the
     buffer is expected to be large enough to hold a full header.
     """
     # 24 bits payload length (little endian)
     # 16 bits request ID
     # 8 bits stream ID
     # 8 bits stream flags
     # 4 bits frame type
     # 4 bits frame flags
     # ... payload
     framelength = data[0] + 256 * data[1] + 16384 * data[2]
     requestid, streamid, streamflags = struct.unpack_from(r'<HBB', data, 3)
     typeflags = data[7]
 
     frametype = (typeflags & 0xf0) >> 4
     frameflags = typeflags & 0x0f
 
     return frameheader(framelength, requestid, streamid, streamflags,
                        frametype, frameflags)
 
 def readframe(fh):
     """Read a unified framing protocol frame from a file object.
 
     Returns a 3-tuple of (type, flags, payload) for the decoded frame or
     None if no frame is available. May raise if a malformed frame is
     seen.
     """
     header = bytearray(FRAME_HEADER_SIZE)
 
     readcount = fh.readinto(header)
 
     if readcount == 0:
         return None
 
     if readcount != FRAME_HEADER_SIZE:
         raise error.Abort(_('received incomplete frame: got %d bytes: %s') %
                           (readcount, header))
 
     h = parseheader(header)
 
     payload = fh.read(h.length)
     if len(payload) != h.length:
         raise error.Abort(_('frame length error: expected %d; got %d') %
                           (h.length, len(payload)))
 
     return frame(h.requestid, h.streamid, h.streamflags, h.typeid, h.flags,
                  payload)
 
 def createcommandframes(stream, requestid, cmd, args, datafh=None,
                         maxframesize=DEFAULT_MAX_FRAME_SIZE,
                         redirect=None):
     """Create frames necessary to transmit a request to run a command.
 
     This is a generator of bytearrays. Each item represents a frame
     ready to be sent over the wire to a peer.
     """
     data = {b'name': cmd}
     if args:
         data[b'args'] = args
 
     if redirect:
         data[b'redirect'] = redirect
 
     data = b''.join(cborutil.streamencode(data))
 
     offset = 0
 
     while True:
         flags = 0
 
         # Must set new or continuation flag.
         if not offset:
             flags |= FLAG_COMMAND_REQUEST_NEW
         else:
             flags |= FLAG_COMMAND_REQUEST_CONTINUATION
 
         # Data frames is set on all frames.
         if datafh:
             flags |= FLAG_COMMAND_REQUEST_EXPECT_DATA
 
         payload = data[offset:offset + maxframesize]
         offset += len(payload)
 
         if len(payload) == maxframesize and offset < len(data):
             flags |= FLAG_COMMAND_REQUEST_MORE_FRAMES
 
         yield stream.makeframe(requestid=requestid,
                                typeid=FRAME_TYPE_COMMAND_REQUEST,
                                flags=flags,
                                payload=payload)
 
         if not (flags & FLAG_COMMAND_REQUEST_MORE_FRAMES):
             break
 
     if datafh:
         while True:
             data = datafh.read(DEFAULT_MAX_FRAME_SIZE)
 
             done = False
             if len(data) == DEFAULT_MAX_FRAME_SIZE:
                 flags = FLAG_COMMAND_DATA_CONTINUATION
             else:
                 flags = FLAG_COMMAND_DATA_EOS
                 assert datafh.read(1) == b''
                 done = True
 
             yield stream.makeframe(requestid=requestid,
                                    typeid=FRAME_TYPE_COMMAND_DATA,
                                    flags=flags,
                                    payload=data)
 
             if done:
                 break
 
 def createcommandresponseframesfrombytes(stream, requestid, data,
                                          maxframesize=DEFAULT_MAX_FRAME_SIZE):
     """Create a raw frame to send a bytes response from static bytes input.
 
     Returns a generator of bytearrays.
     """
     # Automatically send the overall CBOR response map.
     overall = b''.join(cborutil.streamencode({b'status': b'ok'}))
     if len(overall) > maxframesize:
         raise error.ProgrammingError('not yet implemented')
 
     # Simple case where we can fit the full response in a single frame.
     if len(overall) + len(data) <= maxframesize:
         flags = FLAG_COMMAND_RESPONSE_EOS
         yield stream.makeframe(requestid=requestid,
                                typeid=FRAME_TYPE_COMMAND_RESPONSE,
                                flags=flags,
                                payload=overall + data)
         return
 
     # It's easier to send the overall CBOR map in its own frame than to track
     # offsets.
     yield stream.makeframe(requestid=requestid,
                            typeid=FRAME_TYPE_COMMAND_RESPONSE,
                            flags=FLAG_COMMAND_RESPONSE_CONTINUATION,
                            payload=overall)
 
     offset = 0
     while True:
         chunk = data[offset:offset + maxframesize]
         offset += len(chunk)
         done = offset == len(data)
 
         if done:
             flags = FLAG_COMMAND_RESPONSE_EOS
         else:
             flags = FLAG_COMMAND_RESPONSE_CONTINUATION
 
         yield stream.makeframe(requestid=requestid,
                                typeid=FRAME_TYPE_COMMAND_RESPONSE,
                                flags=flags,
                                payload=chunk)
 
         if done:
             break
 
 def createbytesresponseframesfromgen(stream, requestid, gen,
                                      maxframesize=DEFAULT_MAX_FRAME_SIZE):
     """Generator of frames from a generator of byte chunks.
 
     This assumes that another frame will follow whatever this emits. i.e.
     this always emits the continuation flag and never emits the end-of-stream
     flag.
     """
     cb = util.chunkbuffer(gen)
     flags = FLAG_COMMAND_RESPONSE_CONTINUATION
 
     while True:
         chunk = cb.read(maxframesize)
         if not chunk:
             break
 
         yield stream.makeframe(requestid=requestid,
                                typeid=FRAME_TYPE_COMMAND_RESPONSE,
                                flags=flags,
                                payload=chunk)
 
         flags |= FLAG_COMMAND_RESPONSE_CONTINUATION
 
 def createcommandresponseokframe(stream, requestid):
     overall = b''.join(cborutil.streamencode({b'status': b'ok'}))
 
     return stream.makeframe(requestid=requestid,
                             typeid=FRAME_TYPE_COMMAND_RESPONSE,
                             flags=FLAG_COMMAND_RESPONSE_CONTINUATION,
                             payload=overall)
 
 def createcommandresponseeosframe(stream, requestid):
     """Create an empty payload frame representing command end-of-stream."""
     return stream.makeframe(requestid=requestid,
                             typeid=FRAME_TYPE_COMMAND_RESPONSE,
                             flags=FLAG_COMMAND_RESPONSE_EOS,
                             payload=b'')
 
 def createalternatelocationresponseframe(stream, requestid, location):
     data = {
         b'status': b'redirect',
         b'location': {
             b'url': location.url,
             b'mediatype': location.mediatype,
         }
     }
 
     for a in (r'size', r'fullhashes', r'fullhashseed', r'serverdercerts',
               r'servercadercerts'):
         value = getattr(location, a)
         if value is not None:
             data[b'location'][pycompat.bytestr(a)] = value
 
     return stream.makeframe(requestid=requestid,
                             typeid=FRAME_TYPE_COMMAND_RESPONSE,
                             flags=FLAG_COMMAND_RESPONSE_CONTINUATION,
                             payload=b''.join(cborutil.streamencode(data)))
 
 def createcommanderrorresponse(stream, requestid, message, args=None):
     # TODO should this be using a list of {'msg': ..., 'args': {}} so atom
     # formatting works consistently?
     m = {
         b'status': b'error',
         b'error': {
             b'message': message,
         }
     }
 
     if args:
         m[b'error'][b'args'] = args
 
     overall = b''.join(cborutil.streamencode(m))
 
     yield stream.makeframe(requestid=requestid,
                            typeid=FRAME_TYPE_COMMAND_RESPONSE,
                            flags=FLAG_COMMAND_RESPONSE_EOS,
                            payload=overall)
 
 def createerrorframe(stream, requestid, msg, errtype):
     # TODO properly handle frame size limits.
     assert len(msg) <= DEFAULT_MAX_FRAME_SIZE
 
     payload = b''.join(cborutil.streamencode({
         b'type': errtype,
         b'message': [{b'msg': msg}],
     }))
 
     yield stream.makeframe(requestid=requestid,
                            typeid=FRAME_TYPE_ERROR_RESPONSE,
                            flags=0,
                            payload=payload)
 
 def createtextoutputframe(stream, requestid, atoms,
                           maxframesize=DEFAULT_MAX_FRAME_SIZE):
     """Create a text output frame to render text to people.
 
     ``atoms`` is a 3-tuple of (formatting string, args, labels).
 
     The formatting string contains ``%s`` tokens to be replaced by the
     corresponding indexed entry in ``args``. ``labels`` is an iterable of
     formatters to be applied at rendering time. In terms of the ``ui``
     class, each atom corresponds to a ``ui.write()``.
     """
     atomdicts = []
 
     for (formatting, args, labels) in atoms:
         # TODO look for localstr, other types here?
 
         if not isinstance(formatting, bytes):
             raise ValueError('must use bytes formatting strings')
         for arg in args:
             if not isinstance(arg, bytes):
                 raise ValueError('must use bytes for arguments')
         for label in labels:
             if not isinstance(label, bytes):
                 raise ValueError('must use bytes for labels')
 
         # Formatting string must be ASCII.
         formatting = formatting.decode(r'ascii', r'replace').encode(r'ascii')
 
         # Arguments must be UTF-8.
         args = [a.decode(r'utf-8', r'replace').encode(r'utf-8') for a in args]
 
         # Labels must be ASCII.
         labels = [l.decode(r'ascii', r'strict').encode(r'ascii')
                   for l in labels]
 
         atom = {b'msg': formatting}
         if args:
             atom[b'args'] = args
         if labels:
             atom[b'labels'] = labels
 
         atomdicts.append(atom)
 
     payload = b''.join(cborutil.streamencode(atomdicts))
 
     if len(payload) > maxframesize:
         raise ValueError('cannot encode data in a single frame')
 
     yield stream.makeframe(requestid=requestid,
                            typeid=FRAME_TYPE_TEXT_OUTPUT,
                            flags=0,
                            payload=payload)
 
 class bufferingcommandresponseemitter(object):
     """Helper object to emit command response frames intelligently.
 
     Raw command response data is likely emitted in chunks much smaller
     than what can fit in a single frame. This class exists to buffer
     chunks until enough data is available to fit in a single frame.
 
     TODO we'll need something like this when compression is supported.
     So it might make sense to implement this functionality at the stream
     level.
     """
     def __init__(self, stream, requestid, maxframesize=DEFAULT_MAX_FRAME_SIZE):
         self._stream = stream
         self._requestid = requestid
         self._maxsize = maxframesize
         self._chunks = []
         self._chunkssize = 0
 
     def send(self, data):
         """Send new data for emission.
 
         Is a generator of new frames that were derived from the new input.
 
         If the special input ``None`` is received, flushes all buffered
         data to frames.
         """
 
         if data is None:
             for frame in self._flush():
                 yield frame
             return
 
         # There is a ton of potential to do more complicated things here.
         # Our immediate goal is to coalesce small chunks into big frames,
         # not achieve the fewest number of frames possible. So we go with
         # a simple implementation:
         #
         # * If a chunk is too large for a frame, we flush and emit frames
         #   for the new chunk.
         # * If a chunk can be buffered without total buffered size limits
         #   being exceeded, we do that.
         # * If a chunk causes us to go over our buffering limit, we flush
         #   and then buffer the new chunk.
 
         if len(data) > self._maxsize:
             for frame in self._flush():
                 yield frame
 
             # Now emit frames for the big chunk.
             offset = 0
             while True:
                 chunk = data[offset:offset + self._maxsize]
                 offset += len(chunk)
 
                 yield self._stream.makeframe(
                     self._requestid,
                     typeid=FRAME_TYPE_COMMAND_RESPONSE,
                     flags=FLAG_COMMAND_RESPONSE_CONTINUATION,
                     payload=chunk)
 
                 if offset == len(data):
                     return
 
         # If we don't have enough to constitute a full frame, buffer and
         # return.
         if len(data) + self._chunkssize < self._maxsize:
             self._chunks.append(data)
             self._chunkssize += len(data)
             return
 
         # Else flush what we have and buffer the new chunk. We could do
         # something more intelligent here, like break the chunk. Let's
         # keep things simple for now.
         for frame in self._flush():
             yield frame
 
         self._chunks.append(data)
         self._chunkssize = len(data)
 
     def _flush(self):
         payload = b''.join(self._chunks)
         assert len(payload) <= self._maxsize
 
         self._chunks[:] = []
         self._chunkssize = 0
 
         yield self._stream.makeframe(
             self._requestid,
             typeid=FRAME_TYPE_COMMAND_RESPONSE,
             flags=FLAG_COMMAND_RESPONSE_CONTINUATION,
             payload=payload)
 
 class stream(object):
     """Represents a logical unidirectional series of frames."""
 
     def __init__(self, streamid, active=False):
         self.streamid = streamid
         self._active = active
 
     def makeframe(self, requestid, typeid, flags, payload):
         """Create a frame to be sent out over this stream.
 
         Only returns the frame instance. Does not actually send it.
         """
         streamflags = 0
         if not self._active:
             streamflags |= STREAM_FLAG_BEGIN_STREAM
             self._active = True
 
         return makeframe(requestid, self.streamid, streamflags, typeid, flags,
                          payload)
 
     def setdecoder(self, name, extraobjs):
         """Set the decoder for this stream.
 
         Receives the stream profile name and any additional CBOR objects
         decoded from the stream encoding settings frame payloads.
         """
 
 def ensureserverstream(stream):
     if stream.streamid % 2:
         raise error.ProgrammingError('server should only write to even '
                                      'numbered streams; %d is not even' %
                                      stream.streamid)
 
 DEFAULT_PROTOCOL_SETTINGS = {
     'contentencodings': [b'identity'],
 }
 
 class serverreactor(object):
     """Holds state of a server handling frame-based protocol requests.
 
     This class is the "brain" of the unified frame-based protocol server
     component. While the protocol is stateless from the perspective of
     requests/commands, something needs to track which frames have been
     received, what frames to expect, etc. This class is that thing.
 
     Instances are modeled as a state machine of sorts. Instances are also
     reactionary to external events. The point of this class is to encapsulate
     the state of the connection and the exchange of frames, not to perform
     work. Instead, callers tell this class when something occurs, like a
     frame arriving. If that activity is worthy of a follow-up action (say
     *run a command*), the return value of that handler will say so.
 
     I/O and CPU intensive operations are purposefully delegated outside of
     this class.
 
     Consumers are expected to tell instances when events occur. They do so by
     calling the various ``on*`` methods. These methods return a 2-tuple
     describing any follow-up action(s) to take. The first element is the
     name of an action to perform. The second is a data structure (usually
     a dict) specific to that action that contains more information. e.g.
     if the server wants to send frames back to the client, the data structure
     will contain a reference to those frames.
 
     Valid actions that consumers can be instructed to take are:
 
     sendframes
        Indicates that frames should be sent to the client. The ``framegen``
        key contains a generator of frames that should be sent. The server
        assumes that all frames are sent to the client.
 
     error
        Indicates that an error occurred. Consumer should probably abort.
 
     runcommand
        Indicates that the consumer should run a wire protocol command. Details
        of the command to run are given in the data structure.
 
     wantframe
        Indicates that nothing of interest happened and the server is waiting on
        more frames from the client before anything interesting can be done.
 
     noop
        Indicates no additional action is required.
 
     Known Issues
     ------------
 
     There are no limits to the number of partially received commands or their
     size. A malicious client could stream command request data and exhaust the
     server's memory.
 
     Partially received commands are not acted upon when end of input is
     reached. Should the server error if it receives a partial request?
     Should the client send a message to abort a partially transmitted request
     to facilitate graceful shutdown?
 
     Active requests that haven't been responded to aren't tracked. This means
     that if we receive a command and instruct its dispatch, another command
     with its request ID can come in over the wire and there will be a race
     between who responds to what.
     """
 
-    def __init__(self, deferoutput=False):
+    def __init__(self, ui, deferoutput=False):
         """Construct a new server reactor.
 
         ``deferoutput`` can be used to indicate that no output frames should be
         instructed to be sent until input has been exhausted. In this mode,
         events that would normally generate output frames (such as a command
         response being ready) will instead defer instructing the consumer to
         send those frames. This is useful for half-duplex transports where the
         sender cannot receive until all data has been transmitted.
         """
+        self._ui = ui
         self._deferoutput = deferoutput
         self._state = 'initial'
         self._nextoutgoingstreamid = 2
         self._bufferedframegens = []
         # stream id -> stream instance for all active streams from the client.
         self._incomingstreams = {}
         self._outgoingstreams = {}
         # request id -> dict of commands that are actively being received.
         self._receivingcommands = {}
         # Request IDs that have been received and are actively being processed.
         # Once all output for a request has been sent, it is removed from this
         # set.
         self._activecommands = set()
 
         self._protocolsettingsdecoder = None
 
         # Sender protocol settings are optional. Set implied default values.
         self._sendersettings = dict(DEFAULT_PROTOCOL_SETTINGS)
 
     def onframerecv(self, frame):
         """Process a frame that has been received off the wire.
 
         Returns a dict with an ``action`` key that details what action,
         if any, the consumer should take next.
         """
         if not frame.streamid % 2:
             self._state = 'errored'
             return self._makeerrorresult(
                 _('received frame with even numbered stream ID: %d') %
                   frame.streamid)
 
         if frame.streamid not in self._incomingstreams:
             if not frame.streamflags & STREAM_FLAG_BEGIN_STREAM:
                 self._state = 'errored'
                 return self._makeerrorresult(
                     _('received frame on unknown inactive stream without '
                       'beginning of stream flag set'))
 
             self._incomingstreams[frame.streamid] = stream(frame.streamid)
 
         if frame.streamflags & STREAM_FLAG_ENCODING_APPLIED:
             # TODO handle decoding frames
             self._state = 'errored'
             raise error.ProgrammingError('support for decoding stream payloads '
                                          'not yet implemented')
 
         if frame.streamflags & STREAM_FLAG_END_STREAM:
             del self._incomingstreams[frame.streamid]
 
         handlers = {
             'initial': self._onframeinitial,
             'protocol-settings-receiving': self._onframeprotocolsettings,
             'idle': self._onframeidle,
             'command-receiving': self._onframecommandreceiving,
             'errored': self._onframeerrored,
         }
 
         meth = handlers.get(self._state)
         if not meth:
             raise error.ProgrammingError('unhandled state: %s' % self._state)
 
         return meth(frame)
 
     def oncommandresponseready(self, stream, requestid, data):
         """Signal that a bytes response is ready to be sent to the client.
 
         The raw bytes response is passed as an argument.
         """
         ensureserverstream(stream)
 
         def sendframes():
             for frame in createcommandresponseframesfrombytes(stream, requestid,
                                                               data):
                 yield frame
 
             self._activecommands.remove(requestid)
 
         result = sendframes()
 
         if self._deferoutput:
             self._bufferedframegens.append(result)
             return 'noop', {}
         else:
             return 'sendframes', {
                 'framegen': result,
             }
 
     def oncommandresponsereadyobjects(self, stream, requestid, objs):
         """Signal that objects are ready to be sent to the client.
 
         ``objs`` is an iterable of objects (typically a generator) that will
         be encoded via CBOR and added to frames, which will be sent to the
         client.
         """
         ensureserverstream(stream)
 
         # We need to take care over exception handling. Uncaught exceptions
         # when generating frames could lead to premature end of the frame
         # stream and the possibility of the server or client process getting
         # in a bad state.
         #
         # Keep in mind that if ``objs`` is a generator, advancing it could
         # raise exceptions that originated in e.g. wire protocol command
         # functions. That is why we differentiate between exceptions raised
         # when iterating versus other exceptions that occur.
         #
         # In all cases, when the function finishes, the request is fully
         # handled and no new frames for it should be seen.
 
         def sendframes():
             emitted = False
             alternatelocationsent = False
             emitter = bufferingcommandresponseemitter(stream, requestid)
             while True:
                 try:
                     o = next(objs)
                 except StopIteration:
                     for frame in emitter.send(None):
                         yield frame
 
                     if emitted:
                         yield createcommandresponseeosframe(stream, requestid)
                     break
 
                 except error.WireprotoCommandError as e:
                     for frame in createcommanderrorresponse(
                         stream, requestid, e.message, e.messageargs):
                         yield frame
                     break
 
                 except Exception as e:
                     for frame in createerrorframe(
                         stream, requestid, '%s' % stringutil.forcebytestr(e),
                         errtype='server'):
 
                         yield frame
 
                     break
 
                 try:
                     # Alternate location responses can only be the first and
                     # only object in the output stream.
                     if isinstance(o, wireprototypes.alternatelocationresponse):
                         if emitted:
                             raise error.ProgrammingError(
                                 'alternatelocationresponse seen after initial '
                                 'output object')
 
                         yield createalternatelocationresponseframe(
                             stream, requestid, o)
 
                         alternatelocationsent = True
                         emitted = True
                         continue
 
                     if alternatelocationsent:
                         raise error.ProgrammingError(
                             'object follows alternatelocationresponse')
 
                     if not emitted:
                         yield createcommandresponseokframe(stream, requestid)
                         emitted = True
 
                     # Objects emitted by command functions can be serializable
                     # data structures or special types.
                     # TODO consider extracting the content normalization to a
                     # standalone function, as it may be useful for e.g. cachers.
 
                     # A pre-encoded object is sent directly to the emitter.
                     if isinstance(o, wireprototypes.encodedresponse):
                         for frame in emitter.send(o.data):
                             yield frame
 
                     # A regular object is CBOR encoded.
                     else:
                         for chunk in cborutil.streamencode(o):
                             for frame in emitter.send(chunk):
                                 yield frame
 
                 except Exception as e:
                     for frame in createerrorframe(stream, requestid,
                                                   '%s' % e,
                                                   errtype='server'):
                         yield frame
 
                     break
 
             self._activecommands.remove(requestid)
 
         return self._handlesendframes(sendframes())
 
     def oninputeof(self):
         """Signals that end of input has been received.
 
         No more frames will be received. All pending activity should be
         completed.
         """
         # TODO should we do anything about in-flight commands?
 
         if not self._deferoutput or not self._bufferedframegens:
             return 'noop', {}
 
         # If we buffered all our responses, emit those.
         def makegen():
             for gen in self._bufferedframegens:
                 for frame in gen:
                     yield frame
 
         return 'sendframes', {
             'framegen': makegen(),
         }
 
     def _handlesendframes(self, framegen):
         if self._deferoutput:
             self._bufferedframegens.append(framegen)
             return 'noop', {}
         else:
             return 'sendframes', {
                 'framegen': framegen,
             }
 
     def onservererror(self, stream, requestid, msg):
         ensureserverstream(stream)
 
         def sendframes():
             for frame in createerrorframe(stream, requestid, msg,
                                           errtype='server'):
                 yield frame
 
             self._activecommands.remove(requestid)
 
         return self._handlesendframes(sendframes())
 
     def oncommanderror(self, stream, requestid, message, args=None):
         """Called when a command encountered an error before sending output."""
         ensureserverstream(stream)
 
         def sendframes():
             for frame in createcommanderrorresponse(stream, requestid, message,
                                                     args):
                 yield frame
 
             self._activecommands.remove(requestid)
 
         return self._handlesendframes(sendframes())
 
     def makeoutputstream(self):
         """Create a stream to be used for sending data to the client."""
         streamid = self._nextoutgoingstreamid
         self._nextoutgoingstreamid += 2
 
         s = stream(streamid)
         self._outgoingstreams[streamid] = s
 
         return s
 
     def _makeerrorresult(self, msg):
         return 'error', {
             'message': msg,
         }
 
     def _makeruncommandresult(self, requestid):
         entry = self._receivingcommands[requestid]
 
         if not entry['requestdone']:
             self._state = 'errored'
             raise error.ProgrammingError('should not be called without '
                                          'requestdone set')
 
         del self._receivingcommands[requestid]
 
         if self._receivingcommands:
             self._state = 'command-receiving'
         else:
             self._state = 'idle'
 
         # Decode the payloads as CBOR.
         entry['payload'].seek(0)
         request = cborutil.decodeall(entry['payload'].getvalue())[0]
 
         if b'name' not in request:
             self._state = 'errored'
             return self._makeerrorresult(
                 _('command request missing "name" field'))
 
         if b'args' not in request:
             request[b'args'] = {}
 
         assert requestid not in self._activecommands
         self._activecommands.add(requestid)
 
         return 'runcommand', {
             'requestid': requestid,
             'command': request[b'name'],
             'args': request[b'args'],
             'redirect': request.get(b'redirect'),
             'data': entry['data'].getvalue() if entry['data'] else None,
         }
 
     def _makewantframeresult(self):
         return 'wantframe', {
             'state': self._state,
         }
 
     def _validatecommandrequestframe(self, frame):
         new = frame.flags & FLAG_COMMAND_REQUEST_NEW
         continuation = frame.flags & FLAG_COMMAND_REQUEST_CONTINUATION
 
         if new and continuation:
             self._state = 'errored'
             return self._makeerrorresult(
                 _('received command request frame with both new and '
                   'continuation flags set'))
 
         if not new and not continuation:
             self._state = 'errored'
             return self._makeerrorresult(
                 _('received command request frame with neither new nor '
                   'continuation flags set'))
 
     def _onframeinitial(self, frame):
         # Called when we receive a frame when in the "initial" state.
         if frame.typeid == FRAME_TYPE_SENDER_PROTOCOL_SETTINGS:
             self._state = 'protocol-settings-receiving'
             self._protocolsettingsdecoder = cborutil.bufferingdecoder()
             return self._onframeprotocolsettings(frame)
 
         elif frame.typeid == FRAME_TYPE_COMMAND_REQUEST:
             self._state = 'idle'
             return self._onframeidle(frame)
 
         else:
             self._state = 'errored'
             return self._makeerrorresult(
                 _('expected sender protocol settings or command request '
                   'frame; got %d') % frame.typeid)
 
     def _onframeprotocolsettings(self, frame):
         assert self._state == 'protocol-settings-receiving'
         assert self._protocolsettingsdecoder is not None
 
         if frame.typeid != FRAME_TYPE_SENDER_PROTOCOL_SETTINGS:
             self._state = 'errored'
             return self._makeerrorresult(
                 _('expected sender protocol settings frame; got %d') %
                 frame.typeid)
 
         more = frame.flags & FLAG_SENDER_PROTOCOL_SETTINGS_CONTINUATION
         eos = frame.flags & FLAG_SENDER_PROTOCOL_SETTINGS_EOS
 
         if more and eos:
             self._state = 'errored'
             return self._makeerrorresult(
                 _('sender protocol settings frame cannot have both '
                   'continuation and end of stream flags set'))
 
         if not more and not eos:
             self._state = 'errored'
             return self._makeerrorresult(
                 _('sender protocol settings frame must have continuation or '
                   'end of stream flag set'))
 
         # TODO establish limits for maximum amount of data that can be
         # buffered.
         try:
             self._protocolsettingsdecoder.decode(frame.payload)
         except Exception as e:
             self._state = 'errored'
             return self._makeerrorresult(
                 _('error decoding CBOR from sender protocol settings frame: %s')
                 % stringutil.forcebytestr(e))
 
         if more:
             return self._makewantframeresult()
 
         assert eos
 
         decoded = self._protocolsettingsdecoder.getavailable()
         self._protocolsettingsdecoder = None
 
         if not decoded:
             self._state = 'errored'
             return self._makeerrorresult(
                 _('sender protocol settings frame did not contain CBOR data'))
         elif len(decoded) > 1:
             self._state = 'errored'
             return self._makeerrorresult(
                 _('sender protocol settings frame contained multiple CBOR '
                   'values'))
 
         d = decoded[0]
 
         if b'contentencodings' in d:
             self._sendersettings['contentencodings'] = d[b'contentencodings']
 
         self._state = 'idle'
 
         return self._makewantframeresult()
 
     def _onframeidle(self, frame):
         # The only frame type that should be received in this state is a
         # command request.
         if frame.typeid != FRAME_TYPE_COMMAND_REQUEST:
             self._state = 'errored'
             return self._makeerrorresult(
                 _('expected command request frame; got %d') % frame.typeid)
 
         res = self._validatecommandrequestframe(frame)
         if res:
             return res
 
         if frame.requestid in self._receivingcommands:
             self._state = 'errored'
             return self._makeerrorresult(
                 _('request with ID %d already received') % frame.requestid)
 
         if frame.requestid in self._activecommands:
             self._state = 'errored'
             return self._makeerrorresult(
                 _('request with ID %d is already active') % frame.requestid)
 
         new = frame.flags & FLAG_COMMAND_REQUEST_NEW
         moreframes = frame.flags & FLAG_COMMAND_REQUEST_MORE_FRAMES
         expectingdata = frame.flags & FLAG_COMMAND_REQUEST_EXPECT_DATA
 
         if not new:
             self._state = 'errored'
             return self._makeerrorresult(
                 _('received command request frame without new flag set'))
 
         payload = util.bytesio()
         payload.write(frame.payload)
 
         self._receivingcommands[frame.requestid] = {
             'payload': payload,
             'data': None,
             'requestdone': not moreframes,
             'expectingdata': bool(expectingdata),
         }
 
         # This is the final frame for this request. Dispatch it.
         if not moreframes and not expectingdata:
             return self._makeruncommandresult(frame.requestid)
 
         assert moreframes or expectingdata
         self._state = 'command-receiving'
         return self._makewantframeresult()
 
     def _onframecommandreceiving(self, frame):
         if frame.typeid == FRAME_TYPE_COMMAND_REQUEST:
             # Process new command requests as such.
             if frame.flags & FLAG_COMMAND_REQUEST_NEW:
                 return self._onframeidle(frame)
 
             res = self._validatecommandrequestframe(frame)
             if res:
                 return res
 
         # All other frames should be related to a command that is currently
         # receiving but is not active.
         if frame.requestid in self._activecommands:
             self._state = 'errored'
             return self._makeerrorresult(
                 _('received frame for request that is still active: %d') %
                 frame.requestid)
 
         if frame.requestid not in self._receivingcommands:
             self._state = 'errored'
             return self._makeerrorresult(
                 _('received frame for request that is not receiving: %d') %
                   frame.requestid)
 
         entry = self._receivingcommands[frame.requestid]
 
         if frame.typeid == FRAME_TYPE_COMMAND_REQUEST:
             moreframes = frame.flags & FLAG_COMMAND_REQUEST_MORE_FRAMES
             expectingdata = bool(frame.flags & FLAG_COMMAND_REQUEST_EXPECT_DATA)
 
             if entry['requestdone']:
                 self._state = 'errored'
                 return self._makeerrorresult(
                     _('received command request frame when request frames '
                       'were supposedly done'))
 
             if expectingdata != entry['expectingdata']:
                 self._state = 'errored'
                 return self._makeerrorresult(
                     _('mismatch between expect data flag and previous frame'))
 
             entry['payload'].write(frame.payload)
 
             if not moreframes:
                 entry['requestdone'] = True
 
             if not moreframes and not expectingdata:
                 return self._makeruncommandresult(frame.requestid)
 
             return self._makewantframeresult()
 
         elif frame.typeid == FRAME_TYPE_COMMAND_DATA:
             if not entry['expectingdata']:
                 self._state = 'errored'
                 return self._makeerrorresult(_(
                     'received command data frame for request that is not '
                     'expecting data: %d') % frame.requestid)
 
             if entry['data'] is None:
                 entry['data'] = util.bytesio()
 
             return self._handlecommanddataframe(frame, entry)
         else:
             self._state = 'errored'
             return self._makeerrorresult(_(
                 'received unexpected frame type: %d') % frame.typeid)
 
     def _handlecommanddataframe(self, frame, entry):
         assert frame.typeid == FRAME_TYPE_COMMAND_DATA
 
         # TODO support streaming data instead of buffering it.
         entry['data'].write(frame.payload)
 
         if frame.flags & FLAG_COMMAND_DATA_CONTINUATION:
             return self._makewantframeresult()
         elif frame.flags & FLAG_COMMAND_DATA_EOS:
             entry['data'].seek(0)
             return self._makeruncommandresult(frame.requestid)
         else:
             self._state = 'errored'
             return self._makeerrorresult(_('command data frame without '
                                            'flags'))
 
     def _onframeerrored(self, frame):
         return self._makeerrorresult(_('server already errored'))
 
 class commandrequest(object):
     """Represents a request to run a command."""
 
     def __init__(self, requestid, name, args, datafh=None, redirect=None):
         self.requestid = requestid
         self.name = name
         self.args = args
         self.datafh = datafh
         self.redirect = redirect
         self.state = 'pending'
 
 class clientreactor(object):
     """Holds state of a client issuing frame-based protocol requests.
 
     This is like ``serverreactor`` but for client-side state.
 
     Each instance is bound to the lifetime of a connection. For persistent
     connection transports using e.g. TCP sockets and speaking the raw
     framing protocol, there will be a single instance for the lifetime of
     the TCP socket. For transports where there are multiple discrete
     interactions (say tunneled within in HTTP request), there will be a
     separate instance for each distinct interaction.
 
     Consumers are expected to tell instances when events occur by calling
     various methods. These methods return a 2-tuple describing any follow-up
     action(s) to take. The first element is the name of an action to
     perform. The second is a data structure (usually a dict) specific to
     that action that contains more information. e.g. if the reactor wants
     to send frames to the server, the data structure will contain a reference
     to those frames.
 
     Valid actions that consumers can be instructed to take are:
 
     noop
        Indicates no additional action is required.
 
     sendframes
        Indicates that frames should be sent to the server. The ``framegen``
        key contains a generator of frames that should be sent. The reactor
        assumes that all frames in this generator are sent to the server.
 
     error
        Indicates that an error occurred. The ``message`` key contains an
        error message describing the failure.
 
     responsedata
        Indicates a response to a previously-issued command was received.
 
        The ``request`` key contains the ``commandrequest`` instance that
        represents the request this data is for.
 
        The ``data`` key contains the decoded data from the server.
 
        ``expectmore`` and ``eos`` evaluate to True when more response data
        is expected to follow or we're at the end of the response stream,
        respectively.
     """
-    def __init__(self, hasmultiplesend=False, buffersends=True):
+    def __init__(self, ui, hasmultiplesend=False, buffersends=True):
         """Create a new instance.
 
         ``hasmultiplesend`` indicates whether multiple sends are supported
         by the transport. When True, it is possible to send commands immediately
         instead of buffering until the caller signals an intent to finish a
         send operation.
 
         ``buffercommands`` indicates whether sends should be buffered until the
         last request has been issued.
         """
+        self._ui = ui
         self._hasmultiplesend = hasmultiplesend
         self._buffersends = buffersends
 
         self._canissuecommands = True
         self._cansend = True
 
         self._nextrequestid = 1
         # We only support a single outgoing stream for now.
         self._outgoingstream = stream(1)
         self._pendingrequests = collections.deque()
         self._activerequests = {}
         self._incomingstreams = {}
         self._streamsettingsdecoders = {}
 
     def callcommand(self, name, args, datafh=None, redirect=None):
         """Request that a command be executed.
 
         Receives the command name, a dict of arguments to pass to the command,
         and an optional file object containing the raw data for the command.
 
         Returns a 3-tuple of (request, action, action data).
         """
         if not self._canissuecommands:
             raise error.ProgrammingError('cannot issue new commands')
 
         requestid = self._nextrequestid
         self._nextrequestid += 2
 
         request = commandrequest(requestid, name, args, datafh=datafh,
                                  redirect=redirect)
 
         if self._buffersends:
             self._pendingrequests.append(request)
             return request, 'noop', {}
         else:
             if not self._cansend:
                 raise error.ProgrammingError('sends cannot be performed on '
                                              'this instance')
 
             if not self._hasmultiplesend:
                 self._cansend = False
                 self._canissuecommands = False
 
             return request, 'sendframes', {
                 'framegen': self._makecommandframes(request),
             }
 
     def flushcommands(self):
         """Request that all queued commands be sent.
 
         If any commands are buffered, this will instruct the caller to send
         them over the wire. If no commands are buffered it instructs the client
         to no-op.
 
         If instances aren't configured for multiple sends, no new command
         requests are allowed after this is called.
         """
         if not self._pendingrequests:
             return 'noop', {}
 
         if not self._cansend:
             raise error.ProgrammingError('sends cannot be performed on this '
                                          'instance')
 
         # If the instance only allows sending once, mark that we have fired
         # our one shot.
         if not self._hasmultiplesend:
             self._canissuecommands = False
             self._cansend = False
 
         def makeframes():
             while self._pendingrequests:
                 request = self._pendingrequests.popleft()
                 for frame in self._makecommandframes(request):
                     yield frame
 
         return 'sendframes', {
             'framegen': makeframes(),
         }
 
     def _makecommandframes(self, request):
         """Emit frames to issue a command request.
 
         As a side-effect, update request accounting to reflect its changed
         state.
         """
         self._activerequests[request.requestid] = request
         request.state = 'sending'
 
         res = createcommandframes(self._outgoingstream,
                                   request.requestid,
                                   request.name,
                                   request.args,
                                   datafh=request.datafh,
                                   redirect=request.redirect)
 
         for frame in res:
             yield frame
 
         request.state = 'sent'
 
     def onframerecv(self, frame):
         """Process a frame that has been received off the wire.
 
         Returns a 2-tuple of (action, meta) describing further action the
         caller needs to take as a result of receiving this frame.
         """
         if frame.streamid % 2:
             return 'error', {
                 'message': (
                     _('received frame with odd numbered stream ID: %d') %
                     frame.streamid),
             }
 
         if frame.streamid not in self._incomingstreams:
             if not frame.streamflags & STREAM_FLAG_BEGIN_STREAM:
                 return 'error', {
                     'message': _('received frame on unknown stream '
                                  'without beginning of stream flag set'),
                 }
 
             self._incomingstreams[frame.streamid] = stream(frame.streamid)
 
         if frame.streamflags & STREAM_FLAG_ENCODING_APPLIED:
             raise error.ProgrammingError('support for decoding stream '
                                          'payloads not yet implemneted')
 
         if frame.streamflags & STREAM_FLAG_END_STREAM:
             del self._incomingstreams[frame.streamid]
 
         if frame.typeid == FRAME_TYPE_STREAM_SETTINGS:
             return self._onstreamsettingsframe(frame)
 
         if frame.requestid not in self._activerequests:
             return 'error', {
                 'message': (_('received frame for inactive request ID: %d') %
                             frame.requestid),
             }
 
         request = self._activerequests[frame.requestid]
         request.state = 'receiving'
 
         handlers = {
             FRAME_TYPE_COMMAND_RESPONSE: self._oncommandresponseframe,
             FRAME_TYPE_ERROR_RESPONSE: self._onerrorresponseframe,
         }
 
         meth = handlers.get(frame.typeid)
         if not meth:
             raise error.ProgrammingError('unhandled frame type: %d' %
                                          frame.typeid)
 
         return meth(request, frame)
 
     def _onstreamsettingsframe(self, frame):
         assert frame.typeid == FRAME_TYPE_STREAM_SETTINGS
 
         more = frame.flags & FLAG_STREAM_ENCODING_SETTINGS_CONTINUATION
         eos = frame.flags & FLAG_STREAM_ENCODING_SETTINGS_EOS
 
         if more and eos:
             return 'error', {
                 'message': (_('stream encoding settings frame cannot have both '
                               'continuation and end of stream flags set')),
             }
 
         if not more and not eos:
             return 'error', {
                 'message': _('stream encoding settings frame must have '
                              'continuation or end of stream flag set'),
             }
 
         if frame.streamid not in self._streamsettingsdecoders:
             decoder = cborutil.bufferingdecoder()
             self._streamsettingsdecoders[frame.streamid] = decoder
 
         decoder = self._streamsettingsdecoders[frame.streamid]
 
         try:
             decoder.decode(frame.payload)
         except Exception as e:
             return 'error', {
                 'message': (_('error decoding CBOR from stream encoding '
                              'settings frame: %s') %
                            stringutil.forcebytestr(e)),
             }
 
         if more:
             return 'noop', {}
 
         assert eos
 
         decoded = decoder.getavailable()
         del self._streamsettingsdecoders[frame.streamid]
 
         if not decoded:
             return 'error', {
                 'message': _('stream encoding settings frame did not contain '
                              'CBOR data'),
             }
 
         try:
             self._incomingstreams[frame.streamid].setdecoder(decoded[0],
                                                              decoded[1:])
         except Exception as e:
             return 'error', {
                 'message': (_('error setting stream decoder: %s') %
                             stringutil.forcebytestr(e)),
             }
 
         return 'noop', {}
 
     def _oncommandresponseframe(self, request, frame):
         if frame.flags & FLAG_COMMAND_RESPONSE_EOS:
             request.state = 'received'
             del self._activerequests[request.requestid]
 
         return 'responsedata', {
             'request': request,
             'expectmore': frame.flags & FLAG_COMMAND_RESPONSE_CONTINUATION,
             'eos': frame.flags & FLAG_COMMAND_RESPONSE_EOS,
             'data': frame.payload,
         }
 
     def _onerrorresponseframe(self, request, frame):
         request.state = 'errored'
         del self._activerequests[request.requestid]
 
         # The payload should be a CBOR map.
         m = cborutil.decodeall(frame.payload)[0]
 
         return 'error', {
             'request': request,
             'type': m['type'],
             'message': m['message'],
         }

mercurial/wireprotov2server.py

@@ -1,1187 +1,1187
 # Copyright 21 May 2005 - (c) 2005 Jake Edge <[e-mail]>
 # Copyright 2005-2007 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import contextlib
 import hashlib
 
 from .i18n import _
 from .node import (
     hex,
     nullid,
 )
 from . import (
     discovery,
     encoding,
     error,
     narrowspec,
     pycompat,
     wireprotoframing,
     wireprototypes,
 )
 from .utils import (
     cborutil,
     interfaceutil,
     stringutil,
 )
 
 FRAMINGTYPE = b'application/mercurial-exp-framing-0005'
 
 HTTP_WIREPROTO_V2 = wireprototypes.HTTP_WIREPROTO_V2
 
 COMMANDS = wireprototypes.commanddict()
 
 # Value inserted into cache key computation function. Change the value to
 # force new cache keys for every command request. This should be done when
 # there is a change to how caching works, etc.
 GLOBAL_CACHE_VERSION = 1
 
 def handlehttpv2request(rctx, req, res, checkperm, urlparts):
     from .hgweb import common as hgwebcommon
 
     # URL space looks like: <permissions>/<command>, where <permission> can
     # be ``ro`` or ``rw`` to signal read-only or read-write, respectively.
 
     # Root URL does nothing meaningful... yet.
     if not urlparts:
         res.status = b'200 OK'
         res.headers[b'Content-Type'] = b'text/plain'
         res.setbodybytes(_('HTTP version 2 API handler'))
         return
 
     if len(urlparts) == 1:
         res.status = b'404 Not Found'
         res.headers[b'Content-Type'] = b'text/plain'
         res.setbodybytes(_('do not know how to process %s\n') %
                          req.dispatchpath)
         return
 
     permission, command = urlparts[0:2]
 
     if permission not in (b'ro', b'rw'):
         res.status = b'404 Not Found'
         res.headers[b'Content-Type'] = b'text/plain'
         res.setbodybytes(_('unknown permission: %s') % permission)
         return
 
     if req.method != 'POST':
         res.status = b'405 Method Not Allowed'
         res.headers[b'Allow'] = b'POST'
         res.setbodybytes(_('commands require POST requests'))
         return
 
     # At some point we'll want to use our own API instead of recycling the
     # behavior of version 1 of the wire protocol...
     # TODO return reasonable responses - not responses that overload the
     # HTTP status line message for error reporting.
     try:
         checkperm(rctx, req, 'pull' if permission == b'ro' else 'push')
     except hgwebcommon.ErrorResponse as e:
         res.status = hgwebcommon.statusmessage(e.code, pycompat.bytestr(e))
         for k, v in e.headers:
             res.headers[k] = v
         res.setbodybytes('permission denied')
         return
 
     # We have a special endpoint to reflect the request back at the client.
     if command == b'debugreflect':
         _processhttpv2reflectrequest(rctx.repo.ui, rctx.repo, req, res)
         return
 
     # Extra commands that we handle that aren't really wire protocol
     # commands. Think extra hard before making this hackery available to
     # extension.
     extracommands = {'multirequest'}
 
     if command not in COMMANDS and command not in extracommands:
         res.status = b'404 Not Found'
         res.headers[b'Content-Type'] = b'text/plain'
         res.setbodybytes(_('unknown wire protocol command: %s\n') % command)
         return
 
     repo = rctx.repo
     ui = repo.ui
 
     proto = httpv2protocolhandler(req, ui)
 
     if (not COMMANDS.commandavailable(command, proto)
         and command not in extracommands):
         res.status = b'404 Not Found'
         res.headers[b'Content-Type'] = b'text/plain'
         res.setbodybytes(_('invalid wire protocol command: %s') % command)
         return
 
     # TODO consider cases where proxies may add additional Accept headers.
     if req.headers.get(b'Accept') != FRAMINGTYPE:
         res.status = b'406 Not Acceptable'
         res.headers[b'Content-Type'] = b'text/plain'
         res.setbodybytes(_('client MUST specify Accept header with value: %s\n')
                            % FRAMINGTYPE)
         return
 
     if req.headers.get(b'Content-Type') != FRAMINGTYPE:
         res.status = b'415 Unsupported Media Type'
         # TODO we should send a response with appropriate media type,
         # since client does Accept it.
         res.headers[b'Content-Type'] = b'text/plain'
         res.setbodybytes(_('client MUST send Content-Type header with '
                            'value: %s\n') % FRAMINGTYPE)
         return
 
     _processhttpv2request(ui, repo, req, res, permission, command, proto)
 
 def _processhttpv2reflectrequest(ui, repo, req, res):
     """Reads unified frame protocol request and dumps out state to client.
 
     This special endpoint can be used to help debug the wire protocol.
 
     Instead of routing the request through the normal dispatch mechanism,
     we instead read all frames, decode them, and feed them into our state
     tracker. We then dump the log of all that activity back out to the
     client.
     """
     import json
 
     # Reflection APIs have a history of being abused, accidentally disclosing
     # sensitive data, etc. So we have a config knob.
     if not ui.configbool('experimental', 'web.api.debugreflect'):
         res.status = b'404 Not Found'
         res.headers[b'Content-Type'] = b'text/plain'
         res.setbodybytes(_('debugreflect service not available'))
         return
 
     # We assume we have a unified framing protocol request body.
 
-    reactor = wireprotoframing.serverreactor()
+    reactor = wireprotoframing.serverreactor(ui)
     states = []
 
     while True:
         frame = wireprotoframing.readframe(req.bodyfh)
 
         if not frame:
             states.append(b'received: <no frame>')
             break
 
         states.append(b'received: %d %d %d %s' % (frame.typeid, frame.flags,
                                                   frame.requestid,
                                                   frame.payload))
 
         action, meta = reactor.onframerecv(frame)
         states.append(json.dumps((action, meta), sort_keys=True,
                                  separators=(', ', ': ')))
 
     action, meta = reactor.oninputeof()
     meta['action'] = action
     states.append(json.dumps(meta, sort_keys=True, separators=(', ',': ')))
 
     res.status = b'200 OK'
     res.headers[b'Content-Type'] = b'text/plain'
     res.setbodybytes(b'\n'.join(states))
 
 def _processhttpv2request(ui, repo, req, res, authedperm, reqcommand, proto):
     """Post-validation handler for HTTPv2 requests.
 
     Called when the HTTP request contains unified frame-based protocol
     frames for evaluation.
     """
     # TODO Some HTTP clients are full duplex and can receive data before
     # the entire request is transmitted. Figure out a way to indicate support
     # for that so we can opt into full duplex mode.
-    reactor = wireprotoframing.serverreactor(deferoutput=True)
+    reactor = wireprotoframing.serverreactor(ui, deferoutput=True)
     seencommand = False
 
     outstream = reactor.makeoutputstream()
 
     while True:
         frame = wireprotoframing.readframe(req.bodyfh)
         if not frame:
             break
 
         action, meta = reactor.onframerecv(frame)
 
         if action == 'wantframe':
             # Need more data before we can do anything.
             continue
         elif action == 'runcommand':
             sentoutput = _httpv2runcommand(ui, repo, req, res, authedperm,
                                            reqcommand, reactor, outstream,
                                            meta, issubsequent=seencommand)
 
             if sentoutput:
                 return
 
             seencommand = True
 
         elif action == 'error':
             # TODO define proper error mechanism.
             res.status = b'200 OK'
             res.headers[b'Content-Type'] = b'text/plain'
             res.setbodybytes(meta['message'] + b'\n')
             return
         else:
             raise error.ProgrammingError(
                 'unhandled action from frame processor: %s' % action)
 
     action, meta = reactor.oninputeof()
     if action == 'sendframes':
         # We assume we haven't started sending the response yet. If we're
         # wrong, the response type will raise an exception.
         res.status = b'200 OK'
         res.headers[b'Content-Type'] = FRAMINGTYPE
         res.setbodygen(meta['framegen'])
     elif action == 'noop':
         pass
     else:
         raise error.ProgrammingError('unhandled action from frame processor: %s'
                                      % action)
 
 def _httpv2runcommand(ui, repo, req, res, authedperm, reqcommand, reactor,
                       outstream, command, issubsequent):
     """Dispatch a wire protocol command made from HTTPv2 requests.
 
     The authenticated permission (``authedperm``) along with the original
     command from the URL (``reqcommand``) are passed in.
     """
     # We already validated that the session has permissions to perform the
     # actions in ``authedperm``. In the unified frame protocol, the canonical
     # command to run is expressed in a frame. However, the URL also requested
     # to run a specific command. We need to be careful that the command we
     # run doesn't have permissions requirements greater than what was granted
     # by ``authedperm``.
     #
     # Our rule for this is we only allow one command per HTTP request and
     # that command must match the command in the URL. However, we make
     # an exception for the ``multirequest`` URL. This URL is allowed to
     # execute multiple commands. We double check permissions of each command
     # as it is invoked to ensure there is no privilege escalation.
     # TODO consider allowing multiple commands to regular command URLs
     # iff each command is the same.
 
     proto = httpv2protocolhandler(req, ui, args=command['args'])
 
     if reqcommand == b'multirequest':
         if not COMMANDS.commandavailable(command['command'], proto):
             # TODO proper error mechanism
             res.status = b'200 OK'
             res.headers[b'Content-Type'] = b'text/plain'
             res.setbodybytes(_('wire protocol command not available: %s') %
                              command['command'])
             return True
 
         # TODO don't use assert here, since it may be elided by -O.
         assert authedperm in (b'ro', b'rw')
         wirecommand = COMMANDS[command['command']]
         assert wirecommand.permission in ('push', 'pull')
 
         if authedperm == b'ro' and wirecommand.permission != 'pull':
             # TODO proper error mechanism
             res.status = b'403 Forbidden'
             res.headers[b'Content-Type'] = b'text/plain'
             res.setbodybytes(_('insufficient permissions to execute '
                                'command: %s') % command['command'])
             return True
 
         # TODO should we also call checkperm() here? Maybe not if we're going
         # to overhaul that API. The granted scope from the URL check should
         # be good enough.
 
     else:
         # Don't allow multiple commands outside of ``multirequest`` URL.
         if issubsequent:
             # TODO proper error mechanism
             res.status = b'200 OK'
             res.headers[b'Content-Type'] = b'text/plain'
             res.setbodybytes(_('multiple commands cannot be issued to this '
                                'URL'))
             return True
 
         if reqcommand != command['command']:
             # TODO define proper error mechanism
             res.status = b'200 OK'
             res.headers[b'Content-Type'] = b'text/plain'
             res.setbodybytes(_('command in frame must match command in URL'))
             return True
 
     res.status = b'200 OK'
     res.headers[b'Content-Type'] = FRAMINGTYPE
 
     try:
         objs = dispatch(repo, proto, command['command'], command['redirect'])
 
         action, meta = reactor.oncommandresponsereadyobjects(
             outstream, command['requestid'], objs)
 
     except error.WireprotoCommandError as e:
         action, meta = reactor.oncommanderror(
             outstream, command['requestid'], e.message, e.messageargs)
 
     except Exception as e:
         action, meta = reactor.onservererror(
             outstream, command['requestid'],
             _('exception when invoking command: %s') %
             stringutil.forcebytestr(e))
 
     if action == 'sendframes':
         res.setbodygen(meta['framegen'])
         return True
     elif action == 'noop':
         return False
     else:
         raise error.ProgrammingError('unhandled event from reactor: %s' %
                                      action)
 
 def getdispatchrepo(repo, proto, command):
     return repo.filtered('served')
 
 def dispatch(repo, proto, command, redirect):
     """Run a wire protocol command.
 
     Returns an iterable of objects that will be sent to the client.
     """
     repo = getdispatchrepo(repo, proto, command)
 
     entry = COMMANDS[command]
     func = entry.func
     spec = entry.args
 
     args = proto.getargs(spec)
 
     # There is some duplicate boilerplate code here for calling the command and
     # emitting objects. It is either that or a lot of indented code that looks
     # like a pyramid (since there are a lot of code paths that result in not
     # using the cacher).
     callcommand = lambda: func(repo, proto, **pycompat.strkwargs(args))
 
     # Request is not cacheable. Don't bother instantiating a cacher.
     if not entry.cachekeyfn:
         for o in callcommand():
             yield o
         return
 
     if redirect:
         redirecttargets = redirect[b'targets']
         redirecthashes = redirect[b'hashes']
     else:
         redirecttargets = []
         redirecthashes = []
 
     cacher = makeresponsecacher(repo, proto, command, args,
                                 cborutil.streamencode,
                                 redirecttargets=redirecttargets,
                                 redirecthashes=redirecthashes)
 
     # But we have no cacher. Do default handling.
     if not cacher:
         for o in callcommand():
             yield o
         return
 
     with cacher:
         cachekey = entry.cachekeyfn(repo, proto, cacher, **args)
 
         # No cache key or the cacher doesn't like it. Do default handling.
         if cachekey is None or not cacher.setcachekey(cachekey):
             for o in callcommand():
                 yield o
             return
 
         # Serve it from the cache, if possible.
         cached = cacher.lookup()
 
         if cached:
             for o in cached['objs']:
                 yield o
             return
 
         # Else call the command and feed its output into the cacher, allowing
         # the cacher to buffer/mutate objects as it desires.
         for o in callcommand():
             for o in cacher.onobject(o):
                 yield o
 
         for o in cacher.onfinished():
             yield o
 
 @interfaceutil.implementer(wireprototypes.baseprotocolhandler)
 class httpv2protocolhandler(object):
     def __init__(self, req, ui, args=None):
         self._req = req
         self._ui = ui
         self._args = args
 
     @property
     def name(self):
         return HTTP_WIREPROTO_V2
 
     def getargs(self, args):
         # First look for args that were passed but aren't registered on this
         # command.
         extra = set(self._args) - set(args)
         if extra:
             raise error.WireprotoCommandError(
                 'unsupported argument to command: %s' %
                 ', '.join(sorted(extra)))
 
         # And look for required arguments that are missing.
         missing = {a for a in args if args[a]['required']} - set(self._args)
 
         if missing:
             raise error.WireprotoCommandError(
                 'missing required arguments: %s' % ', '.join(sorted(missing)))
 
         # Now derive the arguments to pass to the command, taking into
         # account the arguments specified by the client.
         data = {}
         for k, meta in sorted(args.items()):
             # This argument wasn't passed by the client.
             if k not in self._args:
                 data[k] = meta['default']()
                 continue
 
             v = self._args[k]
 
             # Sets may be expressed as lists. Silently normalize.
             if meta['type'] == 'set' and isinstance(v, list):
                 v = set(v)
 
             # TODO consider more/stronger type validation.
 
             data[k] = v
 
         return data
 
     def getprotocaps(self):
         # Protocol capabilities are currently not implemented for HTTP V2.
         return set()
 
     def getpayload(self):
         raise NotImplementedError
 
     @contextlib.contextmanager
     def mayberedirectstdio(self):
         raise NotImplementedError
 
     def client(self):
         raise NotImplementedError
 
     def addcapabilities(self, repo, caps):
         return caps
 
     def checkperm(self, perm):
         raise NotImplementedError
 
 def httpv2apidescriptor(req, repo):
     proto = httpv2protocolhandler(req, repo.ui)
 
     return _capabilitiesv2(repo, proto)
 
 def _capabilitiesv2(repo, proto):
     """Obtain the set of capabilities for version 2 transports.
 
     These capabilities are distinct from the capabilities for version 1
     transports.
     """
     caps = {
         'commands': {},
         'framingmediatypes': [FRAMINGTYPE],
         'pathfilterprefixes': set(narrowspec.VALID_PREFIXES),
     }
 
     for command, entry in COMMANDS.items():
         args = {}
 
         for arg, meta in entry.args.items():
             args[arg] = {
                 # TODO should this be a normalized type using CBOR's
                 # terminology?
                 b'type': meta['type'],
                 b'required': meta['required'],
             }
 
             if not meta['required']:
                 args[arg][b'default'] = meta['default']()
 
             if meta['validvalues']:
                 args[arg][b'validvalues'] = meta['validvalues']
 
         caps['commands'][command] = {
             'args': args,
             'permissions': [entry.permission],
         }
 
     caps['rawrepoformats'] = sorted(repo.requirements &
                                     repo.supportedformats)
 
     targets = getadvertisedredirecttargets(repo, proto)
     if targets:
         caps[b'redirect'] = {
             b'targets': [],
             b'hashes': [b'sha256', b'sha1'],
         }
 
         for target in targets:
             entry = {
                 b'name': target['name'],
                 b'protocol': target['protocol'],
                 b'uris': target['uris'],
             }
 
             for key in ('snirequired', 'tlsversions'):
                 if key in target:
                     entry[key] = target[key]
 
             caps[b'redirect'][b'targets'].append(entry)
 
     return proto.addcapabilities(repo, caps)
 
 def getadvertisedredirecttargets(repo, proto):
     """Obtain a list of content redirect targets.
 
     Returns a list containing potential redirect targets that will be
     advertised in capabilities data. Each dict MUST have the following
     keys:
 
     name
        The name of this redirect target. This is the identifier clients use
        to refer to a target. It is transferred as part of every command
        request.
 
     protocol
        Network protocol used by this target. Typically this is the string
        in front of the ``://`` in a URL. e.g. ``https``.
 
     uris
        List of representative URIs for this target. Clients can use the
        URIs to test parsing for compatibility or for ordering preference
        for which target to use.
 
     The following optional keys are recognized:
 
     snirequired
        Bool indicating if Server Name Indication (SNI) is required to
        connect to this target.
 
     tlsversions
        List of bytes indicating which TLS versions are supported by this
        target.
 
     By default, clients reflect the target order advertised by servers
     and servers will use the first client-advertised target when picking
     a redirect target. So targets should be advertised in the order the
     server prefers they be used.
     """
     return []
 
 def wireprotocommand(name, args=None, permission='push', cachekeyfn=None):
     """Decorator to declare a wire protocol command.
 
     ``name`` is the name of the wire protocol command being provided.
 
     ``args`` is a dict defining arguments accepted by the command. Keys are
     the argument name. Values are dicts with the following keys:
 
        ``type``
           The argument data type. Must be one of the following string
           literals: ``bytes``, ``int``, ``list``, ``dict``, ``set``,
           or ``bool``.
 
        ``default``
           A callable returning the default value for this argument. If not
           specified, ``None`` will be the default value.
 
        ``example``
           An example value for this argument.
 
        ``validvalues``
           Set of recognized values for this argument.
 
     ``permission`` defines the permission type needed to run this command.
     Can be ``push`` or ``pull``. These roughly map to read-write and read-only,
     respectively. Default is to assume command requires ``push`` permissions
     because otherwise commands not declaring their permissions could modify
     a repository that is supposed to be read-only.
 
     ``cachekeyfn`` defines an optional callable that can derive the
     cache key for this request.
 
     Wire protocol commands are generators of objects to be serialized and
     sent to the client.
 
     If a command raises an uncaught exception, this will be translated into
     a command error.
 
     All commands can opt in to being cacheable by defining a function
     (``cachekeyfn``) that is called to derive a cache key. This function
     receives the same arguments as the command itself plus a ``cacher``
     argument containing the active cacher for the request and returns a bytes
     containing the key in a cache the response to this command may be cached
     under.
     """
     transports = {k for k, v in wireprototypes.TRANSPORTS.items()
                   if v['version'] == 2}
 
     if permission not in ('push', 'pull'):
         raise error.ProgrammingError('invalid wire protocol permission; '
                                      'got %s; expected "push" or "pull"' %
                                      permission)
 
     if args is None:
         args = {}
 
     if not isinstance(args, dict):
         raise error.ProgrammingError('arguments for version 2 commands '
                                      'must be declared as dicts')
 
     for arg, meta in args.items():
         if arg == '*':
             raise error.ProgrammingError('* argument name not allowed on '
                                          'version 2 commands')
 
         if not isinstance(meta, dict):
             raise error.ProgrammingError('arguments for version 2 commands '
                                          'must declare metadata as a dict')
 
         if 'type' not in meta:
             raise error.ProgrammingError('%s argument for command %s does not '
                                          'declare type field' % (arg, name))
 
         if meta['type'] not in ('bytes', 'int', 'list', 'dict', 'set', 'bool'):
             raise error.ProgrammingError('%s argument for command %s has '
                                          'illegal type: %s' % (arg, name,
                                                                meta['type']))
 
         if 'example' not in meta:
             raise error.ProgrammingError('%s argument for command %s does not '
                                          'declare example field' % (arg, name))
 
         meta['required'] = 'default' not in meta
 
         meta.setdefault('default', lambda: None)
         meta.setdefault('validvalues', None)
 
     def register(func):
         if name in COMMANDS:
             raise error.ProgrammingError('%s command already registered '
                                          'for version 2' % name)
 
         COMMANDS[name] = wireprototypes.commandentry(
             func, args=args, transports=transports, permission=permission,
             cachekeyfn=cachekeyfn)
 
         return func
 
     return register
 
 def makecommandcachekeyfn(command, localversion=None, allargs=False):
     """Construct a cache key derivation function with common features.
 
     By default, the cache key is a hash of:
 
     * The command name.
     * A global cache version number.
     * A local cache version number (passed via ``localversion``).
     * All the arguments passed to the command.
     * The media type used.
     * Wire protocol version string.
     * The repository path.
     """
     if not allargs:
         raise error.ProgrammingError('only allargs=True is currently supported')
 
     if localversion is None:
         raise error.ProgrammingError('must set localversion argument value')
 
     def cachekeyfn(repo, proto, cacher, **args):
         spec = COMMANDS[command]
 
         # Commands that mutate the repo can not be cached.
         if spec.permission == 'push':
             return None
 
         # TODO config option to disable caching.
 
         # Our key derivation strategy is to construct a data structure
         # holding everything that could influence cacheability and to hash
         # the CBOR representation of that. Using CBOR seems like it might
         # be overkill. However, simpler hashing mechanisms are prone to
         # duplicate input issues. e.g. if you just concatenate two values,
         # "foo"+"bar" is identical to "fo"+"obar". Using CBOR provides
         # "padding" between values and prevents these problems.
 
         # Seed the hash with various data.
         state = {
             # To invalidate all cache keys.
             b'globalversion': GLOBAL_CACHE_VERSION,
             # More granular cache key invalidation.
             b'localversion': localversion,
             # Cache keys are segmented by command.
             b'command': pycompat.sysbytes(command),
             # Throw in the media type and API version strings so changes
             # to exchange semantics invalid cache.
             b'mediatype': FRAMINGTYPE,
             b'version': HTTP_WIREPROTO_V2,
             # So same requests for different repos don't share cache keys.
             b'repo': repo.root,
         }
 
         # The arguments passed to us will have already been normalized.
         # Default values will be set, etc. This is important because it
         # means that it doesn't matter if clients send an explicit argument
         # or rely on the default value: it will all normalize to the same
         # set of arguments on the server and therefore the same cache key.
         #
         # Arguments by their very nature must support being encoded to CBOR.
         # And the CBOR encoder is deterministic. So we hash the arguments
         # by feeding the CBOR of their representation into the hasher.
         if allargs:
             state[b'args'] = pycompat.byteskwargs(args)
 
         cacher.adjustcachekeystate(state)
 
         hasher = hashlib.sha1()
         for chunk in cborutil.streamencode(state):
             hasher.update(chunk)
 
         return pycompat.sysbytes(hasher.hexdigest())
 
     return cachekeyfn
 
 def makeresponsecacher(repo, proto, command, args, objencoderfn,
                        redirecttargets, redirecthashes):
     """Construct a cacher for a cacheable command.
 
     Returns an ``iwireprotocolcommandcacher`` instance.
 
     Extensions can monkeypatch this function to provide custom caching
     backends.
     """
     return None
 
 @wireprotocommand('branchmap', permission='pull')
 def branchmapv2(repo, proto):
     yield {encoding.fromlocal(k): v
            for k, v in repo.branchmap().iteritems()}
 
 @wireprotocommand('capabilities', permission='pull')
 def capabilitiesv2(repo, proto):
     yield _capabilitiesv2(repo, proto)
 
 @wireprotocommand(
     'changesetdata',
     args={
         'noderange': {
             'type': 'list',
             'default': lambda: None,
             'example': [[b'0123456...'], [b'abcdef...']],
         },
         'nodes': {
             'type': 'list',
             'default': lambda: None,
             'example': [b'0123456...'],
         },
         'nodesdepth': {
             'type': 'int',
             'default': lambda: None,
             'example': 10,
         },
         'fields': {
             'type': 'set',
             'default': set,
             'example': {b'parents', b'revision'},
             'validvalues': {b'bookmarks', b'parents', b'phase', b'revision'},
         },
     },
     permission='pull')
 def changesetdata(repo, proto, noderange, nodes, nodesdepth, fields):
     # TODO look for unknown fields and abort when they can't be serviced.
     # This could probably be validated by dispatcher using validvalues.
 
     if noderange is None and nodes is None:
         raise error.WireprotoCommandError(
             'noderange or nodes must be defined')
 
     if nodesdepth is not None and nodes is None:
         raise error.WireprotoCommandError(
             'nodesdepth requires the nodes argument')
 
     if noderange is not None:
         if len(noderange) != 2:
             raise error.WireprotoCommandError(
                 'noderange must consist of 2 elements')
 
         if not noderange[1]:
             raise error.WireprotoCommandError(
                 'heads in noderange request cannot be empty')
 
     cl = repo.changelog
     hasnode = cl.hasnode
 
     seen = set()
     outgoing = []
 
     if nodes is not None:
         outgoing = [n for n in nodes if hasnode(n)]
 
         if nodesdepth:
             outgoing = [cl.node(r) for r in
                         repo.revs(b'ancestors(%ln, %d)', outgoing,
                                   nodesdepth - 1)]
 
         seen |= set(outgoing)
 
     if noderange is not None:
         if noderange[0]:
             common = [n for n in noderange[0] if hasnode(n)]
         else:
             common = [nullid]
 
         for n in discovery.outgoing(repo, common, noderange[1]).missing:
             if n not in seen:
                 outgoing.append(n)
             # Don't need to add to seen here because this is the final
             # source of nodes and there should be no duplicates in this
             # list.
 
     seen.clear()
     publishing = repo.publishing()
 
     if outgoing:
         repo.hook('preoutgoing', throw=True, source='serve')
 
     yield {
         b'totalitems': len(outgoing),
     }
 
     # The phases of nodes already transferred to the client may have changed
     # since the client last requested data. We send phase-only records
     # for these revisions, if requested.
     if b'phase' in fields and noderange is not None:
         # TODO skip nodes whose phase will be reflected by a node in the
         # outgoing set. This is purely an optimization to reduce data
         # size.
         for node in noderange[0]:
             yield {
                 b'node': node,
                 b'phase': b'public' if publishing else repo[node].phasestr()
             }
 
     nodebookmarks = {}
     for mark, node in repo._bookmarks.items():
         nodebookmarks.setdefault(node, set()).add(mark)
 
     # It is already topologically sorted by revision number.
     for node in outgoing:
         d = {
             b'node': node,
         }
 
         if b'parents' in fields:
             d[b'parents'] = cl.parents(node)
 
         if b'phase' in fields:
             if publishing:
                 d[b'phase'] = b'public'
             else:
                 ctx = repo[node]
                 d[b'phase'] = ctx.phasestr()
 
         if b'bookmarks' in fields and node in nodebookmarks:
             d[b'bookmarks'] = sorted(nodebookmarks[node])
             del nodebookmarks[node]
 
         followingmeta = []
         followingdata = []
 
         if b'revision' in fields:
             revisiondata = cl.revision(node, raw=True)
             followingmeta.append((b'revision', len(revisiondata)))
             followingdata.append(revisiondata)
 
         # TODO make it possible for extensions to wrap a function or register
         # a handler to service custom fields.
 
         if followingmeta:
             d[b'fieldsfollowing'] = followingmeta
 
         yield d
 
         for extra in followingdata:
             yield extra
 
     # If requested, send bookmarks from nodes that didn't have revision
     # data sent so receiver is aware of any bookmark updates.
     if b'bookmarks' in fields:
         for node, marks in sorted(nodebookmarks.iteritems()):
             yield {
                 b'node': node,
                 b'bookmarks': sorted(marks),
             }
 
 class FileAccessError(Exception):
     """Represents an error accessing a specific file."""
 
     def __init__(self, path, msg, args):
         self.path = path
         self.msg = msg
         self.args = args
 
 def getfilestore(repo, proto, path):
     """Obtain a file storage object for use with wire protocol.
 
     Exists as a standalone function so extensions can monkeypatch to add
     access control.
     """
     # This seems to work even if the file doesn't exist. So catch
     # "empty" files and return an error.
     fl = repo.file(path)
 
     if not len(fl):
         raise FileAccessError(path, 'unknown file: %s', (path,))
 
     return fl
 
 @wireprotocommand(
     'filedata',
     args={
         'haveparents': {
             'type': 'bool',
             'default': lambda: False,
             'example': True,
         },
         'nodes': {
             'type': 'list',
             'example': [b'0123456...'],
         },
         'fields': {
             'type': 'set',
             'default': set,
             'example': {b'parents', b'revision'},
             'validvalues': {b'parents', b'revision'},
         },
         'path': {
             'type': 'bytes',
             'example': b'foo.txt',
         }
     },
     permission='pull',
     # TODO censoring a file revision won't invalidate the cache.
     # Figure out a way to take censoring into account when deriving
     # the cache key.
     cachekeyfn=makecommandcachekeyfn('filedata', 1, allargs=True))
 def filedata(repo, proto, haveparents, nodes, fields, path):
     try:
         # Extensions may wish to access the protocol handler.
         store = getfilestore(repo, proto, path)
     except FileAccessError as e:
         raise error.WireprotoCommandError(e.msg, e.args)
 
     # Validate requested nodes.
     for node in nodes:
         try:
             store.rev(node)
         except error.LookupError:
             raise error.WireprotoCommandError('unknown file node: %s',
                                               (hex(node),))
 
     revisions = store.emitrevisions(nodes,
                                     revisiondata=b'revision' in fields,
                                     assumehaveparentrevisions=haveparents)
 
     yield {
         b'totalitems': len(nodes),
     }
 
     for revision in revisions:
         d = {
             b'node': revision.node,
         }
 
         if b'parents' in fields:
             d[b'parents'] = [revision.p1node, revision.p2node]
 
         followingmeta = []
         followingdata = []
 
         if b'revision' in fields:
             if revision.revision is not None:
                 followingmeta.append((b'revision', len(revision.revision)))
                 followingdata.append(revision.revision)
             else:
                 d[b'deltabasenode'] = revision.basenode
                 followingmeta.append((b'delta', len(revision.delta)))
                 followingdata.append(revision.delta)
 
         if followingmeta:
             d[b'fieldsfollowing'] = followingmeta
 
         yield d
 
         for extra in followingdata:
             yield extra
 
 @wireprotocommand(
     'heads',
     args={
         'publiconly': {
             'type': 'bool',
             'default': lambda: False,
             'example': False,
         },
     },
     permission='pull')
 def headsv2(repo, proto, publiconly):
     if publiconly:
         repo = repo.filtered('immutable')
 
     yield repo.heads()
 
 @wireprotocommand(
     'known',
     args={
         'nodes': {
             'type': 'list',
             'default': list,
             'example': [b'deadbeef'],
         },
     },
     permission='pull')
 def knownv2(repo, proto, nodes):
     result = b''.join(b'1' if n else b'0' for n in repo.known(nodes))
     yield result
 
 @wireprotocommand(
     'listkeys',
     args={
         'namespace': {
             'type': 'bytes',
             'example': b'ns',
         },
     },
     permission='pull')
 def listkeysv2(repo, proto, namespace):
     keys = repo.listkeys(encoding.tolocal(namespace))
     keys = {encoding.fromlocal(k): encoding.fromlocal(v)
             for k, v in keys.iteritems()}
 
     yield keys
 
 @wireprotocommand(
     'lookup',
     args={
         'key': {
             'type': 'bytes',
             'example': b'foo',
         },
     },
     permission='pull')
 def lookupv2(repo, proto, key):
     key = encoding.tolocal(key)
 
     # TODO handle exception.
     node = repo.lookup(key)
 
     yield node
 
 @wireprotocommand(
     'manifestdata',
     args={
         'nodes': {
             'type': 'list',
             'example': [b'0123456...'],
         },
         'haveparents': {
             'type': 'bool',
             'default': lambda: False,
             'example': True,
         },
         'fields': {
             'type': 'set',
             'default': set,
             'example': {b'parents', b'revision'},
             'validvalues': {b'parents', b'revision'},
         },
         'tree': {
             'type': 'bytes',
             'example': b'',
         },
     },
     permission='pull',
     cachekeyfn=makecommandcachekeyfn('manifestdata', 1, allargs=True))
 def manifestdata(repo, proto, haveparents, nodes, fields, tree):
     store = repo.manifestlog.getstorage(tree)
 
     # Validate the node is known and abort on unknown revisions.
     for node in nodes:
         try:
             store.rev(node)
         except error.LookupError:
             raise error.WireprotoCommandError(
                 'unknown node: %s', (node,))
 
     revisions = store.emitrevisions(nodes,
                                     revisiondata=b'revision' in fields,
                                     assumehaveparentrevisions=haveparents)
 
     yield {
         b'totalitems': len(nodes),
     }
 
     for revision in revisions:
         d = {
             b'node': revision.node,
         }
 
         if b'parents' in fields:
             d[b'parents'] = [revision.p1node, revision.p2node]
 
         followingmeta = []
         followingdata = []
 
         if b'revision' in fields:
             if revision.revision is not None:
                 followingmeta.append((b'revision', len(revision.revision)))
                 followingdata.append(revision.revision)
             else:
                 d[b'deltabasenode'] = revision.basenode
                 followingmeta.append((b'delta', len(revision.delta)))
                 followingdata.append(revision.delta)
 
         if followingmeta:
             d[b'fieldsfollowing'] = followingmeta
 
         yield d
 
         for extra in followingdata:
             yield extra
 
 @wireprotocommand(
     'pushkey',
     args={
         'namespace': {
             'type': 'bytes',
             'example': b'ns',
         },
         'key': {
             'type': 'bytes',
             'example': b'key',
         },
         'old': {
             'type': 'bytes',
             'example': b'old',
         },
         'new': {
             'type': 'bytes',
             'example': 'new',
         },
     },
     permission='push')
 def pushkeyv2(repo, proto, namespace, key, old, new):
     # TODO handle ui output redirection
     yield repo.pushkey(encoding.tolocal(namespace),
                        encoding.tolocal(key),
                        encoding.tolocal(old),
                        encoding.tolocal(new))

tests/test-wireproto-clientreactor.py

@@ -1,284 +1,291
 from __future__ import absolute_import
 
 import unittest
 
 from mercurial import (
     error,
+    ui as uimod,
     wireprotoframing as framing,
 )
 from mercurial.utils import (
     cborutil,
 )
 
 ffs = framing.makeframefromhumanstring
 
+globalui = uimod.ui()
+
 def sendframe(reactor, frame):
     """Send a frame bytearray to a reactor."""
     header = framing.parseheader(frame)
     payload = frame[framing.FRAME_HEADER_SIZE:]
     assert len(payload) == header.length
 
     return reactor.onframerecv(framing.frame(header.requestid,
                                              header.streamid,
                                              header.streamflags,
                                              header.typeid,
                                              header.flags,
                                              payload))
 
 class SingleSendTests(unittest.TestCase):
     """A reactor that can only send once rejects subsequent sends."""
 
     if not getattr(unittest.TestCase, 'assertRaisesRegex', False):
         # Python 3.7 deprecates the regex*p* version, but 2.7 lacks
         # the regex version.
         assertRaisesRegex = (# camelcase-required
             unittest.TestCase.assertRaisesRegexp)
 
     def testbasic(self):
-        reactor = framing.clientreactor(hasmultiplesend=False, buffersends=True)
+        reactor = framing.clientreactor(globalui,
+                                        hasmultiplesend=False,
+                                        buffersends=True)
 
         request, action, meta = reactor.callcommand(b'foo', {})
         self.assertEqual(request.state, b'pending')
         self.assertEqual(action, b'noop')
 
         action, meta = reactor.flushcommands()
         self.assertEqual(action, b'sendframes')
 
         for frame in meta[b'framegen']:
             self.assertEqual(request.state, b'sending')
 
         self.assertEqual(request.state, b'sent')
 
         with self.assertRaisesRegex(error.ProgrammingError,
                                      'cannot issue new commands'):
             reactor.callcommand(b'foo', {})
 
         with self.assertRaisesRegex(error.ProgrammingError,
                                      'cannot issue new commands'):
             reactor.callcommand(b'foo', {})
 
 class NoBufferTests(unittest.TestCase):
     """A reactor without send buffering sends requests immediately."""
     def testbasic(self):
-        reactor = framing.clientreactor(hasmultiplesend=True, buffersends=False)
+        reactor = framing.clientreactor(globalui,
+                                        hasmultiplesend=True,
+                                        buffersends=False)
 
         request, action, meta = reactor.callcommand(b'command1', {})
         self.assertEqual(request.requestid, 1)
         self.assertEqual(action, b'sendframes')
 
         self.assertEqual(request.state, b'pending')
 
         for frame in meta[b'framegen']:
             self.assertEqual(request.state, b'sending')
 
         self.assertEqual(request.state, b'sent')
 
         action, meta = reactor.flushcommands()
         self.assertEqual(action, b'noop')
 
         # And we can send another command.
         request, action, meta = reactor.callcommand(b'command2', {})
         self.assertEqual(request.requestid, 3)
         self.assertEqual(action, b'sendframes')
 
         for frame in meta[b'framegen']:
             self.assertEqual(request.state, b'sending')
 
         self.assertEqual(request.state, b'sent')
 
 class BadFrameRecvTests(unittest.TestCase):
     if not getattr(unittest.TestCase, 'assertRaisesRegex', False):
         # Python 3.7 deprecates the regex*p* version, but 2.7 lacks
         # the regex version.
         assertRaisesRegex = (# camelcase-required
             unittest.TestCase.assertRaisesRegexp)
 
     def testoddstream(self):
-        reactor = framing.clientreactor()
+        reactor = framing.clientreactor(globalui)
 
         action, meta = sendframe(reactor, ffs(b'1 1 0 1 0 foo'))
         self.assertEqual(action, b'error')
         self.assertEqual(meta[b'message'],
                          b'received frame with odd numbered stream ID: 1')
 
     def testunknownstream(self):
-        reactor = framing.clientreactor()
+        reactor = framing.clientreactor(globalui)
 
         action, meta = sendframe(reactor, ffs(b'1 0 0 1 0 foo'))
         self.assertEqual(action, b'error')
         self.assertEqual(meta[b'message'],
                          b'received frame on unknown stream without beginning '
                          b'of stream flag set')
 
     def testunhandledframetype(self):
-        reactor = framing.clientreactor(buffersends=False)
+        reactor = framing.clientreactor(globalui, buffersends=False)
 
         request, action, meta = reactor.callcommand(b'foo', {})
         for frame in meta[b'framegen']:
             pass
 
         with self.assertRaisesRegex(error.ProgrammingError,
                                      'unhandled frame type'):
             sendframe(reactor, ffs(b'1 0 stream-begin text-output 0 foo'))
 
 class StreamTests(unittest.TestCase):
     def testmultipleresponseframes(self):
-        reactor = framing.clientreactor(buffersends=False)
+        reactor = framing.clientreactor(globalui, buffersends=False)
 
         request, action, meta = reactor.callcommand(b'foo', {})
 
         self.assertEqual(action, b'sendframes')
         for f in meta[b'framegen']:
             pass
 
         action, meta = sendframe(
             reactor,
             ffs(b'%d 0 stream-begin command-response 0 foo' %
                 request.requestid))
         self.assertEqual(action, b'responsedata')
 
         action, meta = sendframe(
             reactor,
             ffs(b'%d 0 0 command-response eos bar' % request.requestid))
         self.assertEqual(action, b'responsedata')
 
 class RedirectTests(unittest.TestCase):
     def testredirect(self):
-        reactor = framing.clientreactor(buffersends=False)
+        reactor = framing.clientreactor(globalui, buffersends=False)
 
         redirect = {
             b'targets': [b'a', b'b'],
             b'hashes': [b'sha256'],
         }
 
         request, action, meta = reactor.callcommand(
             b'foo', {}, redirect=redirect)
 
         self.assertEqual(action, b'sendframes')
 
         frames = list(meta[b'framegen'])
         self.assertEqual(len(frames), 1)
 
         self.assertEqual(frames[0],
                          ffs(b'1 1 stream-begin command-request new '
                              b"cbor:{b'name': b'foo', "
                              b"b'redirect': {b'targets': [b'a', b'b'], "
                              b"b'hashes': [b'sha256']}}"))
 
 class StreamSettingsTests(unittest.TestCase):
     def testnoflags(self):
-        reactor = framing.clientreactor(buffersends=False)
+        reactor = framing.clientreactor(globalui, buffersends=False)
 
         request, action, meta = reactor.callcommand(b'foo', {})
         for f in meta[b'framegen']:
             pass
 
         action, meta = sendframe(reactor,
             ffs(b'1 2 stream-begin stream-settings 0 '))
 
         self.assertEqual(action, b'error')
         self.assertEqual(meta, {
             b'message': b'stream encoding settings frame must have '
                         b'continuation or end of stream flag set',
         })
 
     def testconflictflags(self):
-        reactor = framing.clientreactor(buffersends=False)
+        reactor = framing.clientreactor(globalui, buffersends=False)
 
         request, action, meta = reactor.callcommand(b'foo', {})
         for f in meta[b'framegen']:
             pass
 
         action, meta = sendframe(reactor,
             ffs(b'1 2 stream-begin stream-settings continuation|eos '))
 
         self.assertEqual(action, b'error')
         self.assertEqual(meta, {
             b'message': b'stream encoding settings frame cannot have both '
                         b'continuation and end of stream flags set',
         })
 
     def testemptypayload(self):
-        reactor = framing.clientreactor(buffersends=False)
+        reactor = framing.clientreactor(globalui, buffersends=False)
 
         request, action, meta = reactor.callcommand(b'foo', {})
         for f in meta[b'framegen']:
             pass
 
         action, meta = sendframe(reactor,
             ffs(b'1 2 stream-begin stream-settings eos '))
 
         self.assertEqual(action, b'error')
         self.assertEqual(meta, {
             b'message': b'stream encoding settings frame did not contain '
                         b'CBOR data'
         })
 
     def testbadcbor(self):
-        reactor = framing.clientreactor(buffersends=False)
+        reactor = framing.clientreactor(globalui, buffersends=False)
 
         request, action, meta = reactor.callcommand(b'foo', {})
         for f in meta[b'framegen']:
             pass
 
         action, meta = sendframe(reactor,
             ffs(b'1 2 stream-begin stream-settings eos badvalue'))
 
         self.assertEqual(action, b'error')
 
     def testsingleobject(self):
-        reactor = framing.clientreactor(buffersends=False)
+        reactor = framing.clientreactor(globalui, buffersends=False)
 
         request, action, meta = reactor.callcommand(b'foo', {})
         for f in meta[b'framegen']:
             pass
 
         action, meta = sendframe(reactor,
             ffs(b'1 2 stream-begin stream-settings eos cbor:b"identity"'))
 
         self.assertEqual(action, b'noop')
         self.assertEqual(meta, {})
 
     def testmultipleobjects(self):
-        reactor = framing.clientreactor(buffersends=False)
+        reactor = framing.clientreactor(globalui, buffersends=False)
 
         request, action, meta = reactor.callcommand(b'foo', {})
         for f in meta[b'framegen']:
             pass
 
         data = b''.join([
             b''.join(cborutil.streamencode(b'identity')),
             b''.join(cborutil.streamencode({b'foo', b'bar'})),
         ])
 
         action, meta = sendframe(reactor,
             ffs(b'1 2 stream-begin stream-settings eos %s' % data))
 
         self.assertEqual(action, b'noop')
         self.assertEqual(meta, {})
 
     def testmultipleframes(self):
-        reactor = framing.clientreactor(buffersends=False)
+        reactor = framing.clientreactor(globalui, buffersends=False)
 
         request, action, meta = reactor.callcommand(b'foo', {})
         for f in meta[b'framegen']:
             pass
 
         data = b''.join(cborutil.streamencode(b'identity'))
 
         action, meta = sendframe(reactor,
             ffs(b'1 2 stream-begin stream-settings continuation %s' %
                 data[0:3]))
 
         self.assertEqual(action, b'noop')
         self.assertEqual(meta, {})
 
         action, meta = sendframe(reactor,
             ffs(b'1 2 0 stream-settings eos %s' % data[3:]))
 
         self.assertEqual(action, b'noop')
         self.assertEqual(meta, {})
 
 if __name__ == '__main__':
     import silenttestrunner
     silenttestrunner.main(__name__)

tests/test-wireproto-serverreactor.py

@@ -1,602 +1,604
 from __future__ import absolute_import, print_function
 
 import unittest
 
 from mercurial.thirdparty import (
     cbor,
 )
 from mercurial import (
+    ui as uimod,
     util,
     wireprotoframing as framing,
 )
 from mercurial.utils import (
     cborutil,
 )
 
 ffs = framing.makeframefromhumanstring
 
 OK = cbor.dumps({b'status': b'ok'})
 
 def makereactor(deferoutput=False):
-    return framing.serverreactor(deferoutput=deferoutput)
+    ui = uimod.ui()
+    return framing.serverreactor(ui, deferoutput=deferoutput)
 
 def sendframes(reactor, gen):
     """Send a generator of frame bytearray to a reactor.
 
     Emits a generator of results from ``onframerecv()`` calls.
     """
     for frame in gen:
         header = framing.parseheader(frame)
         payload = frame[framing.FRAME_HEADER_SIZE:]
         assert len(payload) == header.length
 
         yield reactor.onframerecv(framing.frame(header.requestid,
                                                 header.streamid,
                                                 header.streamflags,
                                                 header.typeid,
                                                 header.flags,
                                                 payload))
 
 def sendcommandframes(reactor, stream, rid, cmd, args, datafh=None):
     """Generate frames to run a command and send them to a reactor."""
     return sendframes(reactor,
                       framing.createcommandframes(stream, rid, cmd, args,
                                                   datafh))
 
 
 class ServerReactorTests(unittest.TestCase):
     def _sendsingleframe(self, reactor, f):
         results = list(sendframes(reactor, [f]))
         self.assertEqual(len(results), 1)
 
         return results[0]
 
     def assertaction(self, res, expected):
         self.assertIsInstance(res, tuple)
         self.assertEqual(len(res), 2)
         self.assertIsInstance(res[1], dict)
         self.assertEqual(res[0], expected)
 
     def assertframesequal(self, frames, framestrings):
         expected = [ffs(s) for s in framestrings]
         self.assertEqual(list(frames), expected)
 
     def test1framecommand(self):
         """Receiving a command in a single frame yields request to run it."""
         reactor = makereactor()
         stream = framing.stream(1)
         results = list(sendcommandframes(reactor, stream, 1, b'mycommand', {}))
         self.assertEqual(len(results), 1)
         self.assertaction(results[0], b'runcommand')
         self.assertEqual(results[0][1], {
             b'requestid': 1,
             b'command': b'mycommand',
             b'args': {},
             b'redirect': None,
             b'data': None,
         })
 
         result = reactor.oninputeof()
         self.assertaction(result, b'noop')
 
     def test1argument(self):
         reactor = makereactor()
         stream = framing.stream(1)
         results = list(sendcommandframes(reactor, stream, 41, b'mycommand',
                                          {b'foo': b'bar'}))
         self.assertEqual(len(results), 1)
         self.assertaction(results[0], b'runcommand')
         self.assertEqual(results[0][1], {
             b'requestid': 41,
             b'command': b'mycommand',
             b'args': {b'foo': b'bar'},
             b'redirect': None,
             b'data': None,
         })
 
     def testmultiarguments(self):
         reactor = makereactor()
         stream = framing.stream(1)
         results = list(sendcommandframes(reactor, stream, 1, b'mycommand',
                                          {b'foo': b'bar', b'biz': b'baz'}))
         self.assertEqual(len(results), 1)
         self.assertaction(results[0], b'runcommand')
         self.assertEqual(results[0][1], {
             b'requestid': 1,
             b'command': b'mycommand',
             b'args': {b'foo': b'bar', b'biz': b'baz'},
             b'redirect': None,
             b'data': None,
         })
 
     def testsimplecommanddata(self):
         reactor = makereactor()
         stream = framing.stream(1)
         results = list(sendcommandframes(reactor, stream, 1, b'mycommand', {},
                                          util.bytesio(b'data!')))
         self.assertEqual(len(results), 2)
         self.assertaction(results[0], b'wantframe')
         self.assertaction(results[1], b'runcommand')
         self.assertEqual(results[1][1], {
             b'requestid': 1,
             b'command': b'mycommand',
             b'args': {},
             b'redirect': None,
             b'data': b'data!',
         })
 
     def testmultipledataframes(self):
         frames = [
             ffs(b'1 1 stream-begin command-request new|have-data '
                 b"cbor:{b'name': b'mycommand'}"),
             ffs(b'1 1 0 command-data continuation data1'),
             ffs(b'1 1 0 command-data continuation data2'),
             ffs(b'1 1 0 command-data eos data3'),
         ]
 
         reactor = makereactor()
         results = list(sendframes(reactor, frames))
         self.assertEqual(len(results), 4)
         for i in range(3):
             self.assertaction(results[i], b'wantframe')
         self.assertaction(results[3], b'runcommand')
         self.assertEqual(results[3][1], {
             b'requestid': 1,
             b'command': b'mycommand',
             b'args': {},
             b'redirect': None,
             b'data': b'data1data2data3',
         })
 
     def testargumentanddata(self):
         frames = [
             ffs(b'1 1 stream-begin command-request new|have-data '
                 b"cbor:{b'name': b'command', b'args': {b'key': b'val',"
                 b"b'foo': b'bar'}}"),
             ffs(b'1 1 0 command-data continuation value1'),
             ffs(b'1 1 0 command-data eos value2'),
         ]
 
         reactor = makereactor()
         results = list(sendframes(reactor, frames))
 
         self.assertaction(results[-1], b'runcommand')
         self.assertEqual(results[-1][1], {
             b'requestid': 1,
             b'command': b'command',
             b'args': {
                 b'key': b'val',
                 b'foo': b'bar',
             },
             b'redirect': None,
             b'data': b'value1value2',
         })
 
     def testnewandcontinuation(self):
         result = self._sendsingleframe(makereactor(),
             ffs(b'1 1 stream-begin command-request new|continuation '))
         self.assertaction(result, b'error')
         self.assertEqual(result[1], {
             b'message': b'received command request frame with both new and '
                         b'continuation flags set',
         })
 
     def testneithernewnorcontinuation(self):
         result = self._sendsingleframe(makereactor(),
             ffs(b'1 1 stream-begin command-request 0 '))
         self.assertaction(result, b'error')
         self.assertEqual(result[1], {
             b'message': b'received command request frame with neither new nor '
                         b'continuation flags set',
         })
 
     def testunexpectedcommanddata(self):
         """Command data frame when not running a command is an error."""
         result = self._sendsingleframe(makereactor(),
             ffs(b'1 1 stream-begin command-data 0 ignored'))
         self.assertaction(result, b'error')
         self.assertEqual(result[1], {
             b'message': b'expected sender protocol settings or command request '
                         b'frame; got 2',
         })
 
     def testunexpectedcommanddatareceiving(self):
         """Same as above except the command is receiving."""
         results = list(sendframes(makereactor(), [
             ffs(b'1 1 stream-begin command-request new|more '
                 b"cbor:{b'name': b'ignored'}"),
             ffs(b'1 1 0 command-data eos ignored'),
         ]))
 
         self.assertaction(results[0], b'wantframe')
         self.assertaction(results[1], b'error')
         self.assertEqual(results[1][1], {
             b'message': b'received command data frame for request that is not '
                         b'expecting data: 1',
         })
 
     def testconflictingrequestidallowed(self):
         """Multiple fully serviced commands with same request ID is allowed."""
         reactor = makereactor()
         results = []
         outstream = reactor.makeoutputstream()
         results.append(self._sendsingleframe(
             reactor, ffs(b'1 1 stream-begin command-request new '
                          b"cbor:{b'name': b'command'}")))
         result = reactor.oncommandresponseready(outstream, 1, b'response1')
         self.assertaction(result, b'sendframes')
         list(result[1][b'framegen'])
         results.append(self._sendsingleframe(
             reactor, ffs(b'1 1 stream-begin command-request new '
                          b"cbor:{b'name': b'command'}")))
         result = reactor.oncommandresponseready(outstream, 1, b'response2')
         self.assertaction(result, b'sendframes')
         list(result[1][b'framegen'])
         results.append(self._sendsingleframe(
             reactor, ffs(b'1 1 stream-begin command-request new '
                          b"cbor:{b'name': b'command'}")))
         result = reactor.oncommandresponseready(outstream, 1, b'response3')
         self.assertaction(result, b'sendframes')
         list(result[1][b'framegen'])
 
         for i in range(3):
             self.assertaction(results[i], b'runcommand')
             self.assertEqual(results[i][1], {
                 b'requestid': 1,
                 b'command': b'command',
                 b'args': {},
                 b'redirect': None,
                 b'data': None,
             })
 
     def testconflictingrequestid(self):
         """Request ID for new command matching in-flight command is illegal."""
         results = list(sendframes(makereactor(), [
             ffs(b'1 1 stream-begin command-request new|more '
                 b"cbor:{b'name': b'command'}"),
             ffs(b'1 1 0 command-request new '
                 b"cbor:{b'name': b'command1'}"),
         ]))
 
         self.assertaction(results[0], b'wantframe')
         self.assertaction(results[1], b'error')
         self.assertEqual(results[1][1], {
             b'message': b'request with ID 1 already received',
         })
 
     def testinterleavedcommands(self):
         cbor1 = cbor.dumps({
             b'name': b'command1',
             b'args': {
                 b'foo': b'bar',
                 b'key1': b'val',
             }
         }, canonical=True)
         cbor3 = cbor.dumps({
             b'name': b'command3',
             b'args': {
                 b'biz': b'baz',
                 b'key': b'val',
             },
         }, canonical=True)
 
         results = list(sendframes(makereactor(), [
             ffs(b'1 1 stream-begin command-request new|more %s' % cbor1[0:6]),
             ffs(b'3 1 0 command-request new|more %s' % cbor3[0:10]),
             ffs(b'1 1 0 command-request continuation|more %s' % cbor1[6:9]),
             ffs(b'3 1 0 command-request continuation|more %s' % cbor3[10:13]),
             ffs(b'3 1 0 command-request continuation %s' % cbor3[13:]),
             ffs(b'1 1 0 command-request continuation %s' % cbor1[9:]),
         ]))
 
         self.assertEqual([t[0] for t in results], [
             b'wantframe',
             b'wantframe',
             b'wantframe',
             b'wantframe',
             b'runcommand',
             b'runcommand',
         ])
 
         self.assertEqual(results[4][1], {
             b'requestid': 3,
             b'command': b'command3',
             b'args': {b'biz': b'baz', b'key': b'val'},
             b'redirect': None,
             b'data': None,
         })
         self.assertEqual(results[5][1], {
             b'requestid': 1,
             b'command': b'command1',
             b'args': {b'foo': b'bar', b'key1': b'val'},
             b'redirect': None,
             b'data': None,
         })
 
     def testmissingcommanddataframe(self):
         # The reactor doesn't currently handle partially received commands.
         # So this test is failing to do anything with request 1.
         frames = [
             ffs(b'1 1 stream-begin command-request new|have-data '
                 b"cbor:{b'name': b'command1'}"),
             ffs(b'3 1 0 command-request new '
                 b"cbor:{b'name': b'command2'}"),
         ]
         results = list(sendframes(makereactor(), frames))
         self.assertEqual(len(results), 2)
         self.assertaction(results[0], b'wantframe')
         self.assertaction(results[1], b'runcommand')
 
     def testmissingcommanddataframeflags(self):
         frames = [
             ffs(b'1 1 stream-begin command-request new|have-data '
                 b"cbor:{b'name': b'command1'}"),
             ffs(b'1 1 0 command-data 0 data'),
         ]
         results = list(sendframes(makereactor(), frames))
         self.assertEqual(len(results), 2)
         self.assertaction(results[0], b'wantframe')
         self.assertaction(results[1], b'error')
         self.assertEqual(results[1][1], {
             b'message': b'command data frame without flags',
         })
 
     def testframefornonreceivingrequest(self):
         """Receiving a frame for a command that is not receiving is illegal."""
         results = list(sendframes(makereactor(), [
             ffs(b'1 1 stream-begin command-request new '
                 b"cbor:{b'name': b'command1'}"),
             ffs(b'3 1 0 command-request new|have-data '
                 b"cbor:{b'name': b'command3'}"),
             ffs(b'5 1 0 command-data eos ignored'),
         ]))
         self.assertaction(results[2], b'error')
         self.assertEqual(results[2][1], {
             b'message': b'received frame for request that is not receiving: 5',
         })
 
     def testsimpleresponse(self):
         """Bytes response to command sends result frames."""
         reactor = makereactor()
         instream = framing.stream(1)
         list(sendcommandframes(reactor, instream, 1, b'mycommand', {}))
 
         outstream = reactor.makeoutputstream()
         result = reactor.oncommandresponseready(outstream, 1, b'response')
         self.assertaction(result, b'sendframes')
         self.assertframesequal(result[1][b'framegen'], [
             b'1 2 stream-begin command-response eos %sresponse' % OK,
         ])
 
     def testmultiframeresponse(self):
         """Bytes response spanning multiple frames is handled."""
         first = b'x' * framing.DEFAULT_MAX_FRAME_SIZE
         second = b'y' * 100
 
         reactor = makereactor()
         instream = framing.stream(1)
         list(sendcommandframes(reactor, instream, 1, b'mycommand', {}))
 
         outstream = reactor.makeoutputstream()
         result = reactor.oncommandresponseready(outstream, 1, first + second)
         self.assertaction(result, b'sendframes')
         self.assertframesequal(result[1][b'framegen'], [
             b'1 2 stream-begin command-response continuation %s' % OK,
             b'1 2 0 command-response continuation %s' % first,
             b'1 2 0 command-response eos %s' % second,
         ])
 
     def testservererror(self):
         reactor = makereactor()
         instream = framing.stream(1)
         list(sendcommandframes(reactor, instream, 1, b'mycommand', {}))
 
         outstream = reactor.makeoutputstream()
         result = reactor.onservererror(outstream, 1, b'some message')
         self.assertaction(result, b'sendframes')
         self.assertframesequal(result[1][b'framegen'], [
             b"1 2 stream-begin error-response 0 "
             b"cbor:{b'type': b'server', "
             b"b'message': [{b'msg': b'some message'}]}",
         ])
 
     def test1commanddeferresponse(self):
         """Responses when in deferred output mode are delayed until EOF."""
         reactor = makereactor(deferoutput=True)
         instream = framing.stream(1)
         results = list(sendcommandframes(reactor, instream, 1, b'mycommand',
                                          {}))
         self.assertEqual(len(results), 1)
         self.assertaction(results[0], b'runcommand')
 
         outstream = reactor.makeoutputstream()
         result = reactor.oncommandresponseready(outstream, 1, b'response')
         self.assertaction(result, b'noop')
         result = reactor.oninputeof()
         self.assertaction(result, b'sendframes')
         self.assertframesequal(result[1][b'framegen'], [
             b'1 2 stream-begin command-response eos %sresponse' % OK,
         ])
 
     def testmultiplecommanddeferresponse(self):
         reactor = makereactor(deferoutput=True)
         instream = framing.stream(1)
         list(sendcommandframes(reactor, instream, 1, b'command1', {}))
         list(sendcommandframes(reactor, instream, 3, b'command2', {}))
 
         outstream = reactor.makeoutputstream()
         result = reactor.oncommandresponseready(outstream, 1, b'response1')
         self.assertaction(result, b'noop')
         result = reactor.oncommandresponseready(outstream, 3, b'response2')
         self.assertaction(result, b'noop')
         result = reactor.oninputeof()
         self.assertaction(result, b'sendframes')
         self.assertframesequal(result[1][b'framegen'], [
             b'1 2 stream-begin command-response eos %sresponse1' % OK,
             b'3 2 0 command-response eos %sresponse2' % OK,
         ])
 
     def testrequestidtracking(self):
         reactor = makereactor(deferoutput=True)
         instream = framing.stream(1)
         list(sendcommandframes(reactor, instream, 1, b'command1', {}))
         list(sendcommandframes(reactor, instream, 3, b'command2', {}))
         list(sendcommandframes(reactor, instream, 5, b'command3', {}))
 
         # Register results for commands out of order.
         outstream = reactor.makeoutputstream()
         reactor.oncommandresponseready(outstream, 3, b'response3')
         reactor.oncommandresponseready(outstream, 1, b'response1')
         reactor.oncommandresponseready(outstream, 5, b'response5')
 
         result = reactor.oninputeof()
         self.assertaction(result, b'sendframes')
         self.assertframesequal(result[1][b'framegen'], [
             b'3 2 stream-begin command-response eos %sresponse3' % OK,
             b'1 2 0 command-response eos %sresponse1' % OK,
             b'5 2 0 command-response eos %sresponse5' % OK,
         ])
 
     def testduplicaterequestonactivecommand(self):
         """Receiving a request ID that matches a request that isn't finished."""
         reactor = makereactor()
         stream = framing.stream(1)
         list(sendcommandframes(reactor, stream, 1, b'command1', {}))
         results = list(sendcommandframes(reactor, stream, 1, b'command1', {}))
 
         self.assertaction(results[0], b'error')
         self.assertEqual(results[0][1], {
             b'message': b'request with ID 1 is already active',
         })
 
     def testduplicaterequestonactivecommandnosend(self):
         """Same as above but we've registered a response but haven't sent it."""
         reactor = makereactor()
         instream = framing.stream(1)
         list(sendcommandframes(reactor, instream, 1, b'command1', {}))
         outstream = reactor.makeoutputstream()
         reactor.oncommandresponseready(outstream, 1, b'response')
 
         # We've registered the response but haven't sent it. From the
         # perspective of the reactor, the command is still active.
 
         results = list(sendcommandframes(reactor, instream, 1, b'command1', {}))
         self.assertaction(results[0], b'error')
         self.assertEqual(results[0][1], {
             b'message': b'request with ID 1 is already active',
         })
 
     def testduplicaterequestaftersend(self):
         """We can use a duplicate request ID after we've sent the response."""
         reactor = makereactor()
         instream = framing.stream(1)
         list(sendcommandframes(reactor, instream, 1, b'command1', {}))
         outstream = reactor.makeoutputstream()
         res = reactor.oncommandresponseready(outstream, 1, b'response')
         list(res[1][b'framegen'])
 
         results = list(sendcommandframes(reactor, instream, 1, b'command1', {}))
         self.assertaction(results[0], b'runcommand')
 
     def testprotocolsettingsnoflags(self):
         result = self._sendsingleframe(
             makereactor(),
             ffs(b'0 1 stream-begin sender-protocol-settings 0 '))
         self.assertaction(result, b'error')
         self.assertEqual(result[1], {
             b'message': b'sender protocol settings frame must have '
                         b'continuation or end of stream flag set',
         })
 
     def testprotocolsettingsconflictflags(self):
         result = self._sendsingleframe(
             makereactor(),
             ffs(b'0 1 stream-begin sender-protocol-settings continuation|eos '))
         self.assertaction(result, b'error')
         self.assertEqual(result[1], {
             b'message': b'sender protocol settings frame cannot have both '
                         b'continuation and end of stream flags set',
         })
 
     def testprotocolsettingsemptypayload(self):
         result = self._sendsingleframe(
             makereactor(),
             ffs(b'0 1 stream-begin sender-protocol-settings eos '))
         self.assertaction(result, b'error')
         self.assertEqual(result[1], {
             b'message': b'sender protocol settings frame did not contain CBOR '
                         b'data',
         })
 
     def testprotocolsettingsmultipleobjects(self):
         result = self._sendsingleframe(
             makereactor(),
             ffs(b'0 1 stream-begin sender-protocol-settings eos '
                 b'\x46foobar\x43foo'))
         self.assertaction(result, b'error')
         self.assertEqual(result[1], {
             b'message': b'sender protocol settings frame contained multiple '
                         b'CBOR values',
         })
 
     def testprotocolsettingscontentencodings(self):
         reactor = makereactor()
 
         result = self._sendsingleframe(
             reactor,
             ffs(b'0 1 stream-begin sender-protocol-settings eos '
                 b'cbor:{b"contentencodings": [b"a", b"b"]}'))
         self.assertaction(result, b'wantframe')
 
         self.assertEqual(reactor._state, b'idle')
         self.assertEqual(reactor._sendersettings[b'contentencodings'],
                          [b'a', b'b'])
 
     def testprotocolsettingsmultipleframes(self):
         reactor = makereactor()
 
         data = b''.join(cborutil.streamencode({
             b'contentencodings': [b'value1', b'value2'],
         }))
 
         results = list(sendframes(reactor, [
             ffs(b'0 1 stream-begin sender-protocol-settings continuation %s' %
                 data[0:5]),
             ffs(b'0 1 0 sender-protocol-settings eos %s' % data[5:]),
         ]))
 
         self.assertEqual(len(results), 2)
 
         self.assertaction(results[0], b'wantframe')
         self.assertaction(results[1], b'wantframe')
 
         self.assertEqual(reactor._state, b'idle')
         self.assertEqual(reactor._sendersettings[b'contentencodings'],
                          [b'value1', b'value2'])
 
     def testprotocolsettingsbadcbor(self):
         result = self._sendsingleframe(
             makereactor(),
             ffs(b'0 1 stream-begin sender-protocol-settings eos badvalue'))
         self.assertaction(result, b'error')
 
     def testprotocolsettingsnoninitial(self):
         # Cannot have protocol settings frames as non-initial frames.
         reactor = makereactor()
 
         stream = framing.stream(1)
         results = list(sendcommandframes(reactor, stream, 1, b'mycommand', {}))
         self.assertEqual(len(results), 1)
         self.assertaction(results[0], b'runcommand')
 
         result = self._sendsingleframe(
             reactor,
             ffs(b'0 1 0 sender-protocol-settings eos '))
         self.assertaction(result, b'error')
         self.assertEqual(result[1], {
             b'message': b'expected command request frame; got 8',
         })
 
 if __name__ == '__main__':
     import silenttestrunner
     silenttestrunner.main(__name__)