wireproto: crude support for version 2 HTTP peer

As part of implementing the server-side bits of the wire protocol
command handlers for version 2, we want a way to easily test those
commands. Currently, we use the "httprequest" action of `hg
debugwireproto`. But this requires explicitly specifying the HTTP
request headers, low-level frame details, and the data structure
to encode with CBOR. That's a lot of boilerplate and a lot of it can
change as the wire protocol evolves.

`hg debugwireproto` has a mechanism to issue commands via the peer
interface. That is *much* easier to use and we prefer to test with
that going forward.

This commit implements enough parts of the peer API to send basic
requests via the HTTP version 2 transport.

The peer code is super hacky. Again, the goal is to facilitate
server testing, not robustly implement a client. The client code
will receive love at a later time.

Differential Revision: https://phab.mercurial-scm.org/D3177

mercurial/debugcommands.py

@@ -2631,8 +2631,8 b' def debugwireproto(ui, repo, path=None, '
 
     ``--peer`` can be used to bypass the handshake protocol and construct a
     peer instance using the specified class type. Valid values are ``raw``,
-    ``ssh1``, and ``ssh2``. ``raw`` instances only allow sending raw data
-    payloads and don't support higher-level command actions.
+    ``http2``, ``ssh1``, and ``ssh2``. ``raw`` instances only allow sending
+    raw data payloads and don't support higher-level command actions.
 
     ``--noreadstderr`` can be used to disable automatic reading from stderr
     of the peer (for SSH connections only). Disabling automatic reading of
@@ -2678,8 +2678,10 b' def debugwireproto(ui, repo, path=None, '
        command listkeys
            namespace bookmarks
 
-    Values are interpreted as Python b'' literals. This allows encoding
-    special byte sequences via backslash escaping.
+    If the value begins with ``eval:``, it will be interpreted as a Python
+    literal expression. Otherwise values are interpreted as Python b'' literals.
+    This allows sending complex types and encoding special byte sequences via
+    backslash escaping.
 
     The following arguments have special meaning:
 
@@ -2803,7 +2805,7 b' def debugwireproto(ui, repo, path=None, '
     if opts['localssh'] and not repo:
         raise error.Abort(_('--localssh requires a repository'))
 
-    if opts['peer'] and opts['peer'] not in ('raw', 'ssh1', 'ssh2'):
+    if opts['peer'] and opts['peer'] not in ('raw', 'http2', 'ssh1', 'ssh2'):
         raise error.Abort(_('invalid value for --peer'),
                           hint=_('valid values are "raw", "ssh1", and "ssh2"'))
 
@@ -2877,18 +2879,20 b' def debugwireproto(ui, repo, path=None, '
             raise error.Abort(_('only http:// paths are currently supported'))
 
         url, authinfo = u.authinfo()
-        openerargs = {}
+        openerargs = {
+            r'useragent': b'Mercurial debugwireproto',
+        }
 
         # Turn pipes/sockets into observers so we can log I/O.
         if ui.verbose:
-            openerargs = {
+            openerargs.update({
                 r'loggingfh': ui,
                 r'loggingname': b's',
                 r'loggingopts': {
                     r'logdata': True,
                     r'logdataapis': False,
                 },
-            }
+            })
 
         if ui.debugflag:
             openerargs[r'loggingopts'][r'logdataapis'] = True
@@ -2901,7 +2905,10 b' def debugwireproto(ui, repo, path=None, '
 
         opener = urlmod.opener(ui, authinfo, **openerargs)
 
-        if opts['peer'] == 'raw':
+        if opts['peer'] == 'http2':
+            ui.write(_('creating http peer for wire protocol version 2\n'))
+            peer = httppeer.httpv2peer(ui, path, opener)
+        elif opts['peer'] == 'raw':
             ui.write(_('using raw connection to peer\n'))
             peer = None
         elif opts['peer']:
@@ -2951,7 +2958,12 b' def debugwireproto(ui, repo, path=None, '
                 else:
                     key, value = fields
 
-                args[key] = stringutil.unescapestr(value)
+                if value.startswith('eval:'):
+                    value = stringutil.evalpythonliteral(value[5:])
+                else:
+                    value = stringutil.unescapestr(value)
+
+                args[key] = value
 
             if batchedcommands is not None:
                 batchedcommands.append((command, args))

mercurial/httppeer.py

@@ -16,6 +16,9 b' import struct'
 import tempfile
 
 from .i18n import _
+from .thirdparty import (
+    cbor,
+)
 from . import (
     bundle2,
     error,
@@ -25,6 +28,8 b' from . import ('
     url as urlmod,
     util,
     wireproto,
+    wireprotoframing,
+    wireprotoserver,
 )
 
 httplib = util.httplib
@@ -467,6 +472,95 b' class httppeer(wireproto.wirepeer):'
     def _abort(self, exception):
         raise exception
 
+# TODO implement interface for version 2 peers
+class httpv2peer(object):
+    def __init__(self, ui, repourl, opener):
+        self.ui = ui
+
+        if repourl.endswith('/'):
+            repourl = repourl[:-1]
+
+        self.url = repourl
+        self._opener = opener
+        # This is an its own attribute to facilitate extensions overriding
+        # the default type.
+        self._requestbuilder = urlreq.request
+
+    def close(self):
+        pass
+
+    # TODO require to be part of a batched primitive, use futures.
+    def _call(self, name, **args):
+        """Call a wire protocol command with arguments."""
+
+        # TODO permissions should come from capabilities results.
+        permission = wireproto.commandsv2[name].permission
+        if permission not in ('push', 'pull'):
+            raise error.ProgrammingError('unknown permission type: %s' %
+                                         permission)
+
+        permission = {
+            'push': 'rw',
+            'pull': 'ro',
+        }[permission]
+
+        url = '%s/api/%s/%s/%s' % (self.url, wireprotoserver.HTTPV2, permission,
+                                   name)
+
+        # TODO modify user-agent to reflect v2.
+        headers = {
+            r'Accept': wireprotoserver.FRAMINGTYPE,
+            r'Content-Type': wireprotoserver.FRAMINGTYPE,
+        }
+
+        # TODO this should be part of a generic peer for the frame-based
+        # protocol.
+        stream = wireprotoframing.stream(1)
+        frames = wireprotoframing.createcommandframes(stream, 1,
+                                                      name, args)
+
+        body = b''.join(map(bytes, frames))
+        req = self._requestbuilder(pycompat.strurl(url), body, headers)
+        req.add_unredirected_header(r'Content-Length', r'%d' % len(body))
+
+        # TODO unify this code with httppeer.
+        try:
+            res = self._opener.open(req)
+        except urlerr.httperror as e:
+            if e.code == 401:
+                raise error.Abort(_('authorization failed'))
+
+            raise
+        except httplib.HTTPException as e:
+            self.ui.traceback()
+            raise IOError(None, e)
+
+        # TODO validate response type, wrap response to handle I/O errors.
+        # TODO more robust frame receiver.
+        results = []
+
+        while True:
+            frame = wireprotoframing.readframe(res)
+            if frame is None:
+                break
+
+            self.ui.note(_('received %r\n') % frame)
+
+            if frame.typeid == wireprotoframing.FRAME_TYPE_BYTES_RESPONSE:
+                if frame.flags & wireprotoframing.FLAG_BYTES_RESPONSE_CBOR:
+                    payload = util.bytesio(frame.payload)
+
+                    decoder = cbor.CBORDecoder(payload)
+                    while payload.tell() + 1 < len(frame.payload):
+                        results.append(decoder.decode())
+                else:
+                    results.append(frame.payload)
+            else:
+                error.ProgrammingError('unhandled frame type: %d' %
+                                       frame.typeid)
+
+        return results
+
 def makepeer(ui, path):
     u = util.url(path)
     if u.query or u.fragment:

tests/test-http-api-httpv2.t

@@ -180,6 +180,36 b' Request to read-only command works out o'
   s>     0\r\n
   s>     \r\n
 
+  $ sendhttpv2peer << EOF
+  > command customreadonly
+  > EOF
+  creating http peer for wire protocol version 2
+  sending customreadonly command
+  s>     POST /api/exp-http-v2-0001/ro/customreadonly HTTP/1.1\r\n
+  s>     Accept-Encoding: identity\r\n
+  s>     accept: application/mercurial-exp-framing-0003\r\n
+  s>     content-type: application/mercurial-exp-framing-0003\r\n
+  s>     content-length: 29\r\n
+  s>     host: $LOCALIP:$HGPORT\r\n (glob)
+  s>     user-agent: Mercurial debugwireproto\r\n
+  s>     \r\n
+  s>     \x15\x00\x00\x01\x00\x01\x01\x11\xa1DnameNcustomreadonly
+  s> makefile('rb', None)
+  s>     HTTP/1.1 200 OK\r\n
+  s>     Server: testing stub value\r\n
+  s>     Date: $HTTP_DATE$\r\n
+  s>     Content-Type: application/mercurial-exp-framing-0003\r\n
+  s>     Transfer-Encoding: chunked\r\n
+  s>     \r\n
+  s>     25\r\n
+  s>     \x1d\x00\x00\x01\x00\x02\x01B
+  s>     customreadonly bytes response
+  s>     \r\n
+  received frame(size=29; request=1; stream=2; streamflags=stream-begin; type=bytes-response; flags=eos)
+  s>     0\r\n
+  s>     \r\n
+  response: [b'customreadonly bytes response']
+
 Request to read-write command fails because server is read-only by default
 
 GET to read-write request yields 405

tests/test-http-protocol.t

@@ -179,7 +179,7 b' Test listkeys for listing namespaces'
   s>     Accept-Encoding: identity\r\n
   s>     accept: application/mercurial-0.1\r\n
   s>     host: $LOCALIP:$HGPORT\r\n (glob)
-  s>     user-agent: mercurial/proto-1.0 (Mercurial *)\r\n (glob)
+  s>     user-agent: Mercurial debugwireproto\r\n
   s>     \r\n
   s> makefile('rb', None)
   s>     HTTP/1.1 200 Script output follows\r\n
@@ -197,7 +197,7 b' Test listkeys for listing namespaces'
   s>     x-hgproto-1: 0.1 0.2 comp=$USUAL_COMPRESSIONS$\r\n
   s>     accept: application/mercurial-0.1\r\n
   s>     host: $LOCALIP:$HGPORT\r\n (glob)
-  s>     user-agent: mercurial/proto-1.0 (Mercurial *)\r\n (glob)
+  s>     user-agent: Mercurial debugwireproto\r\n
   s>     \r\n
   s> makefile('rb', None)
   s>     HTTP/1.1 200 Script output follows\r\n

tests/wireprotohelpers.sh

@@ -5,6 +5,10 b' sendhttpraw() {'
   hg --verbose debugwireproto --peer raw http://$LOCALIP:$HGPORT/
 }
 
+sendhttpv2peer() {
+  hg --verbose debugwireproto --peer http2 http://$LOCALIP:$HGPORT/
+}
+
 cat > dummycommands.py << EOF
 from mercurial import (
     wireprototypes,