tests: add low-level SSH protocol tests

We don't really have good low-level tests for the behavior of the SSH
wire protocol. This commit attempts to establish some.

The added tests consist of a mixture of starting a server
with `hg serve --stdio` and sending bytes to it and using
`hg debugpeer` to go through the official client code. Having
insight into what raw bytes are exchanged as well as what the peer
does is useful.

We also introduce a test extension for modifying the behavior of
the SSH server and peer. For example, we change the server to
not recognize the "hello" command, simulating behavior of <0.9.1
servers.

These tests are generally useful to have. But the impetus for creating
them now is they will be needed for verifying behavior of old clients
and servers when a new SSH protocol is introduced.

Differential Revision: https://phab.mercurial-scm.org/D2026

# no-check-commit because of serve_forever()

tests/sshprotoext.py

@@ -0,0 +1,108 b''
+# sshprotoext.py - Extension to test behavior of SSH protocol
+#
+# Copyright 2018 Gregory Szorc <[e-mail]>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+# This extension replaces the SSH server started via `hg serve --stdio`.
+# The server behaves differently depending on environment variables.
+
+from __future__ import absolute_import
+
+from mercurial import (
+    error,
+    registrar,
+    sshpeer,
+    wireproto,
+    wireprotoserver,
+)
+
+configtable = {}
+configitem = registrar.configitem(configtable)
+
+configitem('sshpeer', 'mode', default=None)
+configitem('sshpeer', 'handshake-mode', default=None)
+
+class bannerserver(wireprotoserver.sshserver):
+    """Server that sends a banner to stdout."""
+    def serve_forever(self):
+        for i in range(10):
+            self._fout.write(b'banner: line %d\n' % i)
+
+        super(bannerserver, self).serve_forever()
+
+class prehelloserver(wireprotoserver.sshserver):
+    """Tests behavior when connecting to <0.9.1 servers.
+
+    The ``hello`` wire protocol command was introduced in Mercurial
+    0.9.1. Modern clients send the ``hello`` command when connecting
+    to SSH servers. This mock server tests behavior of the handshake
+    when ``hello`` is not supported.
+    """
+    def serve_forever(self):
+        l = self._fin.readline()
+        assert l == b'hello\n'
+        # Respond to unknown commands with an empty reply.
+        self._sendresponse(b'')
+        l = self._fin.readline()
+        assert l == b'between\n'
+        rsp = wireproto.dispatch(self._repo, self, b'between')
+        self._handlers[rsp.__class__](self, rsp)
+
+        super(prehelloserver, self).serve_forever()
+
+class extrahandshakecommandspeer(sshpeer.sshpeer):
+    """An ssh peer that sends extra commands as part of initial handshake."""
+    # There isn't a good hook point. So we wrap _callstream() and inject
+    # logic when the peer says "hello".
+    def _callstream(self, cmd, **args):
+        if cmd != b'hello':
+            return super(extrahandshakecommandspeer, self)._callstream(cmd,
+                                                                       **args)
+
+        mode = self._ui.config(b'sshpeer', b'handshake-mode')
+        if mode == b'pre-no-args':
+            self._callstream(b'no-args')
+            return super(extrahandshakecommandspeer, self)._callstream(
+                cmd, **args)
+        elif mode == b'pre-multiple-no-args':
+            self._callstream(b'unknown1')
+            self._callstream(b'unknown2')
+            self._callstream(b'unknown3')
+            return super(extrahandshakecommandspeer, self)._callstream(
+                cmd, **args)
+        else:
+            raise error.ProgrammingError(b'unknown HANDSHAKECOMMANDMODE: %s' %
+                                         mode)
+
+def registercommands():
+    def dummycommand(repo, proto):
+        raise error.ProgrammingError('this should never be called')
+
+    wireproto.wireprotocommand(b'no-args', b'')(dummycommand)
+    wireproto.wireprotocommand(b'unknown1', b'')(dummycommand)
+    wireproto.wireprotocommand(b'unknown2', b'')(dummycommand)
+    wireproto.wireprotocommand(b'unknown3', b'')(dummycommand)
+
+def extsetup(ui):
+    # It's easier for tests to define the server behavior via environment
+    # variables than config options. This is because `hg serve --stdio`
+    # has to be invoked with a certain form for security reasons and
+    # `dummyssh` can't just add `--config` flags to the command line.
+    servermode = ui.environ.get(b'SSHSERVERMODE')
+
+    if servermode == b'banner':
+        wireprotoserver.sshserver = bannerserver
+    elif servermode == b'no-hello':
+        wireprotoserver.sshserver = prehelloserver
+    elif servermode:
+        raise error.ProgrammingError(b'unknown server mode: %s' % servermode)
+
+    peermode = ui.config(b'sshpeer', b'mode')
+
+    if peermode == b'extra-handshake-commands':
+        sshpeer.sshpeer = extrahandshakecommandspeer
+        registercommands()
+    elif peermode:
+        raise error.ProgrammingError(b'unknown peer mode: %s' % peermode)

tests/test-ssh-proto.t

@@ -0,0 +1,398 b''
+  $ cat >> $HGRCPATH << EOF
+  > [ui]
+  > ssh = $PYTHON "$TESTDIR/dummyssh"
+  > [devel]
+  > debug.peer-request = true
+  > [extensions]
+  > sshprotoext = $TESTDIR/sshprotoext.py
+  > EOF
+
+  $ hg init server
+  $ cd server
+  $ echo 0 > foo
+  $ hg -q add foo
+  $ hg commit -m initial
+  $ cd ..
+
+Test a normal behaving server, for sanity
+
+  $ hg --debug debugpeer ssh://user@dummy/server
+  running * "*/tests/dummyssh" 'user@dummy' 'hg -R server serve --stdio' (glob)
+  devel-peer-request: hello
+  sending hello command
+  devel-peer-request: between
+  devel-peer-request:   pairs: 81 bytes
+  sending between command
+  remote: 384
+  remote: capabilities: lookup changegroupsubset branchmap pushkey known getbundle unbundlehash batch streamreqs=generaldelta,revlogv1 $USUAL_BUNDLE2_CAPS_SERVER$ unbundle=HG10GZ,HG10BZ,HG10UN
+  remote: 1
+  url: ssh://user@dummy/server
+  local: no
+  pushable: yes
+
+Server should answer the "hello" command in isolation
+
+  $ hg -R server serve --stdio << EOF
+  > hello
+  > EOF
+  384
+  capabilities: lookup changegroupsubset branchmap pushkey known getbundle unbundlehash batch streamreqs=generaldelta,revlogv1 $USUAL_BUNDLE2_CAPS_SERVER$ unbundle=HG10GZ,HG10BZ,HG10UN
+
+>=0.9.1 clients send a "hello" + "between" for the null range as part of handshake.
+Server should reply with capabilities and should send "1\n\n" as a successful
+reply with empty response to the "between".
+
+  $ hg -R server serve --stdio << EOF
+  > hello
+  > between
+  > pairs 81
+  > 0000000000000000000000000000000000000000-0000000000000000000000000000000000000000
+  > EOF
+  384
+  capabilities: lookup changegroupsubset branchmap pushkey known getbundle unbundlehash batch streamreqs=generaldelta,revlogv1 $USUAL_BUNDLE2_CAPS_SERVER$ unbundle=HG10GZ,HG10BZ,HG10UN
+  1
+  
+
+SSH banner is not printed by default, ignored by clients
+
+  $ SSHSERVERMODE=banner hg debugpeer ssh://user@dummy/server
+  url: ssh://user@dummy/server
+  local: no
+  pushable: yes
+
+--debug will print the banner
+
+  $ SSHSERVERMODE=banner hg --debug debugpeer ssh://user@dummy/server
+  running * "*/tests/dummyssh" 'user@dummy' 'hg -R server serve --stdio' (glob)
+  devel-peer-request: hello
+  sending hello command
+  devel-peer-request: between
+  devel-peer-request:   pairs: 81 bytes
+  sending between command
+  remote: banner: line 0
+  remote: banner: line 1
+  remote: banner: line 2
+  remote: banner: line 3
+  remote: banner: line 4
+  remote: banner: line 5
+  remote: banner: line 6
+  remote: banner: line 7
+  remote: banner: line 8
+  remote: banner: line 9
+  remote: 384
+  remote: capabilities: lookup changegroupsubset branchmap pushkey known getbundle unbundlehash batch streamreqs=generaldelta,revlogv1 $USUAL_BUNDLE2_CAPS_SERVER$ unbundle=HG10GZ,HG10BZ,HG10UN
+  remote: 1
+  url: ssh://user@dummy/server
+  local: no
+  pushable: yes
+
+And test the banner with the raw protocol
+
+  $ SSHSERVERMODE=banner hg -R server serve --stdio << EOF
+  > hello
+  > between
+  > pairs 81
+  > 0000000000000000000000000000000000000000-0000000000000000000000000000000000000000
+  > EOF
+  banner: line 0
+  banner: line 1
+  banner: line 2
+  banner: line 3
+  banner: line 4
+  banner: line 5
+  banner: line 6
+  banner: line 7
+  banner: line 8
+  banner: line 9
+  384
+  capabilities: lookup changegroupsubset branchmap pushkey known getbundle unbundlehash batch streamreqs=generaldelta,revlogv1 $USUAL_BUNDLE2_CAPS_SERVER$ unbundle=HG10GZ,HG10BZ,HG10UN
+  1
+  
+
+Connecting to a <0.9.1 server that doesn't support the hello command
+
+  $ SSHSERVERMODE=no-hello hg --debug debugpeer ssh://user@dummy/server
+  running * "*/tests/dummyssh" 'user@dummy' 'hg -R server serve --stdio' (glob)
+  devel-peer-request: hello
+  sending hello command
+  devel-peer-request: between
+  devel-peer-request:   pairs: 81 bytes
+  sending between command
+  remote: 0
+  remote: 1
+  url: ssh://user@dummy/server
+  local: no
+  pushable: yes
+
+The client should interpret this as no capabilities
+
+  $ SSHSERVERMODE=no-hello hg debugcapabilities ssh://user@dummy/server
+  Main capabilities:
+
+Sending an unknown command to the server results in an empty response to that command
+
+  $ hg -R server serve --stdio << EOF
+  > pre-hello
+  > hello
+  > between
+  > pairs 81
+  > 0000000000000000000000000000000000000000-0000000000000000000000000000000000000000
+  > EOF
+  0
+  384
+  capabilities: lookup changegroupsubset branchmap pushkey known getbundle unbundlehash batch streamreqs=generaldelta,revlogv1 $USUAL_BUNDLE2_CAPS_SERVER$ unbundle=HG10GZ,HG10BZ,HG10UN
+  1
+  
+
+  $ hg --config sshpeer.mode=extra-handshake-commands --config sshpeer.handshake-mode=pre-no-args --debug debugpeer ssh://user@dummy/server
+  running * "*/tests/dummyssh" 'user@dummy' 'hg -R server serve --stdio' (glob)
+  devel-peer-request: no-args
+  sending no-args command
+  devel-peer-request: hello
+  sending hello command
+  devel-peer-request: between
+  devel-peer-request:   pairs: 81 bytes
+  sending between command
+  remote: 0
+  remote: 384
+  remote: capabilities: lookup changegroupsubset branchmap pushkey known getbundle unbundlehash batch streamreqs=generaldelta,revlogv1 $USUAL_BUNDLE2_CAPS_SERVER$ unbundle=HG10GZ,HG10BZ,HG10UN
+  remote: 1
+  url: ssh://user@dummy/server
+  local: no
+  pushable: yes
+
+Send multiple unknown commands before hello
+
+  $ hg -R server serve --stdio << EOF
+  > unknown1
+  > unknown2
+  > unknown3
+  > hello
+  > between
+  > pairs 81
+  > 0000000000000000000000000000000000000000-0000000000000000000000000000000000000000
+  > EOF
+  0
+  0
+  0
+  384
+  capabilities: lookup changegroupsubset branchmap pushkey known getbundle unbundlehash batch streamreqs=generaldelta,revlogv1 $USUAL_BUNDLE2_CAPS_SERVER$ unbundle=HG10GZ,HG10BZ,HG10UN
+  1
+  
+
+  $ hg --config sshpeer.mode=extra-handshake-commands --config sshpeer.handshake-mode=pre-multiple-no-args --debug debugpeer ssh://user@dummy/server
+  running * "*/tests/dummyssh" 'user@dummy' 'hg -R server serve --stdio' (glob)
+  devel-peer-request: unknown1
+  sending unknown1 command
+  devel-peer-request: unknown2
+  sending unknown2 command
+  devel-peer-request: unknown3
+  sending unknown3 command
+  devel-peer-request: hello
+  sending hello command
+  devel-peer-request: between
+  devel-peer-request:   pairs: 81 bytes
+  sending between command
+  remote: 0
+  remote: 0
+  remote: 0
+  remote: 384
+  remote: capabilities: lookup changegroupsubset branchmap pushkey known getbundle unbundlehash batch streamreqs=generaldelta,revlogv1 $USUAL_BUNDLE2_CAPS_SERVER$ unbundle=HG10GZ,HG10BZ,HG10UN
+  remote: 1
+  url: ssh://user@dummy/server
+  local: no
+  pushable: yes
+
+Send an unknown command before hello that has arguments
+
+  $ hg -R server serve --stdio << EOF
+  > with-args
+  > foo 13
+  > value for foo
+  > bar 13
+  > value for bar
+  > hello
+  > between
+  > pairs 81
+  > 0000000000000000000000000000000000000000-0000000000000000000000000000000000000000
+  > EOF
+  0
+  0
+  0
+  0
+  0
+  384
+  capabilities: lookup changegroupsubset branchmap pushkey known getbundle unbundlehash batch streamreqs=generaldelta,revlogv1 $USUAL_BUNDLE2_CAPS_SERVER$ unbundle=HG10GZ,HG10BZ,HG10UN
+  1
+  
+
+Send an unknown command having an argument that looks numeric
+
+  $ hg -R server serve --stdio << EOF
+  > unknown
+  > foo 1
+  > 0
+  > hello
+  > between
+  > pairs 81
+  > 0000000000000000000000000000000000000000-0000000000000000000000000000000000000000
+  > EOF
+  0
+  0
+  0
+  384
+  capabilities: lookup changegroupsubset branchmap pushkey known getbundle unbundlehash batch streamreqs=generaldelta,revlogv1 $USUAL_BUNDLE2_CAPS_SERVER$ unbundle=HG10GZ,HG10BZ,HG10UN
+  1
+  
+
+  $ hg -R server serve --stdio << EOF
+  > unknown
+  > foo 1
+  > 1
+  > hello
+  > between
+  > pairs 81
+  > 0000000000000000000000000000000000000000-0000000000000000000000000000000000000000
+  > EOF
+  0
+  0
+  0
+  384
+  capabilities: lookup changegroupsubset branchmap pushkey known getbundle unbundlehash batch streamreqs=generaldelta,revlogv1 $USUAL_BUNDLE2_CAPS_SERVER$ unbundle=HG10GZ,HG10BZ,HG10UN
+  1
+  
+
+When sending a dict argument value, it is serialized to
+"<arg> <item count>" followed by "<key> <len>\n<value>" for each item
+in the dict.
+
+Dictionary value for unknown command
+
+  $ hg -R server serve --stdio << EOF
+  > unknown
+  > dict 3
+  > key1 3
+  > foo
+  > key2 3
+  > bar
+  > key3 3
+  > baz
+  > hello
+  > EOF
+  0
+  0
+  0
+  0
+  0
+  0
+  0
+  0
+  384
+  capabilities: lookup changegroupsubset branchmap pushkey known getbundle unbundlehash batch streamreqs=generaldelta,revlogv1 $USUAL_BUNDLE2_CAPS_SERVER$ unbundle=HG10GZ,HG10BZ,HG10UN
+
+Incomplete dictionary send
+
+  $ hg -R server serve --stdio << EOF
+  > unknown
+  > dict 3
+  > key1 3
+  > foo
+  > EOF
+  0
+  0
+  0
+  0
+
+Incomplete value send
+
+  $ hg -R server serve --stdio << EOF
+  > unknown
+  > dict 3
+  > key1 3
+  > fo
+  > EOF
+  0
+  0
+  0
+  0
+
+Send a command line with spaces
+
+  $ hg -R server serve --stdio << EOF
+  > unknown withspace
+  > hello
+  > between
+  > pairs 81
+  > 0000000000000000000000000000000000000000-0000000000000000000000000000000000000000
+  > EOF
+  0
+  384
+  capabilities: lookup changegroupsubset branchmap pushkey known getbundle unbundlehash batch streamreqs=generaldelta,revlogv1 $USUAL_BUNDLE2_CAPS_SERVER$ unbundle=HG10GZ,HG10BZ,HG10UN
+  1
+  
+
+  $ hg -R server serve --stdio << EOF
+  > unknown with multiple spaces
+  > hello
+  > between
+  > pairs 81
+  > 0000000000000000000000000000000000000000-0000000000000000000000000000000000000000
+  > EOF
+  0
+  384
+  capabilities: lookup changegroupsubset branchmap pushkey known getbundle unbundlehash batch streamreqs=generaldelta,revlogv1 $USUAL_BUNDLE2_CAPS_SERVER$ unbundle=HG10GZ,HG10BZ,HG10UN
+  1
+  
+
+  $ hg -R server serve --stdio << EOF
+  > unknown with spaces
+  > key 10
+  > some value
+  > hello
+  > between
+  > pairs 81
+  > 0000000000000000000000000000000000000000-0000000000000000000000000000000000000000
+  > EOF
+  0
+  0
+  0
+  384
+  capabilities: lookup changegroupsubset branchmap pushkey known getbundle unbundlehash batch streamreqs=generaldelta,revlogv1 $USUAL_BUNDLE2_CAPS_SERVER$ unbundle=HG10GZ,HG10BZ,HG10UN
+  1
+  
+
+Send an unknown command after the "between"
+
+  $ hg -R server serve --stdio << EOF
+  > hello
+  > between
+  > pairs 81
+  > 0000000000000000000000000000000000000000-0000000000000000000000000000000000000000unknown
+  > EOF
+  384
+  capabilities: lookup changegroupsubset branchmap pushkey known getbundle unbundlehash batch streamreqs=generaldelta,revlogv1 $USUAL_BUNDLE2_CAPS_SERVER$ unbundle=HG10GZ,HG10BZ,HG10UN
+  1
+  
+  0
+
+And one with arguments
+
+  $ hg -R server serve --stdio << EOF
+  > hello
+  > between
+  > pairs 81
+  > 0000000000000000000000000000000000000000-0000000000000000000000000000000000000000unknown
+  > foo 5
+  > value
+  > bar 3
+  > baz
+  > EOF
+  384
+  capabilities: lookup changegroupsubset branchmap pushkey known getbundle unbundlehash batch streamreqs=generaldelta,revlogv1 $USUAL_BUNDLE2_CAPS_SERVER$ unbundle=HG10GZ,HG10BZ,HG10UN
+  1
+  
+  0
+  0
+  0
+  0
+  0