diff --git a/mercurial/configitems.py b/mercurial/configitems.py --- a/mercurial/configitems.py +++ b/mercurial/configitems.py @@ -574,6 +574,9 @@ coreconfigitem('experimental', 'treemani coreconfigitem('experimental', 'update.atomic-file', default=False, ) +coreconfigitem('experimental', 'sshpeer.advertise-v2', + default=False, +) coreconfigitem('extensions', '.*', default=None, generic=True, diff --git a/mercurial/help/internals/wireprotocol.txt b/mercurial/help/internals/wireprotocol.txt --- a/mercurial/help/internals/wireprotocol.txt +++ b/mercurial/help/internals/wireprotocol.txt @@ -218,6 +218,95 @@ part of the response payload and not par after responses. In other words, the length of the response contains the trailing ``\n``. +Clients supporting version 2 of the SSH transport send a line beginning +with ``upgrade`` before the ``hello`` and ``between`` commands. The line +(which isn't a well-formed command line because it doesn't consist of a +single command name) serves to both communicate the client's intent to +switch to transport version 2 (transports are version 1 by default) as +well as to advertise the client's transport-level capabilities so the +server may satisfy that request immediately. + +The upgrade line has the form: + + upgrade + +That is the literal string ``upgrade`` followed by a space, followed by +a randomly generated string, followed by a space, followed by a string +denoting the client's transport capabilities. + +The token can be anything. However, a random UUID is recommended. (Use +of version 4 UUIDs is recommended because version 1 UUIDs can leak the +client's MAC address.) + +The transport capabilities string is a URL/percent encoded string +containing key-value pairs defining the client's transport-level +capabilities. The following capabilities are defined: + +proto + A comma-delimited list of transport protocol versions the client + supports. e.g. ``ssh-v2``. + +If the server does not recognize the ``upgrade`` line, it should issue +an empty response and continue processing the ``hello`` and ``between`` +commands. Here is an example handshake between a version 2 aware client +and a non version 2 aware server: + + c: upgrade 2e82ab3f-9ce3-4b4e-8f8c-6fd1c0e9e23a proto=ssh-v2 + c: hello\n + c: between\n + c: pairs 81\n + c: 0000000000000000000000000000000000000000-0000000000000000000000000000000000000000 + s: 0\n + s: 324\n + s: capabilities: lookup changegroupsubset branchmap pushkey known getbundle ...\n + s: 1\n + s: \n + +(The initial ``0\n`` line from the server indicates an empty response to +the unknown ``upgrade ..`` command/line.) + +If the server recognizes the ``upgrade`` line and is willing to satisfy that +upgrade request, it replies to with a payload of the following form: + + upgraded \n + +This line is the literal string ``upgraded``, a space, the token that was +specified by the client in its ``upgrade ...`` request line, a space, and the +name of the transport protocol that was chosen by the server. The transport +name MUST match one of the names the client specified in the ``proto`` field +of its ``upgrade ...`` request line. + +If a server issues an ``upgraded`` response, it MUST also read and ignore +the lines associated with the ``hello`` and ``between`` command requests +that were issued by the server. It is assumed that the negotiated transport +will respond with equivalent requested information following the transport +handshake. + +All data following the ``\n`` terminating the ``upgraded`` line is the +domain of the negotiated transport. It is common for the data immediately +following to contain additional metadata about the state of the transport and +the server. However, this isn't strictly speaking part of the transport +handshake and isn't covered by this section. + +Here is an example handshake between a version 2 aware client and a version +2 aware server: + + c: upgrade 2e82ab3f-9ce3-4b4e-8f8c-6fd1c0e9e23a proto=ssh-v2 + c: hello\n + c: between\n + c: pairs 81\n + c: 0000000000000000000000000000000000000000-0000000000000000000000000000000000000000 + s: upgraded 2e82ab3f-9ce3-4b4e-8f8c-6fd1c0e9e23a ssh-v2\n + s: + +The client-issued token that is echoed in the response provides a more +resilient mechanism for differentiating *banner* output from Mercurial +output. In version 1, properly formatted banner output could get confused +for Mercurial server output. By submitting a randomly generated token +that is then present in the response, the client can look for that token +in response lines and have reasonable certainty that the line did not +originate from a *banner* message. + SSH Version 1 Transport ----------------------- @@ -281,6 +370,31 @@ response. The server terminates if it receives an empty command (a ``\n`` character). +SSH Version 2 Transport +----------------------- + +**Experimental** + +Version 2 of the SSH transport behaves identically to version 1 of the SSH +transport with the exception of handshake semantics. See above for how +version 2 of the SSH transport is negotiated. + +Immediately following the ``upgraded`` line signaling a switch to version +2 of the SSH protocol, the server automatically sends additional details +about the capabilities of the remote server. This has the form: + + \n + capabilities: ...\n + +e.g. + + s: upgraded 2e82ab3f-9ce3-4b4e-8f8c-6fd1c0e9e23a ssh-v2\n + s: 240\n + s: capabilities: known getbundle batch ...\n + +Following capabilities advertisement, the peers communicate using version +1 of the SSH transport. + Capabilities ============ diff --git a/mercurial/sshpeer.py b/mercurial/sshpeer.py --- a/mercurial/sshpeer.py +++ b/mercurial/sshpeer.py @@ -8,6 +8,7 @@ from __future__ import absolute_import import re +import uuid from .i18n import _ from . import ( @@ -15,6 +16,7 @@ from . import ( pycompat, util, wireproto, + wireprotoserver, ) def _serverquote(s): @@ -162,15 +164,24 @@ def _performhandshake(ui, stdin, stdout, hint = ui.config('ui', 'ssherrorhint') raise error.RepoError(msg, hint=hint) - # The handshake consists of sending 2 wire protocol commands: - # ``hello`` and ``between``. + # The handshake consists of sending wire protocol commands in reverse + # order of protocol implementation and then sniffing for a response + # to one of them. + # + # Those commands (from oldest to newest) are: # - # The ``hello`` command (which was introduced in Mercurial 0.9.1) - # instructs the server to advertise its capabilities. + # ``between`` + # Asks for the set of revisions between a pair of revisions. Command + # present in all Mercurial server implementations. # - # The ``between`` command (which has existed in all Mercurial servers - # for as long as SSH support has existed), asks for the set of revisions - # between a pair of revisions. + # ``hello`` + # Instructs the server to advertise its capabilities. Introduced in + # Mercurial 0.9.1. + # + # ``upgrade`` + # Requests upgrade from default transport protocol version 1 to + # a newer version. Introduced in Mercurial 4.6 as an experimental + # feature. # # The ``between`` command is issued with a request for the null # range. If the remote is a Mercurial server, this request will @@ -186,6 +197,18 @@ def _performhandshake(ui, stdin, stdout, # RFC 822 like lines. Of these, the ``capabilities:`` line contains # the capabilities of the server. # + # The ``upgrade`` command isn't really a command in the traditional + # sense of version 1 of the transport because it isn't using the + # proper mechanism for formatting insteads: instead, it just encodes + # arguments on the line, delimited by spaces. + # + # The ``upgrade`` line looks like ``upgrade ``. + # If the server doesn't support protocol upgrades, it will reply to + # this line with ``0\n``. Otherwise, it emits an + # ``upgraded `` line to both stdout and stderr. + # Content immediately following this line describes additional + # protocol and server state. + # # In addition to the responses to our command requests, the server # may emit "banner" output on stdout. SSH servers are allowed to # print messages to stdout on login. Issuing commands on connection @@ -195,6 +218,14 @@ def _performhandshake(ui, stdin, stdout, requestlog = ui.configbool('devel', 'debug.peer-request') + # Generate a random token to help identify responses to version 2 + # upgrade request. + token = bytes(uuid.uuid4()) + upgradecaps = [ + ('proto', wireprotoserver.SSHV2), + ] + upgradecaps = util.urlreq.urlencode(upgradecaps) + try: pairsarg = '%s-%s' % ('0' * 40, '0' * 40) handshake = [ @@ -204,6 +235,11 @@ def _performhandshake(ui, stdin, stdout, pairsarg, ] + # Request upgrade to version 2 if configured. + if ui.configbool('experimental', 'sshpeer.advertise-v2'): + ui.debug('sending upgrade request: %s %s\n' % (token, upgradecaps)) + handshake.insert(0, 'upgrade %s %s\n' % (token, upgradecaps)) + if requestlog: ui.debug('devel-peer-request: hello\n') ui.debug('sending hello command\n') @@ -217,12 +253,31 @@ def _performhandshake(ui, stdin, stdout, except IOError: badresponse() + # Assume version 1 of wire protocol by default. + protoname = wireprotoserver.SSHV1 + reupgraded = re.compile(b'^upgraded %s (.*)$' % re.escape(token)) + lines = ['', 'dummy'] max_noise = 500 while lines[-1] and max_noise: try: l = stdout.readline() _forwardoutput(ui, stderr) + + # Look for reply to protocol upgrade request. It has a token + # in it, so there should be no false positives. + m = reupgraded.match(l) + if m: + protoname = m.group(1) + ui.debug('protocol upgraded to %s\n' % protoname) + # If an upgrade was handled, the ``hello`` and ``between`` + # requests are ignored. The next output belongs to the + # protocol, so stop scanning lines. + break + + # Otherwise it could be a banner, ``0\n`` response if server + # doesn't support upgrade. + if lines[-1] == '1\n' and l == '\n': break if l: @@ -235,20 +290,39 @@ def _performhandshake(ui, stdin, stdout, badresponse() caps = set() - for l in reversed(lines): - # Look for response to ``hello`` command. Scan from the back so - # we don't misinterpret banner output as the command reply. - if l.startswith('capabilities:'): - caps.update(l[:-1].split(':')[1].split()) - break - # Error if we couldn't find a response to ``hello``. This could - # mean: + # For version 1, we should see a ``capabilities`` line in response to the + # ``hello`` command. + if protoname == wireprotoserver.SSHV1: + for l in reversed(lines): + # Look for response to ``hello`` command. Scan from the back so + # we don't misinterpret banner output as the command reply. + if l.startswith('capabilities:'): + caps.update(l[:-1].split(':')[1].split()) + break + elif protoname == wireprotoserver.SSHV2: + # We see a line with number of bytes to follow and then a value + # looking like ``capabilities: *``. + line = stdout.readline() + try: + valuelen = int(line) + except ValueError: + badresponse() + + capsline = stdout.read(valuelen) + if not capsline.startswith('capabilities: '): + badresponse() + + caps.update(capsline.split(':')[1].split()) + # Trailing newline. + stdout.read(1) + + # Error if we couldn't find capabilities, this means: # # 1. Remote isn't a Mercurial server # 2. Remote is a <0.9.1 Mercurial server # 3. Remote is a future Mercurial server that dropped ``hello`` - # support. + # and other attempted handshake mechanisms. if not caps: badresponse() diff --git a/mercurial/wireprotoserver.py b/mercurial/wireprotoserver.py --- a/mercurial/wireprotoserver.py +++ b/mercurial/wireprotoserver.py @@ -32,6 +32,12 @@ HGTYPE = 'application/mercurial-0.1' HGTYPE2 = 'application/mercurial-0.2' HGERRTYPE = 'application/hg-error' +# Names of the SSH protocol implementations. +SSHV1 = 'ssh-v1' +# This is advertised over the wire. Incremental the counter at the end +# to reflect BC breakages. +SSHV2 = 'exp-ssh-v2-0001' + class abstractserverproto(object): """abstract class that summarizes the protocol API diff --git a/tests/sshprotoext.py b/tests/sshprotoext.py --- a/tests/sshprotoext.py +++ b/tests/sshprotoext.py @@ -53,6 +53,35 @@ class prehelloserver(wireprotoserver.ssh super(prehelloserver, self).serve_forever() +class upgradev2server(wireprotoserver.sshserver): + """Tests behavior for clients that issue upgrade to version 2.""" + def serve_forever(self): + name = wireprotoserver.SSHV2 + l = self._fin.readline() + assert l.startswith(b'upgrade ') + token, caps = l[:-1].split(b' ')[1:] + assert caps == b'proto=%s' % name + + # Filter hello and between requests. + l = self._fin.readline() + assert l == b'hello\n' + l = self._fin.readline() + assert l == b'between\n' + l = self._fin.readline() + assert l == 'pairs 81\n' + self._fin.read(81) + + # Send the upgrade response. + self._fout.write(b'upgraded %s %s\n' % (token, name)) + servercaps = wireproto.capabilities(self._repo, self) + rsp = b'capabilities: %s' % servercaps + self._fout.write(b'%d\n' % len(rsp)) + self._fout.write(rsp) + self._fout.write(b'\n') + self._fout.flush() + + super(upgradev2server, self).serve_forever() + def performhandshake(orig, ui, stdin, stdout, stderr): """Wrapped version of sshpeer._performhandshake to send extra commands.""" mode = ui.config(b'sshpeer', b'handshake-mode') @@ -85,6 +114,8 @@ def extsetup(ui): wireprotoserver.sshserver = bannerserver elif servermode == b'no-hello': wireprotoserver.sshserver = prehelloserver + elif servermode == b'upgradev2': + wireprotoserver.sshserver = upgradev2server elif servermode: raise error.ProgrammingError(b'unknown server mode: %s' % servermode) diff --git a/tests/test-ssh-proto.t b/tests/test-ssh-proto.t --- a/tests/test-ssh-proto.t +++ b/tests/test-ssh-proto.t @@ -388,3 +388,107 @@ And one with arguments 0 0 0 + +Send an upgrade request to a server that doesn't support that command + + $ hg -R server serve --stdio << EOF + > upgrade 2e82ab3f-9ce3-4b4e-8f8c-6fd1c0e9e23a proto=irrelevant1%2Cirrelevant2 + > hello + > between + > pairs 81 + > 0000000000000000000000000000000000000000-0000000000000000000000000000000000000000 + > EOF + 0 + 384 + capabilities: lookup changegroupsubset branchmap pushkey known getbundle unbundlehash batch streamreqs=generaldelta,revlogv1 $USUAL_BUNDLE2_CAPS_SERVER$ unbundle=HG10GZ,HG10BZ,HG10UN + 1 + + + $ hg --config experimental.sshpeer.advertise-v2=true --debug debugpeer ssh://user@dummy/server + running * "*/tests/dummyssh" 'user@dummy' 'hg -R server serve --stdio' (glob) + sending upgrade request: * proto=exp-ssh-v2-0001 (glob) + devel-peer-request: hello + sending hello command + devel-peer-request: between + devel-peer-request: pairs: 81 bytes + sending between command + remote: 0 + remote: 384 + remote: capabilities: lookup changegroupsubset branchmap pushkey known getbundle unbundlehash batch streamreqs=generaldelta,revlogv1 $USUAL_BUNDLE2_CAPS_SERVER$ unbundle=HG10GZ,HG10BZ,HG10UN + remote: 1 + url: ssh://user@dummy/server + local: no + pushable: yes + +Send an upgrade request to a server that supports upgrade + + $ SSHSERVERMODE=upgradev2 hg -R server serve --stdio << EOF + > upgrade this-is-some-token proto=exp-ssh-v2-0001 + > hello + > between + > pairs 81 + > 0000000000000000000000000000000000000000-0000000000000000000000000000000000000000 + > EOF + upgraded this-is-some-token exp-ssh-v2-0001 + 383 + capabilities: lookup changegroupsubset branchmap pushkey known getbundle unbundlehash batch streamreqs=generaldelta,revlogv1 $USUAL_BUNDLE2_CAPS_SERVER$ unbundle=HG10GZ,HG10BZ,HG10UN + + $ SSHSERVERMODE=upgradev2 hg --config experimental.sshpeer.advertise-v2=true --debug debugpeer ssh://user@dummy/server + running * "*/tests/dummyssh" 'user@dummy' 'hg -R server serve --stdio' (glob) + sending upgrade request: * proto=exp-ssh-v2-0001 (glob) + devel-peer-request: hello + sending hello command + devel-peer-request: between + devel-peer-request: pairs: 81 bytes + sending between command + protocol upgraded to exp-ssh-v2-0001 + url: ssh://user@dummy/server + local: no + pushable: yes + +Verify the peer has capabilities + + $ SSHSERVERMODE=upgradev2 hg --config experimental.sshpeer.advertise-v2=true --debug debugcapabilities ssh://user@dummy/server + running * "*/tests/dummyssh" 'user@dummy' 'hg -R server serve --stdio' (glob) + sending upgrade request: * proto=exp-ssh-v2-0001 (glob) + devel-peer-request: hello + sending hello command + devel-peer-request: between + devel-peer-request: pairs: 81 bytes + sending between command + protocol upgraded to exp-ssh-v2-0001 + Main capabilities: + batch + branchmap + $USUAL_BUNDLE2_CAPS_SERVER$ + changegroupsubset + getbundle + known + lookup + pushkey + streamreqs=generaldelta,revlogv1 + unbundle=HG10GZ,HG10BZ,HG10UN + unbundlehash + Bundle2 capabilities: + HG20 + bookmarks + changegroup + 01 + 02 + digests + md5 + sha1 + sha512 + error + abort + unsupportedcontent + pushraced + pushkey + hgtagsfnodes + listkeys + phases + heads + pushkey + remote-changegroup + http + https