##// END OF EJS Templates
sshpeer: initial definition and implementation of new SSH protocol...
sshpeer: initial definition and implementation of new SSH protocol The existing SSH protocol has several design flaws. Future commits will elaborate on these flaws as new features are introduced to combat these flaws. For now, hopefully you can take me for my word that a ground up rewrite of the SSH protocol is needed. This commit lays the foundation for a new SSH protocol by defining a mechanism to upgrade the SSH transport channel away from the default (version 1) protocol to something modern (which we'll call "version 2" for now). This upgrade process is detailed in the internals documentation for the wire protocol. The gist of it is the client sends a request line preceding the "hello" command/line which basically says "I'm requesting an upgrade: here's what I support." If the server recognizes that line, it processes the upgrade request and the transport channel is switched to use the new version of the protocol. If not, it sends an empty response, which is how all Mercurial SSH servers from the beginning of time reacted to unknown commands. The upgrade request is effectively ignored and the client continues to use the existing version of the protocol as if nothing happened. The new version of the SSH protocol is completely identical to version 1 aside from the upgrade dance and the bytes that follow. The immediate bytes that follow the protocol switch are defined to be a length framed "capabilities: " line containing the remote's advertised capabilities. In reality, this looks very similar to what the "hello" response would look like. But it will evolve quickly. The methodology by which the protocol will evolve is important. I'm not going to introduce the new protocol all at once. That would likely lead to endless bike shedding and forward progress would stall. Instead, I intend to tricle out new features and diversions from the existing protocol in small, incremental changes. To support the gradual evolution of the protocol, the on-the-wire advertised protocol name contains an "exp" to denote "experimental" and a 4 digit field to capture the sub-version of the protocol. Whenever we make a BC change to the wire protocol, we can increment this version and lock out all older clients because it will appear as a completely different protocol version. This means we can incur as many breaking changes as we want. We don't have to commit to supporting any one feature or idea for a long period of time. We can even evolve the handshake mechanism, because that is defined as being an implementation detail of the negotiated protocol version! Hopefully this lowers the barrier to accepting changes to the protocol and for experimenting with "radical" ideas during its development. In core, sshpeer received most of the attention. We haven't even implemented the server bits for the new protocol in core yet. Instead, we add very primitive support to our test server, mainly just to exercise the added code paths in sshpeer. Differential Revision: https://phab.mercurial-scm.org/D2061 # no-check-commit because of required foo_bar naming

File last commit:

r35656:edc9330a default
r35994:48a3a928 default
Show More
test-revlog-raw.py
315 lines | 10.9 KiB | text/x-python | PythonLexer
# test revlog interaction about raw data (flagprocessor)
from __future__ import absolute_import, print_function
import sys
from mercurial import (
encoding,
node,
revlog,
transaction,
vfs,
)
# TESTTMP is optional. This makes it convenient to run without run-tests.py
tvfs = vfs.vfs(encoding.environ.get('TESTTMP', b'/tmp'))
# Enable generaldelta otherwise revlog won't use delta as expected by the test
tvfs.options = {'generaldelta': True, 'revlogv1': True}
# The test wants to control whether to use delta explicitly, based on
# "storedeltachains".
revlog.revlog._isgooddeltainfo = lambda self, d, textlen: self.storedeltachains
def abort(msg):
print('abort: %s' % msg)
# Return 0 so run-tests.py could compare the output.
sys.exit()
# Register a revlog processor for flag EXTSTORED.
#
# It simply prepends a fixed header, and replaces '1' to 'i'. So it has
# insertion and replacement, and may be interesting to test revlog's line-based
# deltas.
_extheader = b'E\n'
def readprocessor(self, rawtext):
# True: the returned text could be used to verify hash
text = rawtext[len(_extheader):].replace(b'i', b'1')
return text, True
def writeprocessor(self, text):
# False: the returned rawtext shouldn't be used to verify hash
rawtext = _extheader + text.replace(b'1', b'i')
return rawtext, False
def rawprocessor(self, rawtext):
# False: do not verify hash. Only the content returned by "readprocessor"
# can be used to verify hash.
return False
revlog.addflagprocessor(revlog.REVIDX_EXTSTORED,
(readprocessor, writeprocessor, rawprocessor))
# Utilities about reading and appending revlog
def newtransaction():
# A transaction is required to write revlogs
report = lambda msg: None
return transaction.transaction(report, tvfs, {'plain': tvfs}, b'journal')
def newrevlog(name=b'_testrevlog.i', recreate=False):
if recreate:
tvfs.tryunlink(name)
rlog = revlog.revlog(tvfs, name)
return rlog
def appendrev(rlog, text, tr, isext=False, isdelta=True):
'''Append a revision. If isext is True, set the EXTSTORED flag so flag
processor will be used (and rawtext is different from text). If isdelta is
True, force the revision to be a delta, otherwise it's full text.
'''
nextrev = len(rlog)
p1 = rlog.node(nextrev - 1)
p2 = node.nullid
if isext:
flags = revlog.REVIDX_EXTSTORED
else:
flags = revlog.REVIDX_DEFAULT_FLAGS
# Change storedeltachains temporarily, to override revlog's delta decision
rlog.storedeltachains = isdelta
try:
rlog.addrevision(text, tr, nextrev, p1, p2, flags=flags)
return nextrev
except Exception as ex:
abort('rev %d: failed to append: %s' % (nextrev, ex))
finally:
# Restore storedeltachains. It is always True, see revlog.__init__
rlog.storedeltachains = True
def addgroupcopy(rlog, tr, destname=b'_destrevlog.i', optimaldelta=True):
'''Copy revlog to destname using revlog.addgroup. Return the copied revlog.
This emulates push or pull. They use changegroup. Changegroup requires
repo to work. We don't have a repo, so a dummy changegroup is used.
If optimaldelta is True, use optimized delta parent, so the destination
revlog could probably reuse it. Otherwise it builds sub-optimal delta, and
the destination revlog needs more work to use it.
This exercises some revlog.addgroup (and revlog._addrevision(text=None))
code path, which is not covered by "appendrev" alone.
'''
class dummychangegroup(object):
@staticmethod
def deltachunk(pnode):
pnode = pnode or node.nullid
parentrev = rlog.rev(pnode)
r = parentrev + 1
if r >= len(rlog):
return {}
if optimaldelta:
deltaparent = parentrev
else:
# suboptimal deltaparent
deltaparent = min(0, parentrev)
return {'node': rlog.node(r), 'p1': pnode, 'p2': node.nullid,
'cs': rlog.node(rlog.linkrev(r)), 'flags': rlog.flags(r),
'deltabase': rlog.node(deltaparent),
'delta': rlog.revdiff(deltaparent, r)}
def deltaiter(self):
chain = None
for chunkdata in iter(lambda: self.deltachunk(chain), {}):
node = chunkdata['node']
p1 = chunkdata['p1']
p2 = chunkdata['p2']
cs = chunkdata['cs']
deltabase = chunkdata['deltabase']
delta = chunkdata['delta']
flags = chunkdata['flags']
chain = node
yield (node, p1, p2, cs, deltabase, delta, flags)
def linkmap(lnode):
return rlog.rev(lnode)
dlog = newrevlog(destname, recreate=True)
dummydeltas = dummychangegroup().deltaiter()
dlog.addgroup(dummydeltas, linkmap, tr)
return dlog
def lowlevelcopy(rlog, tr, destname=b'_destrevlog.i'):
'''Like addgroupcopy, but use the low level revlog._addrevision directly.
It exercises some code paths that are hard to reach easily otherwise.
'''
dlog = newrevlog(destname, recreate=True)
for r in rlog:
p1 = rlog.node(r - 1)
p2 = node.nullid
if r == 0:
text = rlog.revision(r, raw=True)
cachedelta = None
else:
# deltaparent is more interesting if it has the EXTSTORED flag.
deltaparent = max([0] + [p for p in range(r - 2) if rlog.flags(p)])
text = None
cachedelta = (deltaparent, rlog.revdiff(deltaparent, r))
flags = rlog.flags(r)
ifh = dfh = None
try:
ifh = dlog.opener(dlog.indexfile, 'a+')
if not dlog._inline:
dfh = dlog.opener(dlog.datafile, 'a+')
dlog._addrevision(rlog.node(r), text, tr, r, p1, p2, flags,
cachedelta, ifh, dfh)
finally:
if dfh is not None:
dfh.close()
if ifh is not None:
ifh.close()
return dlog
# Utilities to generate revisions for testing
def genbits(n):
'''Given a number n, generate (2 ** (n * 2) + 1) numbers in range(2 ** n).
i.e. the generated numbers have a width of n bits.
The combination of two adjacent numbers will cover all possible cases.
That is to say, given any x, y where both x, and y are in range(2 ** n),
there is an x followed immediately by y in the generated sequence.
'''
m = 2 ** n
# Gray Code. See https://en.wikipedia.org/wiki/Gray_code
gray = lambda x: x ^ (x >> 1)
reversegray = dict((gray(i), i) for i in range(m))
# Generate (n * 2) bit gray code, yield lower n bits as X, and look for
# the next unused gray code where higher n bits equal to X.
# For gray codes whose higher bits are X, a[X] of them have been used.
a = [0] * m
# Iterate from 0.
x = 0
yield x
for i in range(m * m):
x = reversegray[x]
y = gray(a[x] + x * m) & (m - 1)
assert a[x] < m
a[x] += 1
x = y
yield x
def gentext(rev):
'''Given a revision number, generate dummy text'''
return b''.join(b'%d\n' % j for j in range(-1, rev % 5))
def writecases(rlog, tr):
'''Write some revisions interested to the test.
The test is interested in 3 properties of a revision:
- Is it a delta or a full text? (isdelta)
This is to catch some delta application issues.
- Does it have a flag of EXTSTORED? (isext)
This is to catch some flag processor issues. Especially when
interacted with revlog deltas.
- Is its text empty? (isempty)
This is less important. It is intended to try to catch some careless
checks like "if text" instead of "if text is None". Note: if flag
processor is involved, raw text may be not empty.
Write 65 revisions. So that all combinations of the above flags for
adjacent revisions are covered. That is to say,
len(set(
(r.delta, r.ext, r.empty, (r+1).delta, (r+1).ext, (r+1).empty)
for r in range(len(rlog) - 1)
)) is 64.
Where "r.delta", "r.ext", and "r.empty" are booleans matching properties
mentioned above.
Return expected [(text, rawtext)].
'''
result = []
for i, x in enumerate(genbits(3)):
isdelta, isext, isempty = bool(x & 1), bool(x & 2), bool(x & 4)
if isempty:
text = b''
else:
text = gentext(i)
rev = appendrev(rlog, text, tr, isext=isext, isdelta=isdelta)
# Verify text, rawtext, and rawsize
if isext:
rawtext = writeprocessor(None, text)[0]
else:
rawtext = text
if rlog.rawsize(rev) != len(rawtext):
abort('rev %d: wrong rawsize' % rev)
if rlog.revision(rev, raw=False) != text:
abort('rev %d: wrong text' % rev)
if rlog.revision(rev, raw=True) != rawtext:
abort('rev %d: wrong rawtext' % rev)
result.append((text, rawtext))
# Verify flags like isdelta, isext work as expected
if bool(rlog.deltaparent(rev) > -1) != isdelta:
abort('rev %d: isdelta is ineffective' % rev)
if bool(rlog.flags(rev)) != isext:
abort('rev %d: isext is ineffective' % rev)
return result
# Main test and checking
def checkrevlog(rlog, expected):
'''Check if revlog has expected contents. expected is [(text, rawtext)]'''
# Test using different access orders. This could expose some issues
# depending on revlog caching (see revlog._cache).
for r0 in range(len(rlog) - 1):
r1 = r0 + 1
for revorder in [[r0, r1], [r1, r0]]:
for raworder in [[True], [False], [True, False], [False, True]]:
nlog = newrevlog()
for rev in revorder:
for raw in raworder:
t = nlog.revision(rev, raw=raw)
if t != expected[rev][int(raw)]:
abort('rev %d: corrupted %stext'
% (rev, raw and 'raw' or ''))
def maintest():
expected = rl = None
with newtransaction() as tr:
rl = newrevlog(recreate=True)
expected = writecases(rl, tr)
checkrevlog(rl, expected)
print('local test passed')
# Copy via revlog.addgroup
rl1 = addgroupcopy(rl, tr)
checkrevlog(rl1, expected)
rl2 = addgroupcopy(rl, tr, optimaldelta=False)
checkrevlog(rl2, expected)
print('addgroupcopy test passed')
# Copy via revlog.clone
rl3 = newrevlog(name='_destrevlog3.i', recreate=True)
rl.clone(tr, rl3)
checkrevlog(rl3, expected)
print('clone test passed')
# Copy via low-level revlog._addrevision
rl4 = lowlevelcopy(rl, tr)
checkrevlog(rl4, expected)
print('lowlevelcopy test passed')
try:
maintest()
except Exception as ex:
abort('crashed: %s' % ex)