Show More
@@ -1,55 +1,138 b'' | |||
|
1 | 1 | # exchangev2.py - repository exchange for wire protocol version 2 |
|
2 | 2 | # |
|
3 | 3 | # Copyright 2018 Gregory Szorc <gregory.szorc@gmail.com> |
|
4 | 4 | # |
|
5 | 5 | # This software may be used and distributed according to the terms of the |
|
6 | 6 | # GNU General Public License version 2 or any later version. |
|
7 | 7 | |
|
8 | 8 | from __future__ import absolute_import |
|
9 | 9 | |
|
10 | import weakref | |
|
11 | ||
|
12 | from .i18n import _ | |
|
10 | 13 | from .node import ( |
|
11 | 14 | nullid, |
|
15 | short, | |
|
12 | 16 | ) |
|
13 | 17 | from . import ( |
|
18 | mdiff, | |
|
19 | pycompat, | |
|
14 | 20 | setdiscovery, |
|
15 | 21 | ) |
|
16 | 22 | |
|
17 | 23 | def pull(pullop): |
|
18 | 24 | """Pull using wire protocol version 2.""" |
|
19 | 25 | repo = pullop.repo |
|
20 | 26 | remote = pullop.remote |
|
27 | tr = pullop.trmanager.transaction() | |
|
21 | 28 | |
|
22 | 29 | # Figure out what needs to be fetched. |
|
23 | 30 | common, fetch, remoteheads = _pullchangesetdiscovery( |
|
24 | 31 | repo, remote, pullop.heads, abortwhenunrelated=pullop.force) |
|
25 | 32 | |
|
33 | pullheads = pullop.heads or remoteheads | |
|
34 | _fetchchangesets(repo, tr, remote, common, fetch, pullheads) | |
|
35 | ||
|
26 | 36 | def _pullchangesetdiscovery(repo, remote, heads, abortwhenunrelated=True): |
|
27 | 37 | """Determine which changesets need to be pulled.""" |
|
28 | 38 | |
|
29 | 39 | if heads: |
|
30 | 40 | knownnode = repo.changelog.hasnode |
|
31 | 41 | if all(knownnode(head) for head in heads): |
|
32 | 42 | return heads, False, heads |
|
33 | 43 | |
|
34 | 44 | # TODO wire protocol version 2 is capable of more efficient discovery |
|
35 | 45 | # than setdiscovery. Consider implementing something better. |
|
36 | 46 | common, fetch, remoteheads = setdiscovery.findcommonheads( |
|
37 | 47 | repo.ui, repo, remote, abortwhenunrelated=abortwhenunrelated) |
|
38 | 48 | |
|
39 | 49 | common = set(common) |
|
40 | 50 | remoteheads = set(remoteheads) |
|
41 | 51 | |
|
42 | 52 | # If a remote head is filtered locally, put it back in the common set. |
|
43 | 53 | # See the comment in exchange._pulldiscoverychangegroup() for more. |
|
44 | 54 | |
|
45 | 55 | if fetch and remoteheads: |
|
46 | 56 | nodemap = repo.unfiltered().changelog.nodemap |
|
47 | 57 | |
|
48 | 58 | common |= {head for head in remoteheads if head in nodemap} |
|
49 | 59 | |
|
50 | 60 | if set(remoteheads).issubset(common): |
|
51 | 61 | fetch = [] |
|
52 | 62 | |
|
53 | 63 | common.discard(nullid) |
|
54 | 64 | |
|
55 | 65 | return common, fetch, remoteheads |
|
66 | ||
|
67 | def _fetchchangesets(repo, tr, remote, common, fetch, remoteheads): | |
|
68 | if not fetch: | |
|
69 | return | |
|
70 | ||
|
71 | # TODO consider adding a step here where we obtain the DAG shape first | |
|
72 | # (or ask the server to slice changesets into chunks for us) so that | |
|
73 | # we can perform multiple fetches in batches. This will facilitate | |
|
74 | # resuming interrupted clones, higher server-side cache hit rates due | |
|
75 | # to smaller segments, etc. | |
|
76 | with remote.commandexecutor() as e: | |
|
77 | objs = e.callcommand(b'changesetdata', { | |
|
78 | b'noderange': [sorted(common), sorted(remoteheads)], | |
|
79 | b'fields': {b'parents', b'revision'}, | |
|
80 | }).result() | |
|
81 | ||
|
82 | # The context manager waits on all response data when exiting. So | |
|
83 | # we need to remain in the context manager in order to stream data. | |
|
84 | return _processchangesetdata(repo, tr, objs) | |
|
85 | ||
|
86 | def _processchangesetdata(repo, tr, objs): | |
|
87 | repo.hook('prechangegroup', throw=True, | |
|
88 | **pycompat.strkwargs(tr.hookargs)) | |
|
89 | ||
|
90 | urepo = repo.unfiltered() | |
|
91 | cl = urepo.changelog | |
|
92 | ||
|
93 | cl.delayupdate(tr) | |
|
94 | ||
|
95 | # The first emitted object is a header describing the data that | |
|
96 | # follows. | |
|
97 | meta = next(objs) | |
|
98 | ||
|
99 | progress = repo.ui.makeprogress(_('changesets'), | |
|
100 | unit=_('chunks'), | |
|
101 | total=meta.get(b'totalitems')) | |
|
102 | ||
|
103 | def linkrev(node): | |
|
104 | repo.ui.debug('add changeset %s\n' % short(node)) | |
|
105 | # Linkrev for changelog is always self. | |
|
106 | return len(cl) | |
|
107 | ||
|
108 | def onchangeset(cl, node): | |
|
109 | progress.increment() | |
|
110 | ||
|
111 | # addgroup() expects a 7-tuple describing revisions. This normalizes | |
|
112 | # the wire data to that format. | |
|
113 | def iterrevisions(): | |
|
114 | for cset in objs: | |
|
115 | assert b'revisionsize' in cset | |
|
116 | data = next(objs) | |
|
117 | ||
|
118 | yield ( | |
|
119 | cset[b'node'], | |
|
120 | cset[b'parents'][0], | |
|
121 | cset[b'parents'][1], | |
|
122 | # Linknode is always itself for changesets. | |
|
123 | cset[b'node'], | |
|
124 | # We always send full revisions. So delta base is not set. | |
|
125 | nullid, | |
|
126 | mdiff.trivialdiffheader(len(data)) + data, | |
|
127 | # Flags not yet supported. | |
|
128 | 0, | |
|
129 | ) | |
|
130 | ||
|
131 | added = cl.addgroup(iterrevisions(), linkrev, weakref.proxy(tr), | |
|
132 | addrevisioncb=onchangeset) | |
|
133 | ||
|
134 | progress.complete() | |
|
135 | ||
|
136 | return { | |
|
137 | 'added': added, | |
|
138 | } |
@@ -1,53 +1,214 b'' | |||
|
1 | 1 | Tests for wire protocol version 2 exchange. |
|
2 | 2 | Tests in this file should be folded into existing tests once protocol |
|
3 | 3 | v2 has enough features that it can be enabled via #testcase in existing |
|
4 | 4 | tests. |
|
5 | 5 | |
|
6 | 6 | $ . $TESTDIR/wireprotohelpers.sh |
|
7 | 7 | $ enablehttpv2client |
|
8 | 8 | |
|
9 | 9 | $ hg init server-simple |
|
10 | 10 | $ enablehttpv2 server-simple |
|
11 | 11 | $ cd server-simple |
|
12 | 12 | $ cat >> .hg/hgrc << EOF |
|
13 | 13 | > [phases] |
|
14 | 14 | > publish = false |
|
15 | 15 | > EOF |
|
16 | 16 | $ echo a0 > a |
|
17 | 17 | $ echo b0 > b |
|
18 | 18 | $ hg -q commit -A -m 'commit 0' |
|
19 | 19 | |
|
20 | 20 | $ echo a1 > a |
|
21 | 21 | $ hg commit -m 'commit 1' |
|
22 | 22 | $ hg phase --public -r . |
|
23 | 23 | $ echo a2 > a |
|
24 | 24 | $ hg commit -m 'commit 2' |
|
25 | 25 | |
|
26 | 26 | $ hg -q up -r 0 |
|
27 | 27 | $ echo b1 > b |
|
28 | 28 | $ hg -q commit -m 'head 2 commit 1' |
|
29 | 29 | $ echo b2 > b |
|
30 | 30 | $ hg -q commit -m 'head 2 commit 2' |
|
31 | 31 | |
|
32 | 32 | $ hg serve -p $HGPORT -d --pid-file hg.pid -E error.log |
|
33 | 33 | $ cat hg.pid > $DAEMON_PIDS |
|
34 | 34 | |
|
35 | 35 | $ cd .. |
|
36 | 36 | |
|
37 | 37 | Test basic clone |
|
38 | 38 | |
|
39 | 39 | $ hg --debug clone -U http://localhost:$HGPORT client-simple |
|
40 | 40 | using http://localhost:$HGPORT/ |
|
41 | 41 | sending capabilities command |
|
42 | 42 | query 1; heads |
|
43 | 43 | sending 2 commands |
|
44 | 44 | sending command heads: {} |
|
45 | 45 | sending command known: { |
|
46 | 46 | 'nodes': [] |
|
47 | 47 | } |
|
48 | 48 | received frame(size=11; request=1; stream=2; streamflags=stream-begin; type=command-response; flags=continuation) |
|
49 | 49 | received frame(size=43; request=1; stream=2; streamflags=; type=command-response; flags=continuation) |
|
50 | 50 | received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos) |
|
51 | 51 | received frame(size=11; request=3; stream=2; streamflags=; type=command-response; flags=continuation) |
|
52 | 52 | received frame(size=1; request=3; stream=2; streamflags=; type=command-response; flags=continuation) |
|
53 | 53 | received frame(size=0; request=3; stream=2; streamflags=; type=command-response; flags=eos) |
|
54 | sending 1 commands | |
|
55 | sending command changesetdata: { | |
|
56 | 'fields': set([ | |
|
57 | 'parents', | |
|
58 | 'revision' | |
|
59 | ]), | |
|
60 | 'noderange': [ | |
|
61 | [], | |
|
62 | [ | |
|
63 | '\xca\xa2\xa4eE\x1d\xd1\xfa\xcd\xa0\xf5\xb1#\x12\xc3UXA\x88\xa1', | |
|
64 | '\xcd%4vk\xec\xe18\xc7\xc1\xaf\xdch%0/\x0fb\xd8\x1f' | |
|
65 | ] | |
|
66 | ] | |
|
67 | } | |
|
68 | received frame(size=11; request=1; stream=2; streamflags=stream-begin; type=command-response; flags=continuation) | |
|
69 | received frame(size=809; request=1; stream=2; streamflags=; type=command-response; flags=continuation) | |
|
70 | received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos) | |
|
71 | add changeset 3390ef850073 | |
|
72 | add changeset 4432d83626e8 | |
|
73 | add changeset cd2534766bec | |
|
74 | add changeset e96ae20f4188 | |
|
75 | add changeset caa2a465451d | |
|
76 | updating the branch cache | |
|
77 | new changesets 3390ef850073:caa2a465451d | |
|
78 | ||
|
79 | All changesets should have been transferred | |
|
80 | ||
|
81 | $ hg -R client-simple debugindex -c | |
|
82 | rev linkrev nodeid p1 p2 | |
|
83 | 0 0 3390ef850073 000000000000 000000000000 | |
|
84 | 1 1 4432d83626e8 3390ef850073 000000000000 | |
|
85 | 2 2 cd2534766bec 4432d83626e8 000000000000 | |
|
86 | 3 3 e96ae20f4188 3390ef850073 000000000000 | |
|
87 | 4 4 caa2a465451d e96ae20f4188 000000000000 | |
|
88 | ||
|
89 | $ hg -R client-simple log -G -T '{rev} {node} {phase}\n' | |
|
90 | o 4 caa2a465451dd1facda0f5b12312c355584188a1 public | |
|
91 | | | |
|
92 | o 3 e96ae20f4188487b9ae4ef3941c27c81143146e5 public | |
|
93 | | | |
|
94 | | o 2 cd2534766bece138c7c1afdc6825302f0f62d81f public | |
|
95 | | | | |
|
96 | | o 1 4432d83626e8a98655f062ec1f2a43b07f7fbbb0 public | |
|
97 | |/ | |
|
98 | o 0 3390ef850073fbc2f0dfff2244342c8e9229013a public | |
|
99 | ||
|
100 | ||
|
101 | Cloning only a specific revision works | |
|
102 | ||
|
103 | $ hg --debug clone -U -r 4432d83626e8 http://localhost:$HGPORT client-singlehead | |
|
104 | using http://localhost:$HGPORT/ | |
|
105 | sending capabilities command | |
|
106 | sending 1 commands | |
|
107 | sending command lookup: { | |
|
108 | 'key': '4432d83626e8' | |
|
109 | } | |
|
110 | received frame(size=11; request=1; stream=2; streamflags=stream-begin; type=command-response; flags=continuation) | |
|
111 | received frame(size=21; request=1; stream=2; streamflags=; type=command-response; flags=continuation) | |
|
112 | received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos) | |
|
113 | query 1; heads | |
|
114 | sending 2 commands | |
|
115 | sending command heads: {} | |
|
116 | sending command known: { | |
|
117 | 'nodes': [] | |
|
118 | } | |
|
119 | received frame(size=11; request=1; stream=2; streamflags=stream-begin; type=command-response; flags=continuation) | |
|
120 | received frame(size=43; request=1; stream=2; streamflags=; type=command-response; flags=continuation) | |
|
121 | received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos) | |
|
122 | received frame(size=11; request=3; stream=2; streamflags=; type=command-response; flags=continuation) | |
|
123 | received frame(size=1; request=3; stream=2; streamflags=; type=command-response; flags=continuation) | |
|
124 | received frame(size=0; request=3; stream=2; streamflags=; type=command-response; flags=eos) | |
|
125 | sending 1 commands | |
|
126 | sending command changesetdata: { | |
|
127 | 'fields': set([ | |
|
128 | 'parents', | |
|
129 | 'revision' | |
|
130 | ]), | |
|
131 | 'noderange': [ | |
|
132 | [], | |
|
133 | [ | |
|
134 | 'D2\xd86&\xe8\xa9\x86U\xf0b\xec\x1f*C\xb0\x7f\x7f\xbb\xb0' | |
|
135 | ] | |
|
136 | ] | |
|
137 | } | |
|
138 | received frame(size=11; request=1; stream=2; streamflags=stream-begin; type=command-response; flags=continuation) | |
|
139 | received frame(size=327; request=1; stream=2; streamflags=; type=command-response; flags=continuation) | |
|
140 | received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos) | |
|
141 | add changeset 3390ef850073 | |
|
142 | add changeset 4432d83626e8 | |
|
143 | updating the branch cache | |
|
144 | new changesets 3390ef850073:4432d83626e8 | |
|
145 | ||
|
146 | $ cd client-singlehead | |
|
147 | ||
|
148 | $ hg log -G -T '{rev} {node} {phase}\n' | |
|
149 | o 1 4432d83626e8a98655f062ec1f2a43b07f7fbbb0 public | |
|
150 | | | |
|
151 | o 0 3390ef850073fbc2f0dfff2244342c8e9229013a public | |
|
152 | ||
|
153 | ||
|
154 | Incremental pull works | |
|
155 | ||
|
156 | $ hg --debug pull | |
|
157 | pulling from http://localhost:$HGPORT/ | |
|
158 | using http://localhost:$HGPORT/ | |
|
159 | sending capabilities command | |
|
160 | query 1; heads | |
|
161 | sending 2 commands | |
|
162 | sending command heads: {} | |
|
163 | sending command known: { | |
|
164 | 'nodes': [ | |
|
165 | 'D2\xd86&\xe8\xa9\x86U\xf0b\xec\x1f*C\xb0\x7f\x7f\xbb\xb0' | |
|
166 | ] | |
|
167 | } | |
|
168 | received frame(size=11; request=1; stream=2; streamflags=stream-begin; type=command-response; flags=continuation) | |
|
169 | received frame(size=43; request=1; stream=2; streamflags=; type=command-response; flags=continuation) | |
|
170 | received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos) | |
|
171 | received frame(size=11; request=3; stream=2; streamflags=; type=command-response; flags=continuation) | |
|
172 | received frame(size=2; request=3; stream=2; streamflags=; type=command-response; flags=continuation) | |
|
173 | received frame(size=0; request=3; stream=2; streamflags=; type=command-response; flags=eos) | |
|
174 | searching for changes | |
|
175 | all local heads known remotely | |
|
176 | sending 1 commands | |
|
177 | sending command changesetdata: { | |
|
178 | 'fields': set([ | |
|
179 | 'parents', | |
|
180 | 'revision' | |
|
181 | ]), | |
|
182 | 'noderange': [ | |
|
183 | [ | |
|
184 | 'D2\xd86&\xe8\xa9\x86U\xf0b\xec\x1f*C\xb0\x7f\x7f\xbb\xb0' | |
|
185 | ], | |
|
186 | [ | |
|
187 | '\xca\xa2\xa4eE\x1d\xd1\xfa\xcd\xa0\xf5\xb1#\x12\xc3UXA\x88\xa1', | |
|
188 | '\xcd%4vk\xec\xe18\xc7\xc1\xaf\xdch%0/\x0fb\xd8\x1f' | |
|
189 | ] | |
|
190 | ] | |
|
191 | } | |
|
192 | received frame(size=11; request=1; stream=2; streamflags=stream-begin; type=command-response; flags=continuation) | |
|
193 | received frame(size=495; request=1; stream=2; streamflags=; type=command-response; flags=continuation) | |
|
194 | received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos) | |
|
195 | add changeset cd2534766bec | |
|
196 | add changeset e96ae20f4188 | |
|
197 | add changeset caa2a465451d | |
|
198 | updating the branch cache | |
|
199 | new changesets cd2534766bec:caa2a465451d | |
|
200 | (run 'hg update' to get a working copy) | |
|
201 | ||
|
202 | $ hg log -G -T '{rev} {node} {phase}\n' | |
|
203 | o 4 caa2a465451dd1facda0f5b12312c355584188a1 public | |
|
204 | | | |
|
205 | o 3 e96ae20f4188487b9ae4ef3941c27c81143146e5 public | |
|
206 | | | |
|
207 | | o 2 cd2534766bece138c7c1afdc6825302f0f62d81f public | |
|
208 | | | | |
|
209 | | o 1 4432d83626e8a98655f062ec1f2a43b07f7fbbb0 public | |
|
210 | |/ | |
|
211 | o 0 3390ef850073fbc2f0dfff2244342c8e9229013a public | |
|
212 | ||
|
213 | ||
|
214 | $ cd .. |
General Comments 0
You need to be logged in to leave comments.
Login now