##// END OF EJS Templates
setdiscovery: simplify by using tiprev directly...
Joerg Sonnenberger -
r47599:e7b4607d default
parent child Browse files
Show More
@@ -1,519 +1,519 b''
1 # setdiscovery.py - improved discovery of common nodeset for mercurial
1 # setdiscovery.py - improved discovery of common nodeset for mercurial
2 #
2 #
3 # Copyright 2010 Benoit Boissinot <bboissin@gmail.com>
3 # Copyright 2010 Benoit Boissinot <bboissin@gmail.com>
4 # and Peter Arrenbrecht <peter@arrenbrecht.ch>
4 # and Peter Arrenbrecht <peter@arrenbrecht.ch>
5 #
5 #
6 # This software may be used and distributed according to the terms of the
6 # This software may be used and distributed according to the terms of the
7 # GNU General Public License version 2 or any later version.
7 # GNU General Public License version 2 or any later version.
8 """
8 """
9 Algorithm works in the following way. You have two repository: local and
9 Algorithm works in the following way. You have two repository: local and
10 remote. They both contains a DAG of changelists.
10 remote. They both contains a DAG of changelists.
11
11
12 The goal of the discovery protocol is to find one set of node *common*,
12 The goal of the discovery protocol is to find one set of node *common*,
13 the set of nodes shared by local and remote.
13 the set of nodes shared by local and remote.
14
14
15 One of the issue with the original protocol was latency, it could
15 One of the issue with the original protocol was latency, it could
16 potentially require lots of roundtrips to discover that the local repo was a
16 potentially require lots of roundtrips to discover that the local repo was a
17 subset of remote (which is a very common case, you usually have few changes
17 subset of remote (which is a very common case, you usually have few changes
18 compared to upstream, while upstream probably had lots of development).
18 compared to upstream, while upstream probably had lots of development).
19
19
20 The new protocol only requires one interface for the remote repo: `known()`,
20 The new protocol only requires one interface for the remote repo: `known()`,
21 which given a set of changelists tells you if they are present in the DAG.
21 which given a set of changelists tells you if they are present in the DAG.
22
22
23 The algorithm then works as follow:
23 The algorithm then works as follow:
24
24
25 - We will be using three sets, `common`, `missing`, `unknown`. Originally
25 - We will be using three sets, `common`, `missing`, `unknown`. Originally
26 all nodes are in `unknown`.
26 all nodes are in `unknown`.
27 - Take a sample from `unknown`, call `remote.known(sample)`
27 - Take a sample from `unknown`, call `remote.known(sample)`
28 - For each node that remote knows, move it and all its ancestors to `common`
28 - For each node that remote knows, move it and all its ancestors to `common`
29 - For each node that remote doesn't know, move it and all its descendants
29 - For each node that remote doesn't know, move it and all its descendants
30 to `missing`
30 to `missing`
31 - Iterate until `unknown` is empty
31 - Iterate until `unknown` is empty
32
32
33 There are a couple optimizations, first is instead of starting with a random
33 There are a couple optimizations, first is instead of starting with a random
34 sample of missing, start by sending all heads, in the case where the local
34 sample of missing, start by sending all heads, in the case where the local
35 repo is a subset, you computed the answer in one round trip.
35 repo is a subset, you computed the answer in one round trip.
36
36
37 Then you can do something similar to the bisecting strategy used when
37 Then you can do something similar to the bisecting strategy used when
38 finding faulty changesets. Instead of random samples, you can try picking
38 finding faulty changesets. Instead of random samples, you can try picking
39 nodes that will maximize the number of nodes that will be
39 nodes that will maximize the number of nodes that will be
40 classified with it (since all ancestors or descendants will be marked as well).
40 classified with it (since all ancestors or descendants will be marked as well).
41 """
41 """
42
42
43 from __future__ import absolute_import
43 from __future__ import absolute_import
44
44
45 import collections
45 import collections
46 import random
46 import random
47
47
48 from .i18n import _
48 from .i18n import _
49 from .node import (
49 from .node import (
50 nullid,
50 nullid,
51 nullrev,
51 nullrev,
52 )
52 )
53 from . import (
53 from . import (
54 error,
54 error,
55 policy,
55 policy,
56 util,
56 util,
57 )
57 )
58
58
59
59
60 def _updatesample(revs, heads, sample, parentfn, quicksamplesize=0):
60 def _updatesample(revs, heads, sample, parentfn, quicksamplesize=0):
61 """update an existing sample to match the expected size
61 """update an existing sample to match the expected size
62
62
63 The sample is updated with revs exponentially distant from each head of the
63 The sample is updated with revs exponentially distant from each head of the
64 <revs> set. (H~1, H~2, H~4, H~8, etc).
64 <revs> set. (H~1, H~2, H~4, H~8, etc).
65
65
66 If a target size is specified, the sampling will stop once this size is
66 If a target size is specified, the sampling will stop once this size is
67 reached. Otherwise sampling will happen until roots of the <revs> set are
67 reached. Otherwise sampling will happen until roots of the <revs> set are
68 reached.
68 reached.
69
69
70 :revs: set of revs we want to discover (if None, assume the whole dag)
70 :revs: set of revs we want to discover (if None, assume the whole dag)
71 :heads: set of DAG head revs
71 :heads: set of DAG head revs
72 :sample: a sample to update
72 :sample: a sample to update
73 :parentfn: a callable to resolve parents for a revision
73 :parentfn: a callable to resolve parents for a revision
74 :quicksamplesize: optional target size of the sample"""
74 :quicksamplesize: optional target size of the sample"""
75 dist = {}
75 dist = {}
76 visit = collections.deque(heads)
76 visit = collections.deque(heads)
77 seen = set()
77 seen = set()
78 factor = 1
78 factor = 1
79 while visit:
79 while visit:
80 curr = visit.popleft()
80 curr = visit.popleft()
81 if curr in seen:
81 if curr in seen:
82 continue
82 continue
83 d = dist.setdefault(curr, 1)
83 d = dist.setdefault(curr, 1)
84 if d > factor:
84 if d > factor:
85 factor *= 2
85 factor *= 2
86 if d == factor:
86 if d == factor:
87 sample.add(curr)
87 sample.add(curr)
88 if quicksamplesize and (len(sample) >= quicksamplesize):
88 if quicksamplesize and (len(sample) >= quicksamplesize):
89 return
89 return
90 seen.add(curr)
90 seen.add(curr)
91
91
92 for p in parentfn(curr):
92 for p in parentfn(curr):
93 if p != nullrev and (not revs or p in revs):
93 if p != nullrev and (not revs or p in revs):
94 dist.setdefault(p, d + 1)
94 dist.setdefault(p, d + 1)
95 visit.append(p)
95 visit.append(p)
96
96
97
97
98 def _limitsample(sample, desiredlen, randomize=True):
98 def _limitsample(sample, desiredlen, randomize=True):
99 """return a random subset of sample of at most desiredlen item.
99 """return a random subset of sample of at most desiredlen item.
100
100
101 If randomize is False, though, a deterministic subset is returned.
101 If randomize is False, though, a deterministic subset is returned.
102 This is meant for integration tests.
102 This is meant for integration tests.
103 """
103 """
104 if len(sample) <= desiredlen:
104 if len(sample) <= desiredlen:
105 return sample
105 return sample
106 if randomize:
106 if randomize:
107 return set(random.sample(sample, desiredlen))
107 return set(random.sample(sample, desiredlen))
108 sample = list(sample)
108 sample = list(sample)
109 sample.sort()
109 sample.sort()
110 return set(sample[:desiredlen])
110 return set(sample[:desiredlen])
111
111
112
112
113 class partialdiscovery(object):
113 class partialdiscovery(object):
114 """an object representing ongoing discovery
114 """an object representing ongoing discovery
115
115
116 Feed with data from the remote repository, this object keep track of the
116 Feed with data from the remote repository, this object keep track of the
117 current set of changeset in various states:
117 current set of changeset in various states:
118
118
119 - common: revs also known remotely
119 - common: revs also known remotely
120 - undecided: revs we don't have information on yet
120 - undecided: revs we don't have information on yet
121 - missing: revs missing remotely
121 - missing: revs missing remotely
122 (all tracked revisions are known locally)
122 (all tracked revisions are known locally)
123 """
123 """
124
124
125 def __init__(self, repo, targetheads, respectsize, randomize=True):
125 def __init__(self, repo, targetheads, respectsize, randomize=True):
126 self._repo = repo
126 self._repo = repo
127 self._targetheads = targetheads
127 self._targetheads = targetheads
128 self._common = repo.changelog.incrementalmissingrevs()
128 self._common = repo.changelog.incrementalmissingrevs()
129 self._undecided = None
129 self._undecided = None
130 self.missing = set()
130 self.missing = set()
131 self._childrenmap = None
131 self._childrenmap = None
132 self._respectsize = respectsize
132 self._respectsize = respectsize
133 self.randomize = randomize
133 self.randomize = randomize
134
134
135 def addcommons(self, commons):
135 def addcommons(self, commons):
136 """register nodes known as common"""
136 """register nodes known as common"""
137 self._common.addbases(commons)
137 self._common.addbases(commons)
138 if self._undecided is not None:
138 if self._undecided is not None:
139 self._common.removeancestorsfrom(self._undecided)
139 self._common.removeancestorsfrom(self._undecided)
140
140
141 def addmissings(self, missings):
141 def addmissings(self, missings):
142 """register some nodes as missing"""
142 """register some nodes as missing"""
143 newmissing = self._repo.revs(b'%ld::%ld', missings, self.undecided)
143 newmissing = self._repo.revs(b'%ld::%ld', missings, self.undecided)
144 if newmissing:
144 if newmissing:
145 self.missing.update(newmissing)
145 self.missing.update(newmissing)
146 self.undecided.difference_update(newmissing)
146 self.undecided.difference_update(newmissing)
147
147
148 def addinfo(self, sample):
148 def addinfo(self, sample):
149 """consume an iterable of (rev, known) tuples"""
149 """consume an iterable of (rev, known) tuples"""
150 common = set()
150 common = set()
151 missing = set()
151 missing = set()
152 for rev, known in sample:
152 for rev, known in sample:
153 if known:
153 if known:
154 common.add(rev)
154 common.add(rev)
155 else:
155 else:
156 missing.add(rev)
156 missing.add(rev)
157 if common:
157 if common:
158 self.addcommons(common)
158 self.addcommons(common)
159 if missing:
159 if missing:
160 self.addmissings(missing)
160 self.addmissings(missing)
161
161
162 def hasinfo(self):
162 def hasinfo(self):
163 """return True is we have any clue about the remote state"""
163 """return True is we have any clue about the remote state"""
164 return self._common.hasbases()
164 return self._common.hasbases()
165
165
166 def iscomplete(self):
166 def iscomplete(self):
167 """True if all the necessary data have been gathered"""
167 """True if all the necessary data have been gathered"""
168 return self._undecided is not None and not self._undecided
168 return self._undecided is not None and not self._undecided
169
169
170 @property
170 @property
171 def undecided(self):
171 def undecided(self):
172 if self._undecided is not None:
172 if self._undecided is not None:
173 return self._undecided
173 return self._undecided
174 self._undecided = set(self._common.missingancestors(self._targetheads))
174 self._undecided = set(self._common.missingancestors(self._targetheads))
175 return self._undecided
175 return self._undecided
176
176
177 def stats(self):
177 def stats(self):
178 return {
178 return {
179 'undecided': len(self.undecided),
179 'undecided': len(self.undecided),
180 }
180 }
181
181
182 def commonheads(self):
182 def commonheads(self):
183 """the heads of the known common set"""
183 """the heads of the known common set"""
184 # heads(common) == heads(common.bases) since common represents
184 # heads(common) == heads(common.bases) since common represents
185 # common.bases and all its ancestors
185 # common.bases and all its ancestors
186 return self._common.basesheads()
186 return self._common.basesheads()
187
187
188 def _parentsgetter(self):
188 def _parentsgetter(self):
189 getrev = self._repo.changelog.index.__getitem__
189 getrev = self._repo.changelog.index.__getitem__
190
190
191 def getparents(r):
191 def getparents(r):
192 return getrev(r)[5:7]
192 return getrev(r)[5:7]
193
193
194 return getparents
194 return getparents
195
195
196 def _childrengetter(self):
196 def _childrengetter(self):
197
197
198 if self._childrenmap is not None:
198 if self._childrenmap is not None:
199 # During discovery, the `undecided` set keep shrinking.
199 # During discovery, the `undecided` set keep shrinking.
200 # Therefore, the map computed for an iteration N will be
200 # Therefore, the map computed for an iteration N will be
201 # valid for iteration N+1. Instead of computing the same
201 # valid for iteration N+1. Instead of computing the same
202 # data over and over we cached it the first time.
202 # data over and over we cached it the first time.
203 return self._childrenmap.__getitem__
203 return self._childrenmap.__getitem__
204
204
205 # _updatesample() essentially does interaction over revisions to look
205 # _updatesample() essentially does interaction over revisions to look
206 # up their children. This lookup is expensive and doing it in a loop is
206 # up their children. This lookup is expensive and doing it in a loop is
207 # quadratic. We precompute the children for all relevant revisions and
207 # quadratic. We precompute the children for all relevant revisions and
208 # make the lookup in _updatesample() a simple dict lookup.
208 # make the lookup in _updatesample() a simple dict lookup.
209 self._childrenmap = children = {}
209 self._childrenmap = children = {}
210
210
211 parentrevs = self._parentsgetter()
211 parentrevs = self._parentsgetter()
212 revs = self.undecided
212 revs = self.undecided
213
213
214 for rev in sorted(revs):
214 for rev in sorted(revs):
215 # Always ensure revision has an entry so we don't need to worry
215 # Always ensure revision has an entry so we don't need to worry
216 # about missing keys.
216 # about missing keys.
217 children[rev] = []
217 children[rev] = []
218 for prev in parentrevs(rev):
218 for prev in parentrevs(rev):
219 if prev == nullrev:
219 if prev == nullrev:
220 continue
220 continue
221 c = children.get(prev)
221 c = children.get(prev)
222 if c is not None:
222 if c is not None:
223 c.append(rev)
223 c.append(rev)
224 return children.__getitem__
224 return children.__getitem__
225
225
226 def takequicksample(self, headrevs, size):
226 def takequicksample(self, headrevs, size):
227 """takes a quick sample of size <size>
227 """takes a quick sample of size <size>
228
228
229 It is meant for initial sampling and focuses on querying heads and close
229 It is meant for initial sampling and focuses on querying heads and close
230 ancestors of heads.
230 ancestors of heads.
231
231
232 :headrevs: set of head revisions in local DAG to consider
232 :headrevs: set of head revisions in local DAG to consider
233 :size: the maximum size of the sample"""
233 :size: the maximum size of the sample"""
234 revs = self.undecided
234 revs = self.undecided
235 if len(revs) <= size:
235 if len(revs) <= size:
236 return list(revs)
236 return list(revs)
237 sample = set(self._repo.revs(b'heads(%ld)', revs))
237 sample = set(self._repo.revs(b'heads(%ld)', revs))
238
238
239 if len(sample) >= size:
239 if len(sample) >= size:
240 return _limitsample(sample, size, randomize=self.randomize)
240 return _limitsample(sample, size, randomize=self.randomize)
241
241
242 _updatesample(
242 _updatesample(
243 None, headrevs, sample, self._parentsgetter(), quicksamplesize=size
243 None, headrevs, sample, self._parentsgetter(), quicksamplesize=size
244 )
244 )
245 return sample
245 return sample
246
246
247 def takefullsample(self, headrevs, size):
247 def takefullsample(self, headrevs, size):
248 revs = self.undecided
248 revs = self.undecided
249 if len(revs) <= size:
249 if len(revs) <= size:
250 return list(revs)
250 return list(revs)
251 repo = self._repo
251 repo = self._repo
252 sample = set(repo.revs(b'heads(%ld)', revs))
252 sample = set(repo.revs(b'heads(%ld)', revs))
253 parentrevs = self._parentsgetter()
253 parentrevs = self._parentsgetter()
254
254
255 # update from heads
255 # update from heads
256 revsheads = sample.copy()
256 revsheads = sample.copy()
257 _updatesample(revs, revsheads, sample, parentrevs)
257 _updatesample(revs, revsheads, sample, parentrevs)
258
258
259 # update from roots
259 # update from roots
260 revsroots = set(repo.revs(b'roots(%ld)', revs))
260 revsroots = set(repo.revs(b'roots(%ld)', revs))
261 childrenrevs = self._childrengetter()
261 childrenrevs = self._childrengetter()
262 _updatesample(revs, revsroots, sample, childrenrevs)
262 _updatesample(revs, revsroots, sample, childrenrevs)
263 assert sample
263 assert sample
264
264
265 if not self._respectsize:
265 if not self._respectsize:
266 size = max(size, min(len(revsroots), len(revsheads)))
266 size = max(size, min(len(revsroots), len(revsheads)))
267
267
268 sample = _limitsample(sample, size, randomize=self.randomize)
268 sample = _limitsample(sample, size, randomize=self.randomize)
269 if len(sample) < size:
269 if len(sample) < size:
270 more = size - len(sample)
270 more = size - len(sample)
271 takefrom = list(revs - sample)
271 takefrom = list(revs - sample)
272 if self.randomize:
272 if self.randomize:
273 sample.update(random.sample(takefrom, more))
273 sample.update(random.sample(takefrom, more))
274 else:
274 else:
275 takefrom.sort()
275 takefrom.sort()
276 sample.update(takefrom[:more])
276 sample.update(takefrom[:more])
277 return sample
277 return sample
278
278
279
279
280 partialdiscovery = policy.importrust(
280 partialdiscovery = policy.importrust(
281 'discovery', member='PartialDiscovery', default=partialdiscovery
281 'discovery', member='PartialDiscovery', default=partialdiscovery
282 )
282 )
283
283
284
284
285 def findcommonheads(
285 def findcommonheads(
286 ui,
286 ui,
287 local,
287 local,
288 remote,
288 remote,
289 abortwhenunrelated=True,
289 abortwhenunrelated=True,
290 ancestorsof=None,
290 ancestorsof=None,
291 audit=None,
291 audit=None,
292 ):
292 ):
293 """Return a tuple (common, anyincoming, remoteheads) used to identify
293 """Return a tuple (common, anyincoming, remoteheads) used to identify
294 missing nodes from or in remote.
294 missing nodes from or in remote.
295
295
296 The audit argument is an optional dictionnary that a caller can pass. it
296 The audit argument is an optional dictionnary that a caller can pass. it
297 will be updated with extra data about the discovery, this is useful for
297 will be updated with extra data about the discovery, this is useful for
298 debug.
298 debug.
299 """
299 """
300
300
301 samplegrowth = float(ui.config(b'devel', b'discovery.grow-sample.rate'))
301 samplegrowth = float(ui.config(b'devel', b'discovery.grow-sample.rate'))
302
302
303 start = util.timer()
303 start = util.timer()
304
304
305 roundtrips = 0
305 roundtrips = 0
306 cl = local.changelog
306 cl = local.changelog
307 clnode = cl.node
307 clnode = cl.node
308 clrev = cl.rev
308 clrev = cl.rev
309
309
310 if ancestorsof is not None:
310 if ancestorsof is not None:
311 ownheads = [clrev(n) for n in ancestorsof]
311 ownheads = [clrev(n) for n in ancestorsof]
312 else:
312 else:
313 ownheads = [rev for rev in cl.headrevs() if rev != nullrev]
313 ownheads = [rev for rev in cl.headrevs() if rev != nullrev]
314
314
315 initial_head_exchange = ui.configbool(b'devel', b'discovery.exchange-heads')
315 initial_head_exchange = ui.configbool(b'devel', b'discovery.exchange-heads')
316 initialsamplesize = ui.configint(b'devel', b'discovery.sample-size.initial')
316 initialsamplesize = ui.configint(b'devel', b'discovery.sample-size.initial')
317 fullsamplesize = ui.configint(b'devel', b'discovery.sample-size')
317 fullsamplesize = ui.configint(b'devel', b'discovery.sample-size')
318 # We also ask remote about all the local heads. That set can be arbitrarily
318 # We also ask remote about all the local heads. That set can be arbitrarily
319 # large, so we used to limit it size to `initialsamplesize`. We no longer
319 # large, so we used to limit it size to `initialsamplesize`. We no longer
320 # do as it proved counter productive. The skipped heads could lead to a
320 # do as it proved counter productive. The skipped heads could lead to a
321 # large "undecided" set, slower to be clarified than if we asked the
321 # large "undecided" set, slower to be clarified than if we asked the
322 # question for all heads right away.
322 # question for all heads right away.
323 #
323 #
324 # We are already fetching all server heads using the `heads` commands,
324 # We are already fetching all server heads using the `heads` commands,
325 # sending a equivalent number of heads the other way should not have a
325 # sending a equivalent number of heads the other way should not have a
326 # significant impact. In addition, it is very likely that we are going to
326 # significant impact. In addition, it is very likely that we are going to
327 # have to issue "known" request for an equivalent amount of revisions in
327 # have to issue "known" request for an equivalent amount of revisions in
328 # order to decide if theses heads are common or missing.
328 # order to decide if theses heads are common or missing.
329 #
329 #
330 # find a detailled analysis below.
330 # find a detailled analysis below.
331 #
331 #
332 # Case A: local and server both has few heads
332 # Case A: local and server both has few heads
333 #
333 #
334 # Ownheads is below initialsamplesize, limit would not have any effect.
334 # Ownheads is below initialsamplesize, limit would not have any effect.
335 #
335 #
336 # Case B: local has few heads and server has many
336 # Case B: local has few heads and server has many
337 #
337 #
338 # Ownheads is below initialsamplesize, limit would not have any effect.
338 # Ownheads is below initialsamplesize, limit would not have any effect.
339 #
339 #
340 # Case C: local and server both has many heads
340 # Case C: local and server both has many heads
341 #
341 #
342 # We now transfert some more data, but not significantly more than is
342 # We now transfert some more data, but not significantly more than is
343 # already transfered to carry the server heads.
343 # already transfered to carry the server heads.
344 #
344 #
345 # Case D: local has many heads, server has few
345 # Case D: local has many heads, server has few
346 #
346 #
347 # D.1 local heads are mostly known remotely
347 # D.1 local heads are mostly known remotely
348 #
348 #
349 # All the known head will have be part of a `known` request at some
349 # All the known head will have be part of a `known` request at some
350 # point for the discovery to finish. Sending them all earlier is
350 # point for the discovery to finish. Sending them all earlier is
351 # actually helping.
351 # actually helping.
352 #
352 #
353 # (This case is fairly unlikely, it requires the numerous heads to all
353 # (This case is fairly unlikely, it requires the numerous heads to all
354 # be merged server side in only a few heads)
354 # be merged server side in only a few heads)
355 #
355 #
356 # D.2 local heads are mostly missing remotely
356 # D.2 local heads are mostly missing remotely
357 #
357 #
358 # To determine that the heads are missing, we'll have to issue `known`
358 # To determine that the heads are missing, we'll have to issue `known`
359 # request for them or one of their ancestors. This amount of `known`
359 # request for them or one of their ancestors. This amount of `known`
360 # request will likely be in the same order of magnitude than the amount
360 # request will likely be in the same order of magnitude than the amount
361 # of local heads.
361 # of local heads.
362 #
362 #
363 # The only case where we can be more efficient using `known` request on
363 # The only case where we can be more efficient using `known` request on
364 # ancestors are case were all the "missing" local heads are based on a
364 # ancestors are case were all the "missing" local heads are based on a
365 # few changeset, also "missing". This means we would have a "complex"
365 # few changeset, also "missing". This means we would have a "complex"
366 # graph (with many heads) attached to, but very independant to a the
366 # graph (with many heads) attached to, but very independant to a the
367 # "simple" graph on the server. This is a fairly usual case and have
367 # "simple" graph on the server. This is a fairly usual case and have
368 # not been met in the wild so far.
368 # not been met in the wild so far.
369 if initial_head_exchange:
369 if initial_head_exchange:
370 if remote.limitedarguments:
370 if remote.limitedarguments:
371 sample = _limitsample(ownheads, initialsamplesize)
371 sample = _limitsample(ownheads, initialsamplesize)
372 # indices between sample and externalized version must match
372 # indices between sample and externalized version must match
373 sample = list(sample)
373 sample = list(sample)
374 else:
374 else:
375 sample = ownheads
375 sample = ownheads
376
376
377 ui.debug(b"query 1; heads\n")
377 ui.debug(b"query 1; heads\n")
378 roundtrips += 1
378 roundtrips += 1
379 with remote.commandexecutor() as e:
379 with remote.commandexecutor() as e:
380 fheads = e.callcommand(b'heads', {})
380 fheads = e.callcommand(b'heads', {})
381 fknown = e.callcommand(
381 fknown = e.callcommand(
382 b'known',
382 b'known',
383 {
383 {
384 b'nodes': [clnode(r) for r in sample],
384 b'nodes': [clnode(r) for r in sample],
385 },
385 },
386 )
386 )
387
387
388 srvheadhashes, yesno = fheads.result(), fknown.result()
388 srvheadhashes, yesno = fheads.result(), fknown.result()
389
389
390 if audit is not None:
390 if audit is not None:
391 audit[b'total-roundtrips'] = 1
391 audit[b'total-roundtrips'] = 1
392
392
393 if cl.tip() == nullid:
393 if cl.tiprev() == nullrev:
394 if srvheadhashes != [nullid]:
394 if srvheadhashes != [nullid]:
395 return [nullid], True, srvheadhashes
395 return [nullid], True, srvheadhashes
396 return [nullid], False, []
396 return [nullid], False, []
397 else:
397 else:
398 # we still need the remote head for the function return
398 # we still need the remote head for the function return
399 with remote.commandexecutor() as e:
399 with remote.commandexecutor() as e:
400 fheads = e.callcommand(b'heads', {})
400 fheads = e.callcommand(b'heads', {})
401 srvheadhashes = fheads.result()
401 srvheadhashes = fheads.result()
402
402
403 # start actual discovery (we note this before the next "if" for
403 # start actual discovery (we note this before the next "if" for
404 # compatibility reasons)
404 # compatibility reasons)
405 ui.status(_(b"searching for changes\n"))
405 ui.status(_(b"searching for changes\n"))
406
406
407 knownsrvheads = [] # revnos of remote heads that are known locally
407 knownsrvheads = [] # revnos of remote heads that are known locally
408 for node in srvheadhashes:
408 for node in srvheadhashes:
409 if node == nullid:
409 if node == nullid:
410 continue
410 continue
411
411
412 try:
412 try:
413 knownsrvheads.append(clrev(node))
413 knownsrvheads.append(clrev(node))
414 # Catches unknown and filtered nodes.
414 # Catches unknown and filtered nodes.
415 except error.LookupError:
415 except error.LookupError:
416 continue
416 continue
417
417
418 if initial_head_exchange:
418 if initial_head_exchange:
419 # early exit if we know all the specified remote heads already
419 # early exit if we know all the specified remote heads already
420 if len(knownsrvheads) == len(srvheadhashes):
420 if len(knownsrvheads) == len(srvheadhashes):
421 ui.debug(b"all remote heads known locally\n")
421 ui.debug(b"all remote heads known locally\n")
422 return srvheadhashes, False, srvheadhashes
422 return srvheadhashes, False, srvheadhashes
423
423
424 if len(sample) == len(ownheads) and all(yesno):
424 if len(sample) == len(ownheads) and all(yesno):
425 ui.note(_(b"all local changesets known remotely\n"))
425 ui.note(_(b"all local changesets known remotely\n"))
426 ownheadhashes = [clnode(r) for r in ownheads]
426 ownheadhashes = [clnode(r) for r in ownheads]
427 return ownheadhashes, True, srvheadhashes
427 return ownheadhashes, True, srvheadhashes
428
428
429 # full blown discovery
429 # full blown discovery
430
430
431 # if the server has a limit to its arguments size, we can't grow the sample.
431 # if the server has a limit to its arguments size, we can't grow the sample.
432 configbool = local.ui.configbool
432 configbool = local.ui.configbool
433 grow_sample = configbool(b'devel', b'discovery.grow-sample')
433 grow_sample = configbool(b'devel', b'discovery.grow-sample')
434 grow_sample = grow_sample and not remote.limitedarguments
434 grow_sample = grow_sample and not remote.limitedarguments
435
435
436 dynamic_sample = configbool(b'devel', b'discovery.grow-sample.dynamic')
436 dynamic_sample = configbool(b'devel', b'discovery.grow-sample.dynamic')
437 hard_limit_sample = not (dynamic_sample or remote.limitedarguments)
437 hard_limit_sample = not (dynamic_sample or remote.limitedarguments)
438
438
439 randomize = ui.configbool(b'devel', b'discovery.randomize')
439 randomize = ui.configbool(b'devel', b'discovery.randomize')
440 disco = partialdiscovery(
440 disco = partialdiscovery(
441 local, ownheads, hard_limit_sample, randomize=randomize
441 local, ownheads, hard_limit_sample, randomize=randomize
442 )
442 )
443 if initial_head_exchange:
443 if initial_head_exchange:
444 # treat remote heads (and maybe own heads) as a first implicit sample
444 # treat remote heads (and maybe own heads) as a first implicit sample
445 # response
445 # response
446 disco.addcommons(knownsrvheads)
446 disco.addcommons(knownsrvheads)
447 disco.addinfo(zip(sample, yesno))
447 disco.addinfo(zip(sample, yesno))
448
448
449 full = not initial_head_exchange
449 full = not initial_head_exchange
450 progress = ui.makeprogress(_(b'searching'), unit=_(b'queries'))
450 progress = ui.makeprogress(_(b'searching'), unit=_(b'queries'))
451 while not disco.iscomplete():
451 while not disco.iscomplete():
452
452
453 if full or disco.hasinfo():
453 if full or disco.hasinfo():
454 if full:
454 if full:
455 ui.note(_(b"sampling from both directions\n"))
455 ui.note(_(b"sampling from both directions\n"))
456 else:
456 else:
457 ui.debug(b"taking initial sample\n")
457 ui.debug(b"taking initial sample\n")
458 samplefunc = disco.takefullsample
458 samplefunc = disco.takefullsample
459 targetsize = fullsamplesize
459 targetsize = fullsamplesize
460 if grow_sample:
460 if grow_sample:
461 fullsamplesize = int(fullsamplesize * samplegrowth)
461 fullsamplesize = int(fullsamplesize * samplegrowth)
462 else:
462 else:
463 # use even cheaper initial sample
463 # use even cheaper initial sample
464 ui.debug(b"taking quick initial sample\n")
464 ui.debug(b"taking quick initial sample\n")
465 samplefunc = disco.takequicksample
465 samplefunc = disco.takequicksample
466 targetsize = initialsamplesize
466 targetsize = initialsamplesize
467 sample = samplefunc(ownheads, targetsize)
467 sample = samplefunc(ownheads, targetsize)
468
468
469 roundtrips += 1
469 roundtrips += 1
470 progress.update(roundtrips)
470 progress.update(roundtrips)
471 stats = disco.stats()
471 stats = disco.stats()
472 ui.debug(
472 ui.debug(
473 b"query %i; still undecided: %i, sample size is: %i\n"
473 b"query %i; still undecided: %i, sample size is: %i\n"
474 % (roundtrips, stats['undecided'], len(sample))
474 % (roundtrips, stats['undecided'], len(sample))
475 )
475 )
476
476
477 # indices between sample and externalized version must match
477 # indices between sample and externalized version must match
478 sample = list(sample)
478 sample = list(sample)
479
479
480 with remote.commandexecutor() as e:
480 with remote.commandexecutor() as e:
481 yesno = e.callcommand(
481 yesno = e.callcommand(
482 b'known',
482 b'known',
483 {
483 {
484 b'nodes': [clnode(r) for r in sample],
484 b'nodes': [clnode(r) for r in sample],
485 },
485 },
486 ).result()
486 ).result()
487
487
488 full = True
488 full = True
489
489
490 disco.addinfo(zip(sample, yesno))
490 disco.addinfo(zip(sample, yesno))
491
491
492 result = disco.commonheads()
492 result = disco.commonheads()
493 elapsed = util.timer() - start
493 elapsed = util.timer() - start
494 progress.complete()
494 progress.complete()
495 ui.debug(b"%d total queries in %.4fs\n" % (roundtrips, elapsed))
495 ui.debug(b"%d total queries in %.4fs\n" % (roundtrips, elapsed))
496 msg = (
496 msg = (
497 b'found %d common and %d unknown server heads,'
497 b'found %d common and %d unknown server heads,'
498 b' %d roundtrips in %.4fs\n'
498 b' %d roundtrips in %.4fs\n'
499 )
499 )
500 missing = set(result) - set(knownsrvheads)
500 missing = set(result) - set(knownsrvheads)
501 ui.log(b'discovery', msg, len(result), len(missing), roundtrips, elapsed)
501 ui.log(b'discovery', msg, len(result), len(missing), roundtrips, elapsed)
502
502
503 if audit is not None:
503 if audit is not None:
504 audit[b'total-roundtrips'] = roundtrips
504 audit[b'total-roundtrips'] = roundtrips
505
505
506 if not result and srvheadhashes != [nullid]:
506 if not result and srvheadhashes != [nullid]:
507 if abortwhenunrelated:
507 if abortwhenunrelated:
508 raise error.Abort(_(b"repository is unrelated"))
508 raise error.Abort(_(b"repository is unrelated"))
509 else:
509 else:
510 ui.warn(_(b"warning: repository is unrelated\n"))
510 ui.warn(_(b"warning: repository is unrelated\n"))
511 return (
511 return (
512 {nullid},
512 {nullid},
513 True,
513 True,
514 srvheadhashes,
514 srvheadhashes,
515 )
515 )
516
516
517 anyincoming = srvheadhashes != [nullid]
517 anyincoming = srvheadhashes != [nullid]
518 result = {clnode(r) for r in result}
518 result = {clnode(r) for r in result}
519 return result, anyincoming, srvheadhashes
519 return result, anyincoming, srvheadhashes
General Comments 0
You need to be logged in to leave comments. Login now