##// END OF EJS Templates
discovery: re-adjust a conditional wrongly changed...
Boris Feld -
r41197:cc1f545c default
parent child Browse files
Show More
@@ -1,340 +1,340 b''
1 1 # setdiscovery.py - improved discovery of common nodeset for mercurial
2 2 #
3 3 # Copyright 2010 Benoit Boissinot <bboissin@gmail.com>
4 4 # and Peter Arrenbrecht <peter@arrenbrecht.ch>
5 5 #
6 6 # This software may be used and distributed according to the terms of the
7 7 # GNU General Public License version 2 or any later version.
8 8 """
9 9 Algorithm works in the following way. You have two repository: local and
10 10 remote. They both contains a DAG of changelists.
11 11
12 12 The goal of the discovery protocol is to find one set of node *common*,
13 13 the set of nodes shared by local and remote.
14 14
15 15 One of the issue with the original protocol was latency, it could
16 16 potentially require lots of roundtrips to discover that the local repo was a
17 17 subset of remote (which is a very common case, you usually have few changes
18 18 compared to upstream, while upstream probably had lots of development).
19 19
20 20 The new protocol only requires one interface for the remote repo: `known()`,
21 21 which given a set of changelists tells you if they are present in the DAG.
22 22
23 23 The algorithm then works as follow:
24 24
25 25 - We will be using three sets, `common`, `missing`, `unknown`. Originally
26 26 all nodes are in `unknown`.
27 27 - Take a sample from `unknown`, call `remote.known(sample)`
28 28 - For each node that remote knows, move it and all its ancestors to `common`
29 29 - For each node that remote doesn't know, move it and all its descendants
30 30 to `missing`
31 31 - Iterate until `unknown` is empty
32 32
33 33 There are a couple optimizations, first is instead of starting with a random
34 34 sample of missing, start by sending all heads, in the case where the local
35 35 repo is a subset, you computed the answer in one round trip.
36 36
37 37 Then you can do something similar to the bisecting strategy used when
38 38 finding faulty changesets. Instead of random samples, you can try picking
39 39 nodes that will maximize the number of nodes that will be
40 40 classified with it (since all ancestors or descendants will be marked as well).
41 41 """
42 42
43 43 from __future__ import absolute_import
44 44
45 45 import collections
46 46 import random
47 47
48 48 from .i18n import _
49 49 from .node import (
50 50 nullid,
51 51 nullrev,
52 52 )
53 53 from . import (
54 54 error,
55 55 util,
56 56 )
57 57
58 58 def _updatesample(revs, heads, sample, parentfn, quicksamplesize=0):
59 59 """update an existing sample to match the expected size
60 60
61 61 The sample is updated with revs exponentially distant from each head of the
62 62 <revs> set. (H~1, H~2, H~4, H~8, etc).
63 63
64 64 If a target size is specified, the sampling will stop once this size is
65 65 reached. Otherwise sampling will happen until roots of the <revs> set are
66 66 reached.
67 67
68 68 :revs: set of revs we want to discover (if None, assume the whole dag)
69 69 :heads: set of DAG head revs
70 70 :sample: a sample to update
71 71 :parentfn: a callable to resolve parents for a revision
72 72 :quicksamplesize: optional target size of the sample"""
73 73 dist = {}
74 74 visit = collections.deque(heads)
75 75 seen = set()
76 76 factor = 1
77 77 while visit:
78 78 curr = visit.popleft()
79 79 if curr in seen:
80 80 continue
81 81 d = dist.setdefault(curr, 1)
82 82 if d > factor:
83 83 factor *= 2
84 84 if d == factor:
85 85 sample.add(curr)
86 86 if quicksamplesize and (len(sample) >= quicksamplesize):
87 87 return
88 88 seen.add(curr)
89 89
90 90 for p in parentfn(curr):
91 91 if p != nullrev and (not revs or p in revs):
92 92 dist.setdefault(p, d + 1)
93 93 visit.append(p)
94 94
95 95 def _takequicksample(repo, headrevs, revs, size):
96 96 """takes a quick sample of size <size>
97 97
98 98 It is meant for initial sampling and focuses on querying heads and close
99 99 ancestors of heads.
100 100
101 101 :dag: a dag object
102 102 :headrevs: set of head revisions in local DAG to consider
103 103 :revs: set of revs to discover
104 104 :size: the maximum size of the sample"""
105 105 if len(revs) <= size:
106 106 return list(revs)
107 107 sample = set(repo.revs('heads(%ld)', revs))
108 108
109 109 if len(sample) >= size:
110 110 return _limitsample(sample, size)
111 111
112 112 _updatesample(None, headrevs, sample, repo.changelog.parentrevs,
113 113 quicksamplesize=size)
114 114 return sample
115 115
116 116 def _takefullsample(repo, headrevs, revs, size):
117 117 if len(revs) <= size:
118 118 return list(revs)
119 119 sample = set(repo.revs('heads(%ld)', revs))
120 120
121 121 # update from heads
122 122 revsheads = set(repo.revs('heads(%ld)', revs))
123 123 _updatesample(revs, revsheads, sample, repo.changelog.parentrevs)
124 124
125 125 # update from roots
126 126 revsroots = set(repo.revs('roots(%ld)', revs))
127 127
128 128 # _updatesample() essentially does interaction over revisions to look up
129 129 # their children. This lookup is expensive and doing it in a loop is
130 130 # quadratic. We precompute the children for all relevant revisions and
131 131 # make the lookup in _updatesample() a simple dict lookup.
132 132 #
133 133 # Because this function can be called multiple times during discovery, we
134 134 # may still perform redundant work and there is room to optimize this by
135 135 # keeping a persistent cache of children across invocations.
136 136 children = {}
137 137
138 138 parentrevs = repo.changelog.parentrevs
139 139 for rev in repo.changelog.revs(start=min(revsroots)):
140 140 # Always ensure revision has an entry so we don't need to worry about
141 141 # missing keys.
142 142 children.setdefault(rev, [])
143 143
144 144 for prev in parentrevs(rev):
145 145 if prev == nullrev:
146 146 continue
147 147
148 148 children.setdefault(prev, []).append(rev)
149 149
150 150 _updatesample(revs, revsroots, sample, children.__getitem__)
151 151 assert sample
152 152 sample = _limitsample(sample, size)
153 if len(sample) <= size:
153 if len(sample) < size:
154 154 more = size - len(sample)
155 155 sample.update(random.sample(list(revs - sample), more))
156 156 return sample
157 157
158 158 def _limitsample(sample, desiredlen):
159 159 """return a random subset of sample of at most desiredlen item"""
160 160 if len(sample) > desiredlen:
161 161 sample = set(random.sample(sample, desiredlen))
162 162 return sample
163 163
164 164 class partialdiscovery(object):
165 165 """an object representing ongoing discovery
166 166
167 167 Feed with data from the remote repository, this object keep track of the
168 168 current set of changeset in various states:
169 169
170 170 - common: own nodes I know we both know
171 171 """
172 172
173 173 def __init__(self, repo):
174 174 self._repo = repo
175 175 self._common = repo.changelog.incrementalmissingrevs()
176 176
177 177 def addcommons(self, commons):
178 178 """registrer nodes known as common"""
179 179 self._common.addbases(commons)
180 180
181 181 def hasinfo(self):
182 182 """return True is we have any clue about the remote state"""
183 183 return self._common.hasbases()
184 184
185 185 def commonheads(self):
186 186 """the heads of the known common set"""
187 187 # heads(common) == heads(common.bases) since common represents
188 188 # common.bases and all its ancestors
189 189 # The presence of nullrev will confuse heads(). So filter it out.
190 190 return set(self._repo.revs('heads(%ld)',
191 191 self._common.bases - {nullrev}))
192 192
193 193 def findcommonheads(ui, local, remote,
194 194 initialsamplesize=100,
195 195 fullsamplesize=200,
196 196 abortwhenunrelated=True,
197 197 ancestorsof=None):
198 198 '''Return a tuple (common, anyincoming, remoteheads) used to identify
199 199 missing nodes from or in remote.
200 200 '''
201 201 start = util.timer()
202 202
203 203 roundtrips = 0
204 204 cl = local.changelog
205 205 clnode = cl.node
206 206 clrev = cl.rev
207 207
208 208 if ancestorsof is not None:
209 209 ownheads = [clrev(n) for n in ancestorsof]
210 210 else:
211 211 ownheads = [rev for rev in cl.headrevs() if rev != nullrev]
212 212
213 213 # early exit if we know all the specified remote heads already
214 214 ui.debug("query 1; heads\n")
215 215 roundtrips += 1
216 216 sample = _limitsample(ownheads, initialsamplesize)
217 217 # indices between sample and externalized version must match
218 218 sample = list(sample)
219 219
220 220 with remote.commandexecutor() as e:
221 221 fheads = e.callcommand('heads', {})
222 222 fknown = e.callcommand('known', {
223 223 'nodes': [clnode(r) for r in sample],
224 224 })
225 225
226 226 srvheadhashes, yesno = fheads.result(), fknown.result()
227 227
228 228 if cl.tip() == nullid:
229 229 if srvheadhashes != [nullid]:
230 230 return [nullid], True, srvheadhashes
231 231 return [nullid], False, []
232 232
233 233 # start actual discovery (we note this before the next "if" for
234 234 # compatibility reasons)
235 235 ui.status(_("searching for changes\n"))
236 236
237 237 srvheads = []
238 238 for node in srvheadhashes:
239 239 if node == nullid:
240 240 continue
241 241
242 242 try:
243 243 srvheads.append(clrev(node))
244 244 # Catches unknown and filtered nodes.
245 245 except error.LookupError:
246 246 continue
247 247
248 248 if len(srvheads) == len(srvheadhashes):
249 249 ui.debug("all remote heads known locally\n")
250 250 return srvheadhashes, False, srvheadhashes
251 251
252 252 if len(sample) == len(ownheads) and all(yesno):
253 253 ui.note(_("all local heads known remotely\n"))
254 254 ownheadhashes = [clnode(r) for r in ownheads]
255 255 return ownheadhashes, True, srvheadhashes
256 256
257 257 # full blown discovery
258 258
259 259 disco = partialdiscovery(local)
260 260 # treat remote heads (and maybe own heads) as a first implicit sample
261 261 # response
262 262 disco.addcommons(srvheads)
263 263 commoninsample = set(n for i, n in enumerate(sample) if yesno[i])
264 264 disco.addcommons(commoninsample)
265 265 # own nodes where I don't know if remote knows them
266 266 undecided = set(disco._common.missingancestors(ownheads))
267 267 # own nodes I know remote lacks
268 268 missing = set()
269 269
270 270 full = False
271 271 progress = ui.makeprogress(_('searching'), unit=_('queries'))
272 272 while undecided:
273 273
274 274 if sample:
275 275 missinginsample = [n for i, n in enumerate(sample) if not yesno[i]]
276 276
277 277 if missing:
278 278 missing.update(local.revs('descendants(%ld) - descendants(%ld)',
279 279 missinginsample, missing))
280 280 else:
281 281 missing.update(local.revs('descendants(%ld)', missinginsample))
282 282
283 283 undecided.difference_update(missing)
284 284
285 285 if not undecided:
286 286 break
287 287
288 288 if full or disco.hasinfo():
289 289 if full:
290 290 ui.note(_("sampling from both directions\n"))
291 291 else:
292 292 ui.debug("taking initial sample\n")
293 293 samplefunc = _takefullsample
294 294 targetsize = fullsamplesize
295 295 else:
296 296 # use even cheaper initial sample
297 297 ui.debug("taking quick initial sample\n")
298 298 samplefunc = _takequicksample
299 299 targetsize = initialsamplesize
300 300 sample = samplefunc(local, ownheads, undecided, targetsize)
301 301
302 302 roundtrips += 1
303 303 progress.update(roundtrips)
304 304 ui.debug("query %i; still undecided: %i, sample size is: %i\n"
305 305 % (roundtrips, len(undecided), len(sample)))
306 306 # indices between sample and externalized version must match
307 307 sample = list(sample)
308 308
309 309 with remote.commandexecutor() as e:
310 310 yesno = e.callcommand('known', {
311 311 'nodes': [clnode(r) for r in sample],
312 312 }).result()
313 313
314 314 full = True
315 315
316 316 if sample:
317 317 commoninsample = set(n for i, n in enumerate(sample) if yesno[i])
318 318 disco.addcommons(commoninsample)
319 319 disco._common.removeancestorsfrom(undecided)
320 320
321 321 result = disco.commonheads()
322 322 elapsed = util.timer() - start
323 323 progress.complete()
324 324 ui.debug("%d total queries in %.4fs\n" % (roundtrips, elapsed))
325 325 msg = ('found %d common and %d unknown server heads,'
326 326 ' %d roundtrips in %.4fs\n')
327 327 missing = set(result) - set(srvheads)
328 328 ui.log('discovery', msg, len(result), len(missing), roundtrips,
329 329 elapsed)
330 330
331 331 if not result and srvheadhashes != [nullid]:
332 332 if abortwhenunrelated:
333 333 raise error.Abort(_("repository is unrelated"))
334 334 else:
335 335 ui.warn(_("warning: repository is unrelated\n"))
336 336 return ({nullid}, True, srvheadhashes,)
337 337
338 338 anyincoming = (srvheadhashes != [nullid])
339 339 result = {clnode(r) for r in result}
340 340 return result, anyincoming, srvheadhashes
General Comments 0
You need to be logged in to leave comments. Login now