##// END OF EJS Templates
setdiscovery: drop shadowed 'undecided' assignment...
Pierre-Yves David -
r23746:4ef2f2fa default
parent child Browse files
Show More
@@ -1,249 +1,248 b''
1 1 # setdiscovery.py - improved discovery of common nodeset for mercurial
2 2 #
3 3 # Copyright 2010 Benoit Boissinot <bboissin@gmail.com>
4 4 # and Peter Arrenbrecht <peter@arrenbrecht.ch>
5 5 #
6 6 # This software may be used and distributed according to the terms of the
7 7 # GNU General Public License version 2 or any later version.
8 8 """
9 9 Algorithm works in the following way. You have two repository: local and
10 10 remote. They both contains a DAG of changelists.
11 11
12 12 The goal of the discovery protocol is to find one set of node *common*,
13 13 the set of nodes shared by local and remote.
14 14
15 15 One of the issue with the original protocol was latency, it could
16 16 potentially require lots of roundtrips to discover that the local repo was a
17 17 subset of remote (which is a very common case, you usually have few changes
18 18 compared to upstream, while upstream probably had lots of development).
19 19
20 20 The new protocol only requires one interface for the remote repo: `known()`,
21 21 which given a set of changelists tells you if they are present in the DAG.
22 22
23 23 The algorithm then works as follow:
24 24
25 25 - We will be using three sets, `common`, `missing`, `unknown`. Originally
26 26 all nodes are in `unknown`.
27 27 - Take a sample from `unknown`, call `remote.known(sample)`
28 28 - For each node that remote knows, move it and all its ancestors to `common`
29 29 - For each node that remote doesn't know, move it and all its descendants
30 30 to `missing`
31 31 - Iterate until `unknown` is empty
32 32
33 33 There are a couple optimizations, first is instead of starting with a random
34 34 sample of missing, start by sending all heads, in the case where the local
35 35 repo is a subset, you computed the answer in one round trip.
36 36
37 37 Then you can do something similar to the bisecting strategy used when
38 38 finding faulty changesets. Instead of random samples, you can try picking
39 39 nodes that will maximize the number of nodes that will be
40 40 classified with it (since all ancestors or descendants will be marked as well).
41 41 """
42 42
43 43 from node import nullid, nullrev
44 44 from i18n import _
45 45 import random
46 46 import util, dagutil
47 47
48 48 def _updatesample(dag, nodes, sample, always, quicksamplesize=0):
49 49 # if nodes is empty we scan the entire graph
50 50 if nodes:
51 51 heads = dag.headsetofconnecteds(nodes)
52 52 else:
53 53 heads = dag.heads()
54 54 dist = {}
55 55 visit = util.deque(heads)
56 56 seen = set()
57 57 factor = 1
58 58 while visit:
59 59 curr = visit.popleft()
60 60 if curr in seen:
61 61 continue
62 62 d = dist.setdefault(curr, 1)
63 63 if d > factor:
64 64 factor *= 2
65 65 if d == factor:
66 66 if curr not in always: # need this check for the early exit below
67 67 sample.add(curr)
68 68 if quicksamplesize and (len(sample) >= quicksamplesize):
69 69 return
70 70 seen.add(curr)
71 71 for p in dag.parents(curr):
72 72 if not nodes or p in nodes:
73 73 dist.setdefault(p, d + 1)
74 74 visit.append(p)
75 75
76 76 def _setupsample(dag, nodes, size):
77 77 if len(nodes) <= size:
78 78 return set(nodes), None, 0
79 79 always = dag.headsetofconnecteds(nodes)
80 80 desiredlen = size - len(always)
81 81 if desiredlen <= 0:
82 82 # This could be bad if there are very many heads, all unknown to the
83 83 # server. We're counting on long request support here.
84 84 return always, None, desiredlen
85 85 return always, set(), desiredlen
86 86
87 87 def _takequicksample(dag, nodes, size, initial):
88 88 always, sample, desiredlen = _setupsample(dag, nodes, size)
89 89 if sample is None:
90 90 return always
91 91 if initial:
92 92 fromset = None
93 93 else:
94 94 fromset = nodes
95 95 _updatesample(dag, fromset, sample, always, quicksamplesize=desiredlen)
96 96 sample.update(always)
97 97 return sample
98 98
99 99 def _takefullsample(dag, nodes, size):
100 100 always, sample, desiredlen = _setupsample(dag, nodes, size)
101 101 if sample is None:
102 102 return always
103 103 # update from heads
104 104 _updatesample(dag, nodes, sample, always)
105 105 # update from roots
106 106 _updatesample(dag.inverse(), nodes, sample, always)
107 107 assert sample
108 108 sample = _limitsample(sample, desiredlen)
109 109 if len(sample) < desiredlen:
110 110 more = desiredlen - len(sample)
111 111 sample.update(random.sample(list(nodes - sample - always), more))
112 112 sample.update(always)
113 113 return sample
114 114
115 115 def _limitsample(sample, desiredlen):
116 116 """return a random subset of sample of at most desiredlen item"""
117 117 if len(sample) > desiredlen:
118 118 sample = set(random.sample(sample, desiredlen))
119 119 return sample
120 120
121 121 def findcommonheads(ui, local, remote,
122 122 initialsamplesize=100,
123 123 fullsamplesize=200,
124 124 abortwhenunrelated=True):
125 125 '''Return a tuple (common, anyincoming, remoteheads) used to identify
126 126 missing nodes from or in remote.
127 127 '''
128 128 roundtrips = 0
129 129 cl = local.changelog
130 130 dag = dagutil.revlogdag(cl)
131 131
132 132 # early exit if we know all the specified remote heads already
133 133 ui.debug("query 1; heads\n")
134 134 roundtrips += 1
135 135 ownheads = dag.heads()
136 136 sample = _limitsample(ownheads, initialsamplesize)
137 137 # indices between sample and externalized version must match
138 138 sample = list(sample)
139 139 if remote.local():
140 140 # stopgap until we have a proper localpeer that supports batch()
141 141 srvheadhashes = remote.heads()
142 142 yesno = remote.known(dag.externalizeall(sample))
143 143 elif remote.capable('batch'):
144 144 batch = remote.batch()
145 145 srvheadhashesref = batch.heads()
146 146 yesnoref = batch.known(dag.externalizeall(sample))
147 147 batch.submit()
148 148 srvheadhashes = srvheadhashesref.value
149 149 yesno = yesnoref.value
150 150 else:
151 151 # compatibility with pre-batch, but post-known remotes during 1.9
152 152 # development
153 153 srvheadhashes = remote.heads()
154 154 sample = []
155 155
156 156 if cl.tip() == nullid:
157 157 if srvheadhashes != [nullid]:
158 158 return [nullid], True, srvheadhashes
159 159 return [nullid], False, []
160 160
161 161 # start actual discovery (we note this before the next "if" for
162 162 # compatibility reasons)
163 163 ui.status(_("searching for changes\n"))
164 164
165 165 srvheads = dag.internalizeall(srvheadhashes, filterunknown=True)
166 166 if len(srvheads) == len(srvheadhashes):
167 167 ui.debug("all remote heads known locally\n")
168 168 return (srvheadhashes, False, srvheadhashes,)
169 169
170 170 if sample and len(ownheads) <= initialsamplesize and util.all(yesno):
171 171 ui.note(_("all local heads known remotely\n"))
172 172 ownheadhashes = dag.externalizeall(ownheads)
173 173 return (ownheadhashes, True, srvheadhashes,)
174 174
175 175 # full blown discovery
176 176
177 # own nodes where I don't know if remote knows them
178 undecided = dag.nodeset()
179 177 # own nodes I know we both know
180 178 # treat remote heads (and maybe own heads) as a first implicit sample
181 179 # response
182 180 common = cl.incrementalmissingrevs(srvheads)
183 181 commoninsample = set(n for i, n in enumerate(sample) if yesno[i])
184 182 common.addbases(commoninsample)
183 # own nodes where I don't know if remote knows them
185 184 undecided = set(common.missingancestors(ownheads))
186 185 # own nodes I know remote lacks
187 186 missing = set()
188 187
189 188 full = False
190 189 while undecided:
191 190
192 191 if sample:
193 192 missinginsample = [n for i, n in enumerate(sample) if not yesno[i]]
194 193 missing.update(dag.descendantset(missinginsample, missing))
195 194
196 195 undecided.difference_update(missing)
197 196
198 197 if not undecided:
199 198 break
200 199
201 200 if full:
202 201 ui.note(_("sampling from both directions\n"))
203 202 sample = _takefullsample(dag, undecided, size=fullsamplesize)
204 203 targetsize = fullsamplesize
205 204 elif common.hasbases():
206 205 # use cheapish initial sample
207 206 ui.debug("taking initial sample\n")
208 207 sample = _takefullsample(dag, undecided, size=fullsamplesize)
209 208 targetsize = fullsamplesize
210 209 else:
211 210 # use even cheaper initial sample
212 211 ui.debug("taking quick initial sample\n")
213 212 sample = _takequicksample(dag, undecided, size=initialsamplesize,
214 213 initial=True)
215 214 targetsize = initialsamplesize
216 215 sample = _limitsample(sample, targetsize)
217 216
218 217 roundtrips += 1
219 218 ui.progress(_('searching'), roundtrips, unit=_('queries'))
220 219 ui.debug("query %i; still undecided: %i, sample size is: %i\n"
221 220 % (roundtrips, len(undecided), len(sample)))
222 221 # indices between sample and externalized version must match
223 222 sample = list(sample)
224 223 yesno = remote.known(dag.externalizeall(sample))
225 224 full = True
226 225
227 226 if sample:
228 227 commoninsample = set(n for i, n in enumerate(sample) if yesno[i])
229 228 common.addbases(commoninsample)
230 229 common.removeancestorsfrom(undecided)
231 230
232 231 # heads(common) == heads(common.bases) since common represents common.bases
233 232 # and all its ancestors
234 233 result = dag.headsetofconnecteds(common.bases)
235 234 # common.bases can include nullrev, but our contract requires us to not
236 235 # return any heads in that case, so discard that
237 236 result.discard(nullrev)
238 237 ui.progress(_('searching'), None)
239 238 ui.debug("%d total queries\n" % roundtrips)
240 239
241 240 if not result and srvheadhashes != [nullid]:
242 241 if abortwhenunrelated:
243 242 raise util.Abort(_("repository is unrelated"))
244 243 else:
245 244 ui.warn(_("warning: repository is unrelated\n"))
246 245 return (set([nullid]), True, srvheadhashes,)
247 246
248 247 anyincoming = (srvheadhashes != [nullid])
249 248 return dag.externalizeall(result), anyincoming, srvheadhashes
General Comments 0
You need to be logged in to leave comments. Login now