##// END OF EJS Templates
setdiscovery: drop unused 'initial' argument for '_takequicksample'...
Pierre-Yves David -
r23806:d6cbbe3b default
parent child Browse files
Show More
@@ -1,246 +1,241
1 1 # setdiscovery.py - improved discovery of common nodeset for mercurial
2 2 #
3 3 # Copyright 2010 Benoit Boissinot <bboissin@gmail.com>
4 4 # and Peter Arrenbrecht <peter@arrenbrecht.ch>
5 5 #
6 6 # This software may be used and distributed according to the terms of the
7 7 # GNU General Public License version 2 or any later version.
8 8 """
9 9 Algorithm works in the following way. You have two repository: local and
10 10 remote. They both contains a DAG of changelists.
11 11
12 12 The goal of the discovery protocol is to find one set of node *common*,
13 13 the set of nodes shared by local and remote.
14 14
15 15 One of the issue with the original protocol was latency, it could
16 16 potentially require lots of roundtrips to discover that the local repo was a
17 17 subset of remote (which is a very common case, you usually have few changes
18 18 compared to upstream, while upstream probably had lots of development).
19 19
20 20 The new protocol only requires one interface for the remote repo: `known()`,
21 21 which given a set of changelists tells you if they are present in the DAG.
22 22
23 23 The algorithm then works as follow:
24 24
25 25 - We will be using three sets, `common`, `missing`, `unknown`. Originally
26 26 all nodes are in `unknown`.
27 27 - Take a sample from `unknown`, call `remote.known(sample)`
28 28 - For each node that remote knows, move it and all its ancestors to `common`
29 29 - For each node that remote doesn't know, move it and all its descendants
30 30 to `missing`
31 31 - Iterate until `unknown` is empty
32 32
33 33 There are a couple optimizations, first is instead of starting with a random
34 34 sample of missing, start by sending all heads, in the case where the local
35 35 repo is a subset, you computed the answer in one round trip.
36 36
37 37 Then you can do something similar to the bisecting strategy used when
38 38 finding faulty changesets. Instead of random samples, you can try picking
39 39 nodes that will maximize the number of nodes that will be
40 40 classified with it (since all ancestors or descendants will be marked as well).
41 41 """
42 42
43 43 from node import nullid, nullrev
44 44 from i18n import _
45 45 import random
46 46 import util, dagutil
47 47
48 48 def _updatesample(dag, nodes, sample, always, quicksamplesize=0):
49 49 # if nodes is empty we scan the entire graph
50 50 if nodes:
51 51 heads = dag.headsetofconnecteds(nodes)
52 52 else:
53 53 heads = dag.heads()
54 54 dist = {}
55 55 visit = util.deque(heads)
56 56 seen = set()
57 57 factor = 1
58 58 while visit:
59 59 curr = visit.popleft()
60 60 if curr in seen:
61 61 continue
62 62 d = dist.setdefault(curr, 1)
63 63 if d > factor:
64 64 factor *= 2
65 65 if d == factor:
66 66 if curr not in always: # need this check for the early exit below
67 67 sample.add(curr)
68 68 if quicksamplesize and (len(sample) >= quicksamplesize):
69 69 return
70 70 seen.add(curr)
71 71 for p in dag.parents(curr):
72 72 if not nodes or p in nodes:
73 73 dist.setdefault(p, d + 1)
74 74 visit.append(p)
75 75
76 76 def _setupsample(dag, nodes, size):
77 77 if len(nodes) <= size:
78 78 return set(nodes), None, 0
79 79 always = dag.headsetofconnecteds(nodes)
80 80 desiredlen = size - len(always)
81 81 if desiredlen <= 0:
82 82 # This could be bad if there are very many heads, all unknown to the
83 83 # server. We're counting on long request support here.
84 84 return always, None, desiredlen
85 85 return always, set(), desiredlen
86 86
87 def _takequicksample(dag, nodes, size, initial):
87 def _takequicksample(dag, nodes, size):
88 88 always, sample, desiredlen = _setupsample(dag, nodes, size)
89 89 if sample is None:
90 90 return always
91 if initial:
92 fromset = None
93 else:
94 fromset = nodes
95 _updatesample(dag, fromset, sample, always, quicksamplesize=desiredlen)
91 _updatesample(dag, None, sample, always, quicksamplesize=desiredlen)
96 92 sample.update(always)
97 93 return sample
98 94
99 95 def _takefullsample(dag, nodes, size):
100 96 always, sample, desiredlen = _setupsample(dag, nodes, size)
101 97 if sample is None:
102 98 return always
103 99 # update from heads
104 100 _updatesample(dag, nodes, sample, always)
105 101 # update from roots
106 102 _updatesample(dag.inverse(), nodes, sample, always)
107 103 assert sample
108 104 sample = _limitsample(sample, desiredlen)
109 105 if len(sample) < desiredlen:
110 106 more = desiredlen - len(sample)
111 107 sample.update(random.sample(list(nodes - sample - always), more))
112 108 sample.update(always)
113 109 return sample
114 110
115 111 def _limitsample(sample, desiredlen):
116 112 """return a random subset of sample of at most desiredlen item"""
117 113 if len(sample) > desiredlen:
118 114 sample = set(random.sample(sample, desiredlen))
119 115 return sample
120 116
121 117 def findcommonheads(ui, local, remote,
122 118 initialsamplesize=100,
123 119 fullsamplesize=200,
124 120 abortwhenunrelated=True):
125 121 '''Return a tuple (common, anyincoming, remoteheads) used to identify
126 122 missing nodes from or in remote.
127 123 '''
128 124 roundtrips = 0
129 125 cl = local.changelog
130 126 dag = dagutil.revlogdag(cl)
131 127
132 128 # early exit if we know all the specified remote heads already
133 129 ui.debug("query 1; heads\n")
134 130 roundtrips += 1
135 131 ownheads = dag.heads()
136 132 sample = _limitsample(ownheads, initialsamplesize)
137 133 # indices between sample and externalized version must match
138 134 sample = list(sample)
139 135 if remote.local():
140 136 # stopgap until we have a proper localpeer that supports batch()
141 137 srvheadhashes = remote.heads()
142 138 yesno = remote.known(dag.externalizeall(sample))
143 139 elif remote.capable('batch'):
144 140 batch = remote.batch()
145 141 srvheadhashesref = batch.heads()
146 142 yesnoref = batch.known(dag.externalizeall(sample))
147 143 batch.submit()
148 144 srvheadhashes = srvheadhashesref.value
149 145 yesno = yesnoref.value
150 146 else:
151 147 # compatibility with pre-batch, but post-known remotes during 1.9
152 148 # development
153 149 srvheadhashes = remote.heads()
154 150 sample = []
155 151
156 152 if cl.tip() == nullid:
157 153 if srvheadhashes != [nullid]:
158 154 return [nullid], True, srvheadhashes
159 155 return [nullid], False, []
160 156
161 157 # start actual discovery (we note this before the next "if" for
162 158 # compatibility reasons)
163 159 ui.status(_("searching for changes\n"))
164 160
165 161 srvheads = dag.internalizeall(srvheadhashes, filterunknown=True)
166 162 if len(srvheads) == len(srvheadhashes):
167 163 ui.debug("all remote heads known locally\n")
168 164 return (srvheadhashes, False, srvheadhashes,)
169 165
170 166 if sample and len(ownheads) <= initialsamplesize and util.all(yesno):
171 167 ui.note(_("all local heads known remotely\n"))
172 168 ownheadhashes = dag.externalizeall(ownheads)
173 169 return (ownheadhashes, True, srvheadhashes,)
174 170
175 171 # full blown discovery
176 172
177 173 # own nodes I know we both know
178 174 # treat remote heads (and maybe own heads) as a first implicit sample
179 175 # response
180 176 common = cl.incrementalmissingrevs(srvheads)
181 177 commoninsample = set(n for i, n in enumerate(sample) if yesno[i])
182 178 common.addbases(commoninsample)
183 179 # own nodes where I don't know if remote knows them
184 180 undecided = set(common.missingancestors(ownheads))
185 181 # own nodes I know remote lacks
186 182 missing = set()
187 183
188 184 full = False
189 185 while undecided:
190 186
191 187 if sample:
192 188 missinginsample = [n for i, n in enumerate(sample) if not yesno[i]]
193 189 missing.update(dag.descendantset(missinginsample, missing))
194 190
195 191 undecided.difference_update(missing)
196 192
197 193 if not undecided:
198 194 break
199 195
200 196 if full or common.hasbases():
201 197 if full:
202 198 ui.note(_("sampling from both directions\n"))
203 199 else:
204 200 ui.debug("taking initial sample\n")
205 201 sample = _takefullsample(dag, undecided, size=fullsamplesize)
206 202 targetsize = fullsamplesize
207 203 else:
208 204 # use even cheaper initial sample
209 205 ui.debug("taking quick initial sample\n")
210 sample = _takequicksample(dag, undecided, size=initialsamplesize,
211 initial=True)
206 sample = _takequicksample(dag, undecided, size=initialsamplesize)
212 207 targetsize = initialsamplesize
213 208 sample = _limitsample(sample, targetsize)
214 209
215 210 roundtrips += 1
216 211 ui.progress(_('searching'), roundtrips, unit=_('queries'))
217 212 ui.debug("query %i; still undecided: %i, sample size is: %i\n"
218 213 % (roundtrips, len(undecided), len(sample)))
219 214 # indices between sample and externalized version must match
220 215 sample = list(sample)
221 216 yesno = remote.known(dag.externalizeall(sample))
222 217 full = True
223 218
224 219 if sample:
225 220 commoninsample = set(n for i, n in enumerate(sample) if yesno[i])
226 221 common.addbases(commoninsample)
227 222 common.removeancestorsfrom(undecided)
228 223
229 224 # heads(common) == heads(common.bases) since common represents common.bases
230 225 # and all its ancestors
231 226 result = dag.headsetofconnecteds(common.bases)
232 227 # common.bases can include nullrev, but our contract requires us to not
233 228 # return any heads in that case, so discard that
234 229 result.discard(nullrev)
235 230 ui.progress(_('searching'), None)
236 231 ui.debug("%d total queries\n" % roundtrips)
237 232
238 233 if not result and srvheadhashes != [nullid]:
239 234 if abortwhenunrelated:
240 235 raise util.Abort(_("repository is unrelated"))
241 236 else:
242 237 ui.warn(_("warning: repository is unrelated\n"))
243 238 return (set([nullid]), True, srvheadhashes,)
244 239
245 240 anyincoming = (srvheadhashes != [nullid])
246 241 return dag.externalizeall(result), anyincoming, srvheadhashes
General Comments 0
You need to be logged in to leave comments. Login now