##// END OF EJS Templates
setdiscovery: drop unused 'initial' argument for '_takequicksample'...
Pierre-Yves David -
r23806:d6cbbe3b default
parent child Browse files
Show More
@@ -1,246 +1,241
1 # setdiscovery.py - improved discovery of common nodeset for mercurial
1 # setdiscovery.py - improved discovery of common nodeset for mercurial
2 #
2 #
3 # Copyright 2010 Benoit Boissinot <bboissin@gmail.com>
3 # Copyright 2010 Benoit Boissinot <bboissin@gmail.com>
4 # and Peter Arrenbrecht <peter@arrenbrecht.ch>
4 # and Peter Arrenbrecht <peter@arrenbrecht.ch>
5 #
5 #
6 # This software may be used and distributed according to the terms of the
6 # This software may be used and distributed according to the terms of the
7 # GNU General Public License version 2 or any later version.
7 # GNU General Public License version 2 or any later version.
8 """
8 """
9 Algorithm works in the following way. You have two repository: local and
9 Algorithm works in the following way. You have two repository: local and
10 remote. They both contains a DAG of changelists.
10 remote. They both contains a DAG of changelists.
11
11
12 The goal of the discovery protocol is to find one set of node *common*,
12 The goal of the discovery protocol is to find one set of node *common*,
13 the set of nodes shared by local and remote.
13 the set of nodes shared by local and remote.
14
14
15 One of the issue with the original protocol was latency, it could
15 One of the issue with the original protocol was latency, it could
16 potentially require lots of roundtrips to discover that the local repo was a
16 potentially require lots of roundtrips to discover that the local repo was a
17 subset of remote (which is a very common case, you usually have few changes
17 subset of remote (which is a very common case, you usually have few changes
18 compared to upstream, while upstream probably had lots of development).
18 compared to upstream, while upstream probably had lots of development).
19
19
20 The new protocol only requires one interface for the remote repo: `known()`,
20 The new protocol only requires one interface for the remote repo: `known()`,
21 which given a set of changelists tells you if they are present in the DAG.
21 which given a set of changelists tells you if they are present in the DAG.
22
22
23 The algorithm then works as follow:
23 The algorithm then works as follow:
24
24
25 - We will be using three sets, `common`, `missing`, `unknown`. Originally
25 - We will be using three sets, `common`, `missing`, `unknown`. Originally
26 all nodes are in `unknown`.
26 all nodes are in `unknown`.
27 - Take a sample from `unknown`, call `remote.known(sample)`
27 - Take a sample from `unknown`, call `remote.known(sample)`
28 - For each node that remote knows, move it and all its ancestors to `common`
28 - For each node that remote knows, move it and all its ancestors to `common`
29 - For each node that remote doesn't know, move it and all its descendants
29 - For each node that remote doesn't know, move it and all its descendants
30 to `missing`
30 to `missing`
31 - Iterate until `unknown` is empty
31 - Iterate until `unknown` is empty
32
32
33 There are a couple optimizations, first is instead of starting with a random
33 There are a couple optimizations, first is instead of starting with a random
34 sample of missing, start by sending all heads, in the case where the local
34 sample of missing, start by sending all heads, in the case where the local
35 repo is a subset, you computed the answer in one round trip.
35 repo is a subset, you computed the answer in one round trip.
36
36
37 Then you can do something similar to the bisecting strategy used when
37 Then you can do something similar to the bisecting strategy used when
38 finding faulty changesets. Instead of random samples, you can try picking
38 finding faulty changesets. Instead of random samples, you can try picking
39 nodes that will maximize the number of nodes that will be
39 nodes that will maximize the number of nodes that will be
40 classified with it (since all ancestors or descendants will be marked as well).
40 classified with it (since all ancestors or descendants will be marked as well).
41 """
41 """
42
42
43 from node import nullid, nullrev
43 from node import nullid, nullrev
44 from i18n import _
44 from i18n import _
45 import random
45 import random
46 import util, dagutil
46 import util, dagutil
47
47
48 def _updatesample(dag, nodes, sample, always, quicksamplesize=0):
48 def _updatesample(dag, nodes, sample, always, quicksamplesize=0):
49 # if nodes is empty we scan the entire graph
49 # if nodes is empty we scan the entire graph
50 if nodes:
50 if nodes:
51 heads = dag.headsetofconnecteds(nodes)
51 heads = dag.headsetofconnecteds(nodes)
52 else:
52 else:
53 heads = dag.heads()
53 heads = dag.heads()
54 dist = {}
54 dist = {}
55 visit = util.deque(heads)
55 visit = util.deque(heads)
56 seen = set()
56 seen = set()
57 factor = 1
57 factor = 1
58 while visit:
58 while visit:
59 curr = visit.popleft()
59 curr = visit.popleft()
60 if curr in seen:
60 if curr in seen:
61 continue
61 continue
62 d = dist.setdefault(curr, 1)
62 d = dist.setdefault(curr, 1)
63 if d > factor:
63 if d > factor:
64 factor *= 2
64 factor *= 2
65 if d == factor:
65 if d == factor:
66 if curr not in always: # need this check for the early exit below
66 if curr not in always: # need this check for the early exit below
67 sample.add(curr)
67 sample.add(curr)
68 if quicksamplesize and (len(sample) >= quicksamplesize):
68 if quicksamplesize and (len(sample) >= quicksamplesize):
69 return
69 return
70 seen.add(curr)
70 seen.add(curr)
71 for p in dag.parents(curr):
71 for p in dag.parents(curr):
72 if not nodes or p in nodes:
72 if not nodes or p in nodes:
73 dist.setdefault(p, d + 1)
73 dist.setdefault(p, d + 1)
74 visit.append(p)
74 visit.append(p)
75
75
76 def _setupsample(dag, nodes, size):
76 def _setupsample(dag, nodes, size):
77 if len(nodes) <= size:
77 if len(nodes) <= size:
78 return set(nodes), None, 0
78 return set(nodes), None, 0
79 always = dag.headsetofconnecteds(nodes)
79 always = dag.headsetofconnecteds(nodes)
80 desiredlen = size - len(always)
80 desiredlen = size - len(always)
81 if desiredlen <= 0:
81 if desiredlen <= 0:
82 # This could be bad if there are very many heads, all unknown to the
82 # This could be bad if there are very many heads, all unknown to the
83 # server. We're counting on long request support here.
83 # server. We're counting on long request support here.
84 return always, None, desiredlen
84 return always, None, desiredlen
85 return always, set(), desiredlen
85 return always, set(), desiredlen
86
86
87 def _takequicksample(dag, nodes, size, initial):
87 def _takequicksample(dag, nodes, size):
88 always, sample, desiredlen = _setupsample(dag, nodes, size)
88 always, sample, desiredlen = _setupsample(dag, nodes, size)
89 if sample is None:
89 if sample is None:
90 return always
90 return always
91 if initial:
91 _updatesample(dag, None, sample, always, quicksamplesize=desiredlen)
92 fromset = None
93 else:
94 fromset = nodes
95 _updatesample(dag, fromset, sample, always, quicksamplesize=desiredlen)
96 sample.update(always)
92 sample.update(always)
97 return sample
93 return sample
98
94
99 def _takefullsample(dag, nodes, size):
95 def _takefullsample(dag, nodes, size):
100 always, sample, desiredlen = _setupsample(dag, nodes, size)
96 always, sample, desiredlen = _setupsample(dag, nodes, size)
101 if sample is None:
97 if sample is None:
102 return always
98 return always
103 # update from heads
99 # update from heads
104 _updatesample(dag, nodes, sample, always)
100 _updatesample(dag, nodes, sample, always)
105 # update from roots
101 # update from roots
106 _updatesample(dag.inverse(), nodes, sample, always)
102 _updatesample(dag.inverse(), nodes, sample, always)
107 assert sample
103 assert sample
108 sample = _limitsample(sample, desiredlen)
104 sample = _limitsample(sample, desiredlen)
109 if len(sample) < desiredlen:
105 if len(sample) < desiredlen:
110 more = desiredlen - len(sample)
106 more = desiredlen - len(sample)
111 sample.update(random.sample(list(nodes - sample - always), more))
107 sample.update(random.sample(list(nodes - sample - always), more))
112 sample.update(always)
108 sample.update(always)
113 return sample
109 return sample
114
110
115 def _limitsample(sample, desiredlen):
111 def _limitsample(sample, desiredlen):
116 """return a random subset of sample of at most desiredlen item"""
112 """return a random subset of sample of at most desiredlen item"""
117 if len(sample) > desiredlen:
113 if len(sample) > desiredlen:
118 sample = set(random.sample(sample, desiredlen))
114 sample = set(random.sample(sample, desiredlen))
119 return sample
115 return sample
120
116
121 def findcommonheads(ui, local, remote,
117 def findcommonheads(ui, local, remote,
122 initialsamplesize=100,
118 initialsamplesize=100,
123 fullsamplesize=200,
119 fullsamplesize=200,
124 abortwhenunrelated=True):
120 abortwhenunrelated=True):
125 '''Return a tuple (common, anyincoming, remoteheads) used to identify
121 '''Return a tuple (common, anyincoming, remoteheads) used to identify
126 missing nodes from or in remote.
122 missing nodes from or in remote.
127 '''
123 '''
128 roundtrips = 0
124 roundtrips = 0
129 cl = local.changelog
125 cl = local.changelog
130 dag = dagutil.revlogdag(cl)
126 dag = dagutil.revlogdag(cl)
131
127
132 # early exit if we know all the specified remote heads already
128 # early exit if we know all the specified remote heads already
133 ui.debug("query 1; heads\n")
129 ui.debug("query 1; heads\n")
134 roundtrips += 1
130 roundtrips += 1
135 ownheads = dag.heads()
131 ownheads = dag.heads()
136 sample = _limitsample(ownheads, initialsamplesize)
132 sample = _limitsample(ownheads, initialsamplesize)
137 # indices between sample and externalized version must match
133 # indices between sample and externalized version must match
138 sample = list(sample)
134 sample = list(sample)
139 if remote.local():
135 if remote.local():
140 # stopgap until we have a proper localpeer that supports batch()
136 # stopgap until we have a proper localpeer that supports batch()
141 srvheadhashes = remote.heads()
137 srvheadhashes = remote.heads()
142 yesno = remote.known(dag.externalizeall(sample))
138 yesno = remote.known(dag.externalizeall(sample))
143 elif remote.capable('batch'):
139 elif remote.capable('batch'):
144 batch = remote.batch()
140 batch = remote.batch()
145 srvheadhashesref = batch.heads()
141 srvheadhashesref = batch.heads()
146 yesnoref = batch.known(dag.externalizeall(sample))
142 yesnoref = batch.known(dag.externalizeall(sample))
147 batch.submit()
143 batch.submit()
148 srvheadhashes = srvheadhashesref.value
144 srvheadhashes = srvheadhashesref.value
149 yesno = yesnoref.value
145 yesno = yesnoref.value
150 else:
146 else:
151 # compatibility with pre-batch, but post-known remotes during 1.9
147 # compatibility with pre-batch, but post-known remotes during 1.9
152 # development
148 # development
153 srvheadhashes = remote.heads()
149 srvheadhashes = remote.heads()
154 sample = []
150 sample = []
155
151
156 if cl.tip() == nullid:
152 if cl.tip() == nullid:
157 if srvheadhashes != [nullid]:
153 if srvheadhashes != [nullid]:
158 return [nullid], True, srvheadhashes
154 return [nullid], True, srvheadhashes
159 return [nullid], False, []
155 return [nullid], False, []
160
156
161 # start actual discovery (we note this before the next "if" for
157 # start actual discovery (we note this before the next "if" for
162 # compatibility reasons)
158 # compatibility reasons)
163 ui.status(_("searching for changes\n"))
159 ui.status(_("searching for changes\n"))
164
160
165 srvheads = dag.internalizeall(srvheadhashes, filterunknown=True)
161 srvheads = dag.internalizeall(srvheadhashes, filterunknown=True)
166 if len(srvheads) == len(srvheadhashes):
162 if len(srvheads) == len(srvheadhashes):
167 ui.debug("all remote heads known locally\n")
163 ui.debug("all remote heads known locally\n")
168 return (srvheadhashes, False, srvheadhashes,)
164 return (srvheadhashes, False, srvheadhashes,)
169
165
170 if sample and len(ownheads) <= initialsamplesize and util.all(yesno):
166 if sample and len(ownheads) <= initialsamplesize and util.all(yesno):
171 ui.note(_("all local heads known remotely\n"))
167 ui.note(_("all local heads known remotely\n"))
172 ownheadhashes = dag.externalizeall(ownheads)
168 ownheadhashes = dag.externalizeall(ownheads)
173 return (ownheadhashes, True, srvheadhashes,)
169 return (ownheadhashes, True, srvheadhashes,)
174
170
175 # full blown discovery
171 # full blown discovery
176
172
177 # own nodes I know we both know
173 # own nodes I know we both know
178 # treat remote heads (and maybe own heads) as a first implicit sample
174 # treat remote heads (and maybe own heads) as a first implicit sample
179 # response
175 # response
180 common = cl.incrementalmissingrevs(srvheads)
176 common = cl.incrementalmissingrevs(srvheads)
181 commoninsample = set(n for i, n in enumerate(sample) if yesno[i])
177 commoninsample = set(n for i, n in enumerate(sample) if yesno[i])
182 common.addbases(commoninsample)
178 common.addbases(commoninsample)
183 # own nodes where I don't know if remote knows them
179 # own nodes where I don't know if remote knows them
184 undecided = set(common.missingancestors(ownheads))
180 undecided = set(common.missingancestors(ownheads))
185 # own nodes I know remote lacks
181 # own nodes I know remote lacks
186 missing = set()
182 missing = set()
187
183
188 full = False
184 full = False
189 while undecided:
185 while undecided:
190
186
191 if sample:
187 if sample:
192 missinginsample = [n for i, n in enumerate(sample) if not yesno[i]]
188 missinginsample = [n for i, n in enumerate(sample) if not yesno[i]]
193 missing.update(dag.descendantset(missinginsample, missing))
189 missing.update(dag.descendantset(missinginsample, missing))
194
190
195 undecided.difference_update(missing)
191 undecided.difference_update(missing)
196
192
197 if not undecided:
193 if not undecided:
198 break
194 break
199
195
200 if full or common.hasbases():
196 if full or common.hasbases():
201 if full:
197 if full:
202 ui.note(_("sampling from both directions\n"))
198 ui.note(_("sampling from both directions\n"))
203 else:
199 else:
204 ui.debug("taking initial sample\n")
200 ui.debug("taking initial sample\n")
205 sample = _takefullsample(dag, undecided, size=fullsamplesize)
201 sample = _takefullsample(dag, undecided, size=fullsamplesize)
206 targetsize = fullsamplesize
202 targetsize = fullsamplesize
207 else:
203 else:
208 # use even cheaper initial sample
204 # use even cheaper initial sample
209 ui.debug("taking quick initial sample\n")
205 ui.debug("taking quick initial sample\n")
210 sample = _takequicksample(dag, undecided, size=initialsamplesize,
206 sample = _takequicksample(dag, undecided, size=initialsamplesize)
211 initial=True)
212 targetsize = initialsamplesize
207 targetsize = initialsamplesize
213 sample = _limitsample(sample, targetsize)
208 sample = _limitsample(sample, targetsize)
214
209
215 roundtrips += 1
210 roundtrips += 1
216 ui.progress(_('searching'), roundtrips, unit=_('queries'))
211 ui.progress(_('searching'), roundtrips, unit=_('queries'))
217 ui.debug("query %i; still undecided: %i, sample size is: %i\n"
212 ui.debug("query %i; still undecided: %i, sample size is: %i\n"
218 % (roundtrips, len(undecided), len(sample)))
213 % (roundtrips, len(undecided), len(sample)))
219 # indices between sample and externalized version must match
214 # indices between sample and externalized version must match
220 sample = list(sample)
215 sample = list(sample)
221 yesno = remote.known(dag.externalizeall(sample))
216 yesno = remote.known(dag.externalizeall(sample))
222 full = True
217 full = True
223
218
224 if sample:
219 if sample:
225 commoninsample = set(n for i, n in enumerate(sample) if yesno[i])
220 commoninsample = set(n for i, n in enumerate(sample) if yesno[i])
226 common.addbases(commoninsample)
221 common.addbases(commoninsample)
227 common.removeancestorsfrom(undecided)
222 common.removeancestorsfrom(undecided)
228
223
229 # heads(common) == heads(common.bases) since common represents common.bases
224 # heads(common) == heads(common.bases) since common represents common.bases
230 # and all its ancestors
225 # and all its ancestors
231 result = dag.headsetofconnecteds(common.bases)
226 result = dag.headsetofconnecteds(common.bases)
232 # common.bases can include nullrev, but our contract requires us to not
227 # common.bases can include nullrev, but our contract requires us to not
233 # return any heads in that case, so discard that
228 # return any heads in that case, so discard that
234 result.discard(nullrev)
229 result.discard(nullrev)
235 ui.progress(_('searching'), None)
230 ui.progress(_('searching'), None)
236 ui.debug("%d total queries\n" % roundtrips)
231 ui.debug("%d total queries\n" % roundtrips)
237
232
238 if not result and srvheadhashes != [nullid]:
233 if not result and srvheadhashes != [nullid]:
239 if abortwhenunrelated:
234 if abortwhenunrelated:
240 raise util.Abort(_("repository is unrelated"))
235 raise util.Abort(_("repository is unrelated"))
241 else:
236 else:
242 ui.warn(_("warning: repository is unrelated\n"))
237 ui.warn(_("warning: repository is unrelated\n"))
243 return (set([nullid]), True, srvheadhashes,)
238 return (set([nullid]), True, srvheadhashes,)
244
239
245 anyincoming = (srvheadhashes != [nullid])
240 anyincoming = (srvheadhashes != [nullid])
246 return dag.externalizeall(result), anyincoming, srvheadhashes
241 return dag.externalizeall(result), anyincoming, srvheadhashes
General Comments 0
You need to be logged in to leave comments. Login now