##// END OF EJS Templates
repoview: cache hidden changesets...
David Soria Parra -
r22151:c0c369ae default
parent child Browse files
Show More
@@ -1,317 +1,329 b''
1 # repoview.py - Filtered view of a localrepo object
1 # repoview.py - Filtered view of a localrepo object
2 #
2 #
3 # Copyright 2012 Pierre-Yves David <pierre-yves.david@ens-lyon.org>
3 # Copyright 2012 Pierre-Yves David <pierre-yves.david@ens-lyon.org>
4 # Logilab SA <contact@logilab.fr>
4 # Logilab SA <contact@logilab.fr>
5 #
5 #
6 # This software may be used and distributed according to the terms of the
6 # This software may be used and distributed according to the terms of the
7 # GNU General Public License version 2 or any later version.
7 # GNU General Public License version 2 or any later version.
8
8
9 import copy
9 import copy
10 import error
10 import error
11 import hashlib
11 import hashlib
12 import phases
12 import phases
13 import util
13 import util
14 import obsolete
14 import obsolete
15 import struct
15 import struct
16 import tags as tagsmod
16 import tags as tagsmod
17 from mercurial.i18n import _
17 from mercurial.i18n import _
18
18
19 def hideablerevs(repo):
19 def hideablerevs(repo):
20 """Revisions candidates to be hidden
20 """Revisions candidates to be hidden
21
21
22 This is a standalone function to help extensions to wrap it."""
22 This is a standalone function to help extensions to wrap it."""
23 return obsolete.getrevs(repo, 'obsolete')
23 return obsolete.getrevs(repo, 'obsolete')
24
24
25 def _getstaticblockers(repo):
25 def _getstaticblockers(repo):
26 """Cacheable revisions blocking hidden changesets from being filtered.
26 """Cacheable revisions blocking hidden changesets from being filtered.
27
27
28 Additional non-cached hidden blockers are computed in _getdynamicblockers.
28 Additional non-cached hidden blockers are computed in _getdynamicblockers.
29 This is a standalone function to help extensions to wrap it."""
29 This is a standalone function to help extensions to wrap it."""
30 assert not repo.changelog.filteredrevs
30 assert not repo.changelog.filteredrevs
31 hideable = hideablerevs(repo)
31 hideable = hideablerevs(repo)
32 blockers = set()
32 blockers = set()
33 if hideable:
33 if hideable:
34 # We use cl to avoid recursive lookup from repo[xxx]
34 # We use cl to avoid recursive lookup from repo[xxx]
35 cl = repo.changelog
35 cl = repo.changelog
36 firsthideable = min(hideable)
36 firsthideable = min(hideable)
37 revs = cl.revs(start=firsthideable)
37 revs = cl.revs(start=firsthideable)
38 tofilter = repo.revs(
38 tofilter = repo.revs(
39 '(%ld) and children(%ld)', list(revs), list(hideable))
39 '(%ld) and children(%ld)', list(revs), list(hideable))
40 blockers.update([r for r in tofilter if r not in hideable])
40 blockers.update([r for r in tofilter if r not in hideable])
41 return blockers
41 return blockers
42
42
43 def _getdynamicblockers(repo):
43 def _getdynamicblockers(repo):
44 """Non-cacheable revisions blocking hidden changesets from being filtered.
44 """Non-cacheable revisions blocking hidden changesets from being filtered.
45
45
46 Get revisions that will block hidden changesets and are likely to change,
46 Get revisions that will block hidden changesets and are likely to change,
47 but unlikely to create hidden blockers. They won't be cached, so be careful
47 but unlikely to create hidden blockers. They won't be cached, so be careful
48 with adding additional computation."""
48 with adding additional computation."""
49
49
50 cl = repo.changelog
50 cl = repo.changelog
51 blockers = set()
51 blockers = set()
52 blockers.update([par.rev() for par in repo[None].parents()])
52 blockers.update([par.rev() for par in repo[None].parents()])
53 blockers.update([cl.rev(bm) for bm in repo._bookmarks.values()])
53 blockers.update([cl.rev(bm) for bm in repo._bookmarks.values()])
54
54
55 tags = {}
55 tags = {}
56 tagsmod.readlocaltags(repo.ui, repo, tags, {})
56 tagsmod.readlocaltags(repo.ui, repo, tags, {})
57 if tags:
57 if tags:
58 rev, nodemap = cl.rev, cl.nodemap
58 rev, nodemap = cl.rev, cl.nodemap
59 blockers.update(rev(t[0]) for t in tags.values() if t[0] in nodemap)
59 blockers.update(rev(t[0]) for t in tags.values() if t[0] in nodemap)
60 return blockers
60 return blockers
61
61
62 cacheversion = 1
62 cacheversion = 1
63 cachefile = 'cache/hidden'
63 cachefile = 'cache/hidden'
64
64
65 def cachehash(repo, hideable):
65 def cachehash(repo, hideable):
66 """return sha1 hash of repository data to identify a valid cache.
66 """return sha1 hash of repository data to identify a valid cache.
67
67
68 We calculate a sha1 of repo heads and the content of the obsstore and write
68 We calculate a sha1 of repo heads and the content of the obsstore and write
69 it to the cache. Upon reading we can easily validate by checking the hash
69 it to the cache. Upon reading we can easily validate by checking the hash
70 against the stored one and discard the cache in case the hashes don't match.
70 against the stored one and discard the cache in case the hashes don't match.
71 """
71 """
72 h = hashlib.sha1()
72 h = hashlib.sha1()
73 h.update(''.join(repo.heads()))
73 h.update(''.join(repo.heads()))
74 h.update(str(hash(frozenset(hideable))))
74 h.update(str(hash(frozenset(hideable))))
75 return h.digest()
75 return h.digest()
76
76
77 def trywritehiddencache(repo, hideable, hidden):
77 def trywritehiddencache(repo, hideable, hidden):
78 """write cache of hidden changesets to disk
78 """write cache of hidden changesets to disk
79
79
80 Will not write the cache if a wlock cannot be obtained lazily.
80 Will not write the cache if a wlock cannot be obtained lazily.
81 The cache consists of a head of 22byte:
81 The cache consists of a head of 22byte:
82 2 byte version number of the cache
82 2 byte version number of the cache
83 20 byte sha1 to validate the cache
83 20 byte sha1 to validate the cache
84 n*4 byte hidden revs
84 n*4 byte hidden revs
85 """
85 """
86 wlock = fh = None
86 wlock = fh = None
87 try:
87 try:
88 wlock = repo.wlock(wait=False)
88 wlock = repo.wlock(wait=False)
89 # write cache to file
89 # write cache to file
90 newhash = cachehash(repo, hideable)
90 newhash = cachehash(repo, hideable)
91 sortedset = sorted(hidden)
91 sortedset = sorted(hidden)
92 data = struct.pack('>%iI' % len(sortedset), *sortedset)
92 data = struct.pack('>%iI' % len(sortedset), *sortedset)
93 fh = repo.vfs.open(cachefile, 'w+b', atomictemp=True)
93 fh = repo.vfs.open(cachefile, 'w+b', atomictemp=True)
94 fh.write(struct.pack(">H", cacheversion))
94 fh.write(struct.pack(">H", cacheversion))
95 fh.write(newhash)
95 fh.write(newhash)
96 fh.write(data)
96 fh.write(data)
97 except (IOError, OSError):
97 except (IOError, OSError):
98 ui.debug('error writing hidden changesets cache')
98 repo.ui.debug('error writing hidden changesets cache')
99 except error.LockHeld:
99 except error.LockHeld:
100 ui.debug('cannot obtain lock to write hidden changesets cache')
100 repo.ui.debug('cannot obtain lock to write hidden changesets cache')
101 finally:
101 finally:
102 if fh:
102 if fh:
103 fh.close()
103 fh.close()
104 if wlock:
104 if wlock:
105 wlock.release()
105 wlock.release()
106
106
107 def tryreadcache(repo, hideable):
107 def tryreadcache(repo, hideable):
108 """read a cache if the cache exists and is valid, otherwise returns None."""
108 """read a cache if the cache exists and is valid, otherwise returns None."""
109 hidden = fh = None
109 hidden = fh = None
110 try:
110 try:
111 if repo.vfs.exists(cachefile):
111 if repo.vfs.exists(cachefile):
112 fh = repo.vfs.open(cachefile, 'rb')
112 fh = repo.vfs.open(cachefile, 'rb')
113 version, = struct.unpack(">H", fh.read(2))
113 version, = struct.unpack(">H", fh.read(2))
114 oldhash = fh.read(20)
114 oldhash = fh.read(20)
115 newhash = cachehash(repo, hideable)
115 newhash = cachehash(repo, hideable)
116 if (cacheversion, oldhash) == (version, newhash):
116 if (cacheversion, oldhash) == (version, newhash):
117 # cache is valid, so we can start reading the hidden revs
117 # cache is valid, so we can start reading the hidden revs
118 data = fh.read()
118 data = fh.read()
119 count = len(data) / 4
119 count = len(data) / 4
120 hidden = frozenset(struct.unpack('>%iI' % count, data))
120 hidden = frozenset(struct.unpack('>%iI' % count, data))
121 return hidden
121 return hidden
122 finally:
122 finally:
123 if fh:
123 if fh:
124 fh.close()
124 fh.close()
125
125
126 def computehidden(repo):
126 def computehidden(repo):
127 """compute the set of hidden revision to filter
127 """compute the set of hidden revision to filter
128
128
129 During most operation hidden should be filtered."""
129 During most operation hidden should be filtered."""
130 assert not repo.changelog.filteredrevs
130 assert not repo.changelog.filteredrevs
131
131 hidden = frozenset()
132 hidden = frozenset()
132 hideable = hideablerevs(repo)
133 hideable = hideablerevs(repo)
133 if hideable:
134 if hideable:
134 cl = repo.changelog
135 cl = repo.changelog
136 hidden = tryreadcache(repo, hideable)
137 if hidden is None:
135 blocked = cl.ancestors(_getstaticblockers(repo), inclusive=True)
138 blocked = cl.ancestors(_getstaticblockers(repo), inclusive=True)
136 hidden = frozenset(r for r in hideable if r not in blocked)
139 hidden = frozenset(r for r in hideable if r not in blocked)
140 trywritehiddencache(repo, hideable, hidden)
141 elif repo.ui.configbool('experimental', 'verifyhiddencache', True):
142 blocked = cl.ancestors(_getstaticblockers(repo), inclusive=True)
143 computed = frozenset(r for r in hideable if r not in blocked)
144 if computed != hidden:
145 trywritehiddencache(repo, hideable, computed)
146 repo.ui.warn(_('Cache inconsistency detected. Please ' +
147 'open an issue on http://bz.selenic.com.\n'))
148 hidden = computed
137
149
138 # check if we have wd parents, bookmarks or tags pointing to hidden
150 # check if we have wd parents, bookmarks or tags pointing to hidden
139 # changesets and remove those.
151 # changesets and remove those.
140 dynamic = hidden & _getdynamicblockers(repo)
152 dynamic = hidden & _getdynamicblockers(repo)
141 if dynamic:
153 if dynamic:
142 blocked = cl.ancestors(dynamic, inclusive=True)
154 blocked = cl.ancestors(dynamic, inclusive=True)
143 hidden = frozenset(r for r in hidden if r not in blocked)
155 hidden = frozenset(r for r in hidden if r not in blocked)
144 return hidden
156 return hidden
145
157
146 def computeunserved(repo):
158 def computeunserved(repo):
147 """compute the set of revision that should be filtered when used a server
159 """compute the set of revision that should be filtered when used a server
148
160
149 Secret and hidden changeset should not pretend to be here."""
161 Secret and hidden changeset should not pretend to be here."""
150 assert not repo.changelog.filteredrevs
162 assert not repo.changelog.filteredrevs
151 # fast path in simple case to avoid impact of non optimised code
163 # fast path in simple case to avoid impact of non optimised code
152 hiddens = filterrevs(repo, 'visible')
164 hiddens = filterrevs(repo, 'visible')
153 if phases.hassecret(repo):
165 if phases.hassecret(repo):
154 cl = repo.changelog
166 cl = repo.changelog
155 secret = phases.secret
167 secret = phases.secret
156 getphase = repo._phasecache.phase
168 getphase = repo._phasecache.phase
157 first = min(cl.rev(n) for n in repo._phasecache.phaseroots[secret])
169 first = min(cl.rev(n) for n in repo._phasecache.phaseroots[secret])
158 revs = cl.revs(start=first)
170 revs = cl.revs(start=first)
159 secrets = set(r for r in revs if getphase(repo, r) >= secret)
171 secrets = set(r for r in revs if getphase(repo, r) >= secret)
160 return frozenset(hiddens | secrets)
172 return frozenset(hiddens | secrets)
161 else:
173 else:
162 return hiddens
174 return hiddens
163
175
164 def computemutable(repo):
176 def computemutable(repo):
165 """compute the set of revision that should be filtered when used a server
177 """compute the set of revision that should be filtered when used a server
166
178
167 Secret and hidden changeset should not pretend to be here."""
179 Secret and hidden changeset should not pretend to be here."""
168 assert not repo.changelog.filteredrevs
180 assert not repo.changelog.filteredrevs
169 # fast check to avoid revset call on huge repo
181 # fast check to avoid revset call on huge repo
170 if util.any(repo._phasecache.phaseroots[1:]):
182 if util.any(repo._phasecache.phaseroots[1:]):
171 getphase = repo._phasecache.phase
183 getphase = repo._phasecache.phase
172 maymutable = filterrevs(repo, 'base')
184 maymutable = filterrevs(repo, 'base')
173 return frozenset(r for r in maymutable if getphase(repo, r))
185 return frozenset(r for r in maymutable if getphase(repo, r))
174 return frozenset()
186 return frozenset()
175
187
176 def computeimpactable(repo):
188 def computeimpactable(repo):
177 """Everything impactable by mutable revision
189 """Everything impactable by mutable revision
178
190
179 The immutable filter still have some chance to get invalidated. This will
191 The immutable filter still have some chance to get invalidated. This will
180 happen when:
192 happen when:
181
193
182 - you garbage collect hidden changeset,
194 - you garbage collect hidden changeset,
183 - public phase is moved backward,
195 - public phase is moved backward,
184 - something is changed in the filtering (this could be fixed)
196 - something is changed in the filtering (this could be fixed)
185
197
186 This filter out any mutable changeset and any public changeset that may be
198 This filter out any mutable changeset and any public changeset that may be
187 impacted by something happening to a mutable revision.
199 impacted by something happening to a mutable revision.
188
200
189 This is achieved by filtered everything with a revision number egal or
201 This is achieved by filtered everything with a revision number egal or
190 higher than the first mutable changeset is filtered."""
202 higher than the first mutable changeset is filtered."""
191 assert not repo.changelog.filteredrevs
203 assert not repo.changelog.filteredrevs
192 cl = repo.changelog
204 cl = repo.changelog
193 firstmutable = len(cl)
205 firstmutable = len(cl)
194 for roots in repo._phasecache.phaseroots[1:]:
206 for roots in repo._phasecache.phaseroots[1:]:
195 if roots:
207 if roots:
196 firstmutable = min(firstmutable, min(cl.rev(r) for r in roots))
208 firstmutable = min(firstmutable, min(cl.rev(r) for r in roots))
197 # protect from nullrev root
209 # protect from nullrev root
198 firstmutable = max(0, firstmutable)
210 firstmutable = max(0, firstmutable)
199 return frozenset(xrange(firstmutable, len(cl)))
211 return frozenset(xrange(firstmutable, len(cl)))
200
212
201 # function to compute filtered set
213 # function to compute filtered set
202 #
214 #
203 # When adding a new filter you MUST update the table at:
215 # When adding a new filter you MUST update the table at:
204 # mercurial.branchmap.subsettable
216 # mercurial.branchmap.subsettable
205 # Otherwise your filter will have to recompute all its branches cache
217 # Otherwise your filter will have to recompute all its branches cache
206 # from scratch (very slow).
218 # from scratch (very slow).
207 filtertable = {'visible': computehidden,
219 filtertable = {'visible': computehidden,
208 'served': computeunserved,
220 'served': computeunserved,
209 'immutable': computemutable,
221 'immutable': computemutable,
210 'base': computeimpactable}
222 'base': computeimpactable}
211
223
212 def filterrevs(repo, filtername):
224 def filterrevs(repo, filtername):
213 """returns set of filtered revision for this filter name"""
225 """returns set of filtered revision for this filter name"""
214 if filtername not in repo.filteredrevcache:
226 if filtername not in repo.filteredrevcache:
215 func = filtertable[filtername]
227 func = filtertable[filtername]
216 repo.filteredrevcache[filtername] = func(repo.unfiltered())
228 repo.filteredrevcache[filtername] = func(repo.unfiltered())
217 return repo.filteredrevcache[filtername]
229 return repo.filteredrevcache[filtername]
218
230
219 class repoview(object):
231 class repoview(object):
220 """Provide a read/write view of a repo through a filtered changelog
232 """Provide a read/write view of a repo through a filtered changelog
221
233
222 This object is used to access a filtered version of a repository without
234 This object is used to access a filtered version of a repository without
223 altering the original repository object itself. We can not alter the
235 altering the original repository object itself. We can not alter the
224 original object for two main reasons:
236 original object for two main reasons:
225 - It prevents the use of a repo with multiple filters at the same time. In
237 - It prevents the use of a repo with multiple filters at the same time. In
226 particular when multiple threads are involved.
238 particular when multiple threads are involved.
227 - It makes scope of the filtering harder to control.
239 - It makes scope of the filtering harder to control.
228
240
229 This object behaves very closely to the original repository. All attribute
241 This object behaves very closely to the original repository. All attribute
230 operations are done on the original repository:
242 operations are done on the original repository:
231 - An access to `repoview.someattr` actually returns `repo.someattr`,
243 - An access to `repoview.someattr` actually returns `repo.someattr`,
232 - A write to `repoview.someattr` actually sets value of `repo.someattr`,
244 - A write to `repoview.someattr` actually sets value of `repo.someattr`,
233 - A deletion of `repoview.someattr` actually drops `someattr`
245 - A deletion of `repoview.someattr` actually drops `someattr`
234 from `repo.__dict__`.
246 from `repo.__dict__`.
235
247
236 The only exception is the `changelog` property. It is overridden to return
248 The only exception is the `changelog` property. It is overridden to return
237 a (surface) copy of `repo.changelog` with some revisions filtered. The
249 a (surface) copy of `repo.changelog` with some revisions filtered. The
238 `filtername` attribute of the view control the revisions that need to be
250 `filtername` attribute of the view control the revisions that need to be
239 filtered. (the fact the changelog is copied is an implementation detail).
251 filtered. (the fact the changelog is copied is an implementation detail).
240
252
241 Unlike attributes, this object intercepts all method calls. This means that
253 Unlike attributes, this object intercepts all method calls. This means that
242 all methods are run on the `repoview` object with the filtered `changelog`
254 all methods are run on the `repoview` object with the filtered `changelog`
243 property. For this purpose the simple `repoview` class must be mixed with
255 property. For this purpose the simple `repoview` class must be mixed with
244 the actual class of the repository. This ensures that the resulting
256 the actual class of the repository. This ensures that the resulting
245 `repoview` object have the very same methods than the repo object. This
257 `repoview` object have the very same methods than the repo object. This
246 leads to the property below.
258 leads to the property below.
247
259
248 repoview.method() --> repo.__class__.method(repoview)
260 repoview.method() --> repo.__class__.method(repoview)
249
261
250 The inheritance has to be done dynamically because `repo` can be of any
262 The inheritance has to be done dynamically because `repo` can be of any
251 subclasses of `localrepo`. Eg: `bundlerepo` or `statichttprepo`.
263 subclasses of `localrepo`. Eg: `bundlerepo` or `statichttprepo`.
252 """
264 """
253
265
254 def __init__(self, repo, filtername):
266 def __init__(self, repo, filtername):
255 object.__setattr__(self, '_unfilteredrepo', repo)
267 object.__setattr__(self, '_unfilteredrepo', repo)
256 object.__setattr__(self, 'filtername', filtername)
268 object.__setattr__(self, 'filtername', filtername)
257 object.__setattr__(self, '_clcachekey', None)
269 object.__setattr__(self, '_clcachekey', None)
258 object.__setattr__(self, '_clcache', None)
270 object.__setattr__(self, '_clcache', None)
259
271
260 # not a propertycache on purpose we shall implement a proper cache later
272 # not a propertycache on purpose we shall implement a proper cache later
261 @property
273 @property
262 def changelog(self):
274 def changelog(self):
263 """return a filtered version of the changeset
275 """return a filtered version of the changeset
264
276
265 this changelog must not be used for writing"""
277 this changelog must not be used for writing"""
266 # some cache may be implemented later
278 # some cache may be implemented later
267 unfi = self._unfilteredrepo
279 unfi = self._unfilteredrepo
268 unfichangelog = unfi.changelog
280 unfichangelog = unfi.changelog
269 revs = filterrevs(unfi, self.filtername)
281 revs = filterrevs(unfi, self.filtername)
270 cl = self._clcache
282 cl = self._clcache
271 newkey = (len(unfichangelog), unfichangelog.tip(), hash(revs))
283 newkey = (len(unfichangelog), unfichangelog.tip(), hash(revs))
272 if cl is not None:
284 if cl is not None:
273 # we need to check curkey too for some obscure reason.
285 # we need to check curkey too for some obscure reason.
274 # MQ test show a corruption of the underlying repo (in _clcache)
286 # MQ test show a corruption of the underlying repo (in _clcache)
275 # without change in the cachekey.
287 # without change in the cachekey.
276 oldfilter = cl.filteredrevs
288 oldfilter = cl.filteredrevs
277 try:
289 try:
278 cl.filterrevs = () # disable filtering for tip
290 cl.filterrevs = () # disable filtering for tip
279 curkey = (len(cl), cl.tip(), hash(oldfilter))
291 curkey = (len(cl), cl.tip(), hash(oldfilter))
280 finally:
292 finally:
281 cl.filteredrevs = oldfilter
293 cl.filteredrevs = oldfilter
282 if newkey != self._clcachekey or newkey != curkey:
294 if newkey != self._clcachekey or newkey != curkey:
283 cl = None
295 cl = None
284 # could have been made None by the previous if
296 # could have been made None by the previous if
285 if cl is None:
297 if cl is None:
286 cl = copy.copy(unfichangelog)
298 cl = copy.copy(unfichangelog)
287 cl.filteredrevs = revs
299 cl.filteredrevs = revs
288 object.__setattr__(self, '_clcache', cl)
300 object.__setattr__(self, '_clcache', cl)
289 object.__setattr__(self, '_clcachekey', newkey)
301 object.__setattr__(self, '_clcachekey', newkey)
290 return cl
302 return cl
291
303
292 def unfiltered(self):
304 def unfiltered(self):
293 """Return an unfiltered version of a repo"""
305 """Return an unfiltered version of a repo"""
294 return self._unfilteredrepo
306 return self._unfilteredrepo
295
307
296 def filtered(self, name):
308 def filtered(self, name):
297 """Return a filtered version of a repository"""
309 """Return a filtered version of a repository"""
298 if name == self.filtername:
310 if name == self.filtername:
299 return self
311 return self
300 return self.unfiltered().filtered(name)
312 return self.unfiltered().filtered(name)
301
313
302 # everything access are forwarded to the proxied repo
314 # everything access are forwarded to the proxied repo
303 def __getattr__(self, attr):
315 def __getattr__(self, attr):
304 return getattr(self._unfilteredrepo, attr)
316 return getattr(self._unfilteredrepo, attr)
305
317
306 def __setattr__(self, attr, value):
318 def __setattr__(self, attr, value):
307 return setattr(self._unfilteredrepo, attr, value)
319 return setattr(self._unfilteredrepo, attr, value)
308
320
309 def __delattr__(self, attr):
321 def __delattr__(self, attr):
310 return delattr(self._unfilteredrepo, attr)
322 return delattr(self._unfilteredrepo, attr)
311
323
312 # The `requirements` attribute is initialized during __init__. But
324 # The `requirements` attribute is initialized during __init__. But
313 # __getattr__ won't be called as it also exists on the class. We need
325 # __getattr__ won't be called as it also exists on the class. We need
314 # explicit forwarding to main repo here
326 # explicit forwarding to main repo here
315 @property
327 @property
316 def requirements(self):
328 def requirements(self):
317 return self._unfilteredrepo.requirements
329 return self._unfilteredrepo.requirements
General Comments 0
You need to be logged in to leave comments. Login now