##// END OF EJS Templates
fsmonitor: hook up state-enter, state-leave signals...
Martijn Pieters -
r28443:49d65663 default
parent child Browse files
Show More
@@ -1,626 +1,694 b''
1 # __init__.py - fsmonitor initialization and overrides
1 # __init__.py - fsmonitor initialization and overrides
2 #
2 #
3 # Copyright 2013-2016 Facebook, Inc.
3 # Copyright 2013-2016 Facebook, Inc.
4 #
4 #
5 # This software may be used and distributed according to the terms of the
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
6 # GNU General Public License version 2 or any later version.
7
7
8 '''Faster status operations with the Watchman file monitor (EXPERIMENTAL)
8 '''Faster status operations with the Watchman file monitor (EXPERIMENTAL)
9
9
10 Integrates the file-watching program Watchman with Mercurial to produce faster
10 Integrates the file-watching program Watchman with Mercurial to produce faster
11 status results.
11 status results.
12
12
13 On a particular Linux system, for a real-world repository with over 400,000
13 On a particular Linux system, for a real-world repository with over 400,000
14 files hosted on ext4, vanilla `hg status` takes 1.3 seconds. On the same
14 files hosted on ext4, vanilla `hg status` takes 1.3 seconds. On the same
15 system, with fsmonitor it takes about 0.3 seconds.
15 system, with fsmonitor it takes about 0.3 seconds.
16
16
17 fsmonitor requires no configuration -- it will tell Watchman about your
17 fsmonitor requires no configuration -- it will tell Watchman about your
18 repository as necessary. You'll need to install Watchman from
18 repository as necessary. You'll need to install Watchman from
19 https://facebook.github.io/watchman/ and make sure it is in your PATH.
19 https://facebook.github.io/watchman/ and make sure it is in your PATH.
20
20
21 The following configuration options exist:
21 The following configuration options exist:
22
22
23 ::
23 ::
24
24
25 [fsmonitor]
25 [fsmonitor]
26 mode = {off, on, paranoid}
26 mode = {off, on, paranoid}
27
27
28 When `mode = off`, fsmonitor will disable itself (similar to not loading the
28 When `mode = off`, fsmonitor will disable itself (similar to not loading the
29 extension at all). When `mode = on`, fsmonitor will be enabled (the default).
29 extension at all). When `mode = on`, fsmonitor will be enabled (the default).
30 When `mode = paranoid`, fsmonitor will query both Watchman and the filesystem,
30 When `mode = paranoid`, fsmonitor will query both Watchman and the filesystem,
31 and ensure that the results are consistent.
31 and ensure that the results are consistent.
32
32
33 ::
33 ::
34
34
35 [fsmonitor]
35 [fsmonitor]
36 timeout = (float)
36 timeout = (float)
37
37
38 A value, in seconds, that determines how long fsmonitor will wait for Watchman
38 A value, in seconds, that determines how long fsmonitor will wait for Watchman
39 to return results. Defaults to `2.0`.
39 to return results. Defaults to `2.0`.
40
40
41 ::
41 ::
42
42
43 [fsmonitor]
43 [fsmonitor]
44 blacklistusers = (list of userids)
44 blacklistusers = (list of userids)
45
45
46 A list of usernames for which fsmonitor will disable itself altogether.
46 A list of usernames for which fsmonitor will disable itself altogether.
47
47
48 ::
48 ::
49
49
50 [fsmonitor]
50 [fsmonitor]
51 walk_on_invalidate = (boolean)
51 walk_on_invalidate = (boolean)
52
52
53 Whether or not to walk the whole repo ourselves when our cached state has been
53 Whether or not to walk the whole repo ourselves when our cached state has been
54 invalidated, for example when Watchman has been restarted or .hgignore rules
54 invalidated, for example when Watchman has been restarted or .hgignore rules
55 have been changed. Walking the repo in that case can result in competing for
55 have been changed. Walking the repo in that case can result in competing for
56 I/O with Watchman. For large repos it is recommended to set this value to
56 I/O with Watchman. For large repos it is recommended to set this value to
57 false. You may wish to set this to true if you have a very fast filesystem
57 false. You may wish to set this to true if you have a very fast filesystem
58 that can outpace the IPC overhead of getting the result data for the full repo
58 that can outpace the IPC overhead of getting the result data for the full repo
59 from Watchman. Defaults to false.
59 from Watchman. Defaults to false.
60
60
61 fsmonitor is incompatible with the largefiles and eol extensions, and
61 fsmonitor is incompatible with the largefiles and eol extensions, and
62 will disable itself if any of those are active.
62 will disable itself if any of those are active.
63
63
64 '''
64 '''
65
65
66 # Platforms Supported
66 # Platforms Supported
67 # ===================
67 # ===================
68 #
68 #
69 # **Linux:** *Stable*. Watchman and fsmonitor are both known to work reliably,
69 # **Linux:** *Stable*. Watchman and fsmonitor are both known to work reliably,
70 # even under severe loads.
70 # even under severe loads.
71 #
71 #
72 # **Mac OS X:** *Stable*. The Mercurial test suite passes with fsmonitor
72 # **Mac OS X:** *Stable*. The Mercurial test suite passes with fsmonitor
73 # turned on, on case-insensitive HFS+. There has been a reasonable amount of
73 # turned on, on case-insensitive HFS+. There has been a reasonable amount of
74 # user testing under normal loads.
74 # user testing under normal loads.
75 #
75 #
76 # **Solaris, BSD:** *Alpha*. watchman and fsmonitor are believed to work, but
76 # **Solaris, BSD:** *Alpha*. watchman and fsmonitor are believed to work, but
77 # very little testing has been done.
77 # very little testing has been done.
78 #
78 #
79 # **Windows:** *Alpha*. Not in a release version of watchman or fsmonitor yet.
79 # **Windows:** *Alpha*. Not in a release version of watchman or fsmonitor yet.
80 #
80 #
81 # Known Issues
81 # Known Issues
82 # ============
82 # ============
83 #
83 #
84 # * fsmonitor will disable itself if any of the following extensions are
84 # * fsmonitor will disable itself if any of the following extensions are
85 # enabled: largefiles, inotify, eol; or if the repository has subrepos.
85 # enabled: largefiles, inotify, eol; or if the repository has subrepos.
86 # * fsmonitor will produce incorrect results if nested repos that are not
86 # * fsmonitor will produce incorrect results if nested repos that are not
87 # subrepos exist. *Workaround*: add nested repo paths to your `.hgignore`.
87 # subrepos exist. *Workaround*: add nested repo paths to your `.hgignore`.
88 #
88 #
89 # The issues related to nested repos and subrepos are probably not fundamental
89 # The issues related to nested repos and subrepos are probably not fundamental
90 # ones. Patches to fix them are welcome.
90 # ones. Patches to fix them are welcome.
91
91
92 from __future__ import absolute_import
92 from __future__ import absolute_import
93
93
94 import os
94 import os
95 import stat
95 import stat
96 import sys
96 import sys
97
97
98 from mercurial import (
98 from mercurial import (
99 context,
99 context,
100 extensions,
100 extensions,
101 localrepo,
101 localrepo,
102 merge,
102 pathutil,
103 pathutil,
103 scmutil,
104 scmutil,
104 util,
105 util,
105 )
106 )
106 from mercurial import match as matchmod
107 from mercurial import match as matchmod
107 from mercurial.i18n import _
108 from mercurial.i18n import _
108
109
109 from . import (
110 from . import (
110 state,
111 state,
111 watchmanclient,
112 watchmanclient,
112 )
113 )
113
114
114 # Note for extension authors: ONLY specify testedwith = 'internal' for
115 # Note for extension authors: ONLY specify testedwith = 'internal' for
115 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
116 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
116 # be specifying the version(s) of Mercurial they are tested with, or
117 # be specifying the version(s) of Mercurial they are tested with, or
117 # leave the attribute unspecified.
118 # leave the attribute unspecified.
118 testedwith = 'internal'
119 testedwith = 'internal'
119
120
120 # This extension is incompatible with the following blacklisted extensions
121 # This extension is incompatible with the following blacklisted extensions
121 # and will disable itself when encountering one of these:
122 # and will disable itself when encountering one of these:
122 _blacklist = ['largefiles', 'eol']
123 _blacklist = ['largefiles', 'eol']
123
124
124 def _handleunavailable(ui, state, ex):
125 def _handleunavailable(ui, state, ex):
125 """Exception handler for Watchman interaction exceptions"""
126 """Exception handler for Watchman interaction exceptions"""
126 if isinstance(ex, watchmanclient.Unavailable):
127 if isinstance(ex, watchmanclient.Unavailable):
127 if ex.warn:
128 if ex.warn:
128 ui.warn(str(ex) + '\n')
129 ui.warn(str(ex) + '\n')
129 if ex.invalidate:
130 if ex.invalidate:
130 state.invalidate()
131 state.invalidate()
131 ui.log('fsmonitor', 'Watchman unavailable: %s\n', ex.msg)
132 ui.log('fsmonitor', 'Watchman unavailable: %s\n', ex.msg)
132 else:
133 else:
133 ui.log('fsmonitor', 'Watchman exception: %s\n', ex)
134 ui.log('fsmonitor', 'Watchman exception: %s\n', ex)
134
135
135 def _hashignore(ignore):
136 def _hashignore(ignore):
136 """Calculate hash for ignore patterns and filenames
137 """Calculate hash for ignore patterns and filenames
137
138
138 If this information changes between Mercurial invocations, we can't
139 If this information changes between Mercurial invocations, we can't
139 rely on Watchman information anymore and have to re-scan the working
140 rely on Watchman information anymore and have to re-scan the working
140 copy.
141 copy.
141
142
142 """
143 """
143 sha1 = util.sha1()
144 sha1 = util.sha1()
144 if util.safehasattr(ignore, 'includepat'):
145 if util.safehasattr(ignore, 'includepat'):
145 sha1.update(ignore.includepat)
146 sha1.update(ignore.includepat)
146 sha1.update('\0\0')
147 sha1.update('\0\0')
147 if util.safehasattr(ignore, 'excludepat'):
148 if util.safehasattr(ignore, 'excludepat'):
148 sha1.update(ignore.excludepat)
149 sha1.update(ignore.excludepat)
149 sha1.update('\0\0')
150 sha1.update('\0\0')
150 if util.safehasattr(ignore, 'patternspat'):
151 if util.safehasattr(ignore, 'patternspat'):
151 sha1.update(ignore.patternspat)
152 sha1.update(ignore.patternspat)
152 sha1.update('\0\0')
153 sha1.update('\0\0')
153 if util.safehasattr(ignore, '_files'):
154 if util.safehasattr(ignore, '_files'):
154 for f in ignore._files:
155 for f in ignore._files:
155 sha1.update(f)
156 sha1.update(f)
156 sha1.update('\0')
157 sha1.update('\0')
157 return sha1.hexdigest()
158 return sha1.hexdigest()
158
159
159 def overridewalk(orig, self, match, subrepos, unknown, ignored, full=True):
160 def overridewalk(orig, self, match, subrepos, unknown, ignored, full=True):
160 '''Replacement for dirstate.walk, hooking into Watchman.
161 '''Replacement for dirstate.walk, hooking into Watchman.
161
162
162 Whenever full is False, ignored is False, and the Watchman client is
163 Whenever full is False, ignored is False, and the Watchman client is
163 available, use Watchman combined with saved state to possibly return only a
164 available, use Watchman combined with saved state to possibly return only a
164 subset of files.'''
165 subset of files.'''
165 def bail():
166 def bail():
166 return orig(match, subrepos, unknown, ignored, full=True)
167 return orig(match, subrepos, unknown, ignored, full=True)
167
168
168 if full or ignored or not self._watchmanclient.available():
169 if full or ignored or not self._watchmanclient.available():
169 return bail()
170 return bail()
170 state = self._fsmonitorstate
171 state = self._fsmonitorstate
171 clock, ignorehash, notefiles = state.get()
172 clock, ignorehash, notefiles = state.get()
172 if not clock:
173 if not clock:
173 if state.walk_on_invalidate:
174 if state.walk_on_invalidate:
174 return bail()
175 return bail()
175 # Initial NULL clock value, see
176 # Initial NULL clock value, see
176 # https://facebook.github.io/watchman/docs/clockspec.html
177 # https://facebook.github.io/watchman/docs/clockspec.html
177 clock = 'c:0:0'
178 clock = 'c:0:0'
178 notefiles = []
179 notefiles = []
179
180
180 def fwarn(f, msg):
181 def fwarn(f, msg):
181 self._ui.warn('%s: %s\n' % (self.pathto(f), msg))
182 self._ui.warn('%s: %s\n' % (self.pathto(f), msg))
182 return False
183 return False
183
184
184 def badtype(mode):
185 def badtype(mode):
185 kind = _('unknown')
186 kind = _('unknown')
186 if stat.S_ISCHR(mode):
187 if stat.S_ISCHR(mode):
187 kind = _('character device')
188 kind = _('character device')
188 elif stat.S_ISBLK(mode):
189 elif stat.S_ISBLK(mode):
189 kind = _('block device')
190 kind = _('block device')
190 elif stat.S_ISFIFO(mode):
191 elif stat.S_ISFIFO(mode):
191 kind = _('fifo')
192 kind = _('fifo')
192 elif stat.S_ISSOCK(mode):
193 elif stat.S_ISSOCK(mode):
193 kind = _('socket')
194 kind = _('socket')
194 elif stat.S_ISDIR(mode):
195 elif stat.S_ISDIR(mode):
195 kind = _('directory')
196 kind = _('directory')
196 return _('unsupported file type (type is %s)') % kind
197 return _('unsupported file type (type is %s)') % kind
197
198
198 ignore = self._ignore
199 ignore = self._ignore
199 dirignore = self._dirignore
200 dirignore = self._dirignore
200 if unknown:
201 if unknown:
201 if _hashignore(ignore) != ignorehash and clock != 'c:0:0':
202 if _hashignore(ignore) != ignorehash and clock != 'c:0:0':
202 # ignore list changed -- can't rely on Watchman state any more
203 # ignore list changed -- can't rely on Watchman state any more
203 if state.walk_on_invalidate:
204 if state.walk_on_invalidate:
204 return bail()
205 return bail()
205 notefiles = []
206 notefiles = []
206 clock = 'c:0:0'
207 clock = 'c:0:0'
207 else:
208 else:
208 # always ignore
209 # always ignore
209 ignore = util.always
210 ignore = util.always
210 dirignore = util.always
211 dirignore = util.always
211
212
212 matchfn = match.matchfn
213 matchfn = match.matchfn
213 matchalways = match.always()
214 matchalways = match.always()
214 dmap = self._map
215 dmap = self._map
215 nonnormalset = getattr(self, '_nonnormalset', None)
216 nonnormalset = getattr(self, '_nonnormalset', None)
216
217
217 copymap = self._copymap
218 copymap = self._copymap
218 getkind = stat.S_IFMT
219 getkind = stat.S_IFMT
219 dirkind = stat.S_IFDIR
220 dirkind = stat.S_IFDIR
220 regkind = stat.S_IFREG
221 regkind = stat.S_IFREG
221 lnkkind = stat.S_IFLNK
222 lnkkind = stat.S_IFLNK
222 join = self._join
223 join = self._join
223 normcase = util.normcase
224 normcase = util.normcase
224 fresh_instance = False
225 fresh_instance = False
225
226
226 exact = skipstep3 = False
227 exact = skipstep3 = False
227 if matchfn == match.exact: # match.exact
228 if matchfn == match.exact: # match.exact
228 exact = True
229 exact = True
229 dirignore = util.always # skip step 2
230 dirignore = util.always # skip step 2
230 elif match.files() and not match.anypats(): # match.match, no patterns
231 elif match.files() and not match.anypats(): # match.match, no patterns
231 skipstep3 = True
232 skipstep3 = True
232
233
233 if not exact and self._checkcase:
234 if not exact and self._checkcase:
234 # note that even though we could receive directory entries, we're only
235 # note that even though we could receive directory entries, we're only
235 # interested in checking if a file with the same name exists. So only
236 # interested in checking if a file with the same name exists. So only
236 # normalize files if possible.
237 # normalize files if possible.
237 normalize = self._normalizefile
238 normalize = self._normalizefile
238 skipstep3 = False
239 skipstep3 = False
239 else:
240 else:
240 normalize = None
241 normalize = None
241
242
242 # step 1: find all explicit files
243 # step 1: find all explicit files
243 results, work, dirsnotfound = self._walkexplicit(match, subrepos)
244 results, work, dirsnotfound = self._walkexplicit(match, subrepos)
244
245
245 skipstep3 = skipstep3 and not (work or dirsnotfound)
246 skipstep3 = skipstep3 and not (work or dirsnotfound)
246 work = [d for d in work if not dirignore(d[0])]
247 work = [d for d in work if not dirignore(d[0])]
247
248
248 if not work and (exact or skipstep3):
249 if not work and (exact or skipstep3):
249 for s in subrepos:
250 for s in subrepos:
250 del results[s]
251 del results[s]
251 del results['.hg']
252 del results['.hg']
252 return results
253 return results
253
254
254 # step 2: query Watchman
255 # step 2: query Watchman
255 try:
256 try:
256 # Use the user-configured timeout for the query.
257 # Use the user-configured timeout for the query.
257 # Add a little slack over the top of the user query to allow for
258 # Add a little slack over the top of the user query to allow for
258 # overheads while transferring the data
259 # overheads while transferring the data
259 self._watchmanclient.settimeout(state.timeout + 0.1)
260 self._watchmanclient.settimeout(state.timeout + 0.1)
260 result = self._watchmanclient.command('query', {
261 result = self._watchmanclient.command('query', {
261 'fields': ['mode', 'mtime', 'size', 'exists', 'name'],
262 'fields': ['mode', 'mtime', 'size', 'exists', 'name'],
262 'since': clock,
263 'since': clock,
263 'expression': [
264 'expression': [
264 'not', [
265 'not', [
265 'anyof', ['dirname', '.hg'],
266 'anyof', ['dirname', '.hg'],
266 ['name', '.hg', 'wholename']
267 ['name', '.hg', 'wholename']
267 ]
268 ]
268 ],
269 ],
269 'sync_timeout': int(state.timeout * 1000),
270 'sync_timeout': int(state.timeout * 1000),
270 'empty_on_fresh_instance': state.walk_on_invalidate,
271 'empty_on_fresh_instance': state.walk_on_invalidate,
271 })
272 })
272 except Exception as ex:
273 except Exception as ex:
273 _handleunavailable(self._ui, state, ex)
274 _handleunavailable(self._ui, state, ex)
274 self._watchmanclient.clearconnection()
275 self._watchmanclient.clearconnection()
275 return bail()
276 return bail()
276 else:
277 else:
277 # We need to propagate the last observed clock up so that we
278 # We need to propagate the last observed clock up so that we
278 # can use it for our next query
279 # can use it for our next query
279 state.setlastclock(result['clock'])
280 state.setlastclock(result['clock'])
280 if result['is_fresh_instance']:
281 if result['is_fresh_instance']:
281 if state.walk_on_invalidate:
282 if state.walk_on_invalidate:
282 state.invalidate()
283 state.invalidate()
283 return bail()
284 return bail()
284 fresh_instance = True
285 fresh_instance = True
285 # Ignore any prior noteable files from the state info
286 # Ignore any prior noteable files from the state info
286 notefiles = []
287 notefiles = []
287
288
288 # for file paths which require normalization and we encounter a case
289 # for file paths which require normalization and we encounter a case
289 # collision, we store our own foldmap
290 # collision, we store our own foldmap
290 if normalize:
291 if normalize:
291 foldmap = dict((normcase(k), k) for k in results)
292 foldmap = dict((normcase(k), k) for k in results)
292
293
293 switch_slashes = os.sep == '\\'
294 switch_slashes = os.sep == '\\'
294 # The order of the results is, strictly speaking, undefined.
295 # The order of the results is, strictly speaking, undefined.
295 # For case changes on a case insensitive filesystem we may receive
296 # For case changes on a case insensitive filesystem we may receive
296 # two entries, one with exists=True and another with exists=False.
297 # two entries, one with exists=True and another with exists=False.
297 # The exists=True entries in the same response should be interpreted
298 # The exists=True entries in the same response should be interpreted
298 # as being happens-after the exists=False entries due to the way that
299 # as being happens-after the exists=False entries due to the way that
299 # Watchman tracks files. We use this property to reconcile deletes
300 # Watchman tracks files. We use this property to reconcile deletes
300 # for name case changes.
301 # for name case changes.
301 for entry in result['files']:
302 for entry in result['files']:
302 fname = entry['name']
303 fname = entry['name']
303 if switch_slashes:
304 if switch_slashes:
304 fname = fname.replace('\\', '/')
305 fname = fname.replace('\\', '/')
305 if normalize:
306 if normalize:
306 normed = normcase(fname)
307 normed = normcase(fname)
307 fname = normalize(fname, True, True)
308 fname = normalize(fname, True, True)
308 foldmap[normed] = fname
309 foldmap[normed] = fname
309 fmode = entry['mode']
310 fmode = entry['mode']
310 fexists = entry['exists']
311 fexists = entry['exists']
311 kind = getkind(fmode)
312 kind = getkind(fmode)
312
313
313 if not fexists:
314 if not fexists:
314 # if marked as deleted and we don't already have a change
315 # if marked as deleted and we don't already have a change
315 # record, mark it as deleted. If we already have an entry
316 # record, mark it as deleted. If we already have an entry
316 # for fname then it was either part of walkexplicit or was
317 # for fname then it was either part of walkexplicit or was
317 # an earlier result that was a case change
318 # an earlier result that was a case change
318 if fname not in results and fname in dmap and (
319 if fname not in results and fname in dmap and (
319 matchalways or matchfn(fname)):
320 matchalways or matchfn(fname)):
320 results[fname] = None
321 results[fname] = None
321 elif kind == dirkind:
322 elif kind == dirkind:
322 if fname in dmap and (matchalways or matchfn(fname)):
323 if fname in dmap and (matchalways or matchfn(fname)):
323 results[fname] = None
324 results[fname] = None
324 elif kind == regkind or kind == lnkkind:
325 elif kind == regkind or kind == lnkkind:
325 if fname in dmap:
326 if fname in dmap:
326 if matchalways or matchfn(fname):
327 if matchalways or matchfn(fname):
327 results[fname] = entry
328 results[fname] = entry
328 elif (matchalways or matchfn(fname)) and not ignore(fname):
329 elif (matchalways or matchfn(fname)) and not ignore(fname):
329 results[fname] = entry
330 results[fname] = entry
330 elif fname in dmap and (matchalways or matchfn(fname)):
331 elif fname in dmap and (matchalways or matchfn(fname)):
331 results[fname] = None
332 results[fname] = None
332
333
333 # step 3: query notable files we don't already know about
334 # step 3: query notable files we don't already know about
334 # XXX try not to iterate over the entire dmap
335 # XXX try not to iterate over the entire dmap
335 if normalize:
336 if normalize:
336 # any notable files that have changed case will already be handled
337 # any notable files that have changed case will already be handled
337 # above, so just check membership in the foldmap
338 # above, so just check membership in the foldmap
338 notefiles = set((normalize(f, True, True) for f in notefiles
339 notefiles = set((normalize(f, True, True) for f in notefiles
339 if normcase(f) not in foldmap))
340 if normcase(f) not in foldmap))
340 visit = set((f for f in notefiles if (f not in results and matchfn(f)
341 visit = set((f for f in notefiles if (f not in results and matchfn(f)
341 and (f in dmap or not ignore(f)))))
342 and (f in dmap or not ignore(f)))))
342
343
343 if nonnormalset is not None and not fresh_instance:
344 if nonnormalset is not None and not fresh_instance:
344 if matchalways:
345 if matchalways:
345 visit.update(f for f in nonnormalset if f not in results)
346 visit.update(f for f in nonnormalset if f not in results)
346 visit.update(f for f in copymap if f not in results)
347 visit.update(f for f in copymap if f not in results)
347 else:
348 else:
348 visit.update(f for f in nonnormalset
349 visit.update(f for f in nonnormalset
349 if f not in results and matchfn(f))
350 if f not in results and matchfn(f))
350 visit.update(f for f in copymap
351 visit.update(f for f in copymap
351 if f not in results and matchfn(f))
352 if f not in results and matchfn(f))
352 else:
353 else:
353 if matchalways:
354 if matchalways:
354 visit.update(f for f, st in dmap.iteritems()
355 visit.update(f for f, st in dmap.iteritems()
355 if (f not in results and
356 if (f not in results and
356 (st[2] < 0 or st[0] != 'n' or fresh_instance)))
357 (st[2] < 0 or st[0] != 'n' or fresh_instance)))
357 visit.update(f for f in copymap if f not in results)
358 visit.update(f for f in copymap if f not in results)
358 else:
359 else:
359 visit.update(f for f, st in dmap.iteritems()
360 visit.update(f for f, st in dmap.iteritems()
360 if (f not in results and
361 if (f not in results and
361 (st[2] < 0 or st[0] != 'n' or fresh_instance)
362 (st[2] < 0 or st[0] != 'n' or fresh_instance)
362 and matchfn(f)))
363 and matchfn(f)))
363 visit.update(f for f in copymap
364 visit.update(f for f in copymap
364 if f not in results and matchfn(f))
365 if f not in results and matchfn(f))
365
366
366 audit = pathutil.pathauditor(self._root).check
367 audit = pathutil.pathauditor(self._root).check
367 auditpass = [f for f in visit if audit(f)]
368 auditpass = [f for f in visit if audit(f)]
368 auditpass.sort()
369 auditpass.sort()
369 auditfail = visit.difference(auditpass)
370 auditfail = visit.difference(auditpass)
370 for f in auditfail:
371 for f in auditfail:
371 results[f] = None
372 results[f] = None
372
373
373 nf = iter(auditpass).next
374 nf = iter(auditpass).next
374 for st in util.statfiles([join(f) for f in auditpass]):
375 for st in util.statfiles([join(f) for f in auditpass]):
375 f = nf()
376 f = nf()
376 if st or f in dmap:
377 if st or f in dmap:
377 results[f] = st
378 results[f] = st
378
379
379 for s in subrepos:
380 for s in subrepos:
380 del results[s]
381 del results[s]
381 del results['.hg']
382 del results['.hg']
382 return results
383 return results
383
384
384 def overridestatus(
385 def overridestatus(
385 orig, self, node1='.', node2=None, match=None, ignored=False,
386 orig, self, node1='.', node2=None, match=None, ignored=False,
386 clean=False, unknown=False, listsubrepos=False):
387 clean=False, unknown=False, listsubrepos=False):
387 listignored = ignored
388 listignored = ignored
388 listclean = clean
389 listclean = clean
389 listunknown = unknown
390 listunknown = unknown
390
391
391 def _cmpsets(l1, l2):
392 def _cmpsets(l1, l2):
392 try:
393 try:
393 if 'FSMONITOR_LOG_FILE' in os.environ:
394 if 'FSMONITOR_LOG_FILE' in os.environ:
394 fn = os.environ['FSMONITOR_LOG_FILE']
395 fn = os.environ['FSMONITOR_LOG_FILE']
395 f = open(fn, 'wb')
396 f = open(fn, 'wb')
396 else:
397 else:
397 fn = 'fsmonitorfail.log'
398 fn = 'fsmonitorfail.log'
398 f = self.opener(fn, 'wb')
399 f = self.opener(fn, 'wb')
399 except (IOError, OSError):
400 except (IOError, OSError):
400 self.ui.warn(_('warning: unable to write to %s\n') % fn)
401 self.ui.warn(_('warning: unable to write to %s\n') % fn)
401 return
402 return
402
403
403 try:
404 try:
404 for i, (s1, s2) in enumerate(zip(l1, l2)):
405 for i, (s1, s2) in enumerate(zip(l1, l2)):
405 if set(s1) != set(s2):
406 if set(s1) != set(s2):
406 f.write('sets at position %d are unequal\n' % i)
407 f.write('sets at position %d are unequal\n' % i)
407 f.write('watchman returned: %s\n' % s1)
408 f.write('watchman returned: %s\n' % s1)
408 f.write('stat returned: %s\n' % s2)
409 f.write('stat returned: %s\n' % s2)
409 finally:
410 finally:
410 f.close()
411 f.close()
411
412
412 if isinstance(node1, context.changectx):
413 if isinstance(node1, context.changectx):
413 ctx1 = node1
414 ctx1 = node1
414 else:
415 else:
415 ctx1 = self[node1]
416 ctx1 = self[node1]
416 if isinstance(node2, context.changectx):
417 if isinstance(node2, context.changectx):
417 ctx2 = node2
418 ctx2 = node2
418 else:
419 else:
419 ctx2 = self[node2]
420 ctx2 = self[node2]
420
421
421 working = ctx2.rev() is None
422 working = ctx2.rev() is None
422 parentworking = working and ctx1 == self['.']
423 parentworking = working and ctx1 == self['.']
423 match = match or matchmod.always(self.root, self.getcwd())
424 match = match or matchmod.always(self.root, self.getcwd())
424
425
425 # Maybe we can use this opportunity to update Watchman's state.
426 # Maybe we can use this opportunity to update Watchman's state.
426 # Mercurial uses workingcommitctx and/or memctx to represent the part of
427 # Mercurial uses workingcommitctx and/or memctx to represent the part of
427 # the workingctx that is to be committed. So don't update the state in
428 # the workingctx that is to be committed. So don't update the state in
428 # that case.
429 # that case.
429 # HG_PENDING is set in the environment when the dirstate is being updated
430 # HG_PENDING is set in the environment when the dirstate is being updated
430 # in the middle of a transaction; we must not update our state in that
431 # in the middle of a transaction; we must not update our state in that
431 # case, or we risk forgetting about changes in the working copy.
432 # case, or we risk forgetting about changes in the working copy.
432 updatestate = (parentworking and match.always() and
433 updatestate = (parentworking and match.always() and
433 not isinstance(ctx2, (context.workingcommitctx,
434 not isinstance(ctx2, (context.workingcommitctx,
434 context.memctx)) and
435 context.memctx)) and
435 'HG_PENDING' not in os.environ)
436 'HG_PENDING' not in os.environ)
436
437
437 try:
438 try:
438 if self._fsmonitorstate.walk_on_invalidate:
439 if self._fsmonitorstate.walk_on_invalidate:
439 # Use a short timeout to query the current clock. If that
440 # Use a short timeout to query the current clock. If that
440 # takes too long then we assume that the service will be slow
441 # takes too long then we assume that the service will be slow
441 # to answer our query.
442 # to answer our query.
442 # walk_on_invalidate indicates that we prefer to walk the
443 # walk_on_invalidate indicates that we prefer to walk the
443 # tree ourselves because we can ignore portions that Watchman
444 # tree ourselves because we can ignore portions that Watchman
444 # cannot and we tend to be faster in the warmer buffer cache
445 # cannot and we tend to be faster in the warmer buffer cache
445 # cases.
446 # cases.
446 self._watchmanclient.settimeout(0.1)
447 self._watchmanclient.settimeout(0.1)
447 else:
448 else:
448 # Give Watchman more time to potentially complete its walk
449 # Give Watchman more time to potentially complete its walk
449 # and return the initial clock. In this mode we assume that
450 # and return the initial clock. In this mode we assume that
450 # the filesystem will be slower than parsing a potentially
451 # the filesystem will be slower than parsing a potentially
451 # very large Watchman result set.
452 # very large Watchman result set.
452 self._watchmanclient.settimeout(
453 self._watchmanclient.settimeout(
453 self._fsmonitorstate.timeout + 0.1)
454 self._fsmonitorstate.timeout + 0.1)
454 startclock = self._watchmanclient.getcurrentclock()
455 startclock = self._watchmanclient.getcurrentclock()
455 except Exception as ex:
456 except Exception as ex:
456 self._watchmanclient.clearconnection()
457 self._watchmanclient.clearconnection()
457 _handleunavailable(self.ui, self._fsmonitorstate, ex)
458 _handleunavailable(self.ui, self._fsmonitorstate, ex)
458 # boo, Watchman failed. bail
459 # boo, Watchman failed. bail
459 return orig(node1, node2, match, listignored, listclean,
460 return orig(node1, node2, match, listignored, listclean,
460 listunknown, listsubrepos)
461 listunknown, listsubrepos)
461
462
462 if updatestate:
463 if updatestate:
463 # We need info about unknown files. This may make things slower the
464 # We need info about unknown files. This may make things slower the
464 # first time, but whatever.
465 # first time, but whatever.
465 stateunknown = True
466 stateunknown = True
466 else:
467 else:
467 stateunknown = listunknown
468 stateunknown = listunknown
468
469
469 r = orig(node1, node2, match, listignored, listclean, stateunknown,
470 r = orig(node1, node2, match, listignored, listclean, stateunknown,
470 listsubrepos)
471 listsubrepos)
471 modified, added, removed, deleted, unknown, ignored, clean = r
472 modified, added, removed, deleted, unknown, ignored, clean = r
472
473
473 if updatestate:
474 if updatestate:
474 notefiles = modified + added + removed + deleted + unknown
475 notefiles = modified + added + removed + deleted + unknown
475 self._fsmonitorstate.set(
476 self._fsmonitorstate.set(
476 self._fsmonitorstate.getlastclock() or startclock,
477 self._fsmonitorstate.getlastclock() or startclock,
477 _hashignore(self.dirstate._ignore),
478 _hashignore(self.dirstate._ignore),
478 notefiles)
479 notefiles)
479
480
480 if not listunknown:
481 if not listunknown:
481 unknown = []
482 unknown = []
482
483
483 # don't do paranoid checks if we're not going to query Watchman anyway
484 # don't do paranoid checks if we're not going to query Watchman anyway
484 full = listclean or match.traversedir is not None
485 full = listclean or match.traversedir is not None
485 if self._fsmonitorstate.mode == 'paranoid' and not full:
486 if self._fsmonitorstate.mode == 'paranoid' and not full:
486 # run status again and fall back to the old walk this time
487 # run status again and fall back to the old walk this time
487 self.dirstate._fsmonitordisable = True
488 self.dirstate._fsmonitordisable = True
488
489
489 # shut the UI up
490 # shut the UI up
490 quiet = self.ui.quiet
491 quiet = self.ui.quiet
491 self.ui.quiet = True
492 self.ui.quiet = True
492 fout, ferr = self.ui.fout, self.ui.ferr
493 fout, ferr = self.ui.fout, self.ui.ferr
493 self.ui.fout = self.ui.ferr = open(os.devnull, 'wb')
494 self.ui.fout = self.ui.ferr = open(os.devnull, 'wb')
494
495
495 try:
496 try:
496 rv2 = orig(
497 rv2 = orig(
497 node1, node2, match, listignored, listclean, listunknown,
498 node1, node2, match, listignored, listclean, listunknown,
498 listsubrepos)
499 listsubrepos)
499 finally:
500 finally:
500 self.dirstate._fsmonitordisable = False
501 self.dirstate._fsmonitordisable = False
501 self.ui.quiet = quiet
502 self.ui.quiet = quiet
502 self.ui.fout, self.ui.ferr = fout, ferr
503 self.ui.fout, self.ui.ferr = fout, ferr
503
504
504 # clean isn't tested since it's set to True above
505 # clean isn't tested since it's set to True above
505 _cmpsets([modified, added, removed, deleted, unknown, ignored, clean],
506 _cmpsets([modified, added, removed, deleted, unknown, ignored, clean],
506 rv2)
507 rv2)
507 modified, added, removed, deleted, unknown, ignored, clean = rv2
508 modified, added, removed, deleted, unknown, ignored, clean = rv2
508
509
509 return scmutil.status(
510 return scmutil.status(
510 modified, added, removed, deleted, unknown, ignored, clean)
511 modified, added, removed, deleted, unknown, ignored, clean)
511
512
512 def makedirstate(cls):
513 def makedirstate(cls):
513 class fsmonitordirstate(cls):
514 class fsmonitordirstate(cls):
514 def _fsmonitorinit(self, fsmonitorstate, watchmanclient):
515 def _fsmonitorinit(self, fsmonitorstate, watchmanclient):
515 # _fsmonitordisable is used in paranoid mode
516 # _fsmonitordisable is used in paranoid mode
516 self._fsmonitordisable = False
517 self._fsmonitordisable = False
517 self._fsmonitorstate = fsmonitorstate
518 self._fsmonitorstate = fsmonitorstate
518 self._watchmanclient = watchmanclient
519 self._watchmanclient = watchmanclient
519
520
520 def walk(self, *args, **kwargs):
521 def walk(self, *args, **kwargs):
521 orig = super(fsmonitordirstate, self).walk
522 orig = super(fsmonitordirstate, self).walk
522 if self._fsmonitordisable:
523 if self._fsmonitordisable:
523 return orig(*args, **kwargs)
524 return orig(*args, **kwargs)
524 return overridewalk(orig, self, *args, **kwargs)
525 return overridewalk(orig, self, *args, **kwargs)
525
526
526 def rebuild(self, *args, **kwargs):
527 def rebuild(self, *args, **kwargs):
527 self._fsmonitorstate.invalidate()
528 self._fsmonitorstate.invalidate()
528 return super(fsmonitordirstate, self).rebuild(*args, **kwargs)
529 return super(fsmonitordirstate, self).rebuild(*args, **kwargs)
529
530
530 def invalidate(self, *args, **kwargs):
531 def invalidate(self, *args, **kwargs):
531 self._fsmonitorstate.invalidate()
532 self._fsmonitorstate.invalidate()
532 return super(fsmonitordirstate, self).invalidate(*args, **kwargs)
533 return super(fsmonitordirstate, self).invalidate(*args, **kwargs)
533
534
534 return fsmonitordirstate
535 return fsmonitordirstate
535
536
536 def wrapdirstate(orig, self):
537 def wrapdirstate(orig, self):
537 ds = orig(self)
538 ds = orig(self)
538 # only override the dirstate when Watchman is available for the repo
539 # only override the dirstate when Watchman is available for the repo
539 if util.safehasattr(self, '_fsmonitorstate'):
540 if util.safehasattr(self, '_fsmonitorstate'):
540 ds.__class__ = makedirstate(ds.__class__)
541 ds.__class__ = makedirstate(ds.__class__)
541 ds._fsmonitorinit(self._fsmonitorstate, self._watchmanclient)
542 ds._fsmonitorinit(self._fsmonitorstate, self._watchmanclient)
542 return ds
543 return ds
543
544
544 def extsetup(ui):
545 def extsetup(ui):
545 wrapfilecache(localrepo.localrepository, 'dirstate', wrapdirstate)
546 wrapfilecache(localrepo.localrepository, 'dirstate', wrapdirstate)
546 if sys.platform == 'darwin':
547 if sys.platform == 'darwin':
547 # An assist for avoiding the dangling-symlink fsevents bug
548 # An assist for avoiding the dangling-symlink fsevents bug
548 extensions.wrapfunction(os, 'symlink', wrapsymlink)
549 extensions.wrapfunction(os, 'symlink', wrapsymlink)
549
550
551 extensions.wrapfunction(merge, 'update', wrapupdate)
552
550 def wrapsymlink(orig, source, link_name):
553 def wrapsymlink(orig, source, link_name):
551 ''' if we create a dangling symlink, also touch the parent dir
554 ''' if we create a dangling symlink, also touch the parent dir
552 to encourage fsevents notifications to work more correctly '''
555 to encourage fsevents notifications to work more correctly '''
553 try:
556 try:
554 return orig(source, link_name)
557 return orig(source, link_name)
555 finally:
558 finally:
556 try:
559 try:
557 os.utime(os.path.dirname(link_name), None)
560 os.utime(os.path.dirname(link_name), None)
558 except OSError:
561 except OSError:
559 pass
562 pass
560
563
564 class state_update(object):
565 ''' This context mananger is responsible for dispatching the state-enter
566 and state-leave signals to the watchman service '''
567
568 def __init__(self, repo, node, distance, partial):
569 self.repo = repo
570 self.node = node
571 self.distance = distance
572 self.partial = partial
573
574 def __enter__(self):
575 self._state('state-enter')
576 return self
577
578 def __exit__(self, type_, value, tb):
579 status = 'ok' if type_ is None else 'failed'
580 self._state('state-leave', status=status)
581
582 def _state(self, cmd, status='ok'):
583 if not util.safehasattr(self.repo, '_watchmanclient'):
584 return
585 try:
586 commithash = self.repo[self.node].hex()
587 self.repo._watchmanclient.command(cmd, {
588 'name': 'hg.update',
589 'metadata': {
590 # the target revision
591 'rev': commithash,
592 # approximate number of commits between current and target
593 'distance': self.distance,
594 # success/failure (only really meaningful for state-leave)
595 'status': status,
596 # whether the working copy parent is changing
597 'partial': self.partial,
598 }})
599 except Exception as e:
600 # Swallow any errors; fire and forget
601 self.repo.ui.log(
602 'watchman', 'Exception %s while running %s\n', e, cmd)
603
604 # Bracket working copy updates with calls to the watchman state-enter
605 # and state-leave commands. This allows clients to perform more intelligent
606 # settling during bulk file change scenarios
607 # https://facebook.github.io/watchman/docs/cmd/subscribe.html#advanced-settling
608 def wrapupdate(orig, repo, node, branchmerge, force, ancestor=None,
609 mergeancestor=False, labels=None, matcher=None, **kwargs):
610
611 distance = 0
612 partial = True
613 if matcher is None or matcher.always():
614 partial = False
615 wc = repo[None]
616 parents = wc.parents()
617 if len(parents) == 2:
618 anc = repo.changelog.ancestor(parents[0].node(), parents[1].node())
619 ancrev = repo[anc].rev()
620 distance = abs(repo[node].rev() - ancrev)
621 elif len(parents) == 1:
622 distance = abs(repo[node].rev() - parents[0].rev())
623
624 with state_update(repo, node, distance, partial):
625 return orig(
626 repo, node, branchmerge, force, ancestor, mergeancestor,
627 labels, matcher, *kwargs)
628
561 def reposetup(ui, repo):
629 def reposetup(ui, repo):
562 # We don't work with largefiles or inotify
630 # We don't work with largefiles or inotify
563 exts = extensions.enabled()
631 exts = extensions.enabled()
564 for ext in _blacklist:
632 for ext in _blacklist:
565 if ext in exts:
633 if ext in exts:
566 ui.warn(_('The fsmonitor extension is incompatible with the %s '
634 ui.warn(_('The fsmonitor extension is incompatible with the %s '
567 'extension and has been disabled.\n') % ext)
635 'extension and has been disabled.\n') % ext)
568 return
636 return
569
637
570 if util.safehasattr(repo, 'dirstate'):
638 if util.safehasattr(repo, 'dirstate'):
571 # We don't work with subrepos either. Note that we can get passed in
639 # We don't work with subrepos either. Note that we can get passed in
572 # e.g. a statichttprepo, which throws on trying to access the substate.
640 # e.g. a statichttprepo, which throws on trying to access the substate.
573 # XXX This sucks.
641 # XXX This sucks.
574 try:
642 try:
575 # if repo[None].substate can cause a dirstate parse, which is too
643 # if repo[None].substate can cause a dirstate parse, which is too
576 # slow. Instead, look for a file called hgsubstate,
644 # slow. Instead, look for a file called hgsubstate,
577 if repo.wvfs.exists('.hgsubstate') or repo.wvfs.exists('.hgsub'):
645 if repo.wvfs.exists('.hgsubstate') or repo.wvfs.exists('.hgsub'):
578 return
646 return
579 except AttributeError:
647 except AttributeError:
580 return
648 return
581
649
582 fsmonitorstate = state.state(repo)
650 fsmonitorstate = state.state(repo)
583 if fsmonitorstate.mode == 'off':
651 if fsmonitorstate.mode == 'off':
584 return
652 return
585
653
586 try:
654 try:
587 client = watchmanclient.client(repo)
655 client = watchmanclient.client(repo)
588 except Exception as ex:
656 except Exception as ex:
589 _handleunavailable(ui, fsmonitorstate, ex)
657 _handleunavailable(ui, fsmonitorstate, ex)
590 return
658 return
591
659
592 repo._fsmonitorstate = fsmonitorstate
660 repo._fsmonitorstate = fsmonitorstate
593 repo._watchmanclient = client
661 repo._watchmanclient = client
594
662
595 # at this point since fsmonitorstate wasn't present, repo.dirstate is
663 # at this point since fsmonitorstate wasn't present, repo.dirstate is
596 # not a fsmonitordirstate
664 # not a fsmonitordirstate
597 repo.dirstate.__class__ = makedirstate(repo.dirstate.__class__)
665 repo.dirstate.__class__ = makedirstate(repo.dirstate.__class__)
598 # nuke the dirstate so that _fsmonitorinit and subsequent configuration
666 # nuke the dirstate so that _fsmonitorinit and subsequent configuration
599 # changes take effect on it
667 # changes take effect on it
600 del repo._filecache['dirstate']
668 del repo._filecache['dirstate']
601 delattr(repo.unfiltered(), 'dirstate')
669 delattr(repo.unfiltered(), 'dirstate')
602
670
603 class fsmonitorrepo(repo.__class__):
671 class fsmonitorrepo(repo.__class__):
604 def status(self, *args, **kwargs):
672 def status(self, *args, **kwargs):
605 orig = super(fsmonitorrepo, self).status
673 orig = super(fsmonitorrepo, self).status
606 return overridestatus(orig, self, *args, **kwargs)
674 return overridestatus(orig, self, *args, **kwargs)
607
675
608 repo.__class__ = fsmonitorrepo
676 repo.__class__ = fsmonitorrepo
609
677
610 def wrapfilecache(cls, propname, wrapper):
678 def wrapfilecache(cls, propname, wrapper):
611 """Wraps a filecache property. These can't be wrapped using the normal
679 """Wraps a filecache property. These can't be wrapped using the normal
612 wrapfunction. This should eventually go into upstream Mercurial.
680 wrapfunction. This should eventually go into upstream Mercurial.
613 """
681 """
614 assert callable(wrapper)
682 assert callable(wrapper)
615 for currcls in cls.__mro__:
683 for currcls in cls.__mro__:
616 if propname in currcls.__dict__:
684 if propname in currcls.__dict__:
617 origfn = currcls.__dict__[propname].func
685 origfn = currcls.__dict__[propname].func
618 assert callable(origfn)
686 assert callable(origfn)
619 def wrap(*args, **kwargs):
687 def wrap(*args, **kwargs):
620 return wrapper(origfn, *args, **kwargs)
688 return wrapper(origfn, *args, **kwargs)
621 currcls.__dict__[propname].func = wrap
689 currcls.__dict__[propname].func = wrap
622 break
690 break
623
691
624 if currcls is object:
692 if currcls is object:
625 raise AttributeError(
693 raise AttributeError(
626 _("type '%s' has no property '%s'") % (cls, propname))
694 _("type '%s' has no property '%s'") % (cls, propname))
General Comments 0
You need to be logged in to leave comments. Login now