##// END OF EJS Templates
wrapfunction: use sysstr instead of bytes as argument in "fsmonitor"...
marmoute -
r51671:ba3add90 default
parent child Browse files
Show More
@@ -1,1016 +1,1016 b''
1 # __init__.py - fsmonitor initialization and overrides
1 # __init__.py - fsmonitor initialization and overrides
2 #
2 #
3 # Copyright 2013-2016 Facebook, Inc.
3 # Copyright 2013-2016 Facebook, Inc.
4 #
4 #
5 # This software may be used and distributed according to the terms of the
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
6 # GNU General Public License version 2 or any later version.
7
7
8 '''Faster status operations with the Watchman file monitor (EXPERIMENTAL)
8 '''Faster status operations with the Watchman file monitor (EXPERIMENTAL)
9
9
10 Integrates the file-watching program Watchman with Mercurial to produce faster
10 Integrates the file-watching program Watchman with Mercurial to produce faster
11 status results.
11 status results.
12
12
13 On a particular Linux system, for a real-world repository with over 400,000
13 On a particular Linux system, for a real-world repository with over 400,000
14 files hosted on ext4, vanilla `hg status` takes 1.3 seconds. On the same
14 files hosted on ext4, vanilla `hg status` takes 1.3 seconds. On the same
15 system, with fsmonitor it takes about 0.3 seconds.
15 system, with fsmonitor it takes about 0.3 seconds.
16
16
17 fsmonitor requires no configuration -- it will tell Watchman about your
17 fsmonitor requires no configuration -- it will tell Watchman about your
18 repository as necessary. You'll need to install Watchman from
18 repository as necessary. You'll need to install Watchman from
19 https://facebook.github.io/watchman/ and make sure it is in your PATH.
19 https://facebook.github.io/watchman/ and make sure it is in your PATH.
20
20
21 fsmonitor is incompatible with the largefiles and eol extensions, and
21 fsmonitor is incompatible with the largefiles and eol extensions, and
22 will disable itself if any of those are active.
22 will disable itself if any of those are active.
23
23
24 The following configuration options exist:
24 The following configuration options exist:
25
25
26 ::
26 ::
27
27
28 [fsmonitor]
28 [fsmonitor]
29 mode = {off, on, paranoid}
29 mode = {off, on, paranoid}
30
30
31 When `mode = off`, fsmonitor will disable itself (similar to not loading the
31 When `mode = off`, fsmonitor will disable itself (similar to not loading the
32 extension at all). When `mode = on`, fsmonitor will be enabled (the default).
32 extension at all). When `mode = on`, fsmonitor will be enabled (the default).
33 When `mode = paranoid`, fsmonitor will query both Watchman and the filesystem,
33 When `mode = paranoid`, fsmonitor will query both Watchman and the filesystem,
34 and ensure that the results are consistent.
34 and ensure that the results are consistent.
35
35
36 ::
36 ::
37
37
38 [fsmonitor]
38 [fsmonitor]
39 timeout = (float)
39 timeout = (float)
40
40
41 A value, in seconds, that determines how long fsmonitor will wait for Watchman
41 A value, in seconds, that determines how long fsmonitor will wait for Watchman
42 to return results. Defaults to `2.0`.
42 to return results. Defaults to `2.0`.
43
43
44 ::
44 ::
45
45
46 [fsmonitor]
46 [fsmonitor]
47 blacklistusers = (list of userids)
47 blacklistusers = (list of userids)
48
48
49 A list of usernames for which fsmonitor will disable itself altogether.
49 A list of usernames for which fsmonitor will disable itself altogether.
50
50
51 ::
51 ::
52
52
53 [fsmonitor]
53 [fsmonitor]
54 walk_on_invalidate = (boolean)
54 walk_on_invalidate = (boolean)
55
55
56 Whether or not to walk the whole repo ourselves when our cached state has been
56 Whether or not to walk the whole repo ourselves when our cached state has been
57 invalidated, for example when Watchman has been restarted or .hgignore rules
57 invalidated, for example when Watchman has been restarted or .hgignore rules
58 have been changed. Walking the repo in that case can result in competing for
58 have been changed. Walking the repo in that case can result in competing for
59 I/O with Watchman. For large repos it is recommended to set this value to
59 I/O with Watchman. For large repos it is recommended to set this value to
60 false. You may wish to set this to true if you have a very fast filesystem
60 false. You may wish to set this to true if you have a very fast filesystem
61 that can outpace the IPC overhead of getting the result data for the full repo
61 that can outpace the IPC overhead of getting the result data for the full repo
62 from Watchman. Defaults to false.
62 from Watchman. Defaults to false.
63
63
64 ::
64 ::
65
65
66 [fsmonitor]
66 [fsmonitor]
67 warn_when_unused = (boolean)
67 warn_when_unused = (boolean)
68
68
69 Whether to print a warning during certain operations when fsmonitor would be
69 Whether to print a warning during certain operations when fsmonitor would be
70 beneficial to performance but isn't enabled.
70 beneficial to performance but isn't enabled.
71
71
72 ::
72 ::
73
73
74 [fsmonitor]
74 [fsmonitor]
75 warn_update_file_count = (integer)
75 warn_update_file_count = (integer)
76 # or when mercurial is built with rust support
76 # or when mercurial is built with rust support
77 warn_update_file_count_rust = (integer)
77 warn_update_file_count_rust = (integer)
78
78
79 If ``warn_when_unused`` is set and fsmonitor isn't enabled, a warning will
79 If ``warn_when_unused`` is set and fsmonitor isn't enabled, a warning will
80 be printed during working directory updates if this many files will be
80 be printed during working directory updates if this many files will be
81 created.
81 created.
82 '''
82 '''
83
83
84 # Platforms Supported
84 # Platforms Supported
85 # ===================
85 # ===================
86 #
86 #
87 # **Linux:** *Stable*. Watchman and fsmonitor are both known to work reliably,
87 # **Linux:** *Stable*. Watchman and fsmonitor are both known to work reliably,
88 # even under severe loads.
88 # even under severe loads.
89 #
89 #
90 # **Mac OS X:** *Stable*. The Mercurial test suite passes with fsmonitor
90 # **Mac OS X:** *Stable*. The Mercurial test suite passes with fsmonitor
91 # turned on, on case-insensitive HFS+. There has been a reasonable amount of
91 # turned on, on case-insensitive HFS+. There has been a reasonable amount of
92 # user testing under normal loads.
92 # user testing under normal loads.
93 #
93 #
94 # **Solaris, BSD:** *Alpha*. watchman and fsmonitor are believed to work, but
94 # **Solaris, BSD:** *Alpha*. watchman and fsmonitor are believed to work, but
95 # very little testing has been done.
95 # very little testing has been done.
96 #
96 #
97 # **Windows:** *Alpha*. Not in a release version of watchman or fsmonitor yet.
97 # **Windows:** *Alpha*. Not in a release version of watchman or fsmonitor yet.
98 #
98 #
99 # Known Issues
99 # Known Issues
100 # ============
100 # ============
101 #
101 #
102 # * fsmonitor will disable itself if any of the following extensions are
102 # * fsmonitor will disable itself if any of the following extensions are
103 # enabled: largefiles, inotify, eol; or if the repository has subrepos.
103 # enabled: largefiles, inotify, eol; or if the repository has subrepos.
104 # * fsmonitor will produce incorrect results if nested repos that are not
104 # * fsmonitor will produce incorrect results if nested repos that are not
105 # subrepos exist. *Workaround*: add nested repo paths to your `.hgignore`.
105 # subrepos exist. *Workaround*: add nested repo paths to your `.hgignore`.
106 #
106 #
107 # The issues related to nested repos and subrepos are probably not fundamental
107 # The issues related to nested repos and subrepos are probably not fundamental
108 # ones. Patches to fix them are welcome.
108 # ones. Patches to fix them are welcome.
109
109
110
110
111 import codecs
111 import codecs
112 import os
112 import os
113 import stat
113 import stat
114 import sys
114 import sys
115 import tempfile
115 import tempfile
116 import weakref
116 import weakref
117
117
118 from mercurial.i18n import _
118 from mercurial.i18n import _
119 from mercurial.node import hex
119 from mercurial.node import hex
120
120
121 from mercurial.pycompat import open
121 from mercurial.pycompat import open
122 from mercurial import (
122 from mercurial import (
123 context,
123 context,
124 encoding,
124 encoding,
125 error,
125 error,
126 extensions,
126 extensions,
127 localrepo,
127 localrepo,
128 merge,
128 merge,
129 pathutil,
129 pathutil,
130 pycompat,
130 pycompat,
131 registrar,
131 registrar,
132 scmutil,
132 scmutil,
133 util,
133 util,
134 )
134 )
135
135
136 # no-check-code because we're accessing private information only public in pure
136 # no-check-code because we're accessing private information only public in pure
137 from mercurial.pure import parsers
137 from mercurial.pure import parsers
138 from mercurial import match as matchmod
138 from mercurial import match as matchmod
139 from mercurial.utils import (
139 from mercurial.utils import (
140 hashutil,
140 hashutil,
141 stringutil,
141 stringutil,
142 )
142 )
143
143
144 from . import (
144 from . import (
145 pywatchman,
145 pywatchman,
146 state,
146 state,
147 watchmanclient,
147 watchmanclient,
148 )
148 )
149
149
150 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
150 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
151 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
151 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
152 # be specifying the version(s) of Mercurial they are tested with, or
152 # be specifying the version(s) of Mercurial they are tested with, or
153 # leave the attribute unspecified.
153 # leave the attribute unspecified.
154 testedwith = b'ships-with-hg-core'
154 testedwith = b'ships-with-hg-core'
155
155
156 configtable = {}
156 configtable = {}
157 configitem = registrar.configitem(configtable)
157 configitem = registrar.configitem(configtable)
158
158
159 configitem(
159 configitem(
160 b'fsmonitor',
160 b'fsmonitor',
161 b'mode',
161 b'mode',
162 default=b'on',
162 default=b'on',
163 )
163 )
164 configitem(
164 configitem(
165 b'fsmonitor',
165 b'fsmonitor',
166 b'walk_on_invalidate',
166 b'walk_on_invalidate',
167 default=False,
167 default=False,
168 )
168 )
169 configitem(
169 configitem(
170 b'fsmonitor',
170 b'fsmonitor',
171 b'timeout',
171 b'timeout',
172 default=b'2',
172 default=b'2',
173 )
173 )
174 configitem(
174 configitem(
175 b'fsmonitor',
175 b'fsmonitor',
176 b'blacklistusers',
176 b'blacklistusers',
177 default=list,
177 default=list,
178 )
178 )
179 configitem(
179 configitem(
180 b'fsmonitor',
180 b'fsmonitor',
181 b'watchman_exe',
181 b'watchman_exe',
182 default=b'watchman',
182 default=b'watchman',
183 )
183 )
184 configitem(
184 configitem(
185 b'fsmonitor',
185 b'fsmonitor',
186 b'verbose',
186 b'verbose',
187 default=True,
187 default=True,
188 experimental=True,
188 experimental=True,
189 )
189 )
190 configitem(
190 configitem(
191 b'experimental',
191 b'experimental',
192 b'fsmonitor.transaction_notify',
192 b'fsmonitor.transaction_notify',
193 default=False,
193 default=False,
194 )
194 )
195
195
196 # This extension is incompatible with the following blacklisted extensions
196 # This extension is incompatible with the following blacklisted extensions
197 # and will disable itself when encountering one of these:
197 # and will disable itself when encountering one of these:
198 _blacklist = [b'largefiles', b'eol']
198 _blacklist = [b'largefiles', b'eol']
199
199
200
200
201 def debuginstall(ui, fm):
201 def debuginstall(ui, fm):
202 fm.write(
202 fm.write(
203 b"fsmonitor-watchman",
203 b"fsmonitor-watchman",
204 _(b"fsmonitor checking for watchman binary... (%s)\n"),
204 _(b"fsmonitor checking for watchman binary... (%s)\n"),
205 ui.configpath(b"fsmonitor", b"watchman_exe"),
205 ui.configpath(b"fsmonitor", b"watchman_exe"),
206 )
206 )
207 root = tempfile.mkdtemp()
207 root = tempfile.mkdtemp()
208 c = watchmanclient.client(ui, root)
208 c = watchmanclient.client(ui, root)
209 err = None
209 err = None
210 try:
210 try:
211 v = c.command(b"version")
211 v = c.command(b"version")
212 fm.write(
212 fm.write(
213 b"fsmonitor-watchman-version",
213 b"fsmonitor-watchman-version",
214 _(b" watchman binary version %s\n"),
214 _(b" watchman binary version %s\n"),
215 pycompat.bytestr(v["version"]),
215 pycompat.bytestr(v["version"]),
216 )
216 )
217 except watchmanclient.Unavailable as e:
217 except watchmanclient.Unavailable as e:
218 err = stringutil.forcebytestr(e)
218 err = stringutil.forcebytestr(e)
219 fm.condwrite(
219 fm.condwrite(
220 err,
220 err,
221 b"fsmonitor-watchman-error",
221 b"fsmonitor-watchman-error",
222 _(b" watchman binary missing or broken: %s\n"),
222 _(b" watchman binary missing or broken: %s\n"),
223 err,
223 err,
224 )
224 )
225 return 1 if err else 0
225 return 1 if err else 0
226
226
227
227
228 def _handleunavailable(ui, state, ex):
228 def _handleunavailable(ui, state, ex):
229 """Exception handler for Watchman interaction exceptions"""
229 """Exception handler for Watchman interaction exceptions"""
230 if isinstance(ex, watchmanclient.Unavailable):
230 if isinstance(ex, watchmanclient.Unavailable):
231 # experimental config: fsmonitor.verbose
231 # experimental config: fsmonitor.verbose
232 if ex.warn and ui.configbool(b'fsmonitor', b'verbose'):
232 if ex.warn and ui.configbool(b'fsmonitor', b'verbose'):
233 if b'illegal_fstypes' not in stringutil.forcebytestr(ex):
233 if b'illegal_fstypes' not in stringutil.forcebytestr(ex):
234 ui.warn(stringutil.forcebytestr(ex) + b'\n')
234 ui.warn(stringutil.forcebytestr(ex) + b'\n')
235 if ex.invalidate:
235 if ex.invalidate:
236 state.invalidate()
236 state.invalidate()
237 # experimental config: fsmonitor.verbose
237 # experimental config: fsmonitor.verbose
238 if ui.configbool(b'fsmonitor', b'verbose'):
238 if ui.configbool(b'fsmonitor', b'verbose'):
239 ui.log(
239 ui.log(
240 b'fsmonitor',
240 b'fsmonitor',
241 b'Watchman unavailable: %s\n',
241 b'Watchman unavailable: %s\n',
242 stringutil.forcebytestr(ex.msg),
242 stringutil.forcebytestr(ex.msg),
243 )
243 )
244 else:
244 else:
245 ui.log(
245 ui.log(
246 b'fsmonitor',
246 b'fsmonitor',
247 b'Watchman exception: %s\n',
247 b'Watchman exception: %s\n',
248 stringutil.forcebytestr(ex),
248 stringutil.forcebytestr(ex),
249 )
249 )
250
250
251
251
252 def _hashignore(ignore):
252 def _hashignore(ignore):
253 """Calculate hash for ignore patterns and filenames
253 """Calculate hash for ignore patterns and filenames
254
254
255 If this information changes between Mercurial invocations, we can't
255 If this information changes between Mercurial invocations, we can't
256 rely on Watchman information anymore and have to re-scan the working
256 rely on Watchman information anymore and have to re-scan the working
257 copy.
257 copy.
258
258
259 """
259 """
260 sha1 = hashutil.sha1()
260 sha1 = hashutil.sha1()
261 sha1.update(pycompat.byterepr(ignore))
261 sha1.update(pycompat.byterepr(ignore))
262 return pycompat.sysbytes(sha1.hexdigest())
262 return pycompat.sysbytes(sha1.hexdigest())
263
263
264
264
265 _watchmanencoding = pywatchman.encoding.get_local_encoding()
265 _watchmanencoding = pywatchman.encoding.get_local_encoding()
266 _fsencoding = sys.getfilesystemencoding() or sys.getdefaultencoding()
266 _fsencoding = sys.getfilesystemencoding() or sys.getdefaultencoding()
267 _fixencoding = codecs.lookup(_watchmanencoding) != codecs.lookup(_fsencoding)
267 _fixencoding = codecs.lookup(_watchmanencoding) != codecs.lookup(_fsencoding)
268
268
269
269
270 def _watchmantofsencoding(path):
270 def _watchmantofsencoding(path):
271 """Fix path to match watchman and local filesystem encoding
271 """Fix path to match watchman and local filesystem encoding
272
272
273 watchman's paths encoding can differ from filesystem encoding. For example,
273 watchman's paths encoding can differ from filesystem encoding. For example,
274 on Windows, it's always utf-8.
274 on Windows, it's always utf-8.
275 """
275 """
276 try:
276 try:
277 decoded = path.decode(_watchmanencoding)
277 decoded = path.decode(_watchmanencoding)
278 except UnicodeDecodeError as e:
278 except UnicodeDecodeError as e:
279 raise error.Abort(
279 raise error.Abort(
280 stringutil.forcebytestr(e), hint=b'watchman encoding error'
280 stringutil.forcebytestr(e), hint=b'watchman encoding error'
281 )
281 )
282
282
283 try:
283 try:
284 encoded = decoded.encode(_fsencoding, 'strict')
284 encoded = decoded.encode(_fsencoding, 'strict')
285 except UnicodeEncodeError as e:
285 except UnicodeEncodeError as e:
286 raise error.Abort(stringutil.forcebytestr(e))
286 raise error.Abort(stringutil.forcebytestr(e))
287
287
288 return encoded
288 return encoded
289
289
290
290
291 def overridewalk(orig, self, match, subrepos, unknown, ignored, full=True):
291 def overridewalk(orig, self, match, subrepos, unknown, ignored, full=True):
292 """Replacement for dirstate.walk, hooking into Watchman.
292 """Replacement for dirstate.walk, hooking into Watchman.
293
293
294 Whenever full is False, ignored is False, and the Watchman client is
294 Whenever full is False, ignored is False, and the Watchman client is
295 available, use Watchman combined with saved state to possibly return only a
295 available, use Watchman combined with saved state to possibly return only a
296 subset of files."""
296 subset of files."""
297
297
298 def bail(reason):
298 def bail(reason):
299 self._ui.debug(b'fsmonitor: fallback to core status, %s\n' % reason)
299 self._ui.debug(b'fsmonitor: fallback to core status, %s\n' % reason)
300 return orig(match, subrepos, unknown, ignored, full=True)
300 return orig(match, subrepos, unknown, ignored, full=True)
301
301
302 if full:
302 if full:
303 return bail(b'full rewalk requested')
303 return bail(b'full rewalk requested')
304 if ignored:
304 if ignored:
305 return bail(b'listing ignored files')
305 return bail(b'listing ignored files')
306 if not self._watchmanclient.available():
306 if not self._watchmanclient.available():
307 return bail(b'client unavailable')
307 return bail(b'client unavailable')
308 state = self._fsmonitorstate
308 state = self._fsmonitorstate
309 clock, ignorehash, notefiles = state.get()
309 clock, ignorehash, notefiles = state.get()
310 if not clock:
310 if not clock:
311 if state.walk_on_invalidate:
311 if state.walk_on_invalidate:
312 return bail(b'no clock')
312 return bail(b'no clock')
313 # Initial NULL clock value, see
313 # Initial NULL clock value, see
314 # https://facebook.github.io/watchman/docs/clockspec.html
314 # https://facebook.github.io/watchman/docs/clockspec.html
315 clock = b'c:0:0'
315 clock = b'c:0:0'
316 notefiles = []
316 notefiles = []
317
317
318 ignore = self._ignore
318 ignore = self._ignore
319 dirignore = self._dirignore
319 dirignore = self._dirignore
320 if unknown:
320 if unknown:
321 if _hashignore(ignore) != ignorehash and clock != b'c:0:0':
321 if _hashignore(ignore) != ignorehash and clock != b'c:0:0':
322 # ignore list changed -- can't rely on Watchman state any more
322 # ignore list changed -- can't rely on Watchman state any more
323 if state.walk_on_invalidate:
323 if state.walk_on_invalidate:
324 return bail(b'ignore rules changed')
324 return bail(b'ignore rules changed')
325 notefiles = []
325 notefiles = []
326 clock = b'c:0:0'
326 clock = b'c:0:0'
327 else:
327 else:
328 # always ignore
328 # always ignore
329 ignore = util.always
329 ignore = util.always
330 dirignore = util.always
330 dirignore = util.always
331
331
332 matchfn = match.matchfn
332 matchfn = match.matchfn
333 matchalways = match.always()
333 matchalways = match.always()
334 dmap = self._map
334 dmap = self._map
335 if util.safehasattr(dmap, b'_map'):
335 if util.safehasattr(dmap, b'_map'):
336 # for better performance, directly access the inner dirstate map if the
336 # for better performance, directly access the inner dirstate map if the
337 # standard dirstate implementation is in use.
337 # standard dirstate implementation is in use.
338 dmap = dmap._map
338 dmap = dmap._map
339
339
340 has_mtime = parsers.DIRSTATE_V2_HAS_MTIME
340 has_mtime = parsers.DIRSTATE_V2_HAS_MTIME
341 mtime_is_ambiguous = parsers.DIRSTATE_V2_MTIME_SECOND_AMBIGUOUS
341 mtime_is_ambiguous = parsers.DIRSTATE_V2_MTIME_SECOND_AMBIGUOUS
342 mask = has_mtime | mtime_is_ambiguous
342 mask = has_mtime | mtime_is_ambiguous
343
343
344 # All entries that may not be clean
344 # All entries that may not be clean
345 nonnormalset = {
345 nonnormalset = {
346 f
346 f
347 for f, e in self._map.items()
347 for f, e in self._map.items()
348 if not e.maybe_clean
348 if not e.maybe_clean
349 # same as "not has_time or has_ambiguous_time", but factored to only
349 # same as "not has_time or has_ambiguous_time", but factored to only
350 # need a single access to flags for performance.
350 # need a single access to flags for performance.
351 # `mask` removes all irrelevant bits, then we flip the `mtime` bit so
351 # `mask` removes all irrelevant bits, then we flip the `mtime` bit so
352 # its `true` value is NOT having a mtime, then check if either bit
352 # its `true` value is NOT having a mtime, then check if either bit
353 # is set.
353 # is set.
354 or bool((e.v2_data()[0] & mask) ^ has_mtime)
354 or bool((e.v2_data()[0] & mask) ^ has_mtime)
355 }
355 }
356
356
357 copymap = self._map.copymap
357 copymap = self._map.copymap
358 getkind = stat.S_IFMT
358 getkind = stat.S_IFMT
359 dirkind = stat.S_IFDIR
359 dirkind = stat.S_IFDIR
360 regkind = stat.S_IFREG
360 regkind = stat.S_IFREG
361 lnkkind = stat.S_IFLNK
361 lnkkind = stat.S_IFLNK
362 join = self._join
362 join = self._join
363 normcase = util.normcase
363 normcase = util.normcase
364 fresh_instance = False
364 fresh_instance = False
365
365
366 exact = skipstep3 = False
366 exact = skipstep3 = False
367 if match.isexact(): # match.exact
367 if match.isexact(): # match.exact
368 exact = True
368 exact = True
369 dirignore = util.always # skip step 2
369 dirignore = util.always # skip step 2
370 elif match.prefix(): # match.match, no patterns
370 elif match.prefix(): # match.match, no patterns
371 skipstep3 = True
371 skipstep3 = True
372
372
373 if not exact and self._checkcase:
373 if not exact and self._checkcase:
374 # note that even though we could receive directory entries, we're only
374 # note that even though we could receive directory entries, we're only
375 # interested in checking if a file with the same name exists. So only
375 # interested in checking if a file with the same name exists. So only
376 # normalize files if possible.
376 # normalize files if possible.
377 normalize = self._normalizefile
377 normalize = self._normalizefile
378 skipstep3 = False
378 skipstep3 = False
379 else:
379 else:
380 normalize = None
380 normalize = None
381
381
382 # step 1: find all explicit files
382 # step 1: find all explicit files
383 results, work, dirsnotfound = self._walkexplicit(match, subrepos)
383 results, work, dirsnotfound = self._walkexplicit(match, subrepos)
384
384
385 skipstep3 = skipstep3 and not (work or dirsnotfound)
385 skipstep3 = skipstep3 and not (work or dirsnotfound)
386 work = [d for d in work if not dirignore(d[0])]
386 work = [d for d in work if not dirignore(d[0])]
387
387
388 if not work and (exact or skipstep3):
388 if not work and (exact or skipstep3):
389 for s in subrepos:
389 for s in subrepos:
390 del results[s]
390 del results[s]
391 del results[b'.hg']
391 del results[b'.hg']
392 return results
392 return results
393
393
394 # step 2: query Watchman
394 # step 2: query Watchman
395 try:
395 try:
396 # Use the user-configured timeout for the query.
396 # Use the user-configured timeout for the query.
397 # Add a little slack over the top of the user query to allow for
397 # Add a little slack over the top of the user query to allow for
398 # overheads while transferring the data
398 # overheads while transferring the data
399 self._watchmanclient.settimeout(state.timeout + 0.1)
399 self._watchmanclient.settimeout(state.timeout + 0.1)
400 result = self._watchmanclient.command(
400 result = self._watchmanclient.command(
401 b'query',
401 b'query',
402 {
402 {
403 b'fields': [b'mode', b'mtime', b'size', b'exists', b'name'],
403 b'fields': [b'mode', b'mtime', b'size', b'exists', b'name'],
404 b'since': clock,
404 b'since': clock,
405 b'expression': [
405 b'expression': [
406 b'not',
406 b'not',
407 [
407 [
408 b'anyof',
408 b'anyof',
409 [b'dirname', b'.hg'],
409 [b'dirname', b'.hg'],
410 [b'name', b'.hg', b'wholename'],
410 [b'name', b'.hg', b'wholename'],
411 ],
411 ],
412 ],
412 ],
413 b'sync_timeout': int(state.timeout * 1000),
413 b'sync_timeout': int(state.timeout * 1000),
414 b'empty_on_fresh_instance': state.walk_on_invalidate,
414 b'empty_on_fresh_instance': state.walk_on_invalidate,
415 },
415 },
416 )
416 )
417 except Exception as ex:
417 except Exception as ex:
418 _handleunavailable(self._ui, state, ex)
418 _handleunavailable(self._ui, state, ex)
419 self._watchmanclient.clearconnection()
419 self._watchmanclient.clearconnection()
420 return bail(b'exception during run')
420 return bail(b'exception during run')
421 else:
421 else:
422 # We need to propagate the last observed clock up so that we
422 # We need to propagate the last observed clock up so that we
423 # can use it for our next query
423 # can use it for our next query
424 state.setlastclock(pycompat.sysbytes(result[b'clock']))
424 state.setlastclock(pycompat.sysbytes(result[b'clock']))
425 if result[b'is_fresh_instance']:
425 if result[b'is_fresh_instance']:
426 if state.walk_on_invalidate:
426 if state.walk_on_invalidate:
427 state.invalidate()
427 state.invalidate()
428 return bail(b'fresh instance')
428 return bail(b'fresh instance')
429 fresh_instance = True
429 fresh_instance = True
430 # Ignore any prior noteable files from the state info
430 # Ignore any prior noteable files from the state info
431 notefiles = []
431 notefiles = []
432
432
433 # for file paths which require normalization and we encounter a case
433 # for file paths which require normalization and we encounter a case
434 # collision, we store our own foldmap
434 # collision, we store our own foldmap
435 if normalize:
435 if normalize:
436 foldmap = {normcase(k): k for k in results}
436 foldmap = {normcase(k): k for k in results}
437
437
438 switch_slashes = pycompat.ossep == b'\\'
438 switch_slashes = pycompat.ossep == b'\\'
439 # The order of the results is, strictly speaking, undefined.
439 # The order of the results is, strictly speaking, undefined.
440 # For case changes on a case insensitive filesystem we may receive
440 # For case changes on a case insensitive filesystem we may receive
441 # two entries, one with exists=True and another with exists=False.
441 # two entries, one with exists=True and another with exists=False.
442 # The exists=True entries in the same response should be interpreted
442 # The exists=True entries in the same response should be interpreted
443 # as being happens-after the exists=False entries due to the way that
443 # as being happens-after the exists=False entries due to the way that
444 # Watchman tracks files. We use this property to reconcile deletes
444 # Watchman tracks files. We use this property to reconcile deletes
445 # for name case changes.
445 # for name case changes.
446 for entry in result[b'files']:
446 for entry in result[b'files']:
447 fname = entry[b'name']
447 fname = entry[b'name']
448
448
449 # Watchman always give us a str. Normalize to bytes on Python 3
449 # Watchman always give us a str. Normalize to bytes on Python 3
450 # using Watchman's encoding, if needed.
450 # using Watchman's encoding, if needed.
451 if not isinstance(fname, bytes):
451 if not isinstance(fname, bytes):
452 fname = fname.encode(_watchmanencoding)
452 fname = fname.encode(_watchmanencoding)
453
453
454 if _fixencoding:
454 if _fixencoding:
455 fname = _watchmantofsencoding(fname)
455 fname = _watchmantofsencoding(fname)
456
456
457 if switch_slashes:
457 if switch_slashes:
458 fname = fname.replace(b'\\', b'/')
458 fname = fname.replace(b'\\', b'/')
459 if normalize:
459 if normalize:
460 normed = normcase(fname)
460 normed = normcase(fname)
461 fname = normalize(fname, True, True)
461 fname = normalize(fname, True, True)
462 foldmap[normed] = fname
462 foldmap[normed] = fname
463 fmode = entry[b'mode']
463 fmode = entry[b'mode']
464 fexists = entry[b'exists']
464 fexists = entry[b'exists']
465 kind = getkind(fmode)
465 kind = getkind(fmode)
466
466
467 if b'/.hg/' in fname or fname.endswith(b'/.hg'):
467 if b'/.hg/' in fname or fname.endswith(b'/.hg'):
468 return bail(b'nested-repo-detected')
468 return bail(b'nested-repo-detected')
469
469
470 if not fexists:
470 if not fexists:
471 # if marked as deleted and we don't already have a change
471 # if marked as deleted and we don't already have a change
472 # record, mark it as deleted. If we already have an entry
472 # record, mark it as deleted. If we already have an entry
473 # for fname then it was either part of walkexplicit or was
473 # for fname then it was either part of walkexplicit or was
474 # an earlier result that was a case change
474 # an earlier result that was a case change
475 if (
475 if (
476 fname not in results
476 fname not in results
477 and fname in dmap
477 and fname in dmap
478 and (matchalways or matchfn(fname))
478 and (matchalways or matchfn(fname))
479 ):
479 ):
480 results[fname] = None
480 results[fname] = None
481 elif kind == dirkind:
481 elif kind == dirkind:
482 if fname in dmap and (matchalways or matchfn(fname)):
482 if fname in dmap and (matchalways or matchfn(fname)):
483 results[fname] = None
483 results[fname] = None
484 elif kind == regkind or kind == lnkkind:
484 elif kind == regkind or kind == lnkkind:
485 if fname in dmap:
485 if fname in dmap:
486 if matchalways or matchfn(fname):
486 if matchalways or matchfn(fname):
487 results[fname] = entry
487 results[fname] = entry
488 elif (matchalways or matchfn(fname)) and not ignore(fname):
488 elif (matchalways or matchfn(fname)) and not ignore(fname):
489 results[fname] = entry
489 results[fname] = entry
490 elif fname in dmap and (matchalways or matchfn(fname)):
490 elif fname in dmap and (matchalways or matchfn(fname)):
491 results[fname] = None
491 results[fname] = None
492
492
493 # step 3: query notable files we don't already know about
493 # step 3: query notable files we don't already know about
494 # XXX try not to iterate over the entire dmap
494 # XXX try not to iterate over the entire dmap
495 if normalize:
495 if normalize:
496 # any notable files that have changed case will already be handled
496 # any notable files that have changed case will already be handled
497 # above, so just check membership in the foldmap
497 # above, so just check membership in the foldmap
498 notefiles = {
498 notefiles = {
499 normalize(f, True, True)
499 normalize(f, True, True)
500 for f in notefiles
500 for f in notefiles
501 if normcase(f) not in foldmap
501 if normcase(f) not in foldmap
502 }
502 }
503 visit = {
503 visit = {
504 f
504 f
505 for f in notefiles
505 for f in notefiles
506 if (f not in results and matchfn(f) and (f in dmap or not ignore(f)))
506 if (f not in results and matchfn(f) and (f in dmap or not ignore(f)))
507 }
507 }
508
508
509 if not fresh_instance:
509 if not fresh_instance:
510 if matchalways:
510 if matchalways:
511 visit.update(f for f in nonnormalset if f not in results)
511 visit.update(f for f in nonnormalset if f not in results)
512 visit.update(f for f in copymap if f not in results)
512 visit.update(f for f in copymap if f not in results)
513 else:
513 else:
514 visit.update(
514 visit.update(
515 f for f in nonnormalset if f not in results and matchfn(f)
515 f for f in nonnormalset if f not in results and matchfn(f)
516 )
516 )
517 visit.update(f for f in copymap if f not in results and matchfn(f))
517 visit.update(f for f in copymap if f not in results and matchfn(f))
518 else:
518 else:
519 if matchalways:
519 if matchalways:
520 visit.update(f for f, st in dmap.items() if f not in results)
520 visit.update(f for f, st in dmap.items() if f not in results)
521 visit.update(f for f in copymap if f not in results)
521 visit.update(f for f in copymap if f not in results)
522 else:
522 else:
523 visit.update(
523 visit.update(
524 f for f, st in dmap.items() if f not in results and matchfn(f)
524 f for f, st in dmap.items() if f not in results and matchfn(f)
525 )
525 )
526 visit.update(f for f in copymap if f not in results and matchfn(f))
526 visit.update(f for f in copymap if f not in results and matchfn(f))
527
527
528 audit = pathutil.pathauditor(self._root, cached=True).check
528 audit = pathutil.pathauditor(self._root, cached=True).check
529 auditpass = [f for f in visit if audit(f)]
529 auditpass = [f for f in visit if audit(f)]
530 auditpass.sort()
530 auditpass.sort()
531 auditfail = visit.difference(auditpass)
531 auditfail = visit.difference(auditpass)
532 for f in auditfail:
532 for f in auditfail:
533 results[f] = None
533 results[f] = None
534
534
535 nf = iter(auditpass)
535 nf = iter(auditpass)
536 for st in util.statfiles([join(f) for f in auditpass]):
536 for st in util.statfiles([join(f) for f in auditpass]):
537 f = next(nf)
537 f = next(nf)
538 if st or f in dmap:
538 if st or f in dmap:
539 results[f] = st
539 results[f] = st
540
540
541 for s in subrepos:
541 for s in subrepos:
542 del results[s]
542 del results[s]
543 del results[b'.hg']
543 del results[b'.hg']
544 return results
544 return results
545
545
546
546
547 def overridestatus(
547 def overridestatus(
548 orig,
548 orig,
549 self,
549 self,
550 node1=b'.',
550 node1=b'.',
551 node2=None,
551 node2=None,
552 match=None,
552 match=None,
553 ignored=False,
553 ignored=False,
554 clean=False,
554 clean=False,
555 unknown=False,
555 unknown=False,
556 listsubrepos=False,
556 listsubrepos=False,
557 ):
557 ):
558 listignored = ignored
558 listignored = ignored
559 listclean = clean
559 listclean = clean
560 listunknown = unknown
560 listunknown = unknown
561
561
562 def _cmpsets(l1, l2):
562 def _cmpsets(l1, l2):
563 try:
563 try:
564 if b'FSMONITOR_LOG_FILE' in encoding.environ:
564 if b'FSMONITOR_LOG_FILE' in encoding.environ:
565 fn = encoding.environ[b'FSMONITOR_LOG_FILE']
565 fn = encoding.environ[b'FSMONITOR_LOG_FILE']
566 f = open(fn, b'wb')
566 f = open(fn, b'wb')
567 else:
567 else:
568 fn = b'fsmonitorfail.log'
568 fn = b'fsmonitorfail.log'
569 f = self.vfs.open(fn, b'wb')
569 f = self.vfs.open(fn, b'wb')
570 except (IOError, OSError):
570 except (IOError, OSError):
571 self.ui.warn(_(b'warning: unable to write to %s\n') % fn)
571 self.ui.warn(_(b'warning: unable to write to %s\n') % fn)
572 return
572 return
573
573
574 try:
574 try:
575 for i, (s1, s2) in enumerate(zip(l1, l2)):
575 for i, (s1, s2) in enumerate(zip(l1, l2)):
576 if set(s1) != set(s2):
576 if set(s1) != set(s2):
577 f.write(b'sets at position %d are unequal\n' % i)
577 f.write(b'sets at position %d are unequal\n' % i)
578 f.write(b'watchman returned: %r\n' % s1)
578 f.write(b'watchman returned: %r\n' % s1)
579 f.write(b'stat returned: %r\n' % s2)
579 f.write(b'stat returned: %r\n' % s2)
580 finally:
580 finally:
581 f.close()
581 f.close()
582
582
583 if isinstance(node1, context.changectx):
583 if isinstance(node1, context.changectx):
584 ctx1 = node1
584 ctx1 = node1
585 else:
585 else:
586 ctx1 = self[node1]
586 ctx1 = self[node1]
587 if isinstance(node2, context.changectx):
587 if isinstance(node2, context.changectx):
588 ctx2 = node2
588 ctx2 = node2
589 else:
589 else:
590 ctx2 = self[node2]
590 ctx2 = self[node2]
591
591
592 working = ctx2.rev() is None
592 working = ctx2.rev() is None
593 parentworking = working and ctx1 == self[b'.']
593 parentworking = working and ctx1 == self[b'.']
594 match = match or matchmod.always()
594 match = match or matchmod.always()
595
595
596 # Maybe we can use this opportunity to update Watchman's state.
596 # Maybe we can use this opportunity to update Watchman's state.
597 # Mercurial uses workingcommitctx and/or memctx to represent the part of
597 # Mercurial uses workingcommitctx and/or memctx to represent the part of
598 # the workingctx that is to be committed. So don't update the state in
598 # the workingctx that is to be committed. So don't update the state in
599 # that case.
599 # that case.
600 # HG_PENDING is set in the environment when the dirstate is being updated
600 # HG_PENDING is set in the environment when the dirstate is being updated
601 # in the middle of a transaction; we must not update our state in that
601 # in the middle of a transaction; we must not update our state in that
602 # case, or we risk forgetting about changes in the working copy.
602 # case, or we risk forgetting about changes in the working copy.
603 updatestate = (
603 updatestate = (
604 parentworking
604 parentworking
605 and match.always()
605 and match.always()
606 and not isinstance(ctx2, (context.workingcommitctx, context.memctx))
606 and not isinstance(ctx2, (context.workingcommitctx, context.memctx))
607 and b'HG_PENDING' not in encoding.environ
607 and b'HG_PENDING' not in encoding.environ
608 )
608 )
609
609
610 try:
610 try:
611 if self._fsmonitorstate.walk_on_invalidate:
611 if self._fsmonitorstate.walk_on_invalidate:
612 # Use a short timeout to query the current clock. If that
612 # Use a short timeout to query the current clock. If that
613 # takes too long then we assume that the service will be slow
613 # takes too long then we assume that the service will be slow
614 # to answer our query.
614 # to answer our query.
615 # walk_on_invalidate indicates that we prefer to walk the
615 # walk_on_invalidate indicates that we prefer to walk the
616 # tree ourselves because we can ignore portions that Watchman
616 # tree ourselves because we can ignore portions that Watchman
617 # cannot and we tend to be faster in the warmer buffer cache
617 # cannot and we tend to be faster in the warmer buffer cache
618 # cases.
618 # cases.
619 self._watchmanclient.settimeout(0.1)
619 self._watchmanclient.settimeout(0.1)
620 else:
620 else:
621 # Give Watchman more time to potentially complete its walk
621 # Give Watchman more time to potentially complete its walk
622 # and return the initial clock. In this mode we assume that
622 # and return the initial clock. In this mode we assume that
623 # the filesystem will be slower than parsing a potentially
623 # the filesystem will be slower than parsing a potentially
624 # very large Watchman result set.
624 # very large Watchman result set.
625 self._watchmanclient.settimeout(self._fsmonitorstate.timeout + 0.1)
625 self._watchmanclient.settimeout(self._fsmonitorstate.timeout + 0.1)
626 startclock = self._watchmanclient.getcurrentclock()
626 startclock = self._watchmanclient.getcurrentclock()
627 except Exception as ex:
627 except Exception as ex:
628 self._watchmanclient.clearconnection()
628 self._watchmanclient.clearconnection()
629 _handleunavailable(self.ui, self._fsmonitorstate, ex)
629 _handleunavailable(self.ui, self._fsmonitorstate, ex)
630 # boo, Watchman failed. bail
630 # boo, Watchman failed. bail
631 return orig(
631 return orig(
632 node1,
632 node1,
633 node2,
633 node2,
634 match,
634 match,
635 listignored,
635 listignored,
636 listclean,
636 listclean,
637 listunknown,
637 listunknown,
638 listsubrepos,
638 listsubrepos,
639 )
639 )
640
640
641 if updatestate:
641 if updatestate:
642 # We need info about unknown files. This may make things slower the
642 # We need info about unknown files. This may make things slower the
643 # first time, but whatever.
643 # first time, but whatever.
644 stateunknown = True
644 stateunknown = True
645 else:
645 else:
646 stateunknown = listunknown
646 stateunknown = listunknown
647
647
648 if updatestate:
648 if updatestate:
649 ps = poststatus(startclock)
649 ps = poststatus(startclock)
650 self.addpostdsstatus(ps)
650 self.addpostdsstatus(ps)
651
651
652 r = orig(
652 r = orig(
653 node1, node2, match, listignored, listclean, stateunknown, listsubrepos
653 node1, node2, match, listignored, listclean, stateunknown, listsubrepos
654 )
654 )
655 modified, added, removed, deleted, unknown, ignored, clean = r
655 modified, added, removed, deleted, unknown, ignored, clean = r
656
656
657 if not listunknown:
657 if not listunknown:
658 unknown = []
658 unknown = []
659
659
660 # don't do paranoid checks if we're not going to query Watchman anyway
660 # don't do paranoid checks if we're not going to query Watchman anyway
661 full = listclean or match.traversedir is not None
661 full = listclean or match.traversedir is not None
662 if self._fsmonitorstate.mode == b'paranoid' and not full:
662 if self._fsmonitorstate.mode == b'paranoid' and not full:
663 # run status again and fall back to the old walk this time
663 # run status again and fall back to the old walk this time
664 self.dirstate._fsmonitordisable = True
664 self.dirstate._fsmonitordisable = True
665
665
666 # shut the UI up
666 # shut the UI up
667 quiet = self.ui.quiet
667 quiet = self.ui.quiet
668 self.ui.quiet = True
668 self.ui.quiet = True
669 fout, ferr = self.ui.fout, self.ui.ferr
669 fout, ferr = self.ui.fout, self.ui.ferr
670 self.ui.fout = self.ui.ferr = open(os.devnull, b'wb')
670 self.ui.fout = self.ui.ferr = open(os.devnull, b'wb')
671
671
672 try:
672 try:
673 rv2 = orig(
673 rv2 = orig(
674 node1,
674 node1,
675 node2,
675 node2,
676 match,
676 match,
677 listignored,
677 listignored,
678 listclean,
678 listclean,
679 listunknown,
679 listunknown,
680 listsubrepos,
680 listsubrepos,
681 )
681 )
682 finally:
682 finally:
683 self.dirstate._fsmonitordisable = False
683 self.dirstate._fsmonitordisable = False
684 self.ui.quiet = quiet
684 self.ui.quiet = quiet
685 self.ui.fout, self.ui.ferr = fout, ferr
685 self.ui.fout, self.ui.ferr = fout, ferr
686
686
687 # clean isn't tested since it's set to True above
687 # clean isn't tested since it's set to True above
688 with self.wlock():
688 with self.wlock():
689 _cmpsets(
689 _cmpsets(
690 [modified, added, removed, deleted, unknown, ignored, clean],
690 [modified, added, removed, deleted, unknown, ignored, clean],
691 rv2,
691 rv2,
692 )
692 )
693 modified, added, removed, deleted, unknown, ignored, clean = rv2
693 modified, added, removed, deleted, unknown, ignored, clean = rv2
694
694
695 return scmutil.status(
695 return scmutil.status(
696 modified, added, removed, deleted, unknown, ignored, clean
696 modified, added, removed, deleted, unknown, ignored, clean
697 )
697 )
698
698
699
699
700 class poststatus:
700 class poststatus:
701 def __init__(self, startclock):
701 def __init__(self, startclock):
702 self._startclock = pycompat.sysbytes(startclock)
702 self._startclock = pycompat.sysbytes(startclock)
703
703
704 def __call__(self, wctx, status):
704 def __call__(self, wctx, status):
705 clock = wctx.repo()._fsmonitorstate.getlastclock() or self._startclock
705 clock = wctx.repo()._fsmonitorstate.getlastclock() or self._startclock
706 hashignore = _hashignore(wctx.repo().dirstate._ignore)
706 hashignore = _hashignore(wctx.repo().dirstate._ignore)
707 notefiles = (
707 notefiles = (
708 status.modified
708 status.modified
709 + status.added
709 + status.added
710 + status.removed
710 + status.removed
711 + status.deleted
711 + status.deleted
712 + status.unknown
712 + status.unknown
713 )
713 )
714 wctx.repo()._fsmonitorstate.set(clock, hashignore, notefiles)
714 wctx.repo()._fsmonitorstate.set(clock, hashignore, notefiles)
715
715
716
716
717 def makedirstate(repo, dirstate):
717 def makedirstate(repo, dirstate):
718 class fsmonitordirstate(dirstate.__class__):
718 class fsmonitordirstate(dirstate.__class__):
719 def _fsmonitorinit(self, repo):
719 def _fsmonitorinit(self, repo):
720 # _fsmonitordisable is used in paranoid mode
720 # _fsmonitordisable is used in paranoid mode
721 self._fsmonitordisable = False
721 self._fsmonitordisable = False
722 self._fsmonitorstate = repo._fsmonitorstate
722 self._fsmonitorstate = repo._fsmonitorstate
723 self._watchmanclient = repo._watchmanclient
723 self._watchmanclient = repo._watchmanclient
724 self._repo = weakref.proxy(repo)
724 self._repo = weakref.proxy(repo)
725
725
726 def walk(self, *args, **kwargs):
726 def walk(self, *args, **kwargs):
727 orig = super(fsmonitordirstate, self).walk
727 orig = super(fsmonitordirstate, self).walk
728 if self._fsmonitordisable:
728 if self._fsmonitordisable:
729 return orig(*args, **kwargs)
729 return orig(*args, **kwargs)
730 return overridewalk(orig, self, *args, **kwargs)
730 return overridewalk(orig, self, *args, **kwargs)
731
731
732 def rebuild(self, *args, **kwargs):
732 def rebuild(self, *args, **kwargs):
733 self._fsmonitorstate.invalidate()
733 self._fsmonitorstate.invalidate()
734 return super(fsmonitordirstate, self).rebuild(*args, **kwargs)
734 return super(fsmonitordirstate, self).rebuild(*args, **kwargs)
735
735
736 def invalidate(self, *args, **kwargs):
736 def invalidate(self, *args, **kwargs):
737 self._fsmonitorstate.invalidate()
737 self._fsmonitorstate.invalidate()
738 return super(fsmonitordirstate, self).invalidate(*args, **kwargs)
738 return super(fsmonitordirstate, self).invalidate(*args, **kwargs)
739
739
740 dirstate.__class__ = fsmonitordirstate
740 dirstate.__class__ = fsmonitordirstate
741 dirstate._fsmonitorinit(repo)
741 dirstate._fsmonitorinit(repo)
742
742
743
743
744 def wrapdirstate(orig, self):
744 def wrapdirstate(orig, self):
745 ds = orig(self)
745 ds = orig(self)
746 # only override the dirstate when Watchman is available for the repo
746 # only override the dirstate when Watchman is available for the repo
747 if util.safehasattr(self, b'_fsmonitorstate'):
747 if util.safehasattr(self, b'_fsmonitorstate'):
748 makedirstate(self, ds)
748 makedirstate(self, ds)
749 return ds
749 return ds
750
750
751
751
752 def extsetup(ui):
752 def extsetup(ui):
753 extensions.wrapfilecache(
753 extensions.wrapfilecache(
754 localrepo.localrepository, b'dirstate', wrapdirstate
754 localrepo.localrepository, b'dirstate', wrapdirstate
755 )
755 )
756 if pycompat.isdarwin:
756 if pycompat.isdarwin:
757 # An assist for avoiding the dangling-symlink fsevents bug
757 # An assist for avoiding the dangling-symlink fsevents bug
758 extensions.wrapfunction(os, b'symlink', wrapsymlink)
758 extensions.wrapfunction(os, 'symlink', wrapsymlink)
759
759
760 extensions.wrapfunction(merge, b'_update', wrapupdate)
760 extensions.wrapfunction(merge, '_update', wrapupdate)
761
761
762
762
763 def wrapsymlink(orig, source, link_name):
763 def wrapsymlink(orig, source, link_name):
764 """if we create a dangling symlink, also touch the parent dir
764 """if we create a dangling symlink, also touch the parent dir
765 to encourage fsevents notifications to work more correctly"""
765 to encourage fsevents notifications to work more correctly"""
766 try:
766 try:
767 return orig(source, link_name)
767 return orig(source, link_name)
768 finally:
768 finally:
769 try:
769 try:
770 os.utime(os.path.dirname(link_name), None)
770 os.utime(os.path.dirname(link_name), None)
771 except OSError:
771 except OSError:
772 pass
772 pass
773
773
774
774
775 class state_update:
775 class state_update:
776 """This context manager is responsible for dispatching the state-enter
776 """This context manager is responsible for dispatching the state-enter
777 and state-leave signals to the watchman service. The enter and leave
777 and state-leave signals to the watchman service. The enter and leave
778 methods can be invoked manually (for scenarios where context manager
778 methods can be invoked manually (for scenarios where context manager
779 semantics are not possible). If parameters oldnode and newnode are None,
779 semantics are not possible). If parameters oldnode and newnode are None,
780 they will be populated based on current working copy in enter and
780 they will be populated based on current working copy in enter and
781 leave, respectively. Similarly, if the distance is none, it will be
781 leave, respectively. Similarly, if the distance is none, it will be
782 calculated based on the oldnode and newnode in the leave method."""
782 calculated based on the oldnode and newnode in the leave method."""
783
783
784 def __init__(
784 def __init__(
785 self,
785 self,
786 repo,
786 repo,
787 name,
787 name,
788 oldnode=None,
788 oldnode=None,
789 newnode=None,
789 newnode=None,
790 distance=None,
790 distance=None,
791 partial=False,
791 partial=False,
792 ):
792 ):
793 self.repo = repo.unfiltered()
793 self.repo = repo.unfiltered()
794 self.name = name
794 self.name = name
795 self.oldnode = oldnode
795 self.oldnode = oldnode
796 self.newnode = newnode
796 self.newnode = newnode
797 self.distance = distance
797 self.distance = distance
798 self.partial = partial
798 self.partial = partial
799 self._lock = None
799 self._lock = None
800 self.need_leave = False
800 self.need_leave = False
801
801
802 def __enter__(self):
802 def __enter__(self):
803 self.enter()
803 self.enter()
804
804
805 def enter(self):
805 def enter(self):
806 # Make sure we have a wlock prior to sending notifications to watchman.
806 # Make sure we have a wlock prior to sending notifications to watchman.
807 # We don't want to race with other actors. In the update case,
807 # We don't want to race with other actors. In the update case,
808 # merge.update is going to take the wlock almost immediately. We are
808 # merge.update is going to take the wlock almost immediately. We are
809 # effectively extending the lock around several short sanity checks.
809 # effectively extending the lock around several short sanity checks.
810 if self.oldnode is None:
810 if self.oldnode is None:
811 self.oldnode = self.repo[b'.'].node()
811 self.oldnode = self.repo[b'.'].node()
812
812
813 if self.repo.currentwlock() is None:
813 if self.repo.currentwlock() is None:
814 if util.safehasattr(self.repo, b'wlocknostateupdate'):
814 if util.safehasattr(self.repo, b'wlocknostateupdate'):
815 self._lock = self.repo.wlocknostateupdate()
815 self._lock = self.repo.wlocknostateupdate()
816 else:
816 else:
817 self._lock = self.repo.wlock()
817 self._lock = self.repo.wlock()
818 self.need_leave = self._state(b'state-enter', hex(self.oldnode))
818 self.need_leave = self._state(b'state-enter', hex(self.oldnode))
819 return self
819 return self
820
820
821 def __exit__(self, type_, value, tb):
821 def __exit__(self, type_, value, tb):
822 abort = True if type_ else False
822 abort = True if type_ else False
823 self.exit(abort=abort)
823 self.exit(abort=abort)
824
824
825 def exit(self, abort=False):
825 def exit(self, abort=False):
826 try:
826 try:
827 if self.need_leave:
827 if self.need_leave:
828 status = b'failed' if abort else b'ok'
828 status = b'failed' if abort else b'ok'
829 if self.newnode is None:
829 if self.newnode is None:
830 self.newnode = self.repo[b'.'].node()
830 self.newnode = self.repo[b'.'].node()
831 if self.distance is None:
831 if self.distance is None:
832 self.distance = calcdistance(
832 self.distance = calcdistance(
833 self.repo, self.oldnode, self.newnode
833 self.repo, self.oldnode, self.newnode
834 )
834 )
835 self._state(b'state-leave', hex(self.newnode), status=status)
835 self._state(b'state-leave', hex(self.newnode), status=status)
836 finally:
836 finally:
837 self.need_leave = False
837 self.need_leave = False
838 if self._lock:
838 if self._lock:
839 self._lock.release()
839 self._lock.release()
840
840
841 def _state(self, cmd, commithash, status=b'ok'):
841 def _state(self, cmd, commithash, status=b'ok'):
842 if not util.safehasattr(self.repo, b'_watchmanclient'):
842 if not util.safehasattr(self.repo, b'_watchmanclient'):
843 return False
843 return False
844 try:
844 try:
845 self.repo._watchmanclient.command(
845 self.repo._watchmanclient.command(
846 cmd,
846 cmd,
847 {
847 {
848 b'name': self.name,
848 b'name': self.name,
849 b'metadata': {
849 b'metadata': {
850 # the target revision
850 # the target revision
851 b'rev': commithash,
851 b'rev': commithash,
852 # approximate number of commits between current and target
852 # approximate number of commits between current and target
853 b'distance': self.distance if self.distance else 0,
853 b'distance': self.distance if self.distance else 0,
854 # success/failure (only really meaningful for state-leave)
854 # success/failure (only really meaningful for state-leave)
855 b'status': status,
855 b'status': status,
856 # whether the working copy parent is changing
856 # whether the working copy parent is changing
857 b'partial': self.partial,
857 b'partial': self.partial,
858 },
858 },
859 },
859 },
860 )
860 )
861 return True
861 return True
862 except Exception as e:
862 except Exception as e:
863 # Swallow any errors; fire and forget
863 # Swallow any errors; fire and forget
864 self.repo.ui.log(
864 self.repo.ui.log(
865 b'watchman', b'Exception %s while running %s\n', e, cmd
865 b'watchman', b'Exception %s while running %s\n', e, cmd
866 )
866 )
867 return False
867 return False
868
868
869
869
870 # Estimate the distance between two nodes
870 # Estimate the distance between two nodes
871 def calcdistance(repo, oldnode, newnode):
871 def calcdistance(repo, oldnode, newnode):
872 anc = repo.changelog.ancestor(oldnode, newnode)
872 anc = repo.changelog.ancestor(oldnode, newnode)
873 ancrev = repo[anc].rev()
873 ancrev = repo[anc].rev()
874 distance = abs(repo[oldnode].rev() - ancrev) + abs(
874 distance = abs(repo[oldnode].rev() - ancrev) + abs(
875 repo[newnode].rev() - ancrev
875 repo[newnode].rev() - ancrev
876 )
876 )
877 return distance
877 return distance
878
878
879
879
880 # Bracket working copy updates with calls to the watchman state-enter
880 # Bracket working copy updates with calls to the watchman state-enter
881 # and state-leave commands. This allows clients to perform more intelligent
881 # and state-leave commands. This allows clients to perform more intelligent
882 # settling during bulk file change scenarios
882 # settling during bulk file change scenarios
883 # https://facebook.github.io/watchman/docs/cmd/subscribe.html#advanced-settling
883 # https://facebook.github.io/watchman/docs/cmd/subscribe.html#advanced-settling
884 def wrapupdate(
884 def wrapupdate(
885 orig,
885 orig,
886 repo,
886 repo,
887 node,
887 node,
888 branchmerge,
888 branchmerge,
889 force,
889 force,
890 ancestor=None,
890 ancestor=None,
891 mergeancestor=False,
891 mergeancestor=False,
892 labels=None,
892 labels=None,
893 matcher=None,
893 matcher=None,
894 **kwargs
894 **kwargs
895 ):
895 ):
896
896
897 distance = 0
897 distance = 0
898 partial = True
898 partial = True
899 oldnode = repo[b'.'].node()
899 oldnode = repo[b'.'].node()
900 newnode = repo[node].node()
900 newnode = repo[node].node()
901 if matcher is None or matcher.always():
901 if matcher is None or matcher.always():
902 partial = False
902 partial = False
903 distance = calcdistance(repo.unfiltered(), oldnode, newnode)
903 distance = calcdistance(repo.unfiltered(), oldnode, newnode)
904
904
905 with state_update(
905 with state_update(
906 repo,
906 repo,
907 name=b"hg.update",
907 name=b"hg.update",
908 oldnode=oldnode,
908 oldnode=oldnode,
909 newnode=newnode,
909 newnode=newnode,
910 distance=distance,
910 distance=distance,
911 partial=partial,
911 partial=partial,
912 ):
912 ):
913 return orig(
913 return orig(
914 repo,
914 repo,
915 node,
915 node,
916 branchmerge,
916 branchmerge,
917 force,
917 force,
918 ancestor,
918 ancestor,
919 mergeancestor,
919 mergeancestor,
920 labels,
920 labels,
921 matcher,
921 matcher,
922 **kwargs
922 **kwargs
923 )
923 )
924
924
925
925
926 def repo_has_depth_one_nested_repo(repo):
926 def repo_has_depth_one_nested_repo(repo):
927 for f in repo.wvfs.listdir():
927 for f in repo.wvfs.listdir():
928 if os.path.isdir(os.path.join(repo.root, f, b'.hg')):
928 if os.path.isdir(os.path.join(repo.root, f, b'.hg')):
929 msg = b'fsmonitor: sub-repository %r detected, fsmonitor disabled\n'
929 msg = b'fsmonitor: sub-repository %r detected, fsmonitor disabled\n'
930 repo.ui.debug(msg % f)
930 repo.ui.debug(msg % f)
931 return True
931 return True
932 return False
932 return False
933
933
934
934
935 def reposetup(ui, repo):
935 def reposetup(ui, repo):
936 # We don't work with largefiles or inotify
936 # We don't work with largefiles or inotify
937 exts = extensions.enabled()
937 exts = extensions.enabled()
938 for ext in _blacklist:
938 for ext in _blacklist:
939 if ext in exts:
939 if ext in exts:
940 ui.warn(
940 ui.warn(
941 _(
941 _(
942 b'The fsmonitor extension is incompatible with the %s '
942 b'The fsmonitor extension is incompatible with the %s '
943 b'extension and has been disabled.\n'
943 b'extension and has been disabled.\n'
944 )
944 )
945 % ext
945 % ext
946 )
946 )
947 return
947 return
948
948
949 if repo.local():
949 if repo.local():
950 # We don't work with subrepos either.
950 # We don't work with subrepos either.
951 #
951 #
952 # if repo[None].substate can cause a dirstate parse, which is too
952 # if repo[None].substate can cause a dirstate parse, which is too
953 # slow. Instead, look for a file called hgsubstate,
953 # slow. Instead, look for a file called hgsubstate,
954 if repo.wvfs.exists(b'.hgsubstate') or repo.wvfs.exists(b'.hgsub'):
954 if repo.wvfs.exists(b'.hgsubstate') or repo.wvfs.exists(b'.hgsub'):
955 return
955 return
956
956
957 if repo_has_depth_one_nested_repo(repo):
957 if repo_has_depth_one_nested_repo(repo):
958 return
958 return
959
959
960 fsmonitorstate = state.state(repo)
960 fsmonitorstate = state.state(repo)
961 if fsmonitorstate.mode == b'off':
961 if fsmonitorstate.mode == b'off':
962 return
962 return
963
963
964 try:
964 try:
965 client = watchmanclient.client(repo.ui, repo.root)
965 client = watchmanclient.client(repo.ui, repo.root)
966 except Exception as ex:
966 except Exception as ex:
967 _handleunavailable(ui, fsmonitorstate, ex)
967 _handleunavailable(ui, fsmonitorstate, ex)
968 return
968 return
969
969
970 repo._fsmonitorstate = fsmonitorstate
970 repo._fsmonitorstate = fsmonitorstate
971 repo._watchmanclient = client
971 repo._watchmanclient = client
972
972
973 dirstate, cached = localrepo.isfilecached(repo, b'dirstate')
973 dirstate, cached = localrepo.isfilecached(repo, b'dirstate')
974 if cached:
974 if cached:
975 # at this point since fsmonitorstate wasn't present,
975 # at this point since fsmonitorstate wasn't present,
976 # repo.dirstate is not a fsmonitordirstate
976 # repo.dirstate is not a fsmonitordirstate
977 makedirstate(repo, dirstate)
977 makedirstate(repo, dirstate)
978
978
979 class fsmonitorrepo(repo.__class__):
979 class fsmonitorrepo(repo.__class__):
980 def status(self, *args, **kwargs):
980 def status(self, *args, **kwargs):
981 orig = super(fsmonitorrepo, self).status
981 orig = super(fsmonitorrepo, self).status
982 return overridestatus(orig, self, *args, **kwargs)
982 return overridestatus(orig, self, *args, **kwargs)
983
983
984 def wlocknostateupdate(self, *args, **kwargs):
984 def wlocknostateupdate(self, *args, **kwargs):
985 return super(fsmonitorrepo, self).wlock(*args, **kwargs)
985 return super(fsmonitorrepo, self).wlock(*args, **kwargs)
986
986
987 def wlock(self, *args, **kwargs):
987 def wlock(self, *args, **kwargs):
988 l = super(fsmonitorrepo, self).wlock(*args, **kwargs)
988 l = super(fsmonitorrepo, self).wlock(*args, **kwargs)
989 if not ui.configbool(
989 if not ui.configbool(
990 b"experimental", b"fsmonitor.transaction_notify"
990 b"experimental", b"fsmonitor.transaction_notify"
991 ):
991 ):
992 return l
992 return l
993 if l.held != 1:
993 if l.held != 1:
994 return l
994 return l
995 origrelease = l.releasefn
995 origrelease = l.releasefn
996
996
997 def staterelease():
997 def staterelease():
998 if origrelease:
998 if origrelease:
999 origrelease()
999 origrelease()
1000 if l.stateupdate:
1000 if l.stateupdate:
1001 l.stateupdate.exit()
1001 l.stateupdate.exit()
1002 l.stateupdate = None
1002 l.stateupdate = None
1003
1003
1004 try:
1004 try:
1005 l.stateupdate = None
1005 l.stateupdate = None
1006 l.stateupdate = state_update(self, name=b"hg.transaction")
1006 l.stateupdate = state_update(self, name=b"hg.transaction")
1007 l.stateupdate.enter()
1007 l.stateupdate.enter()
1008 l.releasefn = staterelease
1008 l.releasefn = staterelease
1009 except Exception as e:
1009 except Exception as e:
1010 # Swallow any errors; fire and forget
1010 # Swallow any errors; fire and forget
1011 self.ui.log(
1011 self.ui.log(
1012 b'watchman', b'Exception in state update %s\n', e
1012 b'watchman', b'Exception in state update %s\n', e
1013 )
1013 )
1014 return l
1014 return l
1015
1015
1016 repo.__class__ = fsmonitorrepo
1016 repo.__class__ = fsmonitorrepo
General Comments 0
You need to be logged in to leave comments. Login now