##// END OF EJS Templates
fsmonitor: fix traceback about bytes and str incompatibility...
Raphaël Gomès -
r49126:6fb282ce default
parent child Browse files
Show More
@@ -1,1001 +1,1001
1 1 # __init__.py - fsmonitor initialization and overrides
2 2 #
3 3 # Copyright 2013-2016 Facebook, Inc.
4 4 #
5 5 # This software may be used and distributed according to the terms of the
6 6 # GNU General Public License version 2 or any later version.
7 7
8 8 '''Faster status operations with the Watchman file monitor (EXPERIMENTAL)
9 9
10 10 Integrates the file-watching program Watchman with Mercurial to produce faster
11 11 status results.
12 12
13 13 On a particular Linux system, for a real-world repository with over 400,000
14 14 files hosted on ext4, vanilla `hg status` takes 1.3 seconds. On the same
15 15 system, with fsmonitor it takes about 0.3 seconds.
16 16
17 17 fsmonitor requires no configuration -- it will tell Watchman about your
18 18 repository as necessary. You'll need to install Watchman from
19 19 https://facebook.github.io/watchman/ and make sure it is in your PATH.
20 20
21 21 fsmonitor is incompatible with the largefiles and eol extensions, and
22 22 will disable itself if any of those are active.
23 23
24 24 The following configuration options exist:
25 25
26 26 ::
27 27
28 28 [fsmonitor]
29 29 mode = {off, on, paranoid}
30 30
31 31 When `mode = off`, fsmonitor will disable itself (similar to not loading the
32 32 extension at all). When `mode = on`, fsmonitor will be enabled (the default).
33 33 When `mode = paranoid`, fsmonitor will query both Watchman and the filesystem,
34 34 and ensure that the results are consistent.
35 35
36 36 ::
37 37
38 38 [fsmonitor]
39 39 timeout = (float)
40 40
41 41 A value, in seconds, that determines how long fsmonitor will wait for Watchman
42 42 to return results. Defaults to `2.0`.
43 43
44 44 ::
45 45
46 46 [fsmonitor]
47 47 blacklistusers = (list of userids)
48 48
49 49 A list of usernames for which fsmonitor will disable itself altogether.
50 50
51 51 ::
52 52
53 53 [fsmonitor]
54 54 walk_on_invalidate = (boolean)
55 55
56 56 Whether or not to walk the whole repo ourselves when our cached state has been
57 57 invalidated, for example when Watchman has been restarted or .hgignore rules
58 58 have been changed. Walking the repo in that case can result in competing for
59 59 I/O with Watchman. For large repos it is recommended to set this value to
60 60 false. You may wish to set this to true if you have a very fast filesystem
61 61 that can outpace the IPC overhead of getting the result data for the full repo
62 62 from Watchman. Defaults to false.
63 63
64 64 ::
65 65
66 66 [fsmonitor]
67 67 warn_when_unused = (boolean)
68 68
69 69 Whether to print a warning during certain operations when fsmonitor would be
70 70 beneficial to performance but isn't enabled.
71 71
72 72 ::
73 73
74 74 [fsmonitor]
75 75 warn_update_file_count = (integer)
76 76 # or when mercurial is built with rust support
77 77 warn_update_file_count_rust = (integer)
78 78
79 79 If ``warn_when_unused`` is set and fsmonitor isn't enabled, a warning will
80 80 be printed during working directory updates if this many files will be
81 81 created.
82 82 '''
83 83
84 84 # Platforms Supported
85 85 # ===================
86 86 #
87 87 # **Linux:** *Stable*. Watchman and fsmonitor are both known to work reliably,
88 88 # even under severe loads.
89 89 #
90 90 # **Mac OS X:** *Stable*. The Mercurial test suite passes with fsmonitor
91 91 # turned on, on case-insensitive HFS+. There has been a reasonable amount of
92 92 # user testing under normal loads.
93 93 #
94 94 # **Solaris, BSD:** *Alpha*. watchman and fsmonitor are believed to work, but
95 95 # very little testing has been done.
96 96 #
97 97 # **Windows:** *Alpha*. Not in a release version of watchman or fsmonitor yet.
98 98 #
99 99 # Known Issues
100 100 # ============
101 101 #
102 102 # * fsmonitor will disable itself if any of the following extensions are
103 103 # enabled: largefiles, inotify, eol; or if the repository has subrepos.
104 104 # * fsmonitor will produce incorrect results if nested repos that are not
105 105 # subrepos exist. *Workaround*: add nested repo paths to your `.hgignore`.
106 106 #
107 107 # The issues related to nested repos and subrepos are probably not fundamental
108 108 # ones. Patches to fix them are welcome.
109 109
110 110 from __future__ import absolute_import
111 111
112 112 import codecs
113 113 import os
114 114 import stat
115 115 import sys
116 116 import tempfile
117 117 import weakref
118 118
119 119 from mercurial.i18n import _
120 120 from mercurial.node import hex
121 121 from mercurial.pycompat import open
122 122 from mercurial import (
123 123 context,
124 124 encoding,
125 125 error,
126 126 extensions,
127 127 localrepo,
128 128 merge,
129 129 pathutil,
130 130 pycompat,
131 131 registrar,
132 132 scmutil,
133 133 util,
134 134 )
135 135 from mercurial import match as matchmod
136 136 from mercurial.utils import (
137 137 hashutil,
138 138 stringutil,
139 139 )
140 140
141 141 from . import (
142 142 pywatchman,
143 143 state,
144 144 watchmanclient,
145 145 )
146 146
147 147 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
148 148 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
149 149 # be specifying the version(s) of Mercurial they are tested with, or
150 150 # leave the attribute unspecified.
151 151 testedwith = b'ships-with-hg-core'
152 152
153 153 configtable = {}
154 154 configitem = registrar.configitem(configtable)
155 155
156 156 configitem(
157 157 b'fsmonitor',
158 158 b'mode',
159 159 default=b'on',
160 160 )
161 161 configitem(
162 162 b'fsmonitor',
163 163 b'walk_on_invalidate',
164 164 default=False,
165 165 )
166 166 configitem(
167 167 b'fsmonitor',
168 168 b'timeout',
169 169 default=b'2',
170 170 )
171 171 configitem(
172 172 b'fsmonitor',
173 173 b'blacklistusers',
174 174 default=list,
175 175 )
176 176 configitem(
177 177 b'fsmonitor',
178 178 b'watchman_exe',
179 179 default=b'watchman',
180 180 )
181 181 configitem(
182 182 b'fsmonitor',
183 183 b'verbose',
184 184 default=True,
185 185 experimental=True,
186 186 )
187 187 configitem(
188 188 b'experimental',
189 189 b'fsmonitor.transaction_notify',
190 190 default=False,
191 191 )
192 192
193 193 # This extension is incompatible with the following blacklisted extensions
194 194 # and will disable itself when encountering one of these:
195 195 _blacklist = [b'largefiles', b'eol']
196 196
197 197
198 198 def debuginstall(ui, fm):
199 199 fm.write(
200 200 b"fsmonitor-watchman",
201 201 _(b"fsmonitor checking for watchman binary... (%s)\n"),
202 202 ui.configpath(b"fsmonitor", b"watchman_exe"),
203 203 )
204 204 root = tempfile.mkdtemp()
205 205 c = watchmanclient.client(ui, root)
206 206 err = None
207 207 try:
208 208 v = c.command(b"version")
209 209 fm.write(
210 210 b"fsmonitor-watchman-version",
211 211 _(b" watchman binary version %s\n"),
212 212 pycompat.bytestr(v["version"]),
213 213 )
214 214 except watchmanclient.Unavailable as e:
215 215 err = stringutil.forcebytestr(e)
216 216 fm.condwrite(
217 217 err,
218 218 b"fsmonitor-watchman-error",
219 219 _(b" watchman binary missing or broken: %s\n"),
220 220 err,
221 221 )
222 222 return 1 if err else 0
223 223
224 224
225 225 def _handleunavailable(ui, state, ex):
226 226 """Exception handler for Watchman interaction exceptions"""
227 227 if isinstance(ex, watchmanclient.Unavailable):
228 228 # experimental config: fsmonitor.verbose
229 229 if ex.warn and ui.configbool(b'fsmonitor', b'verbose'):
230 230 if b'illegal_fstypes' not in stringutil.forcebytestr(ex):
231 231 ui.warn(stringutil.forcebytestr(ex) + b'\n')
232 232 if ex.invalidate:
233 233 state.invalidate()
234 234 # experimental config: fsmonitor.verbose
235 235 if ui.configbool(b'fsmonitor', b'verbose'):
236 236 ui.log(
237 237 b'fsmonitor',
238 238 b'Watchman unavailable: %s\n',
239 239 stringutil.forcebytestr(ex.msg),
240 240 )
241 241 else:
242 242 ui.log(
243 243 b'fsmonitor',
244 244 b'Watchman exception: %s\n',
245 245 stringutil.forcebytestr(ex),
246 246 )
247 247
248 248
249 249 def _hashignore(ignore):
250 250 """Calculate hash for ignore patterns and filenames
251 251
252 252 If this information changes between Mercurial invocations, we can't
253 253 rely on Watchman information anymore and have to re-scan the working
254 254 copy.
255 255
256 256 """
257 257 sha1 = hashutil.sha1()
258 258 sha1.update(pycompat.byterepr(ignore))
259 259 return pycompat.sysbytes(sha1.hexdigest())
260 260
261 261
262 262 _watchmanencoding = pywatchman.encoding.get_local_encoding()
263 263 _fsencoding = sys.getfilesystemencoding() or sys.getdefaultencoding()
264 264 _fixencoding = codecs.lookup(_watchmanencoding) != codecs.lookup(_fsencoding)
265 265
266 266
267 267 def _watchmantofsencoding(path):
268 268 """Fix path to match watchman and local filesystem encoding
269 269
270 270 watchman's paths encoding can differ from filesystem encoding. For example,
271 271 on Windows, it's always utf-8.
272 272 """
273 273 try:
274 274 decoded = path.decode(_watchmanencoding)
275 275 except UnicodeDecodeError as e:
276 276 raise error.Abort(
277 277 stringutil.forcebytestr(e), hint=b'watchman encoding error'
278 278 )
279 279
280 280 try:
281 281 encoded = decoded.encode(_fsencoding, 'strict')
282 282 except UnicodeEncodeError as e:
283 283 raise error.Abort(stringutil.forcebytestr(e))
284 284
285 285 return encoded
286 286
287 287
288 288 def overridewalk(orig, self, match, subrepos, unknown, ignored, full=True):
289 289 """Replacement for dirstate.walk, hooking into Watchman.
290 290
291 291 Whenever full is False, ignored is False, and the Watchman client is
292 292 available, use Watchman combined with saved state to possibly return only a
293 293 subset of files."""
294 294
295 295 def bail(reason):
296 296 self._ui.debug(b'fsmonitor: fallback to core status, %s\n' % reason)
297 297 return orig(match, subrepos, unknown, ignored, full=True)
298 298
299 299 if full:
300 300 return bail(b'full rewalk requested')
301 301 if ignored:
302 302 return bail(b'listing ignored files')
303 303 if not self._watchmanclient.available():
304 304 return bail(b'client unavailable')
305 305 state = self._fsmonitorstate
306 306 clock, ignorehash, notefiles = state.get()
307 307 if not clock:
308 308 if state.walk_on_invalidate:
309 309 return bail(b'no clock')
310 310 # Initial NULL clock value, see
311 311 # https://facebook.github.io/watchman/docs/clockspec.html
312 312 clock = b'c:0:0'
313 313 notefiles = []
314 314
315 315 ignore = self._ignore
316 316 dirignore = self._dirignore
317 317 if unknown:
318 318 if _hashignore(ignore) != ignorehash and clock != b'c:0:0':
319 319 # ignore list changed -- can't rely on Watchman state any more
320 320 if state.walk_on_invalidate:
321 321 return bail(b'ignore rules changed')
322 322 notefiles = []
323 323 clock = b'c:0:0'
324 324 else:
325 325 # always ignore
326 326 ignore = util.always
327 327 dirignore = util.always
328 328
329 329 matchfn = match.matchfn
330 330 matchalways = match.always()
331 331 dmap = self._map
332 332 if util.safehasattr(dmap, b'_map'):
333 333 # for better performance, directly access the inner dirstate map if the
334 334 # standard dirstate implementation is in use.
335 335 dmap = dmap._map
336 336 nonnormalset = self._map.nonnormalset
337 337
338 338 copymap = self._map.copymap
339 339 getkind = stat.S_IFMT
340 340 dirkind = stat.S_IFDIR
341 341 regkind = stat.S_IFREG
342 342 lnkkind = stat.S_IFLNK
343 343 join = self._join
344 344 normcase = util.normcase
345 345 fresh_instance = False
346 346
347 347 exact = skipstep3 = False
348 348 if match.isexact(): # match.exact
349 349 exact = True
350 350 dirignore = util.always # skip step 2
351 351 elif match.prefix(): # match.match, no patterns
352 352 skipstep3 = True
353 353
354 354 if not exact and self._checkcase:
355 355 # note that even though we could receive directory entries, we're only
356 356 # interested in checking if a file with the same name exists. So only
357 357 # normalize files if possible.
358 358 normalize = self._normalizefile
359 359 skipstep3 = False
360 360 else:
361 361 normalize = None
362 362
363 363 # step 1: find all explicit files
364 364 results, work, dirsnotfound = self._walkexplicit(match, subrepos)
365 365
366 366 skipstep3 = skipstep3 and not (work or dirsnotfound)
367 367 work = [d for d in work if not dirignore(d[0])]
368 368
369 369 if not work and (exact or skipstep3):
370 370 for s in subrepos:
371 371 del results[s]
372 372 del results[b'.hg']
373 373 return results
374 374
375 375 # step 2: query Watchman
376 376 try:
377 377 # Use the user-configured timeout for the query.
378 378 # Add a little slack over the top of the user query to allow for
379 379 # overheads while transferring the data
380 380 self._watchmanclient.settimeout(state.timeout + 0.1)
381 381 result = self._watchmanclient.command(
382 382 b'query',
383 383 {
384 384 b'fields': [b'mode', b'mtime', b'size', b'exists', b'name'],
385 385 b'since': clock,
386 386 b'expression': [
387 387 b'not',
388 388 [
389 389 b'anyof',
390 390 [b'dirname', b'.hg'],
391 391 [b'name', b'.hg', b'wholename'],
392 392 ],
393 393 ],
394 394 b'sync_timeout': int(state.timeout * 1000),
395 395 b'empty_on_fresh_instance': state.walk_on_invalidate,
396 396 },
397 397 )
398 398 except Exception as ex:
399 399 _handleunavailable(self._ui, state, ex)
400 400 self._watchmanclient.clearconnection()
401 401 return bail(b'exception during run')
402 402 else:
403 403 # We need to propagate the last observed clock up so that we
404 404 # can use it for our next query
405 405 state.setlastclock(pycompat.sysbytes(result[b'clock']))
406 406 if result[b'is_fresh_instance']:
407 407 if state.walk_on_invalidate:
408 408 state.invalidate()
409 409 return bail(b'fresh instance')
410 410 fresh_instance = True
411 411 # Ignore any prior noteable files from the state info
412 412 notefiles = []
413 413
414 414 # for file paths which require normalization and we encounter a case
415 415 # collision, we store our own foldmap
416 416 if normalize:
417 417 foldmap = {normcase(k): k for k in results}
418 418
419 419 switch_slashes = pycompat.ossep == b'\\'
420 420 # The order of the results is, strictly speaking, undefined.
421 421 # For case changes on a case insensitive filesystem we may receive
422 422 # two entries, one with exists=True and another with exists=False.
423 423 # The exists=True entries in the same response should be interpreted
424 424 # as being happens-after the exists=False entries due to the way that
425 425 # Watchman tracks files. We use this property to reconcile deletes
426 426 # for name case changes.
427 427 for entry in result[b'files']:
428 428 fname = entry[b'name']
429 429
430 430 # Watchman always give us a str. Normalize to bytes on Python 3
431 431 # using Watchman's encoding, if needed.
432 432 if not isinstance(fname, bytes):
433 433 fname = fname.encode(_watchmanencoding)
434 434
435 435 if _fixencoding:
436 436 fname = _watchmantofsencoding(fname)
437 437
438 438 if switch_slashes:
439 439 fname = fname.replace(b'\\', b'/')
440 440 if normalize:
441 441 normed = normcase(fname)
442 442 fname = normalize(fname, True, True)
443 443 foldmap[normed] = fname
444 444 fmode = entry[b'mode']
445 445 fexists = entry[b'exists']
446 446 kind = getkind(fmode)
447 447
448 448 if b'/.hg/' in fname or fname.endswith(b'/.hg'):
449 449 return bail(b'nested-repo-detected')
450 450
451 451 if not fexists:
452 452 # if marked as deleted and we don't already have a change
453 453 # record, mark it as deleted. If we already have an entry
454 454 # for fname then it was either part of walkexplicit or was
455 455 # an earlier result that was a case change
456 456 if (
457 457 fname not in results
458 458 and fname in dmap
459 459 and (matchalways or matchfn(fname))
460 460 ):
461 461 results[fname] = None
462 462 elif kind == dirkind:
463 463 if fname in dmap and (matchalways or matchfn(fname)):
464 464 results[fname] = None
465 465 elif kind == regkind or kind == lnkkind:
466 466 if fname in dmap:
467 467 if matchalways or matchfn(fname):
468 468 results[fname] = entry
469 469 elif (matchalways or matchfn(fname)) and not ignore(fname):
470 470 results[fname] = entry
471 471 elif fname in dmap and (matchalways or matchfn(fname)):
472 472 results[fname] = None
473 473
474 474 # step 3: query notable files we don't already know about
475 475 # XXX try not to iterate over the entire dmap
476 476 if normalize:
477 477 # any notable files that have changed case will already be handled
478 478 # above, so just check membership in the foldmap
479 479 notefiles = {
480 480 normalize(f, True, True)
481 481 for f in notefiles
482 482 if normcase(f) not in foldmap
483 483 }
484 484 visit = {
485 485 f
486 486 for f in notefiles
487 487 if (f not in results and matchfn(f) and (f in dmap or not ignore(f)))
488 488 }
489 489
490 490 if not fresh_instance:
491 491 if matchalways:
492 492 visit.update(f for f in nonnormalset if f not in results)
493 493 visit.update(f for f in copymap if f not in results)
494 494 else:
495 495 visit.update(
496 496 f for f in nonnormalset if f not in results and matchfn(f)
497 497 )
498 498 visit.update(f for f in copymap if f not in results and matchfn(f))
499 499 else:
500 500 if matchalways:
501 501 visit.update(
502 502 f for f, st in pycompat.iteritems(dmap) if f not in results
503 503 )
504 504 visit.update(f for f in copymap if f not in results)
505 505 else:
506 506 visit.update(
507 507 f
508 508 for f, st in pycompat.iteritems(dmap)
509 509 if f not in results and matchfn(f)
510 510 )
511 511 visit.update(f for f in copymap if f not in results and matchfn(f))
512 512
513 513 audit = pathutil.pathauditor(self._root, cached=True).check
514 514 auditpass = [f for f in visit if audit(f)]
515 515 auditpass.sort()
516 516 auditfail = visit.difference(auditpass)
517 517 for f in auditfail:
518 518 results[f] = None
519 519
520 520 nf = iter(auditpass)
521 521 for st in util.statfiles([join(f) for f in auditpass]):
522 522 f = next(nf)
523 523 if st or f in dmap:
524 524 results[f] = st
525 525
526 526 for s in subrepos:
527 527 del results[s]
528 528 del results[b'.hg']
529 529 return results
530 530
531 531
532 532 def overridestatus(
533 533 orig,
534 534 self,
535 535 node1=b'.',
536 536 node2=None,
537 537 match=None,
538 538 ignored=False,
539 539 clean=False,
540 540 unknown=False,
541 541 listsubrepos=False,
542 542 ):
543 543 listignored = ignored
544 544 listclean = clean
545 545 listunknown = unknown
546 546
547 547 def _cmpsets(l1, l2):
548 548 try:
549 549 if b'FSMONITOR_LOG_FILE' in encoding.environ:
550 550 fn = encoding.environ[b'FSMONITOR_LOG_FILE']
551 551 f = open(fn, b'wb')
552 552 else:
553 553 fn = b'fsmonitorfail.log'
554 554 f = self.vfs.open(fn, b'wb')
555 555 except (IOError, OSError):
556 556 self.ui.warn(_(b'warning: unable to write to %s\n') % fn)
557 557 return
558 558
559 559 try:
560 560 for i, (s1, s2) in enumerate(zip(l1, l2)):
561 561 if set(s1) != set(s2):
562 562 f.write(b'sets at position %d are unequal\n' % i)
563 f.write(b'watchman returned: %s\n' % s1)
564 f.write(b'stat returned: %s\n' % s2)
563 f.write(b'watchman returned: %r\n' % s1)
564 f.write(b'stat returned: %r\n' % s2)
565 565 finally:
566 566 f.close()
567 567
568 568 if isinstance(node1, context.changectx):
569 569 ctx1 = node1
570 570 else:
571 571 ctx1 = self[node1]
572 572 if isinstance(node2, context.changectx):
573 573 ctx2 = node2
574 574 else:
575 575 ctx2 = self[node2]
576 576
577 577 working = ctx2.rev() is None
578 578 parentworking = working and ctx1 == self[b'.']
579 579 match = match or matchmod.always()
580 580
581 581 # Maybe we can use this opportunity to update Watchman's state.
582 582 # Mercurial uses workingcommitctx and/or memctx to represent the part of
583 583 # the workingctx that is to be committed. So don't update the state in
584 584 # that case.
585 585 # HG_PENDING is set in the environment when the dirstate is being updated
586 586 # in the middle of a transaction; we must not update our state in that
587 587 # case, or we risk forgetting about changes in the working copy.
588 588 updatestate = (
589 589 parentworking
590 590 and match.always()
591 591 and not isinstance(ctx2, (context.workingcommitctx, context.memctx))
592 592 and b'HG_PENDING' not in encoding.environ
593 593 )
594 594
595 595 try:
596 596 if self._fsmonitorstate.walk_on_invalidate:
597 597 # Use a short timeout to query the current clock. If that
598 598 # takes too long then we assume that the service will be slow
599 599 # to answer our query.
600 600 # walk_on_invalidate indicates that we prefer to walk the
601 601 # tree ourselves because we can ignore portions that Watchman
602 602 # cannot and we tend to be faster in the warmer buffer cache
603 603 # cases.
604 604 self._watchmanclient.settimeout(0.1)
605 605 else:
606 606 # Give Watchman more time to potentially complete its walk
607 607 # and return the initial clock. In this mode we assume that
608 608 # the filesystem will be slower than parsing a potentially
609 609 # very large Watchman result set.
610 610 self._watchmanclient.settimeout(self._fsmonitorstate.timeout + 0.1)
611 611 startclock = self._watchmanclient.getcurrentclock()
612 612 except Exception as ex:
613 613 self._watchmanclient.clearconnection()
614 614 _handleunavailable(self.ui, self._fsmonitorstate, ex)
615 615 # boo, Watchman failed. bail
616 616 return orig(
617 617 node1,
618 618 node2,
619 619 match,
620 620 listignored,
621 621 listclean,
622 622 listunknown,
623 623 listsubrepos,
624 624 )
625 625
626 626 if updatestate:
627 627 # We need info about unknown files. This may make things slower the
628 628 # first time, but whatever.
629 629 stateunknown = True
630 630 else:
631 631 stateunknown = listunknown
632 632
633 633 if updatestate:
634 634 ps = poststatus(startclock)
635 635 self.addpostdsstatus(ps)
636 636
637 637 r = orig(
638 638 node1, node2, match, listignored, listclean, stateunknown, listsubrepos
639 639 )
640 640 modified, added, removed, deleted, unknown, ignored, clean = r
641 641
642 642 if not listunknown:
643 643 unknown = []
644 644
645 645 # don't do paranoid checks if we're not going to query Watchman anyway
646 646 full = listclean or match.traversedir is not None
647 647 if self._fsmonitorstate.mode == b'paranoid' and not full:
648 648 # run status again and fall back to the old walk this time
649 649 self.dirstate._fsmonitordisable = True
650 650
651 651 # shut the UI up
652 652 quiet = self.ui.quiet
653 653 self.ui.quiet = True
654 654 fout, ferr = self.ui.fout, self.ui.ferr
655 655 self.ui.fout = self.ui.ferr = open(os.devnull, b'wb')
656 656
657 657 try:
658 658 rv2 = orig(
659 659 node1,
660 660 node2,
661 661 match,
662 662 listignored,
663 663 listclean,
664 664 listunknown,
665 665 listsubrepos,
666 666 )
667 667 finally:
668 668 self.dirstate._fsmonitordisable = False
669 669 self.ui.quiet = quiet
670 670 self.ui.fout, self.ui.ferr = fout, ferr
671 671
672 672 # clean isn't tested since it's set to True above
673 673 with self.wlock():
674 674 _cmpsets(
675 675 [modified, added, removed, deleted, unknown, ignored, clean],
676 676 rv2,
677 677 )
678 678 modified, added, removed, deleted, unknown, ignored, clean = rv2
679 679
680 680 return scmutil.status(
681 681 modified, added, removed, deleted, unknown, ignored, clean
682 682 )
683 683
684 684
685 685 class poststatus(object):
686 686 def __init__(self, startclock):
687 687 self._startclock = pycompat.sysbytes(startclock)
688 688
689 689 def __call__(self, wctx, status):
690 690 clock = wctx.repo()._fsmonitorstate.getlastclock() or self._startclock
691 691 hashignore = _hashignore(wctx.repo().dirstate._ignore)
692 692 notefiles = (
693 693 status.modified
694 694 + status.added
695 695 + status.removed
696 696 + status.deleted
697 697 + status.unknown
698 698 )
699 699 wctx.repo()._fsmonitorstate.set(clock, hashignore, notefiles)
700 700
701 701
702 702 def makedirstate(repo, dirstate):
703 703 class fsmonitordirstate(dirstate.__class__):
704 704 def _fsmonitorinit(self, repo):
705 705 # _fsmonitordisable is used in paranoid mode
706 706 self._fsmonitordisable = False
707 707 self._fsmonitorstate = repo._fsmonitorstate
708 708 self._watchmanclient = repo._watchmanclient
709 709 self._repo = weakref.proxy(repo)
710 710
711 711 def walk(self, *args, **kwargs):
712 712 orig = super(fsmonitordirstate, self).walk
713 713 if self._fsmonitordisable:
714 714 return orig(*args, **kwargs)
715 715 return overridewalk(orig, self, *args, **kwargs)
716 716
717 717 def rebuild(self, *args, **kwargs):
718 718 self._fsmonitorstate.invalidate()
719 719 return super(fsmonitordirstate, self).rebuild(*args, **kwargs)
720 720
721 721 def invalidate(self, *args, **kwargs):
722 722 self._fsmonitorstate.invalidate()
723 723 return super(fsmonitordirstate, self).invalidate(*args, **kwargs)
724 724
725 725 dirstate.__class__ = fsmonitordirstate
726 726 dirstate._fsmonitorinit(repo)
727 727
728 728
729 729 def wrapdirstate(orig, self):
730 730 ds = orig(self)
731 731 # only override the dirstate when Watchman is available for the repo
732 732 if util.safehasattr(self, b'_fsmonitorstate'):
733 733 makedirstate(self, ds)
734 734 return ds
735 735
736 736
737 737 def extsetup(ui):
738 738 extensions.wrapfilecache(
739 739 localrepo.localrepository, b'dirstate', wrapdirstate
740 740 )
741 741 if pycompat.isdarwin:
742 742 # An assist for avoiding the dangling-symlink fsevents bug
743 743 extensions.wrapfunction(os, b'symlink', wrapsymlink)
744 744
745 745 extensions.wrapfunction(merge, b'_update', wrapupdate)
746 746
747 747
748 748 def wrapsymlink(orig, source, link_name):
749 749 """if we create a dangling symlink, also touch the parent dir
750 750 to encourage fsevents notifications to work more correctly"""
751 751 try:
752 752 return orig(source, link_name)
753 753 finally:
754 754 try:
755 755 os.utime(os.path.dirname(link_name), None)
756 756 except OSError:
757 757 pass
758 758
759 759
760 760 class state_update(object):
761 761 """This context manager is responsible for dispatching the state-enter
762 762 and state-leave signals to the watchman service. The enter and leave
763 763 methods can be invoked manually (for scenarios where context manager
764 764 semantics are not possible). If parameters oldnode and newnode are None,
765 765 they will be populated based on current working copy in enter and
766 766 leave, respectively. Similarly, if the distance is none, it will be
767 767 calculated based on the oldnode and newnode in the leave method."""
768 768
769 769 def __init__(
770 770 self,
771 771 repo,
772 772 name,
773 773 oldnode=None,
774 774 newnode=None,
775 775 distance=None,
776 776 partial=False,
777 777 ):
778 778 self.repo = repo.unfiltered()
779 779 self.name = name
780 780 self.oldnode = oldnode
781 781 self.newnode = newnode
782 782 self.distance = distance
783 783 self.partial = partial
784 784 self._lock = None
785 785 self.need_leave = False
786 786
787 787 def __enter__(self):
788 788 self.enter()
789 789
790 790 def enter(self):
791 791 # Make sure we have a wlock prior to sending notifications to watchman.
792 792 # We don't want to race with other actors. In the update case,
793 793 # merge.update is going to take the wlock almost immediately. We are
794 794 # effectively extending the lock around several short sanity checks.
795 795 if self.oldnode is None:
796 796 self.oldnode = self.repo[b'.'].node()
797 797
798 798 if self.repo.currentwlock() is None:
799 799 if util.safehasattr(self.repo, b'wlocknostateupdate'):
800 800 self._lock = self.repo.wlocknostateupdate()
801 801 else:
802 802 self._lock = self.repo.wlock()
803 803 self.need_leave = self._state(b'state-enter', hex(self.oldnode))
804 804 return self
805 805
806 806 def __exit__(self, type_, value, tb):
807 807 abort = True if type_ else False
808 808 self.exit(abort=abort)
809 809
810 810 def exit(self, abort=False):
811 811 try:
812 812 if self.need_leave:
813 813 status = b'failed' if abort else b'ok'
814 814 if self.newnode is None:
815 815 self.newnode = self.repo[b'.'].node()
816 816 if self.distance is None:
817 817 self.distance = calcdistance(
818 818 self.repo, self.oldnode, self.newnode
819 819 )
820 820 self._state(b'state-leave', hex(self.newnode), status=status)
821 821 finally:
822 822 self.need_leave = False
823 823 if self._lock:
824 824 self._lock.release()
825 825
826 826 def _state(self, cmd, commithash, status=b'ok'):
827 827 if not util.safehasattr(self.repo, b'_watchmanclient'):
828 828 return False
829 829 try:
830 830 self.repo._watchmanclient.command(
831 831 cmd,
832 832 {
833 833 b'name': self.name,
834 834 b'metadata': {
835 835 # the target revision
836 836 b'rev': commithash,
837 837 # approximate number of commits between current and target
838 838 b'distance': self.distance if self.distance else 0,
839 839 # success/failure (only really meaningful for state-leave)
840 840 b'status': status,
841 841 # whether the working copy parent is changing
842 842 b'partial': self.partial,
843 843 },
844 844 },
845 845 )
846 846 return True
847 847 except Exception as e:
848 848 # Swallow any errors; fire and forget
849 849 self.repo.ui.log(
850 850 b'watchman', b'Exception %s while running %s\n', e, cmd
851 851 )
852 852 return False
853 853
854 854
855 855 # Estimate the distance between two nodes
856 856 def calcdistance(repo, oldnode, newnode):
857 857 anc = repo.changelog.ancestor(oldnode, newnode)
858 858 ancrev = repo[anc].rev()
859 859 distance = abs(repo[oldnode].rev() - ancrev) + abs(
860 860 repo[newnode].rev() - ancrev
861 861 )
862 862 return distance
863 863
864 864
865 865 # Bracket working copy updates with calls to the watchman state-enter
866 866 # and state-leave commands. This allows clients to perform more intelligent
867 867 # settling during bulk file change scenarios
868 868 # https://facebook.github.io/watchman/docs/cmd/subscribe.html#advanced-settling
869 869 def wrapupdate(
870 870 orig,
871 871 repo,
872 872 node,
873 873 branchmerge,
874 874 force,
875 875 ancestor=None,
876 876 mergeancestor=False,
877 877 labels=None,
878 878 matcher=None,
879 879 **kwargs
880 880 ):
881 881
882 882 distance = 0
883 883 partial = True
884 884 oldnode = repo[b'.'].node()
885 885 newnode = repo[node].node()
886 886 if matcher is None or matcher.always():
887 887 partial = False
888 888 distance = calcdistance(repo.unfiltered(), oldnode, newnode)
889 889
890 890 with state_update(
891 891 repo,
892 892 name=b"hg.update",
893 893 oldnode=oldnode,
894 894 newnode=newnode,
895 895 distance=distance,
896 896 partial=partial,
897 897 ):
898 898 return orig(
899 899 repo,
900 900 node,
901 901 branchmerge,
902 902 force,
903 903 ancestor,
904 904 mergeancestor,
905 905 labels,
906 906 matcher,
907 907 **kwargs
908 908 )
909 909
910 910
911 911 def repo_has_depth_one_nested_repo(repo):
912 912 for f in repo.wvfs.listdir():
913 913 if os.path.isdir(os.path.join(repo.root, f, b'.hg')):
914 914 msg = b'fsmonitor: sub-repository %r detected, fsmonitor disabled\n'
915 915 repo.ui.debug(msg % f)
916 916 return True
917 917 return False
918 918
919 919
920 920 def reposetup(ui, repo):
921 921 # We don't work with largefiles or inotify
922 922 exts = extensions.enabled()
923 923 for ext in _blacklist:
924 924 if ext in exts:
925 925 ui.warn(
926 926 _(
927 927 b'The fsmonitor extension is incompatible with the %s '
928 928 b'extension and has been disabled.\n'
929 929 )
930 930 % ext
931 931 )
932 932 return
933 933
934 934 if repo.local():
935 935 # We don't work with subrepos either.
936 936 #
937 937 # if repo[None].substate can cause a dirstate parse, which is too
938 938 # slow. Instead, look for a file called hgsubstate,
939 939 if repo.wvfs.exists(b'.hgsubstate') or repo.wvfs.exists(b'.hgsub'):
940 940 return
941 941
942 942 if repo_has_depth_one_nested_repo(repo):
943 943 return
944 944
945 945 fsmonitorstate = state.state(repo)
946 946 if fsmonitorstate.mode == b'off':
947 947 return
948 948
949 949 try:
950 950 client = watchmanclient.client(repo.ui, repo.root)
951 951 except Exception as ex:
952 952 _handleunavailable(ui, fsmonitorstate, ex)
953 953 return
954 954
955 955 repo._fsmonitorstate = fsmonitorstate
956 956 repo._watchmanclient = client
957 957
958 958 dirstate, cached = localrepo.isfilecached(repo, b'dirstate')
959 959 if cached:
960 960 # at this point since fsmonitorstate wasn't present,
961 961 # repo.dirstate is not a fsmonitordirstate
962 962 makedirstate(repo, dirstate)
963 963
964 964 class fsmonitorrepo(repo.__class__):
965 965 def status(self, *args, **kwargs):
966 966 orig = super(fsmonitorrepo, self).status
967 967 return overridestatus(orig, self, *args, **kwargs)
968 968
969 969 def wlocknostateupdate(self, *args, **kwargs):
970 970 return super(fsmonitorrepo, self).wlock(*args, **kwargs)
971 971
972 972 def wlock(self, *args, **kwargs):
973 973 l = super(fsmonitorrepo, self).wlock(*args, **kwargs)
974 974 if not ui.configbool(
975 975 b"experimental", b"fsmonitor.transaction_notify"
976 976 ):
977 977 return l
978 978 if l.held != 1:
979 979 return l
980 980 origrelease = l.releasefn
981 981
982 982 def staterelease():
983 983 if origrelease:
984 984 origrelease()
985 985 if l.stateupdate:
986 986 l.stateupdate.exit()
987 987 l.stateupdate = None
988 988
989 989 try:
990 990 l.stateupdate = None
991 991 l.stateupdate = state_update(self, name=b"hg.transaction")
992 992 l.stateupdate.enter()
993 993 l.releasefn = staterelease
994 994 except Exception as e:
995 995 # Swallow any errors; fire and forget
996 996 self.ui.log(
997 997 b'watchman', b'Exception in state update %s\n', e
998 998 )
999 999 return l
1000 1000
1001 1001 repo.__class__ = fsmonitorrepo
General Comments 0
You need to be logged in to leave comments. Login now