##// END OF EJS Templates
wrapfunction: use sysstr instead of bytes as argument in "fsmonitor"...
marmoute -
r51671:ba3add90 default
parent child Browse files
Show More
@@ -1,1016 +1,1016 b''
1 1 # __init__.py - fsmonitor initialization and overrides
2 2 #
3 3 # Copyright 2013-2016 Facebook, Inc.
4 4 #
5 5 # This software may be used and distributed according to the terms of the
6 6 # GNU General Public License version 2 or any later version.
7 7
8 8 '''Faster status operations with the Watchman file monitor (EXPERIMENTAL)
9 9
10 10 Integrates the file-watching program Watchman with Mercurial to produce faster
11 11 status results.
12 12
13 13 On a particular Linux system, for a real-world repository with over 400,000
14 14 files hosted on ext4, vanilla `hg status` takes 1.3 seconds. On the same
15 15 system, with fsmonitor it takes about 0.3 seconds.
16 16
17 17 fsmonitor requires no configuration -- it will tell Watchman about your
18 18 repository as necessary. You'll need to install Watchman from
19 19 https://facebook.github.io/watchman/ and make sure it is in your PATH.
20 20
21 21 fsmonitor is incompatible with the largefiles and eol extensions, and
22 22 will disable itself if any of those are active.
23 23
24 24 The following configuration options exist:
25 25
26 26 ::
27 27
28 28 [fsmonitor]
29 29 mode = {off, on, paranoid}
30 30
31 31 When `mode = off`, fsmonitor will disable itself (similar to not loading the
32 32 extension at all). When `mode = on`, fsmonitor will be enabled (the default).
33 33 When `mode = paranoid`, fsmonitor will query both Watchman and the filesystem,
34 34 and ensure that the results are consistent.
35 35
36 36 ::
37 37
38 38 [fsmonitor]
39 39 timeout = (float)
40 40
41 41 A value, in seconds, that determines how long fsmonitor will wait for Watchman
42 42 to return results. Defaults to `2.0`.
43 43
44 44 ::
45 45
46 46 [fsmonitor]
47 47 blacklistusers = (list of userids)
48 48
49 49 A list of usernames for which fsmonitor will disable itself altogether.
50 50
51 51 ::
52 52
53 53 [fsmonitor]
54 54 walk_on_invalidate = (boolean)
55 55
56 56 Whether or not to walk the whole repo ourselves when our cached state has been
57 57 invalidated, for example when Watchman has been restarted or .hgignore rules
58 58 have been changed. Walking the repo in that case can result in competing for
59 59 I/O with Watchman. For large repos it is recommended to set this value to
60 60 false. You may wish to set this to true if you have a very fast filesystem
61 61 that can outpace the IPC overhead of getting the result data for the full repo
62 62 from Watchman. Defaults to false.
63 63
64 64 ::
65 65
66 66 [fsmonitor]
67 67 warn_when_unused = (boolean)
68 68
69 69 Whether to print a warning during certain operations when fsmonitor would be
70 70 beneficial to performance but isn't enabled.
71 71
72 72 ::
73 73
74 74 [fsmonitor]
75 75 warn_update_file_count = (integer)
76 76 # or when mercurial is built with rust support
77 77 warn_update_file_count_rust = (integer)
78 78
79 79 If ``warn_when_unused`` is set and fsmonitor isn't enabled, a warning will
80 80 be printed during working directory updates if this many files will be
81 81 created.
82 82 '''
83 83
84 84 # Platforms Supported
85 85 # ===================
86 86 #
87 87 # **Linux:** *Stable*. Watchman and fsmonitor are both known to work reliably,
88 88 # even under severe loads.
89 89 #
90 90 # **Mac OS X:** *Stable*. The Mercurial test suite passes with fsmonitor
91 91 # turned on, on case-insensitive HFS+. There has been a reasonable amount of
92 92 # user testing under normal loads.
93 93 #
94 94 # **Solaris, BSD:** *Alpha*. watchman and fsmonitor are believed to work, but
95 95 # very little testing has been done.
96 96 #
97 97 # **Windows:** *Alpha*. Not in a release version of watchman or fsmonitor yet.
98 98 #
99 99 # Known Issues
100 100 # ============
101 101 #
102 102 # * fsmonitor will disable itself if any of the following extensions are
103 103 # enabled: largefiles, inotify, eol; or if the repository has subrepos.
104 104 # * fsmonitor will produce incorrect results if nested repos that are not
105 105 # subrepos exist. *Workaround*: add nested repo paths to your `.hgignore`.
106 106 #
107 107 # The issues related to nested repos and subrepos are probably not fundamental
108 108 # ones. Patches to fix them are welcome.
109 109
110 110
111 111 import codecs
112 112 import os
113 113 import stat
114 114 import sys
115 115 import tempfile
116 116 import weakref
117 117
118 118 from mercurial.i18n import _
119 119 from mercurial.node import hex
120 120
121 121 from mercurial.pycompat import open
122 122 from mercurial import (
123 123 context,
124 124 encoding,
125 125 error,
126 126 extensions,
127 127 localrepo,
128 128 merge,
129 129 pathutil,
130 130 pycompat,
131 131 registrar,
132 132 scmutil,
133 133 util,
134 134 )
135 135
136 136 # no-check-code because we're accessing private information only public in pure
137 137 from mercurial.pure import parsers
138 138 from mercurial import match as matchmod
139 139 from mercurial.utils import (
140 140 hashutil,
141 141 stringutil,
142 142 )
143 143
144 144 from . import (
145 145 pywatchman,
146 146 state,
147 147 watchmanclient,
148 148 )
149 149
150 150 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
151 151 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
152 152 # be specifying the version(s) of Mercurial they are tested with, or
153 153 # leave the attribute unspecified.
154 154 testedwith = b'ships-with-hg-core'
155 155
156 156 configtable = {}
157 157 configitem = registrar.configitem(configtable)
158 158
159 159 configitem(
160 160 b'fsmonitor',
161 161 b'mode',
162 162 default=b'on',
163 163 )
164 164 configitem(
165 165 b'fsmonitor',
166 166 b'walk_on_invalidate',
167 167 default=False,
168 168 )
169 169 configitem(
170 170 b'fsmonitor',
171 171 b'timeout',
172 172 default=b'2',
173 173 )
174 174 configitem(
175 175 b'fsmonitor',
176 176 b'blacklistusers',
177 177 default=list,
178 178 )
179 179 configitem(
180 180 b'fsmonitor',
181 181 b'watchman_exe',
182 182 default=b'watchman',
183 183 )
184 184 configitem(
185 185 b'fsmonitor',
186 186 b'verbose',
187 187 default=True,
188 188 experimental=True,
189 189 )
190 190 configitem(
191 191 b'experimental',
192 192 b'fsmonitor.transaction_notify',
193 193 default=False,
194 194 )
195 195
196 196 # This extension is incompatible with the following blacklisted extensions
197 197 # and will disable itself when encountering one of these:
198 198 _blacklist = [b'largefiles', b'eol']
199 199
200 200
201 201 def debuginstall(ui, fm):
202 202 fm.write(
203 203 b"fsmonitor-watchman",
204 204 _(b"fsmonitor checking for watchman binary... (%s)\n"),
205 205 ui.configpath(b"fsmonitor", b"watchman_exe"),
206 206 )
207 207 root = tempfile.mkdtemp()
208 208 c = watchmanclient.client(ui, root)
209 209 err = None
210 210 try:
211 211 v = c.command(b"version")
212 212 fm.write(
213 213 b"fsmonitor-watchman-version",
214 214 _(b" watchman binary version %s\n"),
215 215 pycompat.bytestr(v["version"]),
216 216 )
217 217 except watchmanclient.Unavailable as e:
218 218 err = stringutil.forcebytestr(e)
219 219 fm.condwrite(
220 220 err,
221 221 b"fsmonitor-watchman-error",
222 222 _(b" watchman binary missing or broken: %s\n"),
223 223 err,
224 224 )
225 225 return 1 if err else 0
226 226
227 227
228 228 def _handleunavailable(ui, state, ex):
229 229 """Exception handler for Watchman interaction exceptions"""
230 230 if isinstance(ex, watchmanclient.Unavailable):
231 231 # experimental config: fsmonitor.verbose
232 232 if ex.warn and ui.configbool(b'fsmonitor', b'verbose'):
233 233 if b'illegal_fstypes' not in stringutil.forcebytestr(ex):
234 234 ui.warn(stringutil.forcebytestr(ex) + b'\n')
235 235 if ex.invalidate:
236 236 state.invalidate()
237 237 # experimental config: fsmonitor.verbose
238 238 if ui.configbool(b'fsmonitor', b'verbose'):
239 239 ui.log(
240 240 b'fsmonitor',
241 241 b'Watchman unavailable: %s\n',
242 242 stringutil.forcebytestr(ex.msg),
243 243 )
244 244 else:
245 245 ui.log(
246 246 b'fsmonitor',
247 247 b'Watchman exception: %s\n',
248 248 stringutil.forcebytestr(ex),
249 249 )
250 250
251 251
252 252 def _hashignore(ignore):
253 253 """Calculate hash for ignore patterns and filenames
254 254
255 255 If this information changes between Mercurial invocations, we can't
256 256 rely on Watchman information anymore and have to re-scan the working
257 257 copy.
258 258
259 259 """
260 260 sha1 = hashutil.sha1()
261 261 sha1.update(pycompat.byterepr(ignore))
262 262 return pycompat.sysbytes(sha1.hexdigest())
263 263
264 264
265 265 _watchmanencoding = pywatchman.encoding.get_local_encoding()
266 266 _fsencoding = sys.getfilesystemencoding() or sys.getdefaultencoding()
267 267 _fixencoding = codecs.lookup(_watchmanencoding) != codecs.lookup(_fsencoding)
268 268
269 269
270 270 def _watchmantofsencoding(path):
271 271 """Fix path to match watchman and local filesystem encoding
272 272
273 273 watchman's paths encoding can differ from filesystem encoding. For example,
274 274 on Windows, it's always utf-8.
275 275 """
276 276 try:
277 277 decoded = path.decode(_watchmanencoding)
278 278 except UnicodeDecodeError as e:
279 279 raise error.Abort(
280 280 stringutil.forcebytestr(e), hint=b'watchman encoding error'
281 281 )
282 282
283 283 try:
284 284 encoded = decoded.encode(_fsencoding, 'strict')
285 285 except UnicodeEncodeError as e:
286 286 raise error.Abort(stringutil.forcebytestr(e))
287 287
288 288 return encoded
289 289
290 290
291 291 def overridewalk(orig, self, match, subrepos, unknown, ignored, full=True):
292 292 """Replacement for dirstate.walk, hooking into Watchman.
293 293
294 294 Whenever full is False, ignored is False, and the Watchman client is
295 295 available, use Watchman combined with saved state to possibly return only a
296 296 subset of files."""
297 297
298 298 def bail(reason):
299 299 self._ui.debug(b'fsmonitor: fallback to core status, %s\n' % reason)
300 300 return orig(match, subrepos, unknown, ignored, full=True)
301 301
302 302 if full:
303 303 return bail(b'full rewalk requested')
304 304 if ignored:
305 305 return bail(b'listing ignored files')
306 306 if not self._watchmanclient.available():
307 307 return bail(b'client unavailable')
308 308 state = self._fsmonitorstate
309 309 clock, ignorehash, notefiles = state.get()
310 310 if not clock:
311 311 if state.walk_on_invalidate:
312 312 return bail(b'no clock')
313 313 # Initial NULL clock value, see
314 314 # https://facebook.github.io/watchman/docs/clockspec.html
315 315 clock = b'c:0:0'
316 316 notefiles = []
317 317
318 318 ignore = self._ignore
319 319 dirignore = self._dirignore
320 320 if unknown:
321 321 if _hashignore(ignore) != ignorehash and clock != b'c:0:0':
322 322 # ignore list changed -- can't rely on Watchman state any more
323 323 if state.walk_on_invalidate:
324 324 return bail(b'ignore rules changed')
325 325 notefiles = []
326 326 clock = b'c:0:0'
327 327 else:
328 328 # always ignore
329 329 ignore = util.always
330 330 dirignore = util.always
331 331
332 332 matchfn = match.matchfn
333 333 matchalways = match.always()
334 334 dmap = self._map
335 335 if util.safehasattr(dmap, b'_map'):
336 336 # for better performance, directly access the inner dirstate map if the
337 337 # standard dirstate implementation is in use.
338 338 dmap = dmap._map
339 339
340 340 has_mtime = parsers.DIRSTATE_V2_HAS_MTIME
341 341 mtime_is_ambiguous = parsers.DIRSTATE_V2_MTIME_SECOND_AMBIGUOUS
342 342 mask = has_mtime | mtime_is_ambiguous
343 343
344 344 # All entries that may not be clean
345 345 nonnormalset = {
346 346 f
347 347 for f, e in self._map.items()
348 348 if not e.maybe_clean
349 349 # same as "not has_time or has_ambiguous_time", but factored to only
350 350 # need a single access to flags for performance.
351 351 # `mask` removes all irrelevant bits, then we flip the `mtime` bit so
352 352 # its `true` value is NOT having a mtime, then check if either bit
353 353 # is set.
354 354 or bool((e.v2_data()[0] & mask) ^ has_mtime)
355 355 }
356 356
357 357 copymap = self._map.copymap
358 358 getkind = stat.S_IFMT
359 359 dirkind = stat.S_IFDIR
360 360 regkind = stat.S_IFREG
361 361 lnkkind = stat.S_IFLNK
362 362 join = self._join
363 363 normcase = util.normcase
364 364 fresh_instance = False
365 365
366 366 exact = skipstep3 = False
367 367 if match.isexact(): # match.exact
368 368 exact = True
369 369 dirignore = util.always # skip step 2
370 370 elif match.prefix(): # match.match, no patterns
371 371 skipstep3 = True
372 372
373 373 if not exact and self._checkcase:
374 374 # note that even though we could receive directory entries, we're only
375 375 # interested in checking if a file with the same name exists. So only
376 376 # normalize files if possible.
377 377 normalize = self._normalizefile
378 378 skipstep3 = False
379 379 else:
380 380 normalize = None
381 381
382 382 # step 1: find all explicit files
383 383 results, work, dirsnotfound = self._walkexplicit(match, subrepos)
384 384
385 385 skipstep3 = skipstep3 and not (work or dirsnotfound)
386 386 work = [d for d in work if not dirignore(d[0])]
387 387
388 388 if not work and (exact or skipstep3):
389 389 for s in subrepos:
390 390 del results[s]
391 391 del results[b'.hg']
392 392 return results
393 393
394 394 # step 2: query Watchman
395 395 try:
396 396 # Use the user-configured timeout for the query.
397 397 # Add a little slack over the top of the user query to allow for
398 398 # overheads while transferring the data
399 399 self._watchmanclient.settimeout(state.timeout + 0.1)
400 400 result = self._watchmanclient.command(
401 401 b'query',
402 402 {
403 403 b'fields': [b'mode', b'mtime', b'size', b'exists', b'name'],
404 404 b'since': clock,
405 405 b'expression': [
406 406 b'not',
407 407 [
408 408 b'anyof',
409 409 [b'dirname', b'.hg'],
410 410 [b'name', b'.hg', b'wholename'],
411 411 ],
412 412 ],
413 413 b'sync_timeout': int(state.timeout * 1000),
414 414 b'empty_on_fresh_instance': state.walk_on_invalidate,
415 415 },
416 416 )
417 417 except Exception as ex:
418 418 _handleunavailable(self._ui, state, ex)
419 419 self._watchmanclient.clearconnection()
420 420 return bail(b'exception during run')
421 421 else:
422 422 # We need to propagate the last observed clock up so that we
423 423 # can use it for our next query
424 424 state.setlastclock(pycompat.sysbytes(result[b'clock']))
425 425 if result[b'is_fresh_instance']:
426 426 if state.walk_on_invalidate:
427 427 state.invalidate()
428 428 return bail(b'fresh instance')
429 429 fresh_instance = True
430 430 # Ignore any prior noteable files from the state info
431 431 notefiles = []
432 432
433 433 # for file paths which require normalization and we encounter a case
434 434 # collision, we store our own foldmap
435 435 if normalize:
436 436 foldmap = {normcase(k): k for k in results}
437 437
438 438 switch_slashes = pycompat.ossep == b'\\'
439 439 # The order of the results is, strictly speaking, undefined.
440 440 # For case changes on a case insensitive filesystem we may receive
441 441 # two entries, one with exists=True and another with exists=False.
442 442 # The exists=True entries in the same response should be interpreted
443 443 # as being happens-after the exists=False entries due to the way that
444 444 # Watchman tracks files. We use this property to reconcile deletes
445 445 # for name case changes.
446 446 for entry in result[b'files']:
447 447 fname = entry[b'name']
448 448
449 449 # Watchman always give us a str. Normalize to bytes on Python 3
450 450 # using Watchman's encoding, if needed.
451 451 if not isinstance(fname, bytes):
452 452 fname = fname.encode(_watchmanencoding)
453 453
454 454 if _fixencoding:
455 455 fname = _watchmantofsencoding(fname)
456 456
457 457 if switch_slashes:
458 458 fname = fname.replace(b'\\', b'/')
459 459 if normalize:
460 460 normed = normcase(fname)
461 461 fname = normalize(fname, True, True)
462 462 foldmap[normed] = fname
463 463 fmode = entry[b'mode']
464 464 fexists = entry[b'exists']
465 465 kind = getkind(fmode)
466 466
467 467 if b'/.hg/' in fname or fname.endswith(b'/.hg'):
468 468 return bail(b'nested-repo-detected')
469 469
470 470 if not fexists:
471 471 # if marked as deleted and we don't already have a change
472 472 # record, mark it as deleted. If we already have an entry
473 473 # for fname then it was either part of walkexplicit or was
474 474 # an earlier result that was a case change
475 475 if (
476 476 fname not in results
477 477 and fname in dmap
478 478 and (matchalways or matchfn(fname))
479 479 ):
480 480 results[fname] = None
481 481 elif kind == dirkind:
482 482 if fname in dmap and (matchalways or matchfn(fname)):
483 483 results[fname] = None
484 484 elif kind == regkind or kind == lnkkind:
485 485 if fname in dmap:
486 486 if matchalways or matchfn(fname):
487 487 results[fname] = entry
488 488 elif (matchalways or matchfn(fname)) and not ignore(fname):
489 489 results[fname] = entry
490 490 elif fname in dmap and (matchalways or matchfn(fname)):
491 491 results[fname] = None
492 492
493 493 # step 3: query notable files we don't already know about
494 494 # XXX try not to iterate over the entire dmap
495 495 if normalize:
496 496 # any notable files that have changed case will already be handled
497 497 # above, so just check membership in the foldmap
498 498 notefiles = {
499 499 normalize(f, True, True)
500 500 for f in notefiles
501 501 if normcase(f) not in foldmap
502 502 }
503 503 visit = {
504 504 f
505 505 for f in notefiles
506 506 if (f not in results and matchfn(f) and (f in dmap or not ignore(f)))
507 507 }
508 508
509 509 if not fresh_instance:
510 510 if matchalways:
511 511 visit.update(f for f in nonnormalset if f not in results)
512 512 visit.update(f for f in copymap if f not in results)
513 513 else:
514 514 visit.update(
515 515 f for f in nonnormalset if f not in results and matchfn(f)
516 516 )
517 517 visit.update(f for f in copymap if f not in results and matchfn(f))
518 518 else:
519 519 if matchalways:
520 520 visit.update(f for f, st in dmap.items() if f not in results)
521 521 visit.update(f for f in copymap if f not in results)
522 522 else:
523 523 visit.update(
524 524 f for f, st in dmap.items() if f not in results and matchfn(f)
525 525 )
526 526 visit.update(f for f in copymap if f not in results and matchfn(f))
527 527
528 528 audit = pathutil.pathauditor(self._root, cached=True).check
529 529 auditpass = [f for f in visit if audit(f)]
530 530 auditpass.sort()
531 531 auditfail = visit.difference(auditpass)
532 532 for f in auditfail:
533 533 results[f] = None
534 534
535 535 nf = iter(auditpass)
536 536 for st in util.statfiles([join(f) for f in auditpass]):
537 537 f = next(nf)
538 538 if st or f in dmap:
539 539 results[f] = st
540 540
541 541 for s in subrepos:
542 542 del results[s]
543 543 del results[b'.hg']
544 544 return results
545 545
546 546
547 547 def overridestatus(
548 548 orig,
549 549 self,
550 550 node1=b'.',
551 551 node2=None,
552 552 match=None,
553 553 ignored=False,
554 554 clean=False,
555 555 unknown=False,
556 556 listsubrepos=False,
557 557 ):
558 558 listignored = ignored
559 559 listclean = clean
560 560 listunknown = unknown
561 561
562 562 def _cmpsets(l1, l2):
563 563 try:
564 564 if b'FSMONITOR_LOG_FILE' in encoding.environ:
565 565 fn = encoding.environ[b'FSMONITOR_LOG_FILE']
566 566 f = open(fn, b'wb')
567 567 else:
568 568 fn = b'fsmonitorfail.log'
569 569 f = self.vfs.open(fn, b'wb')
570 570 except (IOError, OSError):
571 571 self.ui.warn(_(b'warning: unable to write to %s\n') % fn)
572 572 return
573 573
574 574 try:
575 575 for i, (s1, s2) in enumerate(zip(l1, l2)):
576 576 if set(s1) != set(s2):
577 577 f.write(b'sets at position %d are unequal\n' % i)
578 578 f.write(b'watchman returned: %r\n' % s1)
579 579 f.write(b'stat returned: %r\n' % s2)
580 580 finally:
581 581 f.close()
582 582
583 583 if isinstance(node1, context.changectx):
584 584 ctx1 = node1
585 585 else:
586 586 ctx1 = self[node1]
587 587 if isinstance(node2, context.changectx):
588 588 ctx2 = node2
589 589 else:
590 590 ctx2 = self[node2]
591 591
592 592 working = ctx2.rev() is None
593 593 parentworking = working and ctx1 == self[b'.']
594 594 match = match or matchmod.always()
595 595
596 596 # Maybe we can use this opportunity to update Watchman's state.
597 597 # Mercurial uses workingcommitctx and/or memctx to represent the part of
598 598 # the workingctx that is to be committed. So don't update the state in
599 599 # that case.
600 600 # HG_PENDING is set in the environment when the dirstate is being updated
601 601 # in the middle of a transaction; we must not update our state in that
602 602 # case, or we risk forgetting about changes in the working copy.
603 603 updatestate = (
604 604 parentworking
605 605 and match.always()
606 606 and not isinstance(ctx2, (context.workingcommitctx, context.memctx))
607 607 and b'HG_PENDING' not in encoding.environ
608 608 )
609 609
610 610 try:
611 611 if self._fsmonitorstate.walk_on_invalidate:
612 612 # Use a short timeout to query the current clock. If that
613 613 # takes too long then we assume that the service will be slow
614 614 # to answer our query.
615 615 # walk_on_invalidate indicates that we prefer to walk the
616 616 # tree ourselves because we can ignore portions that Watchman
617 617 # cannot and we tend to be faster in the warmer buffer cache
618 618 # cases.
619 619 self._watchmanclient.settimeout(0.1)
620 620 else:
621 621 # Give Watchman more time to potentially complete its walk
622 622 # and return the initial clock. In this mode we assume that
623 623 # the filesystem will be slower than parsing a potentially
624 624 # very large Watchman result set.
625 625 self._watchmanclient.settimeout(self._fsmonitorstate.timeout + 0.1)
626 626 startclock = self._watchmanclient.getcurrentclock()
627 627 except Exception as ex:
628 628 self._watchmanclient.clearconnection()
629 629 _handleunavailable(self.ui, self._fsmonitorstate, ex)
630 630 # boo, Watchman failed. bail
631 631 return orig(
632 632 node1,
633 633 node2,
634 634 match,
635 635 listignored,
636 636 listclean,
637 637 listunknown,
638 638 listsubrepos,
639 639 )
640 640
641 641 if updatestate:
642 642 # We need info about unknown files. This may make things slower the
643 643 # first time, but whatever.
644 644 stateunknown = True
645 645 else:
646 646 stateunknown = listunknown
647 647
648 648 if updatestate:
649 649 ps = poststatus(startclock)
650 650 self.addpostdsstatus(ps)
651 651
652 652 r = orig(
653 653 node1, node2, match, listignored, listclean, stateunknown, listsubrepos
654 654 )
655 655 modified, added, removed, deleted, unknown, ignored, clean = r
656 656
657 657 if not listunknown:
658 658 unknown = []
659 659
660 660 # don't do paranoid checks if we're not going to query Watchman anyway
661 661 full = listclean or match.traversedir is not None
662 662 if self._fsmonitorstate.mode == b'paranoid' and not full:
663 663 # run status again and fall back to the old walk this time
664 664 self.dirstate._fsmonitordisable = True
665 665
666 666 # shut the UI up
667 667 quiet = self.ui.quiet
668 668 self.ui.quiet = True
669 669 fout, ferr = self.ui.fout, self.ui.ferr
670 670 self.ui.fout = self.ui.ferr = open(os.devnull, b'wb')
671 671
672 672 try:
673 673 rv2 = orig(
674 674 node1,
675 675 node2,
676 676 match,
677 677 listignored,
678 678 listclean,
679 679 listunknown,
680 680 listsubrepos,
681 681 )
682 682 finally:
683 683 self.dirstate._fsmonitordisable = False
684 684 self.ui.quiet = quiet
685 685 self.ui.fout, self.ui.ferr = fout, ferr
686 686
687 687 # clean isn't tested since it's set to True above
688 688 with self.wlock():
689 689 _cmpsets(
690 690 [modified, added, removed, deleted, unknown, ignored, clean],
691 691 rv2,
692 692 )
693 693 modified, added, removed, deleted, unknown, ignored, clean = rv2
694 694
695 695 return scmutil.status(
696 696 modified, added, removed, deleted, unknown, ignored, clean
697 697 )
698 698
699 699
700 700 class poststatus:
701 701 def __init__(self, startclock):
702 702 self._startclock = pycompat.sysbytes(startclock)
703 703
704 704 def __call__(self, wctx, status):
705 705 clock = wctx.repo()._fsmonitorstate.getlastclock() or self._startclock
706 706 hashignore = _hashignore(wctx.repo().dirstate._ignore)
707 707 notefiles = (
708 708 status.modified
709 709 + status.added
710 710 + status.removed
711 711 + status.deleted
712 712 + status.unknown
713 713 )
714 714 wctx.repo()._fsmonitorstate.set(clock, hashignore, notefiles)
715 715
716 716
717 717 def makedirstate(repo, dirstate):
718 718 class fsmonitordirstate(dirstate.__class__):
719 719 def _fsmonitorinit(self, repo):
720 720 # _fsmonitordisable is used in paranoid mode
721 721 self._fsmonitordisable = False
722 722 self._fsmonitorstate = repo._fsmonitorstate
723 723 self._watchmanclient = repo._watchmanclient
724 724 self._repo = weakref.proxy(repo)
725 725
726 726 def walk(self, *args, **kwargs):
727 727 orig = super(fsmonitordirstate, self).walk
728 728 if self._fsmonitordisable:
729 729 return orig(*args, **kwargs)
730 730 return overridewalk(orig, self, *args, **kwargs)
731 731
732 732 def rebuild(self, *args, **kwargs):
733 733 self._fsmonitorstate.invalidate()
734 734 return super(fsmonitordirstate, self).rebuild(*args, **kwargs)
735 735
736 736 def invalidate(self, *args, **kwargs):
737 737 self._fsmonitorstate.invalidate()
738 738 return super(fsmonitordirstate, self).invalidate(*args, **kwargs)
739 739
740 740 dirstate.__class__ = fsmonitordirstate
741 741 dirstate._fsmonitorinit(repo)
742 742
743 743
744 744 def wrapdirstate(orig, self):
745 745 ds = orig(self)
746 746 # only override the dirstate when Watchman is available for the repo
747 747 if util.safehasattr(self, b'_fsmonitorstate'):
748 748 makedirstate(self, ds)
749 749 return ds
750 750
751 751
752 752 def extsetup(ui):
753 753 extensions.wrapfilecache(
754 754 localrepo.localrepository, b'dirstate', wrapdirstate
755 755 )
756 756 if pycompat.isdarwin:
757 757 # An assist for avoiding the dangling-symlink fsevents bug
758 extensions.wrapfunction(os, b'symlink', wrapsymlink)
758 extensions.wrapfunction(os, 'symlink', wrapsymlink)
759 759
760 extensions.wrapfunction(merge, b'_update', wrapupdate)
760 extensions.wrapfunction(merge, '_update', wrapupdate)
761 761
762 762
763 763 def wrapsymlink(orig, source, link_name):
764 764 """if we create a dangling symlink, also touch the parent dir
765 765 to encourage fsevents notifications to work more correctly"""
766 766 try:
767 767 return orig(source, link_name)
768 768 finally:
769 769 try:
770 770 os.utime(os.path.dirname(link_name), None)
771 771 except OSError:
772 772 pass
773 773
774 774
775 775 class state_update:
776 776 """This context manager is responsible for dispatching the state-enter
777 777 and state-leave signals to the watchman service. The enter and leave
778 778 methods can be invoked manually (for scenarios where context manager
779 779 semantics are not possible). If parameters oldnode and newnode are None,
780 780 they will be populated based on current working copy in enter and
781 781 leave, respectively. Similarly, if the distance is none, it will be
782 782 calculated based on the oldnode and newnode in the leave method."""
783 783
784 784 def __init__(
785 785 self,
786 786 repo,
787 787 name,
788 788 oldnode=None,
789 789 newnode=None,
790 790 distance=None,
791 791 partial=False,
792 792 ):
793 793 self.repo = repo.unfiltered()
794 794 self.name = name
795 795 self.oldnode = oldnode
796 796 self.newnode = newnode
797 797 self.distance = distance
798 798 self.partial = partial
799 799 self._lock = None
800 800 self.need_leave = False
801 801
802 802 def __enter__(self):
803 803 self.enter()
804 804
805 805 def enter(self):
806 806 # Make sure we have a wlock prior to sending notifications to watchman.
807 807 # We don't want to race with other actors. In the update case,
808 808 # merge.update is going to take the wlock almost immediately. We are
809 809 # effectively extending the lock around several short sanity checks.
810 810 if self.oldnode is None:
811 811 self.oldnode = self.repo[b'.'].node()
812 812
813 813 if self.repo.currentwlock() is None:
814 814 if util.safehasattr(self.repo, b'wlocknostateupdate'):
815 815 self._lock = self.repo.wlocknostateupdate()
816 816 else:
817 817 self._lock = self.repo.wlock()
818 818 self.need_leave = self._state(b'state-enter', hex(self.oldnode))
819 819 return self
820 820
821 821 def __exit__(self, type_, value, tb):
822 822 abort = True if type_ else False
823 823 self.exit(abort=abort)
824 824
825 825 def exit(self, abort=False):
826 826 try:
827 827 if self.need_leave:
828 828 status = b'failed' if abort else b'ok'
829 829 if self.newnode is None:
830 830 self.newnode = self.repo[b'.'].node()
831 831 if self.distance is None:
832 832 self.distance = calcdistance(
833 833 self.repo, self.oldnode, self.newnode
834 834 )
835 835 self._state(b'state-leave', hex(self.newnode), status=status)
836 836 finally:
837 837 self.need_leave = False
838 838 if self._lock:
839 839 self._lock.release()
840 840
841 841 def _state(self, cmd, commithash, status=b'ok'):
842 842 if not util.safehasattr(self.repo, b'_watchmanclient'):
843 843 return False
844 844 try:
845 845 self.repo._watchmanclient.command(
846 846 cmd,
847 847 {
848 848 b'name': self.name,
849 849 b'metadata': {
850 850 # the target revision
851 851 b'rev': commithash,
852 852 # approximate number of commits between current and target
853 853 b'distance': self.distance if self.distance else 0,
854 854 # success/failure (only really meaningful for state-leave)
855 855 b'status': status,
856 856 # whether the working copy parent is changing
857 857 b'partial': self.partial,
858 858 },
859 859 },
860 860 )
861 861 return True
862 862 except Exception as e:
863 863 # Swallow any errors; fire and forget
864 864 self.repo.ui.log(
865 865 b'watchman', b'Exception %s while running %s\n', e, cmd
866 866 )
867 867 return False
868 868
869 869
870 870 # Estimate the distance between two nodes
871 871 def calcdistance(repo, oldnode, newnode):
872 872 anc = repo.changelog.ancestor(oldnode, newnode)
873 873 ancrev = repo[anc].rev()
874 874 distance = abs(repo[oldnode].rev() - ancrev) + abs(
875 875 repo[newnode].rev() - ancrev
876 876 )
877 877 return distance
878 878
879 879
880 880 # Bracket working copy updates with calls to the watchman state-enter
881 881 # and state-leave commands. This allows clients to perform more intelligent
882 882 # settling during bulk file change scenarios
883 883 # https://facebook.github.io/watchman/docs/cmd/subscribe.html#advanced-settling
884 884 def wrapupdate(
885 885 orig,
886 886 repo,
887 887 node,
888 888 branchmerge,
889 889 force,
890 890 ancestor=None,
891 891 mergeancestor=False,
892 892 labels=None,
893 893 matcher=None,
894 894 **kwargs
895 895 ):
896 896
897 897 distance = 0
898 898 partial = True
899 899 oldnode = repo[b'.'].node()
900 900 newnode = repo[node].node()
901 901 if matcher is None or matcher.always():
902 902 partial = False
903 903 distance = calcdistance(repo.unfiltered(), oldnode, newnode)
904 904
905 905 with state_update(
906 906 repo,
907 907 name=b"hg.update",
908 908 oldnode=oldnode,
909 909 newnode=newnode,
910 910 distance=distance,
911 911 partial=partial,
912 912 ):
913 913 return orig(
914 914 repo,
915 915 node,
916 916 branchmerge,
917 917 force,
918 918 ancestor,
919 919 mergeancestor,
920 920 labels,
921 921 matcher,
922 922 **kwargs
923 923 )
924 924
925 925
926 926 def repo_has_depth_one_nested_repo(repo):
927 927 for f in repo.wvfs.listdir():
928 928 if os.path.isdir(os.path.join(repo.root, f, b'.hg')):
929 929 msg = b'fsmonitor: sub-repository %r detected, fsmonitor disabled\n'
930 930 repo.ui.debug(msg % f)
931 931 return True
932 932 return False
933 933
934 934
935 935 def reposetup(ui, repo):
936 936 # We don't work with largefiles or inotify
937 937 exts = extensions.enabled()
938 938 for ext in _blacklist:
939 939 if ext in exts:
940 940 ui.warn(
941 941 _(
942 942 b'The fsmonitor extension is incompatible with the %s '
943 943 b'extension and has been disabled.\n'
944 944 )
945 945 % ext
946 946 )
947 947 return
948 948
949 949 if repo.local():
950 950 # We don't work with subrepos either.
951 951 #
952 952 # if repo[None].substate can cause a dirstate parse, which is too
953 953 # slow. Instead, look for a file called hgsubstate,
954 954 if repo.wvfs.exists(b'.hgsubstate') or repo.wvfs.exists(b'.hgsub'):
955 955 return
956 956
957 957 if repo_has_depth_one_nested_repo(repo):
958 958 return
959 959
960 960 fsmonitorstate = state.state(repo)
961 961 if fsmonitorstate.mode == b'off':
962 962 return
963 963
964 964 try:
965 965 client = watchmanclient.client(repo.ui, repo.root)
966 966 except Exception as ex:
967 967 _handleunavailable(ui, fsmonitorstate, ex)
968 968 return
969 969
970 970 repo._fsmonitorstate = fsmonitorstate
971 971 repo._watchmanclient = client
972 972
973 973 dirstate, cached = localrepo.isfilecached(repo, b'dirstate')
974 974 if cached:
975 975 # at this point since fsmonitorstate wasn't present,
976 976 # repo.dirstate is not a fsmonitordirstate
977 977 makedirstate(repo, dirstate)
978 978
979 979 class fsmonitorrepo(repo.__class__):
980 980 def status(self, *args, **kwargs):
981 981 orig = super(fsmonitorrepo, self).status
982 982 return overridestatus(orig, self, *args, **kwargs)
983 983
984 984 def wlocknostateupdate(self, *args, **kwargs):
985 985 return super(fsmonitorrepo, self).wlock(*args, **kwargs)
986 986
987 987 def wlock(self, *args, **kwargs):
988 988 l = super(fsmonitorrepo, self).wlock(*args, **kwargs)
989 989 if not ui.configbool(
990 990 b"experimental", b"fsmonitor.transaction_notify"
991 991 ):
992 992 return l
993 993 if l.held != 1:
994 994 return l
995 995 origrelease = l.releasefn
996 996
997 997 def staterelease():
998 998 if origrelease:
999 999 origrelease()
1000 1000 if l.stateupdate:
1001 1001 l.stateupdate.exit()
1002 1002 l.stateupdate = None
1003 1003
1004 1004 try:
1005 1005 l.stateupdate = None
1006 1006 l.stateupdate = state_update(self, name=b"hg.transaction")
1007 1007 l.stateupdate.enter()
1008 1008 l.releasefn = staterelease
1009 1009 except Exception as e:
1010 1010 # Swallow any errors; fire and forget
1011 1011 self.ui.log(
1012 1012 b'watchman', b'Exception in state update %s\n', e
1013 1013 )
1014 1014 return l
1015 1015
1016 1016 repo.__class__ = fsmonitorrepo
General Comments 0
You need to be logged in to leave comments. Login now