##// END OF EJS Templates
fsmonitor: normalize clock value to bytes...
Gregory Szorc -
r43715:09ab61c0 stable
parent child Browse files
Show More
@@ -1,988 +1,988 b''
1 1 # __init__.py - fsmonitor initialization and overrides
2 2 #
3 3 # Copyright 2013-2016 Facebook, Inc.
4 4 #
5 5 # This software may be used and distributed according to the terms of the
6 6 # GNU General Public License version 2 or any later version.
7 7
8 8 '''Faster status operations with the Watchman file monitor (EXPERIMENTAL)
9 9
10 10 Integrates the file-watching program Watchman with Mercurial to produce faster
11 11 status results.
12 12
13 13 On a particular Linux system, for a real-world repository with over 400,000
14 14 files hosted on ext4, vanilla `hg status` takes 1.3 seconds. On the same
15 15 system, with fsmonitor it takes about 0.3 seconds.
16 16
17 17 fsmonitor requires no configuration -- it will tell Watchman about your
18 18 repository as necessary. You'll need to install Watchman from
19 19 https://facebook.github.io/watchman/ and make sure it is in your PATH.
20 20
21 21 fsmonitor is incompatible with the largefiles and eol extensions, and
22 22 will disable itself if any of those are active.
23 23
24 24 The following configuration options exist:
25 25
26 26 ::
27 27
28 28 [fsmonitor]
29 29 mode = {off, on, paranoid}
30 30
31 31 When `mode = off`, fsmonitor will disable itself (similar to not loading the
32 32 extension at all). When `mode = on`, fsmonitor will be enabled (the default).
33 33 When `mode = paranoid`, fsmonitor will query both Watchman and the filesystem,
34 34 and ensure that the results are consistent.
35 35
36 36 ::
37 37
38 38 [fsmonitor]
39 39 timeout = (float)
40 40
41 41 A value, in seconds, that determines how long fsmonitor will wait for Watchman
42 42 to return results. Defaults to `2.0`.
43 43
44 44 ::
45 45
46 46 [fsmonitor]
47 47 blacklistusers = (list of userids)
48 48
49 49 A list of usernames for which fsmonitor will disable itself altogether.
50 50
51 51 ::
52 52
53 53 [fsmonitor]
54 54 walk_on_invalidate = (boolean)
55 55
56 56 Whether or not to walk the whole repo ourselves when our cached state has been
57 57 invalidated, for example when Watchman has been restarted or .hgignore rules
58 58 have been changed. Walking the repo in that case can result in competing for
59 59 I/O with Watchman. For large repos it is recommended to set this value to
60 60 false. You may wish to set this to true if you have a very fast filesystem
61 61 that can outpace the IPC overhead of getting the result data for the full repo
62 62 from Watchman. Defaults to false.
63 63
64 64 ::
65 65
66 66 [fsmonitor]
67 67 warn_when_unused = (boolean)
68 68
69 69 Whether to print a warning during certain operations when fsmonitor would be
70 70 beneficial to performance but isn't enabled.
71 71
72 72 ::
73 73
74 74 [fsmonitor]
75 75 warn_update_file_count = (integer)
76 76
77 77 If ``warn_when_unused`` is set and fsmonitor isn't enabled, a warning will
78 78 be printed during working directory updates if this many files will be
79 79 created.
80 80 '''
81 81
82 82 # Platforms Supported
83 83 # ===================
84 84 #
85 85 # **Linux:** *Stable*. Watchman and fsmonitor are both known to work reliably,
86 86 # even under severe loads.
87 87 #
88 88 # **Mac OS X:** *Stable*. The Mercurial test suite passes with fsmonitor
89 89 # turned on, on case-insensitive HFS+. There has been a reasonable amount of
90 90 # user testing under normal loads.
91 91 #
92 92 # **Solaris, BSD:** *Alpha*. watchman and fsmonitor are believed to work, but
93 93 # very little testing has been done.
94 94 #
95 95 # **Windows:** *Alpha*. Not in a release version of watchman or fsmonitor yet.
96 96 #
97 97 # Known Issues
98 98 # ============
99 99 #
100 100 # * fsmonitor will disable itself if any of the following extensions are
101 101 # enabled: largefiles, inotify, eol; or if the repository has subrepos.
102 102 # * fsmonitor will produce incorrect results if nested repos that are not
103 103 # subrepos exist. *Workaround*: add nested repo paths to your `.hgignore`.
104 104 #
105 105 # The issues related to nested repos and subrepos are probably not fundamental
106 106 # ones. Patches to fix them are welcome.
107 107
108 108 from __future__ import absolute_import
109 109
110 110 import codecs
111 111 import hashlib
112 112 import os
113 113 import stat
114 114 import sys
115 115 import tempfile
116 116 import weakref
117 117
118 118 from mercurial.i18n import _
119 119 from mercurial.node import hex
120 120 from mercurial.pycompat import open
121 121 from mercurial import (
122 122 context,
123 123 encoding,
124 124 error,
125 125 extensions,
126 126 localrepo,
127 127 merge,
128 128 pathutil,
129 129 pycompat,
130 130 registrar,
131 131 scmutil,
132 132 util,
133 133 )
134 134 from mercurial import match as matchmod
135 135 from mercurial.utils import stringutil
136 136
137 137 from . import (
138 138 pywatchman,
139 139 state,
140 140 watchmanclient,
141 141 )
142 142
143 143 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
144 144 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
145 145 # be specifying the version(s) of Mercurial they are tested with, or
146 146 # leave the attribute unspecified.
147 147 testedwith = b'ships-with-hg-core'
148 148
149 149 configtable = {}
150 150 configitem = registrar.configitem(configtable)
151 151
152 152 configitem(
153 153 b'fsmonitor', b'mode', default=b'on',
154 154 )
155 155 configitem(
156 156 b'fsmonitor', b'walk_on_invalidate', default=False,
157 157 )
158 158 configitem(
159 159 b'fsmonitor', b'timeout', default=b'2',
160 160 )
161 161 configitem(
162 162 b'fsmonitor', b'blacklistusers', default=list,
163 163 )
164 164 configitem(
165 165 b'fsmonitor', b'watchman_exe', default=b'watchman',
166 166 )
167 167 configitem(
168 168 b'fsmonitor', b'verbose', default=True, experimental=True,
169 169 )
170 170 configitem(
171 171 b'experimental', b'fsmonitor.transaction_notify', default=False,
172 172 )
173 173
174 174 # This extension is incompatible with the following blacklisted extensions
175 175 # and will disable itself when encountering one of these:
176 176 _blacklist = [b'largefiles', b'eol']
177 177
178 178
179 179 def debuginstall(ui, fm):
180 180 fm.write(
181 181 b"fsmonitor-watchman",
182 182 _(b"fsmonitor checking for watchman binary... (%s)\n"),
183 183 ui.configpath(b"fsmonitor", b"watchman_exe"),
184 184 )
185 185 root = tempfile.mkdtemp()
186 186 c = watchmanclient.client(ui, root)
187 187 err = None
188 188 try:
189 189 v = c.command(b"version")
190 190 fm.write(
191 191 b"fsmonitor-watchman-version",
192 192 _(b" watchman binary version %s\n"),
193 193 pycompat.bytestr(v["version"]),
194 194 )
195 195 except watchmanclient.Unavailable as e:
196 196 err = str(e)
197 197 fm.condwrite(
198 198 err,
199 199 b"fsmonitor-watchman-error",
200 200 _(b" watchman binary missing or broken: %s\n"),
201 201 err,
202 202 )
203 203 return 1 if err else 0
204 204
205 205
206 206 def _handleunavailable(ui, state, ex):
207 207 """Exception handler for Watchman interaction exceptions"""
208 208 if isinstance(ex, watchmanclient.Unavailable):
209 209 # experimental config: fsmonitor.verbose
210 210 if ex.warn and ui.configbool(b'fsmonitor', b'verbose'):
211 211 if b'illegal_fstypes' not in stringutil.forcebytestr(ex):
212 212 ui.warn(stringutil.forcebytestr(ex) + b'\n')
213 213 if ex.invalidate:
214 214 state.invalidate()
215 215 # experimental config: fsmonitor.verbose
216 216 if ui.configbool(b'fsmonitor', b'verbose'):
217 217 ui.log(
218 218 b'fsmonitor',
219 219 b'Watchman unavailable: %s\n',
220 220 stringutil.forcebytestr(ex.msg),
221 221 )
222 222 else:
223 223 ui.log(
224 224 b'fsmonitor',
225 225 b'Watchman exception: %s\n',
226 226 stringutil.forcebytestr(ex),
227 227 )
228 228
229 229
230 230 def _hashignore(ignore):
231 231 """Calculate hash for ignore patterns and filenames
232 232
233 233 If this information changes between Mercurial invocations, we can't
234 234 rely on Watchman information anymore and have to re-scan the working
235 235 copy.
236 236
237 237 """
238 238 sha1 = hashlib.sha1()
239 239 sha1.update(pycompat.byterepr(ignore))
240 240 return pycompat.sysbytes(sha1.hexdigest())
241 241
242 242
243 243 _watchmanencoding = pywatchman.encoding.get_local_encoding()
244 244 _fsencoding = sys.getfilesystemencoding() or sys.getdefaultencoding()
245 245 _fixencoding = codecs.lookup(_watchmanencoding) != codecs.lookup(_fsencoding)
246 246
247 247
248 248 def _watchmantofsencoding(path):
249 249 """Fix path to match watchman and local filesystem encoding
250 250
251 251 watchman's paths encoding can differ from filesystem encoding. For example,
252 252 on Windows, it's always utf-8.
253 253 """
254 254 try:
255 255 decoded = path.decode(_watchmanencoding)
256 256 except UnicodeDecodeError as e:
257 257 raise error.Abort(
258 258 stringutil.forcebytestr(e), hint=b'watchman encoding error'
259 259 )
260 260
261 261 try:
262 262 encoded = decoded.encode(_fsencoding, 'strict')
263 263 except UnicodeEncodeError as e:
264 264 raise error.Abort(stringutil.forcebytestr(e))
265 265
266 266 return encoded
267 267
268 268
269 269 def overridewalk(orig, self, match, subrepos, unknown, ignored, full=True):
270 270 '''Replacement for dirstate.walk, hooking into Watchman.
271 271
272 272 Whenever full is False, ignored is False, and the Watchman client is
273 273 available, use Watchman combined with saved state to possibly return only a
274 274 subset of files.'''
275 275
276 276 def bail(reason):
277 277 self._ui.debug(b'fsmonitor: fallback to core status, %s\n' % reason)
278 278 return orig(match, subrepos, unknown, ignored, full=True)
279 279
280 280 if full:
281 281 return bail(b'full rewalk requested')
282 282 if ignored:
283 283 return bail(b'listing ignored files')
284 284 if not self._watchmanclient.available():
285 285 return bail(b'client unavailable')
286 286 state = self._fsmonitorstate
287 287 clock, ignorehash, notefiles = state.get()
288 288 if not clock:
289 289 if state.walk_on_invalidate:
290 290 return bail(b'no clock')
291 291 # Initial NULL clock value, see
292 292 # https://facebook.github.io/watchman/docs/clockspec.html
293 293 clock = b'c:0:0'
294 294 notefiles = []
295 295
296 296 ignore = self._ignore
297 297 dirignore = self._dirignore
298 298 if unknown:
299 299 if _hashignore(ignore) != ignorehash and clock != b'c:0:0':
300 300 # ignore list changed -- can't rely on Watchman state any more
301 301 if state.walk_on_invalidate:
302 302 return bail(b'ignore rules changed')
303 303 notefiles = []
304 304 clock = b'c:0:0'
305 305 else:
306 306 # always ignore
307 307 ignore = util.always
308 308 dirignore = util.always
309 309
310 310 matchfn = match.matchfn
311 311 matchalways = match.always()
312 312 dmap = self._map
313 313 if util.safehasattr(dmap, b'_map'):
314 314 # for better performance, directly access the inner dirstate map if the
315 315 # standard dirstate implementation is in use.
316 316 dmap = dmap._map
317 317 nonnormalset = self._map.nonnormalset
318 318
319 319 copymap = self._map.copymap
320 320 getkind = stat.S_IFMT
321 321 dirkind = stat.S_IFDIR
322 322 regkind = stat.S_IFREG
323 323 lnkkind = stat.S_IFLNK
324 324 join = self._join
325 325 normcase = util.normcase
326 326 fresh_instance = False
327 327
328 328 exact = skipstep3 = False
329 329 if match.isexact(): # match.exact
330 330 exact = True
331 331 dirignore = util.always # skip step 2
332 332 elif match.prefix(): # match.match, no patterns
333 333 skipstep3 = True
334 334
335 335 if not exact and self._checkcase:
336 336 # note that even though we could receive directory entries, we're only
337 337 # interested in checking if a file with the same name exists. So only
338 338 # normalize files if possible.
339 339 normalize = self._normalizefile
340 340 skipstep3 = False
341 341 else:
342 342 normalize = None
343 343
344 344 # step 1: find all explicit files
345 345 results, work, dirsnotfound = self._walkexplicit(match, subrepos)
346 346
347 347 skipstep3 = skipstep3 and not (work or dirsnotfound)
348 348 work = [d for d in work if not dirignore(d[0])]
349 349
350 350 if not work and (exact or skipstep3):
351 351 for s in subrepos:
352 352 del results[s]
353 353 del results[b'.hg']
354 354 return results
355 355
356 356 # step 2: query Watchman
357 357 try:
358 358 # Use the user-configured timeout for the query.
359 359 # Add a little slack over the top of the user query to allow for
360 360 # overheads while transferring the data
361 361 self._watchmanclient.settimeout(state.timeout + 0.1)
362 362 result = self._watchmanclient.command(
363 363 b'query',
364 364 {
365 365 b'fields': [b'mode', b'mtime', b'size', b'exists', b'name'],
366 366 b'since': clock,
367 367 b'expression': [
368 368 b'not',
369 369 [
370 370 b'anyof',
371 371 [b'dirname', b'.hg'],
372 372 [b'name', b'.hg', b'wholename'],
373 373 ],
374 374 ],
375 375 b'sync_timeout': int(state.timeout * 1000),
376 376 b'empty_on_fresh_instance': state.walk_on_invalidate,
377 377 },
378 378 )
379 379 except Exception as ex:
380 380 _handleunavailable(self._ui, state, ex)
381 381 self._watchmanclient.clearconnection()
382 382 return bail(b'exception during run')
383 383 else:
384 384 # We need to propagate the last observed clock up so that we
385 385 # can use it for our next query
386 state.setlastclock(result[b'clock'])
386 state.setlastclock(pycompat.sysbytes(result[b'clock']))
387 387 if result[b'is_fresh_instance']:
388 388 if state.walk_on_invalidate:
389 389 state.invalidate()
390 390 return bail(b'fresh instance')
391 391 fresh_instance = True
392 392 # Ignore any prior noteable files from the state info
393 393 notefiles = []
394 394
395 395 # for file paths which require normalization and we encounter a case
396 396 # collision, we store our own foldmap
397 397 if normalize:
398 398 foldmap = dict((normcase(k), k) for k in results)
399 399
400 400 switch_slashes = pycompat.ossep == b'\\'
401 401 # The order of the results is, strictly speaking, undefined.
402 402 # For case changes on a case insensitive filesystem we may receive
403 403 # two entries, one with exists=True and another with exists=False.
404 404 # The exists=True entries in the same response should be interpreted
405 405 # as being happens-after the exists=False entries due to the way that
406 406 # Watchman tracks files. We use this property to reconcile deletes
407 407 # for name case changes.
408 408 for entry in result[b'files']:
409 409 fname = entry[b'name']
410 410
411 411 # Watchman always give us a str. Normalize to bytes on Python 3
412 412 # using Watchman's encoding, if needed.
413 413 if not isinstance(fname, bytes):
414 414 fname = fname.encode(_watchmanencoding)
415 415
416 416 if _fixencoding:
417 417 fname = _watchmantofsencoding(fname)
418 418
419 419 if switch_slashes:
420 420 fname = fname.replace(b'\\', b'/')
421 421 if normalize:
422 422 normed = normcase(fname)
423 423 fname = normalize(fname, True, True)
424 424 foldmap[normed] = fname
425 425 fmode = entry[b'mode']
426 426 fexists = entry[b'exists']
427 427 kind = getkind(fmode)
428 428
429 429 if b'/.hg/' in fname or fname.endswith(b'/.hg'):
430 430 return bail(b'nested-repo-detected')
431 431
432 432 if not fexists:
433 433 # if marked as deleted and we don't already have a change
434 434 # record, mark it as deleted. If we already have an entry
435 435 # for fname then it was either part of walkexplicit or was
436 436 # an earlier result that was a case change
437 437 if (
438 438 fname not in results
439 439 and fname in dmap
440 440 and (matchalways or matchfn(fname))
441 441 ):
442 442 results[fname] = None
443 443 elif kind == dirkind:
444 444 if fname in dmap and (matchalways or matchfn(fname)):
445 445 results[fname] = None
446 446 elif kind == regkind or kind == lnkkind:
447 447 if fname in dmap:
448 448 if matchalways or matchfn(fname):
449 449 results[fname] = entry
450 450 elif (matchalways or matchfn(fname)) and not ignore(fname):
451 451 results[fname] = entry
452 452 elif fname in dmap and (matchalways or matchfn(fname)):
453 453 results[fname] = None
454 454
455 455 # step 3: query notable files we don't already know about
456 456 # XXX try not to iterate over the entire dmap
457 457 if normalize:
458 458 # any notable files that have changed case will already be handled
459 459 # above, so just check membership in the foldmap
460 460 notefiles = set(
461 461 (
462 462 normalize(f, True, True)
463 463 for f in notefiles
464 464 if normcase(f) not in foldmap
465 465 )
466 466 )
467 467 visit = set(
468 468 (
469 469 f
470 470 for f in notefiles
471 471 if (
472 472 f not in results and matchfn(f) and (f in dmap or not ignore(f))
473 473 )
474 474 )
475 475 )
476 476
477 477 if not fresh_instance:
478 478 if matchalways:
479 479 visit.update(f for f in nonnormalset if f not in results)
480 480 visit.update(f for f in copymap if f not in results)
481 481 else:
482 482 visit.update(
483 483 f for f in nonnormalset if f not in results and matchfn(f)
484 484 )
485 485 visit.update(f for f in copymap if f not in results and matchfn(f))
486 486 else:
487 487 if matchalways:
488 488 visit.update(
489 489 f for f, st in pycompat.iteritems(dmap) if f not in results
490 490 )
491 491 visit.update(f for f in copymap if f not in results)
492 492 else:
493 493 visit.update(
494 494 f
495 495 for f, st in pycompat.iteritems(dmap)
496 496 if f not in results and matchfn(f)
497 497 )
498 498 visit.update(f for f in copymap if f not in results and matchfn(f))
499 499
500 500 audit = pathutil.pathauditor(self._root, cached=True).check
501 501 auditpass = [f for f in visit if audit(f)]
502 502 auditpass.sort()
503 503 auditfail = visit.difference(auditpass)
504 504 for f in auditfail:
505 505 results[f] = None
506 506
507 507 nf = iter(auditpass)
508 508 for st in util.statfiles([join(f) for f in auditpass]):
509 509 f = next(nf)
510 510 if st or f in dmap:
511 511 results[f] = st
512 512
513 513 for s in subrepos:
514 514 del results[s]
515 515 del results[b'.hg']
516 516 return results
517 517
518 518
519 519 def overridestatus(
520 520 orig,
521 521 self,
522 522 node1=b'.',
523 523 node2=None,
524 524 match=None,
525 525 ignored=False,
526 526 clean=False,
527 527 unknown=False,
528 528 listsubrepos=False,
529 529 ):
530 530 listignored = ignored
531 531 listclean = clean
532 532 listunknown = unknown
533 533
534 534 def _cmpsets(l1, l2):
535 535 try:
536 536 if b'FSMONITOR_LOG_FILE' in encoding.environ:
537 537 fn = encoding.environ[b'FSMONITOR_LOG_FILE']
538 538 f = open(fn, b'wb')
539 539 else:
540 540 fn = b'fsmonitorfail.log'
541 541 f = self.vfs.open(fn, b'wb')
542 542 except (IOError, OSError):
543 543 self.ui.warn(_(b'warning: unable to write to %s\n') % fn)
544 544 return
545 545
546 546 try:
547 547 for i, (s1, s2) in enumerate(zip(l1, l2)):
548 548 if set(s1) != set(s2):
549 549 f.write(b'sets at position %d are unequal\n' % i)
550 550 f.write(b'watchman returned: %s\n' % s1)
551 551 f.write(b'stat returned: %s\n' % s2)
552 552 finally:
553 553 f.close()
554 554
555 555 if isinstance(node1, context.changectx):
556 556 ctx1 = node1
557 557 else:
558 558 ctx1 = self[node1]
559 559 if isinstance(node2, context.changectx):
560 560 ctx2 = node2
561 561 else:
562 562 ctx2 = self[node2]
563 563
564 564 working = ctx2.rev() is None
565 565 parentworking = working and ctx1 == self[b'.']
566 566 match = match or matchmod.always()
567 567
568 568 # Maybe we can use this opportunity to update Watchman's state.
569 569 # Mercurial uses workingcommitctx and/or memctx to represent the part of
570 570 # the workingctx that is to be committed. So don't update the state in
571 571 # that case.
572 572 # HG_PENDING is set in the environment when the dirstate is being updated
573 573 # in the middle of a transaction; we must not update our state in that
574 574 # case, or we risk forgetting about changes in the working copy.
575 575 updatestate = (
576 576 parentworking
577 577 and match.always()
578 578 and not isinstance(ctx2, (context.workingcommitctx, context.memctx))
579 579 and b'HG_PENDING' not in encoding.environ
580 580 )
581 581
582 582 try:
583 583 if self._fsmonitorstate.walk_on_invalidate:
584 584 # Use a short timeout to query the current clock. If that
585 585 # takes too long then we assume that the service will be slow
586 586 # to answer our query.
587 587 # walk_on_invalidate indicates that we prefer to walk the
588 588 # tree ourselves because we can ignore portions that Watchman
589 589 # cannot and we tend to be faster in the warmer buffer cache
590 590 # cases.
591 591 self._watchmanclient.settimeout(0.1)
592 592 else:
593 593 # Give Watchman more time to potentially complete its walk
594 594 # and return the initial clock. In this mode we assume that
595 595 # the filesystem will be slower than parsing a potentially
596 596 # very large Watchman result set.
597 597 self._watchmanclient.settimeout(self._fsmonitorstate.timeout + 0.1)
598 598 startclock = self._watchmanclient.getcurrentclock()
599 599 except Exception as ex:
600 600 self._watchmanclient.clearconnection()
601 601 _handleunavailable(self.ui, self._fsmonitorstate, ex)
602 602 # boo, Watchman failed. bail
603 603 return orig(
604 604 node1,
605 605 node2,
606 606 match,
607 607 listignored,
608 608 listclean,
609 609 listunknown,
610 610 listsubrepos,
611 611 )
612 612
613 613 if updatestate:
614 614 # We need info about unknown files. This may make things slower the
615 615 # first time, but whatever.
616 616 stateunknown = True
617 617 else:
618 618 stateunknown = listunknown
619 619
620 620 if updatestate:
621 621 ps = poststatus(startclock)
622 622 self.addpostdsstatus(ps)
623 623
624 624 r = orig(
625 625 node1, node2, match, listignored, listclean, stateunknown, listsubrepos
626 626 )
627 627 modified, added, removed, deleted, unknown, ignored, clean = r
628 628
629 629 if not listunknown:
630 630 unknown = []
631 631
632 632 # don't do paranoid checks if we're not going to query Watchman anyway
633 633 full = listclean or match.traversedir is not None
634 634 if self._fsmonitorstate.mode == b'paranoid' and not full:
635 635 # run status again and fall back to the old walk this time
636 636 self.dirstate._fsmonitordisable = True
637 637
638 638 # shut the UI up
639 639 quiet = self.ui.quiet
640 640 self.ui.quiet = True
641 641 fout, ferr = self.ui.fout, self.ui.ferr
642 642 self.ui.fout = self.ui.ferr = open(os.devnull, b'wb')
643 643
644 644 try:
645 645 rv2 = orig(
646 646 node1,
647 647 node2,
648 648 match,
649 649 listignored,
650 650 listclean,
651 651 listunknown,
652 652 listsubrepos,
653 653 )
654 654 finally:
655 655 self.dirstate._fsmonitordisable = False
656 656 self.ui.quiet = quiet
657 657 self.ui.fout, self.ui.ferr = fout, ferr
658 658
659 659 # clean isn't tested since it's set to True above
660 660 with self.wlock():
661 661 _cmpsets(
662 662 [modified, added, removed, deleted, unknown, ignored, clean],
663 663 rv2,
664 664 )
665 665 modified, added, removed, deleted, unknown, ignored, clean = rv2
666 666
667 667 return scmutil.status(
668 668 modified, added, removed, deleted, unknown, ignored, clean
669 669 )
670 670
671 671
672 672 class poststatus(object):
673 673 def __init__(self, startclock):
674 674 self._startclock = startclock
675 675
676 676 def __call__(self, wctx, status):
677 677 clock = wctx.repo()._fsmonitorstate.getlastclock() or self._startclock
678 678 hashignore = _hashignore(wctx.repo().dirstate._ignore)
679 679 notefiles = (
680 680 status.modified
681 681 + status.added
682 682 + status.removed
683 683 + status.deleted
684 684 + status.unknown
685 685 )
686 686 wctx.repo()._fsmonitorstate.set(clock, hashignore, notefiles)
687 687
688 688
689 689 def makedirstate(repo, dirstate):
690 690 class fsmonitordirstate(dirstate.__class__):
691 691 def _fsmonitorinit(self, repo):
692 692 # _fsmonitordisable is used in paranoid mode
693 693 self._fsmonitordisable = False
694 694 self._fsmonitorstate = repo._fsmonitorstate
695 695 self._watchmanclient = repo._watchmanclient
696 696 self._repo = weakref.proxy(repo)
697 697
698 698 def walk(self, *args, **kwargs):
699 699 orig = super(fsmonitordirstate, self).walk
700 700 if self._fsmonitordisable:
701 701 return orig(*args, **kwargs)
702 702 return overridewalk(orig, self, *args, **kwargs)
703 703
704 704 def rebuild(self, *args, **kwargs):
705 705 self._fsmonitorstate.invalidate()
706 706 return super(fsmonitordirstate, self).rebuild(*args, **kwargs)
707 707
708 708 def invalidate(self, *args, **kwargs):
709 709 self._fsmonitorstate.invalidate()
710 710 return super(fsmonitordirstate, self).invalidate(*args, **kwargs)
711 711
712 712 dirstate.__class__ = fsmonitordirstate
713 713 dirstate._fsmonitorinit(repo)
714 714
715 715
716 716 def wrapdirstate(orig, self):
717 717 ds = orig(self)
718 718 # only override the dirstate when Watchman is available for the repo
719 719 if util.safehasattr(self, b'_fsmonitorstate'):
720 720 makedirstate(self, ds)
721 721 return ds
722 722
723 723
724 724 def extsetup(ui):
725 725 extensions.wrapfilecache(
726 726 localrepo.localrepository, b'dirstate', wrapdirstate
727 727 )
728 728 if pycompat.isdarwin:
729 729 # An assist for avoiding the dangling-symlink fsevents bug
730 730 extensions.wrapfunction(os, b'symlink', wrapsymlink)
731 731
732 732 extensions.wrapfunction(merge, b'update', wrapupdate)
733 733
734 734
735 735 def wrapsymlink(orig, source, link_name):
736 736 ''' if we create a dangling symlink, also touch the parent dir
737 737 to encourage fsevents notifications to work more correctly '''
738 738 try:
739 739 return orig(source, link_name)
740 740 finally:
741 741 try:
742 742 os.utime(os.path.dirname(link_name), None)
743 743 except OSError:
744 744 pass
745 745
746 746
747 747 class state_update(object):
748 748 ''' This context manager is responsible for dispatching the state-enter
749 749 and state-leave signals to the watchman service. The enter and leave
750 750 methods can be invoked manually (for scenarios where context manager
751 751 semantics are not possible). If parameters oldnode and newnode are None,
752 752 they will be populated based on current working copy in enter and
753 753 leave, respectively. Similarly, if the distance is none, it will be
754 754 calculated based on the oldnode and newnode in the leave method.'''
755 755
756 756 def __init__(
757 757 self,
758 758 repo,
759 759 name,
760 760 oldnode=None,
761 761 newnode=None,
762 762 distance=None,
763 763 partial=False,
764 764 ):
765 765 self.repo = repo.unfiltered()
766 766 self.name = name
767 767 self.oldnode = oldnode
768 768 self.newnode = newnode
769 769 self.distance = distance
770 770 self.partial = partial
771 771 self._lock = None
772 772 self.need_leave = False
773 773
774 774 def __enter__(self):
775 775 self.enter()
776 776
777 777 def enter(self):
778 778 # Make sure we have a wlock prior to sending notifications to watchman.
779 779 # We don't want to race with other actors. In the update case,
780 780 # merge.update is going to take the wlock almost immediately. We are
781 781 # effectively extending the lock around several short sanity checks.
782 782 if self.oldnode is None:
783 783 self.oldnode = self.repo[b'.'].node()
784 784
785 785 if self.repo.currentwlock() is None:
786 786 if util.safehasattr(self.repo, b'wlocknostateupdate'):
787 787 self._lock = self.repo.wlocknostateupdate()
788 788 else:
789 789 self._lock = self.repo.wlock()
790 790 self.need_leave = self._state(b'state-enter', hex(self.oldnode))
791 791 return self
792 792
793 793 def __exit__(self, type_, value, tb):
794 794 abort = True if type_ else False
795 795 self.exit(abort=abort)
796 796
797 797 def exit(self, abort=False):
798 798 try:
799 799 if self.need_leave:
800 800 status = b'failed' if abort else b'ok'
801 801 if self.newnode is None:
802 802 self.newnode = self.repo[b'.'].node()
803 803 if self.distance is None:
804 804 self.distance = calcdistance(
805 805 self.repo, self.oldnode, self.newnode
806 806 )
807 807 self._state(b'state-leave', hex(self.newnode), status=status)
808 808 finally:
809 809 self.need_leave = False
810 810 if self._lock:
811 811 self._lock.release()
812 812
813 813 def _state(self, cmd, commithash, status=b'ok'):
814 814 if not util.safehasattr(self.repo, b'_watchmanclient'):
815 815 return False
816 816 try:
817 817 self.repo._watchmanclient.command(
818 818 cmd,
819 819 {
820 820 b'name': self.name,
821 821 b'metadata': {
822 822 # the target revision
823 823 b'rev': commithash,
824 824 # approximate number of commits between current and target
825 825 b'distance': self.distance if self.distance else 0,
826 826 # success/failure (only really meaningful for state-leave)
827 827 b'status': status,
828 828 # whether the working copy parent is changing
829 829 b'partial': self.partial,
830 830 },
831 831 },
832 832 )
833 833 return True
834 834 except Exception as e:
835 835 # Swallow any errors; fire and forget
836 836 self.repo.ui.log(
837 837 b'watchman', b'Exception %s while running %s\n', e, cmd
838 838 )
839 839 return False
840 840
841 841
842 842 # Estimate the distance between two nodes
843 843 def calcdistance(repo, oldnode, newnode):
844 844 anc = repo.changelog.ancestor(oldnode, newnode)
845 845 ancrev = repo[anc].rev()
846 846 distance = abs(repo[oldnode].rev() - ancrev) + abs(
847 847 repo[newnode].rev() - ancrev
848 848 )
849 849 return distance
850 850
851 851
852 852 # Bracket working copy updates with calls to the watchman state-enter
853 853 # and state-leave commands. This allows clients to perform more intelligent
854 854 # settling during bulk file change scenarios
855 855 # https://facebook.github.io/watchman/docs/cmd/subscribe.html#advanced-settling
856 856 def wrapupdate(
857 857 orig,
858 858 repo,
859 859 node,
860 860 branchmerge,
861 861 force,
862 862 ancestor=None,
863 863 mergeancestor=False,
864 864 labels=None,
865 865 matcher=None,
866 866 **kwargs
867 867 ):
868 868
869 869 distance = 0
870 870 partial = True
871 871 oldnode = repo[b'.'].node()
872 872 newnode = repo[node].node()
873 873 if matcher is None or matcher.always():
874 874 partial = False
875 875 distance = calcdistance(repo.unfiltered(), oldnode, newnode)
876 876
877 877 with state_update(
878 878 repo,
879 879 name=b"hg.update",
880 880 oldnode=oldnode,
881 881 newnode=newnode,
882 882 distance=distance,
883 883 partial=partial,
884 884 ):
885 885 return orig(
886 886 repo,
887 887 node,
888 888 branchmerge,
889 889 force,
890 890 ancestor,
891 891 mergeancestor,
892 892 labels,
893 893 matcher,
894 894 **kwargs
895 895 )
896 896
897 897
898 898 def repo_has_depth_one_nested_repo(repo):
899 899 for f in repo.wvfs.listdir():
900 900 if os.path.isdir(os.path.join(repo.root, f, b'.hg')):
901 901 msg = b'fsmonitor: sub-repository %r detected, fsmonitor disabled\n'
902 902 repo.ui.debug(msg % f)
903 903 return True
904 904 return False
905 905
906 906
907 907 def reposetup(ui, repo):
908 908 # We don't work with largefiles or inotify
909 909 exts = extensions.enabled()
910 910 for ext in _blacklist:
911 911 if ext in exts:
912 912 ui.warn(
913 913 _(
914 914 b'The fsmonitor extension is incompatible with the %s '
915 915 b'extension and has been disabled.\n'
916 916 )
917 917 % ext
918 918 )
919 919 return
920 920
921 921 if repo.local():
922 922 # We don't work with subrepos either.
923 923 #
924 924 # if repo[None].substate can cause a dirstate parse, which is too
925 925 # slow. Instead, look for a file called hgsubstate,
926 926 if repo.wvfs.exists(b'.hgsubstate') or repo.wvfs.exists(b'.hgsub'):
927 927 return
928 928
929 929 if repo_has_depth_one_nested_repo(repo):
930 930 return
931 931
932 932 fsmonitorstate = state.state(repo)
933 933 if fsmonitorstate.mode == b'off':
934 934 return
935 935
936 936 try:
937 937 client = watchmanclient.client(repo.ui, repo.root)
938 938 except Exception as ex:
939 939 _handleunavailable(ui, fsmonitorstate, ex)
940 940 return
941 941
942 942 repo._fsmonitorstate = fsmonitorstate
943 943 repo._watchmanclient = client
944 944
945 945 dirstate, cached = localrepo.isfilecached(repo, b'dirstate')
946 946 if cached:
947 947 # at this point since fsmonitorstate wasn't present,
948 948 # repo.dirstate is not a fsmonitordirstate
949 949 makedirstate(repo, dirstate)
950 950
951 951 class fsmonitorrepo(repo.__class__):
952 952 def status(self, *args, **kwargs):
953 953 orig = super(fsmonitorrepo, self).status
954 954 return overridestatus(orig, self, *args, **kwargs)
955 955
956 956 def wlocknostateupdate(self, *args, **kwargs):
957 957 return super(fsmonitorrepo, self).wlock(*args, **kwargs)
958 958
959 959 def wlock(self, *args, **kwargs):
960 960 l = super(fsmonitorrepo, self).wlock(*args, **kwargs)
961 961 if not ui.configbool(
962 962 b"experimental", b"fsmonitor.transaction_notify"
963 963 ):
964 964 return l
965 965 if l.held != 1:
966 966 return l
967 967 origrelease = l.releasefn
968 968
969 969 def staterelease():
970 970 if origrelease:
971 971 origrelease()
972 972 if l.stateupdate:
973 973 l.stateupdate.exit()
974 974 l.stateupdate = None
975 975
976 976 try:
977 977 l.stateupdate = None
978 978 l.stateupdate = state_update(self, name=b"hg.transaction")
979 979 l.stateupdate.enter()
980 980 l.releasefn = staterelease
981 981 except Exception as e:
982 982 # Swallow any errors; fire and forget
983 983 self.ui.log(
984 984 b'watchman', b'Exception in state update %s\n', e
985 985 )
986 986 return l
987 987
988 988 repo.__class__ = fsmonitorrepo
General Comments 0
You need to be logged in to leave comments. Login now