__init__.py
986 lines
| 31.8 KiB
| text/x-python
|
PythonLexer
|
r28433 | # __init__.py - fsmonitor initialization and overrides | ||
# | ||||
# Copyright 2013-2016 Facebook, Inc. | ||||
# | ||||
# This software may be used and distributed according to the terms of the | ||||
# GNU General Public License version 2 or any later version. | ||||
'''Faster status operations with the Watchman file monitor (EXPERIMENTAL) | ||||
Integrates the file-watching program Watchman with Mercurial to produce faster | ||||
status results. | ||||
On a particular Linux system, for a real-world repository with over 400,000 | ||||
files hosted on ext4, vanilla `hg status` takes 1.3 seconds. On the same | ||||
system, with fsmonitor it takes about 0.3 seconds. | ||||
fsmonitor requires no configuration -- it will tell Watchman about your | ||||
repository as necessary. You'll need to install Watchman from | ||||
https://facebook.github.io/watchman/ and make sure it is in your PATH. | ||||
|
r34886 | fsmonitor is incompatible with the largefiles and eol extensions, and | ||
will disable itself if any of those are active. | ||||
|
r28433 | The following configuration options exist: | ||
:: | ||||
[fsmonitor] | ||||
mode = {off, on, paranoid} | ||||
When `mode = off`, fsmonitor will disable itself (similar to not loading the | ||||
extension at all). When `mode = on`, fsmonitor will be enabled (the default). | ||||
When `mode = paranoid`, fsmonitor will query both Watchman and the filesystem, | ||||
and ensure that the results are consistent. | ||||
:: | ||||
[fsmonitor] | ||||
timeout = (float) | ||||
A value, in seconds, that determines how long fsmonitor will wait for Watchman | ||||
to return results. Defaults to `2.0`. | ||||
:: | ||||
[fsmonitor] | ||||
blacklistusers = (list of userids) | ||||
A list of usernames for which fsmonitor will disable itself altogether. | ||||
:: | ||||
[fsmonitor] | ||||
walk_on_invalidate = (boolean) | ||||
Whether or not to walk the whole repo ourselves when our cached state has been | ||||
invalidated, for example when Watchman has been restarted or .hgignore rules | ||||
have been changed. Walking the repo in that case can result in competing for | ||||
I/O with Watchman. For large repos it is recommended to set this value to | ||||
false. You may wish to set this to true if you have a very fast filesystem | ||||
that can outpace the IPC overhead of getting the result data for the full repo | ||||
from Watchman. Defaults to false. | ||||
|
r34886 | :: | ||
[fsmonitor] | ||||
warn_when_unused = (boolean) | ||||
Whether to print a warning during certain operations when fsmonitor would be | ||||
beneficial to performance but isn't enabled. | ||||
|
r28433 | |||
|
r34886 | :: | ||
[fsmonitor] | ||||
warn_update_file_count = (integer) | ||||
|
r46034 | # or when mercurial is built with rust support | ||
warn_update_file_count_rust = (integer) | ||||
|
r34886 | |||
If ``warn_when_unused`` is set and fsmonitor isn't enabled, a warning will | ||||
be printed during working directory updates if this many files will be | ||||
created. | ||||
|
r28433 | ''' | ||
# Platforms Supported | ||||
# =================== | ||||
# | ||||
# **Linux:** *Stable*. Watchman and fsmonitor are both known to work reliably, | ||||
# even under severe loads. | ||||
# | ||||
# **Mac OS X:** *Stable*. The Mercurial test suite passes with fsmonitor | ||||
# turned on, on case-insensitive HFS+. There has been a reasonable amount of | ||||
# user testing under normal loads. | ||||
# | ||||
# **Solaris, BSD:** *Alpha*. watchman and fsmonitor are believed to work, but | ||||
# very little testing has been done. | ||||
# | ||||
# **Windows:** *Alpha*. Not in a release version of watchman or fsmonitor yet. | ||||
# | ||||
# Known Issues | ||||
# ============ | ||||
# | ||||
# * fsmonitor will disable itself if any of the following extensions are | ||||
# enabled: largefiles, inotify, eol; or if the repository has subrepos. | ||||
# * fsmonitor will produce incorrect results if nested repos that are not | ||||
# subrepos exist. *Workaround*: add nested repo paths to your `.hgignore`. | ||||
# | ||||
# The issues related to nested repos and subrepos are probably not fundamental | ||||
# ones. Patches to fix them are welcome. | ||||
from __future__ import absolute_import | ||||
|
r31846 | import codecs | ||
|
r28433 | import os | ||
import stat | ||||
|
r31846 | import sys | ||
|
r42879 | import tempfile | ||
|
r34566 | import weakref | ||
|
r28433 | |||
|
r29205 | from mercurial.i18n import _ | ||
|
r43346 | from mercurial.node import hex | ||
|
r43355 | from mercurial.pycompat import open | ||
|
r28433 | from mercurial import ( | ||
context, | ||||
|
r30666 | encoding, | ||
|
r31846 | error, | ||
|
r28433 | extensions, | ||
localrepo, | ||||
|
r28443 | merge, | ||
|
r28433 | pathutil, | ||
|
r30666 | pycompat, | ||
|
r34464 | registrar, | ||
|
r28433 | scmutil, | ||
util, | ||||
) | ||||
from mercurial import match as matchmod | ||||
|
r44519 | from mercurial.utils import ( | ||
hashutil, | ||||
stringutil, | ||||
) | ||||
|
r28433 | |||
from . import ( | ||||
|
r31846 | pywatchman, | ||
|
r28433 | state, | ||
watchmanclient, | ||||
) | ||||
|
r29841 | # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for | ||
|
r28433 | # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should | ||
# be specifying the version(s) of Mercurial they are tested with, or | ||||
# leave the attribute unspecified. | ||||
|
r43347 | testedwith = b'ships-with-hg-core' | ||
|
r28433 | |||
|
r34464 | configtable = {} | ||
configitem = registrar.configitem(configtable) | ||||
|
r43346 | configitem( | ||
|
r43347 | b'fsmonitor', b'mode', default=b'on', | ||
|
r34464 | ) | ||
|
r43346 | configitem( | ||
|
r43347 | b'fsmonitor', b'walk_on_invalidate', default=False, | ||
|
r34464 | ) | ||
|
r43346 | configitem( | ||
|
r43347 | b'fsmonitor', b'timeout', default=b'2', | ||
|
r34464 | ) | ||
|
r43346 | configitem( | ||
|
r43347 | b'fsmonitor', b'blacklistusers', default=list, | ||
|
r34464 | ) | ||
|
r43346 | configitem( | ||
|
r43347 | b'fsmonitor', b'watchman_exe', default=b'watchman', | ||
|
r42134 | ) | ||
|
r43346 | configitem( | ||
|
r43347 | b'fsmonitor', b'verbose', default=True, experimental=True, | ||
|
r41629 | ) | ||
|
r43346 | configitem( | ||
|
r43347 | b'experimental', b'fsmonitor.transaction_notify', default=False, | ||
|
r35314 | ) | ||
|
r34464 | |||
|
r28433 | # This extension is incompatible with the following blacklisted extensions | ||
# and will disable itself when encountering one of these: | ||||
|
r43347 | _blacklist = [b'largefiles', b'eol'] | ||
|
r28433 | |||
|
r43346 | |||
|
r42879 | def debuginstall(ui, fm): | ||
|
r43346 | fm.write( | ||
|
r43347 | b"fsmonitor-watchman", | ||
_(b"fsmonitor checking for watchman binary... (%s)\n"), | ||||
ui.configpath(b"fsmonitor", b"watchman_exe"), | ||||
|
r43346 | ) | ||
|
r42879 | root = tempfile.mkdtemp() | ||
c = watchmanclient.client(ui, root) | ||||
err = None | ||||
try: | ||||
|
r43347 | v = c.command(b"version") | ||
|
r43346 | fm.write( | ||
|
r43347 | b"fsmonitor-watchman-version", | ||
_(b" watchman binary version %s\n"), | ||||
|
r43707 | pycompat.bytestr(v["version"]), | ||
|
r43346 | ) | ||
|
r42879 | except watchmanclient.Unavailable as e: | ||
|
r43721 | err = stringutil.forcebytestr(e) | ||
|
r43346 | fm.condwrite( | ||
err, | ||||
|
r43347 | b"fsmonitor-watchman-error", | ||
_(b" watchman binary missing or broken: %s\n"), | ||||
|
r43346 | err, | ||
) | ||||
|
r42879 | return 1 if err else 0 | ||
|
r43346 | |||
|
r28433 | def _handleunavailable(ui, state, ex): | ||
"""Exception handler for Watchman interaction exceptions""" | ||||
if isinstance(ex, watchmanclient.Unavailable): | ||||
|
r41758 | # experimental config: fsmonitor.verbose | ||
|
r43347 | if ex.warn and ui.configbool(b'fsmonitor', b'verbose'): | ||
|
r43679 | if b'illegal_fstypes' not in stringutil.forcebytestr(ex): | ||
ui.warn(stringutil.forcebytestr(ex) + b'\n') | ||||
|
r28433 | if ex.invalidate: | ||
state.invalidate() | ||||
|
r41758 | # experimental config: fsmonitor.verbose | ||
|
r43347 | if ui.configbool(b'fsmonitor', b'verbose'): | ||
|
r43708 | ui.log( | ||
b'fsmonitor', | ||||
b'Watchman unavailable: %s\n', | ||||
stringutil.forcebytestr(ex.msg), | ||||
) | ||||
|
r28433 | else: | ||
|
r43708 | ui.log( | ||
b'fsmonitor', | ||||
b'Watchman exception: %s\n', | ||||
stringutil.forcebytestr(ex), | ||||
) | ||||
|
r28433 | |||
|
r43346 | |||
|
r28433 | def _hashignore(ignore): | ||
"""Calculate hash for ignore patterns and filenames | ||||
If this information changes between Mercurial invocations, we can't | ||||
rely on Watchman information anymore and have to re-scan the working | ||||
copy. | ||||
""" | ||||
|
r44519 | sha1 = hashutil.sha1() | ||
|
r43711 | sha1.update(pycompat.byterepr(ignore)) | ||
return pycompat.sysbytes(sha1.hexdigest()) | ||||
|
r28433 | |||
|
r43346 | |||
|
r31846 | _watchmanencoding = pywatchman.encoding.get_local_encoding() | ||
_fsencoding = sys.getfilesystemencoding() or sys.getdefaultencoding() | ||||
_fixencoding = codecs.lookup(_watchmanencoding) != codecs.lookup(_fsencoding) | ||||
|
r43346 | |||
|
r31846 | def _watchmantofsencoding(path): | ||
"""Fix path to match watchman and local filesystem encoding | ||||
watchman's paths encoding can differ from filesystem encoding. For example, | ||||
on Windows, it's always utf-8. | ||||
""" | ||||
try: | ||||
decoded = path.decode(_watchmanencoding) | ||||
except UnicodeDecodeError as e: | ||||
|
r43691 | raise error.Abort( | ||
stringutil.forcebytestr(e), hint=b'watchman encoding error' | ||||
) | ||||
|
r31846 | |||
try: | ||||
encoded = decoded.encode(_fsencoding, 'strict') | ||||
except UnicodeEncodeError as e: | ||||
|
r43691 | raise error.Abort(stringutil.forcebytestr(e)) | ||
|
r31846 | |||
return encoded | ||||
|
r43346 | |||
|
r28433 | def overridewalk(orig, self, match, subrepos, unknown, ignored, full=True): | ||
'''Replacement for dirstate.walk, hooking into Watchman. | ||||
Whenever full is False, ignored is False, and the Watchman client is | ||||
available, use Watchman combined with saved state to possibly return only a | ||||
subset of files.''' | ||||
|
r43346 | |||
|
r35146 | def bail(reason): | ||
|
r43347 | self._ui.debug(b'fsmonitor: fallback to core status, %s\n' % reason) | ||
|
r28433 | return orig(match, subrepos, unknown, ignored, full=True) | ||
|
r35146 | if full: | ||
|
r43347 | return bail(b'full rewalk requested') | ||
|
r35146 | if ignored: | ||
|
r43347 | return bail(b'listing ignored files') | ||
|
r35146 | if not self._watchmanclient.available(): | ||
|
r43347 | return bail(b'client unavailable') | ||
|
r28433 | state = self._fsmonitorstate | ||
clock, ignorehash, notefiles = state.get() | ||||
if not clock: | ||||
if state.walk_on_invalidate: | ||||
|
r43347 | return bail(b'no clock') | ||
|
r28433 | # Initial NULL clock value, see | ||
# https://facebook.github.io/watchman/docs/clockspec.html | ||||
|
r43347 | clock = b'c:0:0' | ||
|
r28433 | notefiles = [] | ||
ignore = self._ignore | ||||
dirignore = self._dirignore | ||||
if unknown: | ||||
|
r43347 | if _hashignore(ignore) != ignorehash and clock != b'c:0:0': | ||
|
r28433 | # ignore list changed -- can't rely on Watchman state any more | ||
if state.walk_on_invalidate: | ||||
|
r43347 | return bail(b'ignore rules changed') | ||
|
r28433 | notefiles = [] | ||
|
r43347 | clock = b'c:0:0' | ||
|
r28433 | else: | ||
# always ignore | ||||
ignore = util.always | ||||
dirignore = util.always | ||||
matchfn = match.matchfn | ||||
matchalways = match.always() | ||||
|
r35084 | dmap = self._map | ||
|
r43347 | if util.safehasattr(dmap, b'_map'): | ||
|
r35084 | # for better performance, directly access the inner dirstate map if the | ||
# standard dirstate implementation is in use. | ||||
dmap = dmap._map | ||||
|
r34898 | nonnormalset = self._map.nonnormalset | ||
|
r28433 | |||
|
r34463 | copymap = self._map.copymap | ||
|
r28433 | getkind = stat.S_IFMT | ||
dirkind = stat.S_IFDIR | ||||
regkind = stat.S_IFREG | ||||
lnkkind = stat.S_IFLNK | ||||
join = self._join | ||||
normcase = util.normcase | ||||
fresh_instance = False | ||||
exact = skipstep3 = False | ||||
|
r32322 | if match.isexact(): # match.exact | ||
|
r28433 | exact = True | ||
dirignore = util.always # skip step 2 | ||||
|
r32322 | elif match.prefix(): # match.match, no patterns | ||
|
r28433 | skipstep3 = True | ||
if not exact and self._checkcase: | ||||
# note that even though we could receive directory entries, we're only | ||||
# interested in checking if a file with the same name exists. So only | ||||
# normalize files if possible. | ||||
normalize = self._normalizefile | ||||
skipstep3 = False | ||||
else: | ||||
normalize = None | ||||
# step 1: find all explicit files | ||||
results, work, dirsnotfound = self._walkexplicit(match, subrepos) | ||||
skipstep3 = skipstep3 and not (work or dirsnotfound) | ||||
work = [d for d in work if not dirignore(d[0])] | ||||
if not work and (exact or skipstep3): | ||||
for s in subrepos: | ||||
del results[s] | ||||
|
r43347 | del results[b'.hg'] | ||
|
r28433 | return results | ||
# step 2: query Watchman | ||||
try: | ||||
# Use the user-configured timeout for the query. | ||||
# Add a little slack over the top of the user query to allow for | ||||
# overheads while transferring the data | ||||
self._watchmanclient.settimeout(state.timeout + 0.1) | ||||
|
r43346 | result = self._watchmanclient.command( | ||
|
r43347 | b'query', | ||
|
r43346 | { | ||
|
r43347 | b'fields': [b'mode', b'mtime', b'size', b'exists', b'name'], | ||
b'since': clock, | ||||
b'expression': [ | ||||
b'not', | ||||
[ | ||||
b'anyof', | ||||
[b'dirname', b'.hg'], | ||||
[b'name', b'.hg', b'wholename'], | ||||
], | ||||
|
r43346 | ], | ||
|
r43347 | b'sync_timeout': int(state.timeout * 1000), | ||
b'empty_on_fresh_instance': state.walk_on_invalidate, | ||||
|
r43346 | }, | ||
) | ||||
|
r28433 | except Exception as ex: | ||
_handleunavailable(self._ui, state, ex) | ||||
self._watchmanclient.clearconnection() | ||||
|
r43347 | return bail(b'exception during run') | ||
|
r28433 | else: | ||
# We need to propagate the last observed clock up so that we | ||||
# can use it for our next query | ||||
|
r43715 | state.setlastclock(pycompat.sysbytes(result[b'clock'])) | ||
|
r43347 | if result[b'is_fresh_instance']: | ||
|
r28433 | if state.walk_on_invalidate: | ||
state.invalidate() | ||||
|
r43347 | return bail(b'fresh instance') | ||
|
r28433 | fresh_instance = True | ||
# Ignore any prior noteable files from the state info | ||||
notefiles = [] | ||||
# for file paths which require normalization and we encounter a case | ||||
# collision, we store our own foldmap | ||||
if normalize: | ||||
|
r44937 | foldmap = {normcase(k): k for k in results} | ||
|
r28433 | |||
|
r43347 | switch_slashes = pycompat.ossep == b'\\' | ||
|
r28433 | # The order of the results is, strictly speaking, undefined. | ||
# For case changes on a case insensitive filesystem we may receive | ||||
# two entries, one with exists=True and another with exists=False. | ||||
# The exists=True entries in the same response should be interpreted | ||||
# as being happens-after the exists=False entries due to the way that | ||||
# Watchman tracks files. We use this property to reconcile deletes | ||||
# for name case changes. | ||||
|
r43347 | for entry in result[b'files']: | ||
fname = entry[b'name'] | ||||
|
r43713 | |||
# Watchman always give us a str. Normalize to bytes on Python 3 | ||||
# using Watchman's encoding, if needed. | ||||
if not isinstance(fname, bytes): | ||||
fname = fname.encode(_watchmanencoding) | ||||
|
r31846 | if _fixencoding: | ||
fname = _watchmantofsencoding(fname) | ||||
|
r43713 | |||
|
r28433 | if switch_slashes: | ||
|
r43347 | fname = fname.replace(b'\\', b'/') | ||
|
r28433 | if normalize: | ||
normed = normcase(fname) | ||||
fname = normalize(fname, True, True) | ||||
foldmap[normed] = fname | ||||
|
r43347 | fmode = entry[b'mode'] | ||
fexists = entry[b'exists'] | ||||
|
r28433 | kind = getkind(fmode) | ||
|
r43347 | if b'/.hg/' in fname or fname.endswith(b'/.hg'): | ||
return bail(b'nested-repo-detected') | ||||
|
r41630 | |||
|
r28433 | if not fexists: | ||
# if marked as deleted and we don't already have a change | ||||
# record, mark it as deleted. If we already have an entry | ||||
# for fname then it was either part of walkexplicit or was | ||||
# an earlier result that was a case change | ||||
|
r43346 | if ( | ||
fname not in results | ||||
and fname in dmap | ||||
and (matchalways or matchfn(fname)) | ||||
): | ||||
|
r28433 | results[fname] = None | ||
elif kind == dirkind: | ||||
if fname in dmap and (matchalways or matchfn(fname)): | ||||
results[fname] = None | ||||
elif kind == regkind or kind == lnkkind: | ||||
if fname in dmap: | ||||
if matchalways or matchfn(fname): | ||||
results[fname] = entry | ||||
elif (matchalways or matchfn(fname)) and not ignore(fname): | ||||
results[fname] = entry | ||||
elif fname in dmap and (matchalways or matchfn(fname)): | ||||
results[fname] = None | ||||
# step 3: query notable files we don't already know about | ||||
# XXX try not to iterate over the entire dmap | ||||
if normalize: | ||||
# any notable files that have changed case will already be handled | ||||
# above, so just check membership in the foldmap | ||||
|
r44937 | notefiles = { | ||
normalize(f, True, True) | ||||
|
r43346 | for f in notefiles | ||
|
r44937 | if normcase(f) not in foldmap | ||
} | ||||
visit = { | ||||
f | ||||
for f in notefiles | ||||
if (f not in results and matchfn(f) and (f in dmap or not ignore(f))) | ||||
} | ||||
|
r28433 | |||
|
r34898 | if not fresh_instance: | ||
|
r28433 | if matchalways: | ||
visit.update(f for f in nonnormalset if f not in results) | ||||
visit.update(f for f in copymap if f not in results) | ||||
else: | ||||
|
r43346 | visit.update( | ||
f for f in nonnormalset if f not in results and matchfn(f) | ||||
) | ||||
visit.update(f for f in copymap if f not in results and matchfn(f)) | ||||
|
r28433 | else: | ||
if matchalways: | ||||
|
r43375 | visit.update( | ||
f for f, st in pycompat.iteritems(dmap) if f not in results | ||||
) | ||||
|
r28433 | visit.update(f for f in copymap if f not in results) | ||
else: | ||||
|
r43346 | visit.update( | ||
f | ||||
|
r43375 | for f, st in pycompat.iteritems(dmap) | ||
|
r43346 | if f not in results and matchfn(f) | ||
) | ||||
visit.update(f for f in copymap if f not in results and matchfn(f)) | ||||
|
r28433 | |||
|
r33722 | audit = pathutil.pathauditor(self._root, cached=True).check | ||
|
r28433 | auditpass = [f for f in visit if audit(f)] | ||
auditpass.sort() | ||||
auditfail = visit.difference(auditpass) | ||||
for f in auditfail: | ||||
results[f] = None | ||||
|
r43714 | nf = iter(auditpass) | ||
|
r28433 | for st in util.statfiles([join(f) for f in auditpass]): | ||
|
r43714 | f = next(nf) | ||
|
r28433 | if st or f in dmap: | ||
results[f] = st | ||||
for s in subrepos: | ||||
del results[s] | ||||
|
r43347 | del results[b'.hg'] | ||
|
r28433 | return results | ||
|
r43346 | |||
|
r28433 | def overridestatus( | ||
|
r43346 | orig, | ||
self, | ||||
|
r43347 | node1=b'.', | ||
|
r43346 | node2=None, | ||
match=None, | ||||
ignored=False, | ||||
clean=False, | ||||
unknown=False, | ||||
listsubrepos=False, | ||||
): | ||||
|
r28433 | listignored = ignored | ||
listclean = clean | ||||
listunknown = unknown | ||||
def _cmpsets(l1, l2): | ||||
try: | ||||
|
r43347 | if b'FSMONITOR_LOG_FILE' in encoding.environ: | ||
fn = encoding.environ[b'FSMONITOR_LOG_FILE'] | ||||
f = open(fn, b'wb') | ||||
|
r28433 | else: | ||
|
r43347 | fn = b'fsmonitorfail.log' | ||
f = self.vfs.open(fn, b'wb') | ||||
|
r28433 | except (IOError, OSError): | ||
|
r43347 | self.ui.warn(_(b'warning: unable to write to %s\n') % fn) | ||
|
r28433 | return | ||
try: | ||||
for i, (s1, s2) in enumerate(zip(l1, l2)): | ||||
if set(s1) != set(s2): | ||||
|
r43347 | f.write(b'sets at position %d are unequal\n' % i) | ||
f.write(b'watchman returned: %s\n' % s1) | ||||
f.write(b'stat returned: %s\n' % s2) | ||||
|
r28433 | finally: | ||
f.close() | ||||
if isinstance(node1, context.changectx): | ||||
ctx1 = node1 | ||||
else: | ||||
ctx1 = self[node1] | ||||
if isinstance(node2, context.changectx): | ||||
ctx2 = node2 | ||||
else: | ||||
ctx2 = self[node2] | ||||
working = ctx2.rev() is None | ||||
|
r43347 | parentworking = working and ctx1 == self[b'.'] | ||
|
r41825 | match = match or matchmod.always() | ||
|
r28433 | |||
# Maybe we can use this opportunity to update Watchman's state. | ||||
# Mercurial uses workingcommitctx and/or memctx to represent the part of | ||||
# the workingctx that is to be committed. So don't update the state in | ||||
# that case. | ||||
# HG_PENDING is set in the environment when the dirstate is being updated | ||||
# in the middle of a transaction; we must not update our state in that | ||||
# case, or we risk forgetting about changes in the working copy. | ||||
|
r43346 | updatestate = ( | ||
parentworking | ||||
and match.always() | ||||
and not isinstance(ctx2, (context.workingcommitctx, context.memctx)) | ||||
|
r43347 | and b'HG_PENDING' not in encoding.environ | ||
|
r43346 | ) | ||
|
r28433 | |||
try: | ||||
if self._fsmonitorstate.walk_on_invalidate: | ||||
# Use a short timeout to query the current clock. If that | ||||
# takes too long then we assume that the service will be slow | ||||
# to answer our query. | ||||
# walk_on_invalidate indicates that we prefer to walk the | ||||
# tree ourselves because we can ignore portions that Watchman | ||||
# cannot and we tend to be faster in the warmer buffer cache | ||||
# cases. | ||||
self._watchmanclient.settimeout(0.1) | ||||
else: | ||||
# Give Watchman more time to potentially complete its walk | ||||
# and return the initial clock. In this mode we assume that | ||||
# the filesystem will be slower than parsing a potentially | ||||
# very large Watchman result set. | ||||
|
r43346 | self._watchmanclient.settimeout(self._fsmonitorstate.timeout + 0.1) | ||
|
r28433 | startclock = self._watchmanclient.getcurrentclock() | ||
except Exception as ex: | ||||
self._watchmanclient.clearconnection() | ||||
_handleunavailable(self.ui, self._fsmonitorstate, ex) | ||||
# boo, Watchman failed. bail | ||||
|
r43346 | return orig( | ||
node1, | ||||
node2, | ||||
match, | ||||
listignored, | ||||
listclean, | ||||
listunknown, | ||||
listsubrepos, | ||||
) | ||||
|
r28433 | |||
if updatestate: | ||||
# We need info about unknown files. This may make things slower the | ||||
# first time, but whatever. | ||||
stateunknown = True | ||||
else: | ||||
stateunknown = listunknown | ||||
|
r32815 | if updatestate: | ||
ps = poststatus(startclock) | ||||
self.addpostdsstatus(ps) | ||||
|
r43346 | r = orig( | ||
node1, node2, match, listignored, listclean, stateunknown, listsubrepos | ||||
) | ||||
|
r28433 | modified, added, removed, deleted, unknown, ignored, clean = r | ||
if not listunknown: | ||||
unknown = [] | ||||
# don't do paranoid checks if we're not going to query Watchman anyway | ||||
full = listclean or match.traversedir is not None | ||||
|
r43347 | if self._fsmonitorstate.mode == b'paranoid' and not full: | ||
|
r28433 | # run status again and fall back to the old walk this time | ||
self.dirstate._fsmonitordisable = True | ||||
# shut the UI up | ||||
quiet = self.ui.quiet | ||||
self.ui.quiet = True | ||||
fout, ferr = self.ui.fout, self.ui.ferr | ||||
|
r43347 | self.ui.fout = self.ui.ferr = open(os.devnull, b'wb') | ||
|
r28433 | |||
try: | ||||
rv2 = orig( | ||||
|
r43346 | node1, | ||
node2, | ||||
match, | ||||
listignored, | ||||
listclean, | ||||
listunknown, | ||||
listsubrepos, | ||||
) | ||||
|
r28433 | finally: | ||
self.dirstate._fsmonitordisable = False | ||||
self.ui.quiet = quiet | ||||
self.ui.fout, self.ui.ferr = fout, ferr | ||||
# clean isn't tested since it's set to True above | ||||
|
r40245 | with self.wlock(): | ||
_cmpsets( | ||||
[modified, added, removed, deleted, unknown, ignored, clean], | ||||
|
r43346 | rv2, | ||
) | ||||
|
r28433 | modified, added, removed, deleted, unknown, ignored, clean = rv2 | ||
return scmutil.status( | ||||
|
r43346 | modified, added, removed, deleted, unknown, ignored, clean | ||
) | ||||
|
r28433 | |||
|
r32815 | class poststatus(object): | ||
def __init__(self, startclock): | ||||
|
r45372 | self._startclock = pycompat.sysbytes(startclock) | ||
|
r32815 | |||
def __call__(self, wctx, status): | ||||
clock = wctx.repo()._fsmonitorstate.getlastclock() or self._startclock | ||||
hashignore = _hashignore(wctx.repo().dirstate._ignore) | ||||
|
r43346 | notefiles = ( | ||
status.modified | ||||
+ status.added | ||||
+ status.removed | ||||
+ status.deleted | ||||
+ status.unknown | ||||
) | ||||
|
r32815 | wctx.repo()._fsmonitorstate.set(clock, hashignore, notefiles) | ||
|
r43346 | |||
|
r33386 | def makedirstate(repo, dirstate): | ||
class fsmonitordirstate(dirstate.__class__): | ||||
|
r34566 | def _fsmonitorinit(self, repo): | ||
|
r28433 | # _fsmonitordisable is used in paranoid mode | ||
self._fsmonitordisable = False | ||||
|
r34566 | self._fsmonitorstate = repo._fsmonitorstate | ||
self._watchmanclient = repo._watchmanclient | ||||
self._repo = weakref.proxy(repo) | ||||
|
r28433 | |||
def walk(self, *args, **kwargs): | ||||
orig = super(fsmonitordirstate, self).walk | ||||
if self._fsmonitordisable: | ||||
return orig(*args, **kwargs) | ||||
return overridewalk(orig, self, *args, **kwargs) | ||||
def rebuild(self, *args, **kwargs): | ||||
self._fsmonitorstate.invalidate() | ||||
return super(fsmonitordirstate, self).rebuild(*args, **kwargs) | ||||
def invalidate(self, *args, **kwargs): | ||||
self._fsmonitorstate.invalidate() | ||||
return super(fsmonitordirstate, self).invalidate(*args, **kwargs) | ||||
|
r33386 | dirstate.__class__ = fsmonitordirstate | ||
|
r34566 | dirstate._fsmonitorinit(repo) | ||
|
r28433 | |||
|
r43346 | |||
|
r28433 | def wrapdirstate(orig, self): | ||
ds = orig(self) | ||||
# only override the dirstate when Watchman is available for the repo | ||||
|
r43347 | if util.safehasattr(self, b'_fsmonitorstate'): | ||
|
r33386 | makedirstate(self, ds) | ||
|
r28433 | return ds | ||
|
r43346 | |||
|
r28433 | def extsetup(ui): | ||
|
r32722 | extensions.wrapfilecache( | ||
|
r43347 | localrepo.localrepository, b'dirstate', wrapdirstate | ||
|
r43346 | ) | ||
|
r34648 | if pycompat.isdarwin: | ||
|
r28433 | # An assist for avoiding the dangling-symlink fsevents bug | ||
|
r43347 | extensions.wrapfunction(os, b'symlink', wrapsymlink) | ||
|
r28433 | |||
|
r43347 | extensions.wrapfunction(merge, b'update', wrapupdate) | ||
|
r28443 | |||
|
r43346 | |||
|
r28433 | def wrapsymlink(orig, source, link_name): | ||
''' if we create a dangling symlink, also touch the parent dir | ||||
to encourage fsevents notifications to work more correctly ''' | ||||
try: | ||||
return orig(source, link_name) | ||||
finally: | ||||
try: | ||||
os.utime(os.path.dirname(link_name), None) | ||||
except OSError: | ||||
pass | ||||
|
r43346 | |||
|
r28443 | class state_update(object): | ||
|
r30332 | ''' This context manager is responsible for dispatching the state-enter | ||
|
r34566 | and state-leave signals to the watchman service. The enter and leave | ||
methods can be invoked manually (for scenarios where context manager | ||||
semantics are not possible). If parameters oldnode and newnode are None, | ||||
they will be populated based on current working copy in enter and | ||||
leave, respectively. Similarly, if the distance is none, it will be | ||||
calculated based on the oldnode and newnode in the leave method.''' | ||||
|
r28443 | |||
|
r43346 | def __init__( | ||
self, | ||||
repo, | ||||
name, | ||||
oldnode=None, | ||||
newnode=None, | ||||
distance=None, | ||||
partial=False, | ||||
): | ||||
|
r34566 | self.repo = repo.unfiltered() | ||
self.name = name | ||||
self.oldnode = oldnode | ||||
self.newnode = newnode | ||||
|
r28443 | self.distance = distance | ||
self.partial = partial | ||||
|
r32334 | self._lock = None | ||
|
r32335 | self.need_leave = False | ||
|
r28443 | |||
def __enter__(self): | ||||
|
r34566 | self.enter() | ||
def enter(self): | ||||
|
r35314 | # Make sure we have a wlock prior to sending notifications to watchman. | ||
# We don't want to race with other actors. In the update case, | ||||
# merge.update is going to take the wlock almost immediately. We are | ||||
# effectively extending the lock around several short sanity checks. | ||||
|
r34566 | if self.oldnode is None: | ||
|
r43347 | self.oldnode = self.repo[b'.'].node() | ||
|
r35314 | |||
if self.repo.currentwlock() is None: | ||||
|
r43347 | if util.safehasattr(self.repo, b'wlocknostateupdate'): | ||
|
r35314 | self._lock = self.repo.wlocknostateupdate() | ||
else: | ||||
self._lock = self.repo.wlock() | ||||
|
r43347 | self.need_leave = self._state(b'state-enter', hex(self.oldnode)) | ||
|
r28443 | return self | ||
def __exit__(self, type_, value, tb): | ||||
|
r34566 | abort = True if type_ else False | ||
self.exit(abort=abort) | ||||
def exit(self, abort=False): | ||||
|
r32334 | try: | ||
|
r32335 | if self.need_leave: | ||
|
r43347 | status = b'failed' if abort else b'ok' | ||
|
r34566 | if self.newnode is None: | ||
|
r43347 | self.newnode = self.repo[b'.'].node() | ||
|
r34566 | if self.distance is None: | ||
self.distance = calcdistance( | ||||
|
r43346 | self.repo, self.oldnode, self.newnode | ||
) | ||||
|
r43347 | self._state(b'state-leave', hex(self.newnode), status=status) | ||
|
r32334 | finally: | ||
|
r34566 | self.need_leave = False | ||
|
r32334 | if self._lock: | ||
self._lock.release() | ||||
|
r28443 | |||
|
r43347 | def _state(self, cmd, commithash, status=b'ok'): | ||
if not util.safehasattr(self.repo, b'_watchmanclient'): | ||||
|
r32335 | return False | ||
|
r28443 | try: | ||
|
r43346 | self.repo._watchmanclient.command( | ||
cmd, | ||||
{ | ||||
|
r43347 | b'name': self.name, | ||
b'metadata': { | ||||
|
r43346 | # the target revision | ||
|
r43347 | b'rev': commithash, | ||
|
r43346 | # approximate number of commits between current and target | ||
|
r43347 | b'distance': self.distance if self.distance else 0, | ||
|
r43346 | # success/failure (only really meaningful for state-leave) | ||
|
r43347 | b'status': status, | ||
|
r43346 | # whether the working copy parent is changing | ||
|
r43347 | b'partial': self.partial, | ||
|
r43346 | }, | ||
}, | ||||
) | ||||
|
r32335 | return True | ||
|
r28443 | except Exception as e: | ||
# Swallow any errors; fire and forget | ||||
self.repo.ui.log( | ||||
|
r43347 | b'watchman', b'Exception %s while running %s\n', e, cmd | ||
|
r43346 | ) | ||
|
r32335 | return False | ||
|
r28443 | |||
|
r43346 | |||
|
r34565 | # Estimate the distance between two nodes | ||
def calcdistance(repo, oldnode, newnode): | ||||
anc = repo.changelog.ancestor(oldnode, newnode) | ||||
ancrev = repo[anc].rev() | ||||
|
r43346 | distance = abs(repo[oldnode].rev() - ancrev) + abs( | ||
repo[newnode].rev() - ancrev | ||||
) | ||||
|
r34565 | return distance | ||
|
r43346 | |||
|
r28443 | # Bracket working copy updates with calls to the watchman state-enter | ||
# and state-leave commands. This allows clients to perform more intelligent | ||||
# settling during bulk file change scenarios | ||||
# https://facebook.github.io/watchman/docs/cmd/subscribe.html#advanced-settling | ||||
|
r43346 | def wrapupdate( | ||
orig, | ||||
repo, | ||||
node, | ||||
branchmerge, | ||||
force, | ||||
ancestor=None, | ||||
mergeancestor=False, | ||||
labels=None, | ||||
matcher=None, | ||||
**kwargs | ||||
): | ||||
|
r28443 | |||
distance = 0 | ||||
partial = True | ||||
|
r43347 | oldnode = repo[b'.'].node() | ||
|
r34566 | newnode = repo[node].node() | ||
|
r28443 | if matcher is None or matcher.always(): | ||
partial = False | ||||
|
r34566 | distance = calcdistance(repo.unfiltered(), oldnode, newnode) | ||
|
r28443 | |||
|
r43346 | with state_update( | ||
repo, | ||||
|
r43347 | name=b"hg.update", | ||
|
r43346 | oldnode=oldnode, | ||
newnode=newnode, | ||||
distance=distance, | ||||
partial=partial, | ||||
): | ||||
|
r28443 | return orig( | ||
|
r43346 | repo, | ||
node, | ||||
branchmerge, | ||||
force, | ||||
ancestor, | ||||
mergeancestor, | ||||
labels, | ||||
matcher, | ||||
**kwargs | ||||
) | ||||
|
r28443 | |||
|
r41630 | def repo_has_depth_one_nested_repo(repo): | ||
for f in repo.wvfs.listdir(): | ||||
|
r43347 | if os.path.isdir(os.path.join(repo.root, f, b'.hg')): | ||
msg = b'fsmonitor: sub-repository %r detected, fsmonitor disabled\n' | ||||
|
r41630 | repo.ui.debug(msg % f) | ||
return True | ||||
return False | ||||
|
r43346 | |||
|
r28433 | def reposetup(ui, repo): | ||
# We don't work with largefiles or inotify | ||||
exts = extensions.enabled() | ||||
for ext in _blacklist: | ||||
if ext in exts: | ||||
|
r43346 | ui.warn( | ||
_( | ||||
|
r43347 | b'The fsmonitor extension is incompatible with the %s ' | ||
b'extension and has been disabled.\n' | ||||
|
r43346 | ) | ||
% ext | ||||
) | ||||
|
r28433 | return | ||
|
r33385 | if repo.local(): | ||
# We don't work with subrepos either. | ||||
# | ||||
# if repo[None].substate can cause a dirstate parse, which is too | ||||
# slow. Instead, look for a file called hgsubstate, | ||||
|
r43347 | if repo.wvfs.exists(b'.hgsubstate') or repo.wvfs.exists(b'.hgsub'): | ||
|
r28433 | return | ||
|
r41630 | if repo_has_depth_one_nested_repo(repo): | ||
return | ||||
|
r28433 | fsmonitorstate = state.state(repo) | ||
|
r43347 | if fsmonitorstate.mode == b'off': | ||
|
r28433 | return | ||
try: | ||||
|
r43709 | client = watchmanclient.client(repo.ui, repo.root) | ||
|
r28433 | except Exception as ex: | ||
_handleunavailable(ui, fsmonitorstate, ex) | ||||
return | ||||
repo._fsmonitorstate = fsmonitorstate | ||||
repo._watchmanclient = client | ||||
|
r43347 | dirstate, cached = localrepo.isfilecached(repo, b'dirstate') | ||
|
r33387 | if cached: | ||
# at this point since fsmonitorstate wasn't present, | ||||
# repo.dirstate is not a fsmonitordirstate | ||||
makedirstate(repo, dirstate) | ||||
|
r28433 | |||
class fsmonitorrepo(repo.__class__): | ||||
def status(self, *args, **kwargs): | ||||
orig = super(fsmonitorrepo, self).status | ||||
return overridestatus(orig, self, *args, **kwargs) | ||||
|
r35314 | def wlocknostateupdate(self, *args, **kwargs): | ||
return super(fsmonitorrepo, self).wlock(*args, **kwargs) | ||||
def wlock(self, *args, **kwargs): | ||||
l = super(fsmonitorrepo, self).wlock(*args, **kwargs) | ||||
if not ui.configbool( | ||||
|
r43347 | b"experimental", b"fsmonitor.transaction_notify" | ||
|
r43346 | ): | ||
|
r35314 | return l | ||
if l.held != 1: | ||||
return l | ||||
origrelease = l.releasefn | ||||
def staterelease(): | ||||
if origrelease: | ||||
origrelease() | ||||
if l.stateupdate: | ||||
l.stateupdate.exit() | ||||
l.stateupdate = None | ||||
try: | ||||
l.stateupdate = None | ||||
|
r43347 | l.stateupdate = state_update(self, name=b"hg.transaction") | ||
|
r35314 | l.stateupdate.enter() | ||
l.releasefn = staterelease | ||||
except Exception as e: | ||||
# Swallow any errors; fire and forget | ||||
|
r43347 | self.ui.log( | ||
b'watchman', b'Exception in state update %s\n', e | ||||
) | ||||
|
r35314 | return l | ||
|
r28433 | repo.__class__ = fsmonitorrepo | ||