fsmonitor: new experimental extension

Extension to plug into a Watchman daemon, speeding up hg status calls by
relying on OS events to tell us what files have changed.

Originally developed at https://bitbucket.org/facebook/hgwatchman

hgext/fsmonitor/__init__.py

@@ -0,0 +1,626 b''
+# __init__.py - fsmonitor initialization and overrides
+#
+# Copyright 2013-2016 Facebook, Inc.
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+'''Faster status operations with the Watchman file monitor (EXPERIMENTAL)
+
+Integrates the file-watching program Watchman with Mercurial to produce faster
+status results.
+
+On a particular Linux system, for a real-world repository with over 400,000
+files hosted on ext4, vanilla `hg status` takes 1.3 seconds. On the same
+system, with fsmonitor it takes about 0.3 seconds.
+
+fsmonitor requires no configuration -- it will tell Watchman about your
+repository as necessary. You'll need to install Watchman from
+https://facebook.github.io/watchman/ and make sure it is in your PATH.
+
+The following configuration options exist:
+
+::
+
+    [fsmonitor]
+    mode = {off, on, paranoid}
+
+When `mode = off`, fsmonitor will disable itself (similar to not loading the
+extension at all). When `mode = on`, fsmonitor will be enabled (the default).
+When `mode = paranoid`, fsmonitor will query both Watchman and the filesystem,
+and ensure that the results are consistent.
+
+::
+
+    [fsmonitor]
+    timeout = (float)
+
+A value, in seconds, that determines how long fsmonitor will wait for Watchman
+to return results. Defaults to `2.0`.
+
+::
+
+    [fsmonitor]
+    blacklistusers = (list of userids)
+
+A list of usernames for which fsmonitor will disable itself altogether.
+
+::
+
+    [fsmonitor]
+    walk_on_invalidate = (boolean)
+
+Whether or not to walk the whole repo ourselves when our cached state has been
+invalidated, for example when Watchman has been restarted or .hgignore rules
+have been changed. Walking the repo in that case can result in competing for
+I/O with Watchman. For large repos it is recommended to set this value to
+false. You may wish to set this to true if you have a very fast filesystem
+that can outpace the IPC overhead of getting the result data for the full repo
+from Watchman. Defaults to false.
+
+fsmonitor is incompatible with the largefiles and eol extensions, and
+will disable itself if any of those are active.
+
+'''
+
+# Platforms Supported
+# ===================
+#
+# **Linux:** *Stable*. Watchman and fsmonitor are both known to work reliably,
+#   even under severe loads.
+#
+# **Mac OS X:** *Stable*. The Mercurial test suite passes with fsmonitor
+#   turned on, on case-insensitive HFS+. There has been a reasonable amount of
+#   user testing under normal loads.
+#
+# **Solaris, BSD:** *Alpha*. watchman and fsmonitor are believed to work, but
+#   very little testing has been done.
+#
+# **Windows:** *Alpha*. Not in a release version of watchman or fsmonitor yet.
+#
+# Known Issues
+# ============
+#
+# * fsmonitor will disable itself if any of the following extensions are
+#   enabled: largefiles, inotify, eol; or if the repository has subrepos.
+# * fsmonitor will produce incorrect results if nested repos that are not
+#   subrepos exist. *Workaround*: add nested repo paths to your `.hgignore`.
+#
+# The issues related to nested repos and subrepos are probably not fundamental
+# ones. Patches to fix them are welcome.
+
+from __future__ import absolute_import
+
+import os
+import stat
+import sys
+
+from mercurial import (
+    context,
+    extensions,
+    localrepo,
+    pathutil,
+    scmutil,
+    util,
+)
+from mercurial import match as matchmod
+from mercurial.i18n import _
+
+from . import (
+    state,
+    watchmanclient,
+)
+
+# Note for extension authors: ONLY specify testedwith = 'internal' for
+# extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
+# be specifying the version(s) of Mercurial they are tested with, or
+# leave the attribute unspecified.
+testedwith = 'internal'
+
+# This extension is incompatible with the following blacklisted extensions
+# and will disable itself when encountering one of these:
+_blacklist = ['largefiles', 'eol']
+
+def _handleunavailable(ui, state, ex):
+    """Exception handler for Watchman interaction exceptions"""
+    if isinstance(ex, watchmanclient.Unavailable):
+        if ex.warn:
+            ui.warn(str(ex) + '\n')
+        if ex.invalidate:
+            state.invalidate()
+        ui.log('fsmonitor', 'Watchman unavailable: %s\n', ex.msg)
+    else:
+        ui.log('fsmonitor', 'Watchman exception: %s\n', ex)
+
+def _hashignore(ignore):
+    """Calculate hash for ignore patterns and filenames
+
+    If this information changes between Mercurial invocations, we can't
+    rely on Watchman information anymore and have to re-scan the working
+    copy.
+
+    """
+    sha1 = util.sha1()
+    if util.safehasattr(ignore, 'includepat'):
+        sha1.update(ignore.includepat)
+    sha1.update('\0\0')
+    if util.safehasattr(ignore, 'excludepat'):
+        sha1.update(ignore.excludepat)
+    sha1.update('\0\0')
+    if util.safehasattr(ignore, 'patternspat'):
+        sha1.update(ignore.patternspat)
+    sha1.update('\0\0')
+    if util.safehasattr(ignore, '_files'):
+        for f in ignore._files:
+            sha1.update(f)
+    sha1.update('\0')
+    return sha1.hexdigest()
+
+def overridewalk(orig, self, match, subrepos, unknown, ignored, full=True):
+    '''Replacement for dirstate.walk, hooking into Watchman.
+
+    Whenever full is False, ignored is False, and the Watchman client is
+    available, use Watchman combined with saved state to possibly return only a
+    subset of files.'''
+    def bail():
+        return orig(match, subrepos, unknown, ignored, full=True)
+
+    if full or ignored or not self._watchmanclient.available():
+        return bail()
+    state = self._fsmonitorstate
+    clock, ignorehash, notefiles = state.get()
+    if not clock:
+        if state.walk_on_invalidate:
+            return bail()
+        # Initial NULL clock value, see
+        # https://facebook.github.io/watchman/docs/clockspec.html
+        clock = 'c:0:0'
+        notefiles = []
+
+    def fwarn(f, msg):
+        self._ui.warn('%s: %s\n' % (self.pathto(f), msg))
+        return False
+
+    def badtype(mode):
+        kind = _('unknown')
+        if stat.S_ISCHR(mode):
+            kind = _('character device')
+        elif stat.S_ISBLK(mode):
+            kind = _('block device')
+        elif stat.S_ISFIFO(mode):
+            kind = _('fifo')
+        elif stat.S_ISSOCK(mode):
+            kind = _('socket')
+        elif stat.S_ISDIR(mode):
+            kind = _('directory')
+        return _('unsupported file type (type is %s)') % kind
+
+    ignore = self._ignore
+    dirignore = self._dirignore
+    if unknown:
+        if _hashignore(ignore) != ignorehash and clock != 'c:0:0':
+            # ignore list changed -- can't rely on Watchman state any more
+            if state.walk_on_invalidate:
+                return bail()
+            notefiles = []
+            clock = 'c:0:0'
+    else:
+        # always ignore
+        ignore = util.always
+        dirignore = util.always
+
+    matchfn = match.matchfn
+    matchalways = match.always()
+    dmap = self._map
+    nonnormalset = getattr(self, '_nonnormalset', None)
+
+    copymap = self._copymap
+    getkind = stat.S_IFMT
+    dirkind = stat.S_IFDIR
+    regkind = stat.S_IFREG
+    lnkkind = stat.S_IFLNK
+    join = self._join
+    normcase = util.normcase
+    fresh_instance = False
+
+    exact = skipstep3 = False
+    if matchfn == match.exact:  # match.exact
+        exact = True
+        dirignore = util.always  # skip step 2
+    elif match.files() and not match.anypats():  # match.match, no patterns
+        skipstep3 = True
+
+    if not exact and self._checkcase:
+        # note that even though we could receive directory entries, we're only
+        # interested in checking if a file with the same name exists. So only
+        # normalize files if possible.
+        normalize = self._normalizefile
+        skipstep3 = False
+    else:
+        normalize = None
+
+    # step 1: find all explicit files
+    results, work, dirsnotfound = self._walkexplicit(match, subrepos)
+
+    skipstep3 = skipstep3 and not (work or dirsnotfound)
+    work = [d for d in work if not dirignore(d[0])]
+
+    if not work and (exact or skipstep3):
+        for s in subrepos:
+            del results[s]
+        del results['.hg']
+        return results
+
+    # step 2: query Watchman
+    try:
+        # Use the user-configured timeout for the query.
+        # Add a little slack over the top of the user query to allow for
+        # overheads while transferring the data
+        self._watchmanclient.settimeout(state.timeout + 0.1)
+        result = self._watchmanclient.command('query', {
+            'fields': ['mode', 'mtime', 'size', 'exists', 'name'],
+            'since': clock,
+            'expression': [
+                'not', [
+                    'anyof', ['dirname', '.hg'],
+                    ['name', '.hg', 'wholename']
+                ]
+            ],
+            'sync_timeout': int(state.timeout * 1000),
+            'empty_on_fresh_instance': state.walk_on_invalidate,
+        })
+    except Exception as ex:
+        _handleunavailable(self._ui, state, ex)
+        self._watchmanclient.clearconnection()
+        return bail()
+    else:
+        # We need to propagate the last observed clock up so that we
+        # can use it for our next query
+        state.setlastclock(result['clock'])
+        if result['is_fresh_instance']:
+            if state.walk_on_invalidate:
+                state.invalidate()
+                return bail()
+            fresh_instance = True
+            # Ignore any prior noteable files from the state info
+            notefiles = []
+
+    # for file paths which require normalization and we encounter a case
+    # collision, we store our own foldmap
+    if normalize:
+        foldmap = dict((normcase(k), k) for k in results)
+
+    switch_slashes = os.sep == '\\'
+    # The order of the results is, strictly speaking, undefined.
+    # For case changes on a case insensitive filesystem we may receive
+    # two entries, one with exists=True and another with exists=False.
+    # The exists=True entries in the same response should be interpreted
+    # as being happens-after the exists=False entries due to the way that
+    # Watchman tracks files.  We use this property to reconcile deletes
+    # for name case changes.
+    for entry in result['files']:
+        fname = entry['name']
+        if switch_slashes:
+            fname = fname.replace('\\', '/')
+        if normalize:
+            normed = normcase(fname)
+            fname = normalize(fname, True, True)
+            foldmap[normed] = fname
+        fmode = entry['mode']
+        fexists = entry['exists']
+        kind = getkind(fmode)
+
+        if not fexists:
+            # if marked as deleted and we don't already have a change
+            # record, mark it as deleted.  If we already have an entry
+            # for fname then it was either part of walkexplicit or was
+            # an earlier result that was a case change
+            if fname not in results and fname in dmap and (
+                    matchalways or matchfn(fname)):
+                results[fname] = None
+        elif kind == dirkind:
+            if fname in dmap and (matchalways or matchfn(fname)):
+                results[fname] = None
+        elif kind == regkind or kind == lnkkind:
+            if fname in dmap:
+                if matchalways or matchfn(fname):
+                    results[fname] = entry
+            elif (matchalways or matchfn(fname)) and not ignore(fname):
+                results[fname] = entry
+        elif fname in dmap and (matchalways or matchfn(fname)):
+            results[fname] = None
+
+    # step 3: query notable files we don't already know about
+    # XXX try not to iterate over the entire dmap
+    if normalize:
+        # any notable files that have changed case will already be handled
+        # above, so just check membership in the foldmap
+        notefiles = set((normalize(f, True, True) for f in notefiles
+                         if normcase(f) not in foldmap))
+    visit = set((f for f in notefiles if (f not in results and matchfn(f)
+                                          and (f in dmap or not ignore(f)))))
+
+    if nonnormalset is not None and not fresh_instance:
+        if matchalways:
+            visit.update(f for f in nonnormalset if f not in results)
+            visit.update(f for f in copymap if f not in results)
+        else:
+            visit.update(f for f in nonnormalset
+                         if f not in results and matchfn(f))
+            visit.update(f for f in copymap
+                         if f not in results and matchfn(f))
+    else:
+        if matchalways:
+            visit.update(f for f, st in dmap.iteritems()
+                         if (f not in results and
+                             (st[2] < 0 or st[0] != 'n' or fresh_instance)))
+            visit.update(f for f in copymap if f not in results)
+        else:
+            visit.update(f for f, st in dmap.iteritems()
+                         if (f not in results and
+                             (st[2] < 0 or st[0] != 'n' or fresh_instance)
+                             and matchfn(f)))
+            visit.update(f for f in copymap
+                         if f not in results and matchfn(f))
+
+    audit = pathutil.pathauditor(self._root).check
+    auditpass = [f for f in visit if audit(f)]
+    auditpass.sort()
+    auditfail = visit.difference(auditpass)
+    for f in auditfail:
+        results[f] = None
+
+    nf = iter(auditpass).next
+    for st in util.statfiles([join(f) for f in auditpass]):
+        f = nf()
+        if st or f in dmap:
+            results[f] = st
+
+    for s in subrepos:
+        del results[s]
+    del results['.hg']
+    return results
+
+def overridestatus(
+        orig, self, node1='.', node2=None, match=None, ignored=False,
+        clean=False, unknown=False, listsubrepos=False):
+    listignored = ignored
+    listclean = clean
+    listunknown = unknown
+
+    def _cmpsets(l1, l2):
+        try:
+            if 'FSMONITOR_LOG_FILE' in os.environ:
+                fn = os.environ['FSMONITOR_LOG_FILE']
+                f = open(fn, 'wb')
+            else:
+                fn = 'fsmonitorfail.log'
+                f = self.opener(fn, 'wb')
+        except (IOError, OSError):
+            self.ui.warn(_('warning: unable to write to %s\n') % fn)
+            return
+
+        try:
+            for i, (s1, s2) in enumerate(zip(l1, l2)):
+                if set(s1) != set(s2):
+                    f.write('sets at position %d are unequal\n' % i)
+                    f.write('watchman returned: %s\n' % s1)
+                    f.write('stat returned: %s\n' % s2)
+        finally:
+            f.close()
+
+    if isinstance(node1, context.changectx):
+        ctx1 = node1
+    else:
+        ctx1 = self[node1]
+    if isinstance(node2, context.changectx):
+        ctx2 = node2
+    else:
+        ctx2 = self[node2]
+
+    working = ctx2.rev() is None
+    parentworking = working and ctx1 == self['.']
+    match = match or matchmod.always(self.root, self.getcwd())
+
+    # Maybe we can use this opportunity to update Watchman's state.
+    # Mercurial uses workingcommitctx and/or memctx to represent the part of
+    # the workingctx that is to be committed. So don't update the state in
+    # that case.
+    # HG_PENDING is set in the environment when the dirstate is being updated
+    # in the middle of a transaction; we must not update our state in that
+    # case, or we risk forgetting about changes in the working copy.
+    updatestate = (parentworking and match.always() and
+                   not isinstance(ctx2, (context.workingcommitctx,
+                                         context.memctx)) and
+                   'HG_PENDING' not in os.environ)
+
+    try:
+        if self._fsmonitorstate.walk_on_invalidate:
+            # Use a short timeout to query the current clock.  If that
+            # takes too long then we assume that the service will be slow
+            # to answer our query.
+            # walk_on_invalidate indicates that we prefer to walk the
+            # tree ourselves because we can ignore portions that Watchman
+            # cannot and we tend to be faster in the warmer buffer cache
+            # cases.
+            self._watchmanclient.settimeout(0.1)
+        else:
+            # Give Watchman more time to potentially complete its walk
+            # and return the initial clock.  In this mode we assume that
+            # the filesystem will be slower than parsing a potentially
+            # very large Watchman result set.
+            self._watchmanclient.settimeout(
+                self._fsmonitorstate.timeout + 0.1)
+        startclock = self._watchmanclient.getcurrentclock()
+    except Exception as ex:
+        self._watchmanclient.clearconnection()
+        _handleunavailable(self.ui, self._fsmonitorstate, ex)
+        # boo, Watchman failed. bail
+        return orig(node1, node2, match, listignored, listclean,
+                    listunknown, listsubrepos)
+
+    if updatestate:
+        # We need info about unknown files. This may make things slower the
+        # first time, but whatever.
+        stateunknown = True
+    else:
+        stateunknown = listunknown
+
+    r = orig(node1, node2, match, listignored, listclean, stateunknown,
+             listsubrepos)
+    modified, added, removed, deleted, unknown, ignored, clean = r
+
+    if updatestate:
+        notefiles = modified + added + removed + deleted + unknown
+        self._fsmonitorstate.set(
+            self._fsmonitorstate.getlastclock() or startclock,
+            _hashignore(self.dirstate._ignore),
+            notefiles)
+
+    if not listunknown:
+        unknown = []
+
+    # don't do paranoid checks if we're not going to query Watchman anyway
+    full = listclean or match.traversedir is not None
+    if self._fsmonitorstate.mode == 'paranoid' and not full:
+        # run status again and fall back to the old walk this time
+        self.dirstate._fsmonitordisable = True
+
+        # shut the UI up
+        quiet = self.ui.quiet
+        self.ui.quiet = True
+        fout, ferr = self.ui.fout, self.ui.ferr
+        self.ui.fout = self.ui.ferr = open(os.devnull, 'wb')
+
+        try:
+            rv2 = orig(
+                node1, node2, match, listignored, listclean, listunknown,
+                listsubrepos)
+        finally:
+            self.dirstate._fsmonitordisable = False
+            self.ui.quiet = quiet
+            self.ui.fout, self.ui.ferr = fout, ferr
+
+        # clean isn't tested since it's set to True above
+        _cmpsets([modified, added, removed, deleted, unknown, ignored, clean],
+                 rv2)
+        modified, added, removed, deleted, unknown, ignored, clean = rv2
+
+    return scmutil.status(
+        modified, added, removed, deleted, unknown, ignored, clean)
+
+def makedirstate(cls):
+    class fsmonitordirstate(cls):
+        def _fsmonitorinit(self, fsmonitorstate, watchmanclient):
+            # _fsmonitordisable is used in paranoid mode
+            self._fsmonitordisable = False
+            self._fsmonitorstate = fsmonitorstate
+            self._watchmanclient = watchmanclient
+
+        def walk(self, *args, **kwargs):
+            orig = super(fsmonitordirstate, self).walk
+            if self._fsmonitordisable:
+                return orig(*args, **kwargs)
+            return overridewalk(orig, self, *args, **kwargs)
+
+        def rebuild(self, *args, **kwargs):
+            self._fsmonitorstate.invalidate()
+            return super(fsmonitordirstate, self).rebuild(*args, **kwargs)
+
+        def invalidate(self, *args, **kwargs):
+            self._fsmonitorstate.invalidate()
+            return super(fsmonitordirstate, self).invalidate(*args, **kwargs)
+
+    return fsmonitordirstate
+
+def wrapdirstate(orig, self):
+    ds = orig(self)
+    # only override the dirstate when Watchman is available for the repo
+    if util.safehasattr(self, '_fsmonitorstate'):
+        ds.__class__ = makedirstate(ds.__class__)
+        ds._fsmonitorinit(self._fsmonitorstate, self._watchmanclient)
+    return ds
+
+def extsetup(ui):
+    wrapfilecache(localrepo.localrepository, 'dirstate', wrapdirstate)
+    if sys.platform == 'darwin':
+        # An assist for avoiding the dangling-symlink fsevents bug
+        extensions.wrapfunction(os, 'symlink', wrapsymlink)
+
+def wrapsymlink(orig, source, link_name):
+    ''' if we create a dangling symlink, also touch the parent dir
+    to encourage fsevents notifications to work more correctly '''
+    try:
+        return orig(source, link_name)
+    finally:
+        try:
+            os.utime(os.path.dirname(link_name), None)
+        except OSError:
+            pass
+
+def reposetup(ui, repo):
+    # We don't work with largefiles or inotify
+    exts = extensions.enabled()
+    for ext in _blacklist:
+        if ext in exts:
+            ui.warn(_('The fsmonitor extension is incompatible with the %s '
+                      'extension and has been disabled.\n') % ext)
+            return
+
+    if util.safehasattr(repo, 'dirstate'):
+        # We don't work with subrepos either. Note that we can get passed in
+        # e.g. a statichttprepo, which throws on trying to access the substate.
+        # XXX This sucks.
+        try:
+            # if repo[None].substate can cause a dirstate parse, which is too
+            # slow. Instead, look for a file called hgsubstate,
+            if repo.wvfs.exists('.hgsubstate') or repo.wvfs.exists('.hgsub'):
+                return
+        except AttributeError:
+            return
+
+        fsmonitorstate = state.state(repo)
+        if fsmonitorstate.mode == 'off':
+            return
+
+        try:
+            client = watchmanclient.client(repo)
+        except Exception as ex:
+            _handleunavailable(ui, fsmonitorstate, ex)
+            return
+
+        repo._fsmonitorstate = fsmonitorstate
+        repo._watchmanclient = client
+
+        # at this point since fsmonitorstate wasn't present, repo.dirstate is
+        # not a fsmonitordirstate
+        repo.dirstate.__class__ = makedirstate(repo.dirstate.__class__)
+        # nuke the dirstate so that _fsmonitorinit and subsequent configuration
+        # changes take effect on it
+        del repo._filecache['dirstate']
+        delattr(repo.unfiltered(), 'dirstate')
+
+        class fsmonitorrepo(repo.__class__):
+            def status(self, *args, **kwargs):
+                orig = super(fsmonitorrepo, self).status
+                return overridestatus(orig, self, *args, **kwargs)
+
+        repo.__class__ = fsmonitorrepo
+
+def wrapfilecache(cls, propname, wrapper):
+    """Wraps a filecache property. These can't be wrapped using the normal
+    wrapfunction. This should eventually go into upstream Mercurial.
+    """
+    assert callable(wrapper)
+    for currcls in cls.__mro__:
+        if propname in currcls.__dict__:
+            origfn = currcls.__dict__[propname].func
+            assert callable(origfn)
+            def wrap(*args, **kwargs):
+                return wrapper(origfn, *args, **kwargs)
+            currcls.__dict__[propname].func = wrap
+            break
+
+    if currcls is object:
+        raise AttributeError(
+            _("type '%s' has no property '%s'") % (cls, propname))

hgext/fsmonitor/state.py

@@ -0,0 +1,115 b''
+# state.py - fsmonitor persistent state
+#
+# Copyright 2013-2016 Facebook, Inc.
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+from __future__ import absolute_import
+
+import errno
+import os
+import socket
+import struct
+
+from mercurial import pathutil
+from mercurial.i18n import _
+
+_version = 4
+_versionformat = ">I"
+
+class state(object):
+    def __init__(self, repo):
+        self._opener = repo.opener
+        self._ui = repo.ui
+        self._rootdir = pathutil.normasprefix(repo.root)
+        self._lastclock = None
+
+        self.mode = self._ui.config('fsmonitor', 'mode', default='on')
+        self.walk_on_invalidate = self._ui.configbool(
+            'fsmonitor', 'walk_on_invalidate', False)
+        self.timeout = float(self._ui.config(
+            'fsmonitor', 'timeout', default='2'))
+
+    def get(self):
+        try:
+            file = self._opener('fsmonitor.state', 'rb')
+        except IOError as inst:
+            if inst.errno != errno.ENOENT:
+                raise
+            return None, None, None
+
+        versionbytes = file.read(4)
+        if len(versionbytes) < 4:
+            self._ui.log(
+                'fsmonitor', 'fsmonitor: state file only has %d bytes, '
+                'nuking state\n' % len(versionbytes))
+            self.invalidate()
+            return None, None, None
+        try:
+            diskversion = struct.unpack(_versionformat, versionbytes)[0]
+            if diskversion != _version:
+                # different version, nuke state and start over
+                self._ui.log(
+                    'fsmonitor', 'fsmonitor: version switch from %d to '
+                    '%d, nuking state\n' % (diskversion, _version))
+                self.invalidate()
+                return None, None, None
+
+            state = file.read().split('\0')
+            # state = hostname\0clock\0ignorehash\0 + list of files, each
+            # followed by a \0
+            diskhostname = state[0]
+            hostname = socket.gethostname()
+            if diskhostname != hostname:
+                # file got moved to a different host
+                self._ui.log('fsmonitor', 'fsmonitor: stored hostname "%s" '
+                             'different from current "%s", nuking state\n' %
+                             (diskhostname, hostname))
+                self.invalidate()
+                return None, None, None
+
+            clock = state[1]
+            ignorehash = state[2]
+            # discard the value after the last \0
+            notefiles = state[3:-1]
+
+        finally:
+            file.close()
+
+        return clock, ignorehash, notefiles
+
+    def set(self, clock, ignorehash, notefiles):
+        if clock is None:
+            self.invalidate()
+            return
+
+        try:
+            file = self._opener('fsmonitor.state', 'wb')
+        except (IOError, OSError):
+            self._ui.warn(_("warning: unable to write out fsmonitor state\n"))
+            return
+
+        try:
+            file.write(struct.pack(_versionformat, _version))
+            file.write(socket.gethostname() + '\0')
+            file.write(clock + '\0')
+            file.write(ignorehash + '\0')
+            if notefiles:
+                file.write('\0'.join(notefiles))
+                file.write('\0')
+        finally:
+            file.close()
+
+    def invalidate(self):
+        try:
+            os.unlink(os.path.join(self._rootdir, '.hg', 'fsmonitor.state'))
+        except OSError as inst:
+            if inst.errno != errno.ENOENT:
+                raise
+
+    def setlastclock(self, clock):
+        self._lastclock = clock
+
+    def getlastclock(self):
+        return self._lastclock

hgext/fsmonitor/watchmanclient.py

@@ -0,0 +1,109 b''
+# watchmanclient.py - Watchman client for the fsmonitor extension
+#
+# Copyright 2013-2016 Facebook, Inc.
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+from __future__ import absolute_import
+
+import getpass
+
+from mercurial import util
+
+from . import pywatchman
+
+class Unavailable(Exception):
+    def __init__(self, msg, warn=True, invalidate=False):
+        self.msg = msg
+        self.warn = warn
+        if self.msg == 'timed out waiting for response':
+            self.warn = False
+        self.invalidate = invalidate
+
+    def __str__(self):
+        if self.warn:
+            return 'warning: Watchman unavailable: %s' % self.msg
+        else:
+            return 'Watchman unavailable: %s' % self.msg
+
+class WatchmanNoRoot(Unavailable):
+    def __init__(self, root, msg):
+        self.root = root
+        super(WatchmanNoRoot, self).__init__(msg)
+
+class client(object):
+    def __init__(self, repo, timeout=1.0):
+        err = None
+        if not self._user:
+            err = "couldn't get user"
+            warn = True
+        if self._user in repo.ui.configlist('fsmonitor', 'blacklistusers'):
+            err = 'user %s in blacklist' % self._user
+            warn = False
+
+        if err:
+            raise Unavailable(err, warn)
+
+        self._timeout = timeout
+        self._watchmanclient = None
+        self._root = repo.root
+        self._ui = repo.ui
+        self._firsttime = True
+
+    def settimeout(self, timeout):
+        self._timeout = timeout
+        if self._watchmanclient is not None:
+            self._watchmanclient.setTimeout(timeout)
+
+    def getcurrentclock(self):
+        result = self.command('clock')
+        if not util.safehasattr(result, 'clock'):
+            raise Unavailable('clock result is missing clock value',
+                              invalidate=True)
+        return result.clock
+
+    def clearconnection(self):
+        self._watchmanclient = None
+
+    def available(self):
+        return self._watchmanclient is not None or self._firsttime
+
+    @util.propertycache
+    def _user(self):
+        try:
+            return getpass.getuser()
+        except KeyError:
+            # couldn't figure out our user
+            return None
+
+    def _command(self, *args):
+        watchmanargs = (args[0], self._root) + args[1:]
+        try:
+            if self._watchmanclient is None:
+                self._firsttime = False
+                self._watchmanclient = pywatchman.client(
+                    timeout=self._timeout,
+                    useImmutableBser=True)
+            return self._watchmanclient.query(*watchmanargs)
+        except pywatchman.CommandError as ex:
+            if ex.msg.startswith('unable to resolve root'):
+                raise WatchmanNoRoot(self._root, ex.msg)
+            raise Unavailable(ex.msg)
+        except pywatchman.WatchmanError as ex:
+            raise Unavailable(str(ex))
+
+    def command(self, *args):
+        try:
+            try:
+                return self._command(*args)
+            except WatchmanNoRoot:
+                # this 'watch' command can also raise a WatchmanNoRoot if
+                # watchman refuses to accept this root
+                self._command('watch')
+                return self._command(*args)
+        except Unavailable:
+            # this is in an outer scope to catch Unavailable form any of the
+            # above _command calls
+            self._watchmanclient = None
+            raise

tests/blacklists/fsmonitor

@@ -0,0 +1,52 b''
+# Blacklist for a full testsuite run with fsmonitor enabled.
+# Use with
+#     run-tests --blacklist=blacklists/fsmonitor \
+#         --extra-config="extensions.fsmonitor="
+# The following tests all fail because they either use extensions that conflict
+# with fsmonitor, use subrepositories, or don't anticipate the extra file in
+# the .hg directory that fsmonitor adds.
+test-basic.t
+test-blackbox.t
+test-check-commit.t
+test-commandserver.t
+test-copy.t
+test-debugextensions.t
+test-eol-add.t
+test-eol-clone.t
+test-eol-hook.t
+test-eol-patch.t
+test-eol-tag.t
+test-eol-update.t
+test-eol.t
+test-eolfilename.t
+test-extension.t
+test-fncache.t
+test-hardlinks.t
+test-help.t
+test-inherit-mode.t
+test-issue3084.t
+test-largefiles-cache.t
+test-largefiles-misc.t
+test-largefiles-small-disk.t
+test-largefiles-update.t
+test-largefiles-wireproto.t
+test-largefiles.t
+test-lfconvert.t
+test-merge-tools.t
+test-nested-repo.t
+test-permissions.t
+test-push-warn.t
+test-subrepo-deep-nested-change.t
+test-subrepo-recursion.t
+test-subrepo.t
+test-tags.t
+
+# The following tests remain enabled; they fail *too*, but only because they
+# occasionally use blacklisted extensions and don't anticipate the warning
+# generated.
+#test-log.t
+#test-hook.t
+#test-rename.t
+#test-histedit-fold.t
+#test-fileset-generated.t
+#test-init.t